Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

flowfile_core/flowfile/code_generator/code_generator.py

@@ -1,15 +1,28 @@
-from typing import List, Dict, Optional, Set, Tuple
+import inspect
+import typing
 import polars as pl
-
 from pl_fuzzy_frame_match.models import FuzzyMapping
 
-from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.configs import logger
+from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, convert_pl_type_to_string
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
+from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_core.flowfile.util.execution_orderer import determine_execution_order
 from flowfile_core.schemas import input_schema, transform_schema
-from flowfile_core.configs import logger
+
+
+class UnsupportedNodeError(Exception):
+    """Raised when code generation encounters a node type that cannot be converted to standalone code."""
+
+    def __init__(self, node_type: str, node_id: int, reason: str):
+        self.node_type = node_type
+        self.node_id = node_id
+        self.reason = reason
+        super().__init__(
+            f"Cannot generate code for node '{node_type}' (node_id={node_id}): {reason}"
+        )
 
 
 class FlowGraphToPolarsConverter:
@@ -19,20 +32,25 @@ class FlowGraphToPolarsConverter:
     This class takes a FlowGraph instance and generates standalone Python code
     that uses only Polars, without any Flowfile dependencies.
     """
+
     flow_graph: FlowGraph
-    node_var_mapping: Dict[int, str]
-    imports: Set[str]
-    code_lines: List[str]
-    output_nodes: List[Tuple[int, str]] = []
-    last_node_var: Optional[str] = None
+    node_var_mapping: dict[int, str]
+    imports: set[str]
+    code_lines: list[str]
+    output_nodes: list[tuple[int, str]] = []
+    last_node_var: str | None = None
+    unsupported_nodes: list[tuple[int, str, str]]  # List of (node_id, node_type, reason)
+    custom_node_classes: dict[str, str]  # Maps custom node class name to source code
 
     def __init__(self, flow_graph: FlowGraph):
         self.flow_graph = flow_graph
-        self.node_var_mapping: Dict[int, str] = {}  # Maps node_id to variable name
-        self.imports: Set[str] = {"import polars as pl"}
-        self.code_lines: List[str] = []
+        self.node_var_mapping: dict[int, str] = {}  # Maps node_id to variable name
+        self.imports: set[str] = {"import polars as pl"}
+        self.code_lines: list[str] = []
         self.output_nodes = []
         self.last_node_var = None
+        self.unsupported_nodes = []
+        self.custom_node_classes = {}
 
     def convert(self) -> str:
         """
@@ -40,23 +58,41 @@ class FlowGraphToPolarsConverter:
 
         Returns:
             str: Complete Python code that can be executed standalone
+
+        Raises:
+            UnsupportedNodeError: If the graph contains nodes that cannot be converted
+                to standalone code (e.g., database nodes, explore_data, external_source).
         """
         # Get execution order
         execution_order = determine_execution_order(
             all_nodes=[node for node in self.flow_graph.nodes if node.is_correct],
-            flow_starts=self.flow_graph._flow_starts + self.flow_graph.get_implicit_starter_nodes()
+            flow_starts=self.flow_graph._flow_starts + self.flow_graph.get_implicit_starter_nodes(),
         )
 
         # Generate code for each node in order
         for node in execution_order:
             self._generate_node_code(node)
 
+        # Check for unsupported nodes and raise an error with all of them listed
+        if self.unsupported_nodes:
+            error_messages = []
+            for node_id, node_type, reason in self.unsupported_nodes:
+                error_messages.append(f"  - Node {node_id} ({node_type}): {reason}")
+            raise UnsupportedNodeError(
+                node_type=self.unsupported_nodes[0][1],
+                node_id=self.unsupported_nodes[0][0],
+                reason=(
+                    f"The flow contains {len(self.unsupported_nodes)} node(s) that cannot be converted to code:\n"
+                    + "\n".join(error_messages)
+                ),
+            )
+
         # Combine everything
         return self._build_final_code()
 
     def handle_output_node(self, node: FlowNode, var_name: str) -> None:
         settings = node.setting_input
-        if hasattr(settings, 'is_flow_output') and settings.is_flow_output:
+        if hasattr(settings, "is_flow_output") and settings.is_flow_output:
             self.output_nodes.append((node.node_id, var_name))
 
     def _generate_node_code(self, node: FlowNode) -> None:
@@ -74,75 +110,79 @@ class FlowGraphToPolarsConverter:
             self.last_node_var = var_name
         # Get input variable names
         input_vars = self._get_input_vars(node)
+
+        # Check if this is a user-defined node
+        if isinstance(settings, input_schema.UserDefinedNode) or getattr(settings, "is_user_defined", False):
+            self._handle_user_defined(node, var_name, input_vars)
+            return
+
         # Route to appropriate handler based on node type
         handler = getattr(self, f"_handle_{node_type}", None)
         if handler:
             handler(settings, var_name, input_vars)
         else:
-            self._add_comment(f"# TODO: Implement handler for node type: {node_type}")
-            raise Exception(f"No handler implemented for node type: {node_type}")
-
-    def _get_input_vars(self, node: FlowNode) -> Dict[str, str]:
+            # Unknown node type - add to unsupported list
+            self.unsupported_nodes.append((
+                node.node_id,
+                node_type,
+                f"No code generator implemented for node type '{node_type}'"
+            ))
+            self._add_comment(f"# WARNING: Cannot generate code for node type '{node_type}' (node_id={node.node_id})")
+            self._add_comment(f"# This node type is not supported for code export")
+
+    def _get_input_vars(self, node: FlowNode) -> dict[str, str]:
         """Get input variable names for a node."""
         input_vars = {}
 
         if node.node_inputs.main_inputs:
             if len(node.node_inputs.main_inputs) == 1:
-                input_vars['main'] = self.node_var_mapping.get(
-                    node.node_inputs.main_inputs[0].node_id, 'df'
-                )
+                input_vars["main"] = self.node_var_mapping.get(node.node_inputs.main_inputs[0].node_id, "df")
             else:
                 for i, input_node in enumerate(node.node_inputs.main_inputs):
-                    input_vars[f'main_{i}'] = self.node_var_mapping.get(
-                        input_node.node_id, f'df_{i}'
-                    )
+                    input_vars[f"main_{i}"] = self.node_var_mapping.get(input_node.node_id, f"df_{i}")
 
         if node.node_inputs.left_input:
-            input_vars['left'] = self.node_var_mapping.get(
-                node.node_inputs.left_input.node_id, 'df_left'
-            )
+            input_vars["left"] = self.node_var_mapping.get(node.node_inputs.left_input.node_id, "df_left")
 
         if node.node_inputs.right_input:
-            input_vars['right'] = self.node_var_mapping.get(
-                node.node_inputs.right_input.node_id, 'df_right'
-            )
+            input_vars["right"] = self.node_var_mapping.get(node.node_inputs.right_input.node_id, "df_right")
 
         return input_vars
 
     def _handle_csv_read(self, file_settings: input_schema.ReceivedTable, var_name: str):
-        if file_settings.table_settings.encoding.lower() in ('utf-8', 'utf8'):
+        if file_settings.table_settings.encoding.lower() in ("utf-8", "utf8"):
             encoding = "utf8-lossy"
             self._add_code(f"{var_name} = pl.scan_csv(")
             self._add_code(f'    "{file_settings.abs_file_path}",')
             self._add_code(f'    separator="{file_settings.table_settings.delimiter}",')
-            self._add_code(f'    has_header={file_settings.table_settings.has_headers},')
-            self._add_code(f'    ignore_errors={file_settings.table_settings.ignore_errors},')
+            self._add_code(f"    has_header={file_settings.table_settings.has_headers},")
+            self._add_code(f"    ignore_errors={file_settings.table_settings.ignore_errors},")
             self._add_code(f'    encoding="{encoding}",')
-            self._add_code(f'    skip_rows={file_settings.table_settings.starting_from_line},')
+            self._add_code(f"    skip_rows={file_settings.table_settings.starting_from_line},")
             self._add_code(")")
         else:
             self._add_code(f"{var_name} = pl.read_csv(")
             self._add_code(f'    "{file_settings.abs_file_path}",')
             self._add_code(f'    separator="{file_settings.table_settings.delimiter}",')
-            self._add_code(f'    has_header={file_settings.table_settings.has_headers},')
-            self._add_code(f'    ignore_errors={file_settings.table_settings.ignore_errors},')
+            self._add_code(f"    has_header={file_settings.table_settings.has_headers},")
+            self._add_code(f"    ignore_errors={file_settings.table_settings.ignore_errors},")
             if file_settings.table_settings.encoding:
                 self._add_code(f'    encoding="{file_settings.table_settings.encoding}",')
-            self._add_code(f'    skip_rows={file_settings.table_settings.starting_from_line},')
+            self._add_code(f"    skip_rows={file_settings.table_settings.starting_from_line},")
             self._add_code(").lazy()")
 
-    def _handle_cloud_storage_reader(self, settings: input_schema.NodeCloudStorageReader, var_name: str, input_vars: Dict[str, str]):
+    def _handle_cloud_storage_reader(
+        self, settings: input_schema.NodeCloudStorageReader, var_name: str, input_vars: dict[str, str]
+    ):
         cloud_read_settings = settings.cloud_storage_settings
-        self.imports.add(
-            "import flowfile as ff"
-        )
+        self.imports.add("import flowfile as ff")
         if cloud_read_settings.file_format == "csv":
             self._add_code(f"{var_name} = ff.scan_csv_from_cloud_storage(")
             self._add_code(f'    "{cloud_read_settings.resource_path}",')
             self._add_code(f'    connection_name="{cloud_read_settings.connection_name}",')
             self._add_code(f'    scan_mode="{cloud_read_settings.scan_mode}",')
             self._add_code(f'    delimiter="{cloud_read_settings.csv_delimiter}",')
-            self._add_code(f'    has_header={cloud_read_settings.csv_has_header},')
+            self._add_code(f"    has_header={cloud_read_settings.csv_has_header},")
             self._add_code(f'    encoding="{cloud_read_settings.csv_encoding}",')
 
         elif cloud_read_settings.file_format == "parquet":
@@ -162,22 +202,22 @@ class FlowGraphToPolarsConverter:
             self._add_code(f'    "{cloud_read_settings.resource_path}",')
             self._add_code(f'    connection_name="{cloud_read_settings.connection_name}",')
             self._add_code(f'    scan_mode="{cloud_read_settings.scan_mode}",')
-            self._add_code(f'    version_id={cloud_read_settings.delta_version},')
+            self._add_code(f"    version_id={cloud_read_settings.delta_version},")
         else:
             return
         self._add_code(").data")
 
-    def _handle_read(self, settings: input_schema.NodeRead, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_read(self, settings: input_schema.NodeRead, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle file reading nodes."""
         file_settings = settings.received_file
 
-        if file_settings.file_type == 'csv':
+        if file_settings.file_type == "csv":
             self._handle_csv_read(file_settings, var_name)
 
-        elif file_settings.file_type == 'parquet':
+        elif file_settings.file_type == "parquet":
             self._add_code(f'{var_name} = pl.scan_parquet("{file_settings.abs_file_path}")')
 
-        elif file_settings.file_type in ('xlsx', 'excel'):
+        elif file_settings.file_type in ("xlsx", "excel"):
             self._add_code(f"{var_name} = pl.read_excel(")
             self._add_code(f'    "{file_settings.abs_file_path}",')
             if file_settings.table_settings.sheet_name:
@@ -187,12 +227,18 @@ class FlowGraphToPolarsConverter:
         self._add_code("")
 
     @staticmethod
-    def _generate_pl_schema_with_typing(flowfile_schema: List[FlowfileColumn]) -> str:
-        polars_schema_str = "pl.Schema([" + ", ".join(f'("{flowfile_column.column_name}", pl.{flowfile_column.data_type})'
-                                   for flowfile_column in flowfile_schema) + "])"
+    def _generate_pl_schema_with_typing(flowfile_schema: list[FlowfileColumn]) -> str:
+        polars_schema_str = (
+            "pl.Schema(["
+            + ", ".join(
+                f'("{flowfile_column.column_name}", pl.{flowfile_column.data_type})'
+                for flowfile_column in flowfile_schema
+            )
+            + "])"
+        )
         return polars_schema_str
 
-    def get_manual_schema_input(self, flowfile_schema: List[FlowfileColumn]) -> str:
+    def get_manual_schema_input(self, flowfile_schema: list[FlowfileColumn]) -> str:
         polars_schema_str = self._generate_pl_schema_with_typing(flowfile_schema)
         is_valid_pl_schema = self._validate_pl_schema(polars_schema_str)
         if is_valid_pl_schema:
@@ -210,19 +256,23 @@ class FlowGraphToPolarsConverter:
             logger.error(f"Invalid Polars schema: {e}")
             return False
 
-    def _handle_manual_input(self, settings: input_schema.NodeManualInput, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_manual_input(
+        self, settings: input_schema.NodeManualInput, var_name: str, input_vars: dict[str, str]
+    ) -> None:
         """Handle manual data input nodes."""
         data = settings.raw_data_format.data
-        flowfile_schema = list(FlowfileColumn.create_from_minimal_field_info(c) for c in settings.raw_data_format.columns)
+        flowfile_schema = list(
+            FlowfileColumn.create_from_minimal_field_info(c) for c in settings.raw_data_format.columns
+        )
         schema = self.get_manual_schema_input(flowfile_schema)
         self._add_code(f"{var_name} = pl.LazyFrame({data}, schema={schema}, strict=False)")
         self._add_code("")
 
-    def _handle_filter(self, settings: input_schema.NodeFilter, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_filter(self, settings: input_schema.NodeFilter, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle filter nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
 
-        if settings.filter_input.filter_type == 'advanced':
+        if settings.filter_input.is_advanced():
             # Parse the advanced filter expression
             self.imports.add(
                 "from polars_expr_transformer.process.polars_expr_transformer import simple_function_to_expr"
@@ -233,28 +283,33 @@ class FlowGraphToPolarsConverter:
         else:
             # Handle basic filter
             basic = settings.filter_input.basic_filter
-            filter_expr = self._create_basic_filter_expr(basic)
-            self._add_code(f"{var_name} = {input_df}.filter({filter_expr})")
+            if basic is not None:
+                filter_expr = self._create_basic_filter_expr(basic)
+                self._add_code(f"{var_name} = {input_df}.filter({filter_expr})")
+            else:
+                self._add_code(f"{var_name} = {input_df}  # No filter applied")
         self._add_code("")
 
-    def _handle_record_count(self, settings: input_schema.NodeRecordCount, var_name: str, input_vars: Dict[str, str]):
-        input_df = input_vars.get('main', 'df')
+    def _handle_record_count(self, settings: input_schema.NodeRecordCount, var_name: str, input_vars: dict[str, str]):
+        input_df = input_vars.get("main", "df")
         self._add_code(f"{var_name} = {input_df}.select(pl.len().alias('number_of_records'))")
 
-    def _handle_graph_solver(self, settings: input_schema.NodeGraphSolver, var_name: str, input_vars: Dict[str, str]):
-        input_df = input_vars.get('main', 'df')
+    def _handle_graph_solver(self, settings: input_schema.NodeGraphSolver, var_name: str, input_vars: dict[str, str]):
+        input_df = input_vars.get("main", "df")
         from_col_name = settings.graph_solver_input.col_from
         to_col_name = settings.graph_solver_input.col_to
         output_col_name = settings.graph_solver_input.output_column_name
-        self._add_code(f'{var_name} = {input_df}.with_columns(graph_solver(pl.col("{from_col_name}"), '
-                       f'pl.col("{to_col_name}"))'
-                       f'.alias("{output_col_name}"))')
+        self._add_code(
+            f'{var_name} = {input_df}.with_columns(graph_solver(pl.col("{from_col_name}"), '
+            f'pl.col("{to_col_name}"))'
+            f'.alias("{output_col_name}"))'
+        )
         self._add_code("")
         self.imports.add("from polars_grouper import graph_solver")
 
-    def _handle_select(self, settings: input_schema.NodeSelect, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_select(self, settings: input_schema.NodeSelect, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle select/rename nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         # Get columns to keep and renames
         select_exprs = []
         for select_input in settings.select_input:
@@ -266,7 +321,7 @@ class FlowGraphToPolarsConverter:
 
                 if (select_input.data_type_change or select_input.is_altered) and select_input.data_type:
                     polars_dtype = self._get_polars_dtype(select_input.data_type)
-                    expr = f'{expr}.cast({polars_dtype})'
+                    expr = f"{expr}.cast({polars_dtype})"
 
                 select_exprs.append(expr)
 
@@ -279,7 +334,7 @@ class FlowGraphToPolarsConverter:
             self._add_code(f"{var_name} = {input_df}")
         self._add_code("")
 
-    def _handle_join(self, settings: input_schema.NodeJoin, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_join(self, settings: input_schema.NodeJoin, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle join nodes by routing to appropriate join type handler.
 
         This is the main entry point for processing join operations. It determines
@@ -293,8 +348,8 @@ class FlowGraphToPolarsConverter:
         Returns:
             None: Modifies internal state by adding generated code
         """
-        left_df = input_vars.get('main', input_vars.get('main_0', 'df_left'))
-        right_df = input_vars.get('right', input_vars.get('main_1', 'df_right'))
+        left_df = input_vars.get("main", input_vars.get("main_0", "df_left"))
+        right_df = input_vars.get("right", input_vars.get("main_1", "df_right"))
         # Ensure left and right DataFrames are distinct
         if left_df == right_df:
             right_df = "df_right"
@@ -305,8 +360,9 @@ class FlowGraphToPolarsConverter:
         else:
             self._handle_standard_join(settings, var_name, left_df, right_df)
 
-    def _handle_semi_anti_join(self, settings: input_schema.NodeJoin, var_name: str, left_df: str,
-                               right_df: str) -> None:
+    def _handle_semi_anti_join(
+        self, settings: input_schema.NodeJoin, var_name: str, left_df: str, right_df: str
+    ) -> None:
         """Handle semi and anti joins which only return rows from the left DataFrame.
 
         Semi joins return rows from left DataFrame that have matches in right.
@@ -333,8 +389,9 @@ class FlowGraphToPolarsConverter:
         self._add_code("    )")
         self._add_code(")")
 
-    def _handle_standard_join(self, settings: input_schema.NodeJoin, var_name: str, left_df: str,
-                              right_df: str) -> None:
+    def _handle_standard_join(
+        self, settings: input_schema.NodeJoin, var_name: str, left_df: str, right_df: str
+    ) -> None:
         """Handle standard joins (left, right, inner, outer) with full column management.
 
         Standard joins may include columns from both DataFrames and require careful
@@ -370,12 +427,11 @@ class FlowGraphToPolarsConverter:
         )
         # Execute the join
         self._execute_join_with_post_processing(
-            settings, var_name, left_df, right_df, left_on, right_on,
-            after_join_drop_cols, reverse_action
+            settings, var_name, left_df, right_df, left_on, right_on, after_join_drop_cols, reverse_action
         )
 
     @staticmethod
-    def _get_join_keys(settings: transform_schema.JoinInputManager) -> Tuple[List[str], List[str]]:
+    def _get_join_keys(settings: transform_schema.JoinInputManager) -> tuple[list[str], list[str]]:
         """Extract join keys based on join type.
 
         Different join types require different handling of join keys:
@@ -397,8 +453,9 @@ class FlowGraphToPolarsConverter:
 
         return left_on, right_on
 
-    def _apply_pre_join_transformations(self, settings: transform_schema.JoinInputManager, left_df: str, right_df: str) -> Tuple[
-        str, str]:
+    def _apply_pre_join_transformations(
+        self, settings: transform_schema.JoinInputManager, left_df: str, right_df: str
+    ) -> tuple[str, str]:
         """Apply column renames and drops before the join operation.
 
         Pre-join transformations prepare DataFrames by:
@@ -419,8 +476,7 @@ class FlowGraphToPolarsConverter:
         right_renames = {
             column.old_name: column.new_name
             for column in settings.right_select.renames
-            if
-            column.old_name != column.new_name and not column.join_key or settings.how in ("outer", "right")
+            if column.old_name != column.new_name and not column.join_key or settings.how in ("outer", "right")
         }
 
         left_renames = {
@@ -430,13 +486,11 @@ class FlowGraphToPolarsConverter:
         }
 
         left_drop_columns = [
-            column.old_name for column in settings.left_select.renames
-            if not column.keep and not column.join_key
+            column.old_name for column in settings.left_select.renames if not column.keep and not column.join_key
         ]
 
         right_drop_columns = [
-            column.old_name for column in settings.right_select.renames
-            if not column.keep and not column.join_key
+            column.old_name for column in settings.right_select.renames if not column.keep and not column.join_key
         ]
 
         # Apply transformations
@@ -451,9 +505,14 @@ class FlowGraphToPolarsConverter:
 
         return left_df, right_df
 
-    def _handle_join_key_transformations(self, settings: transform_schema.JoinInputManager, left_df: str, right_df: str,
-                                         left_on: List[str], right_on: List[str]) \
-            -> Tuple[List[str], List[str], Optional[Dict], List[str]]:
+    def _handle_join_key_transformations(
+        self,
+        settings: transform_schema.JoinInputManager,
+        left_df: str,
+        right_df: str,
+        left_on: list[str],
+        right_on: list[str],
+    ) -> tuple[list[str], list[str], dict | None, list[str]]:
         """Route to appropriate join-specific key transformation handler.
 
         Different join types require different strategies for handling join keys
@@ -484,9 +543,9 @@ class FlowGraphToPolarsConverter:
         else:
             return left_on, right_on, None, []
 
-    def _handle_left_inner_join_keys(self, settings: transform_schema.JoinInputManager, right_df: str,
-                                     left_on: List[str], right_on: List[str]) -> Tuple[
-        List[str], List[str], Dict, List[str]]:
+    def _handle_left_inner_join_keys(
+        self, settings: transform_schema.JoinInputManager, right_df: str, left_on: list[str], right_on: list[str]
+    ) -> tuple[list[str], list[str], dict, list[str]]:
         """Handle key transformations for left and inner joins.
 
         For left/inner joins:
@@ -510,27 +569,26 @@ class FlowGraphToPolarsConverter:
         left_join_keys_to_keep = [jk.new_name for jk in settings.left_select.join_key_selects if jk.keep]
         join_key_duplication_command = [
             f'pl.col("{rjk.old_name}").alias("__DROP__{rjk.new_name}__DROP__")'
-            for rjk in settings.right_select.join_key_selects if rjk.keep
+            for rjk in settings.right_select.join_key_selects
+            if rjk.keep
         ]
 
         reverse_action = {
             f"__DROP__{rjk.new_name}__DROP__": rjk.new_name
-            for rjk in settings.right_select.join_key_selects if rjk.keep
+            for rjk in settings.right_select.join_key_selects
+            if rjk.keep
         }
 
         if join_key_duplication_command:
             self._add_code(f"{right_df} = {right_df}.with_columns([{', '.join(join_key_duplication_command)}])")
 
-        after_join_drop_cols = [
-            k.new_name for k in settings.left_select.join_key_selects
-            if not k.keep
-        ]
+        after_join_drop_cols = [k.new_name for k in settings.left_select.join_key_selects if not k.keep]
 
         return left_on, right_on, reverse_action, after_join_drop_cols
 
-    def _handle_right_join_keys(self, settings: transform_schema.JoinInputManager, left_df: str,
-                                left_on: List[str], right_on: List[str]) -> Tuple[
-        List[str], List[str], None, List[str]]:
+    def _handle_right_join_keys(
+        self, settings: transform_schema.JoinInputManager, left_df: str, left_on: list[str], right_on: list[str]
+    ) -> tuple[list[str], list[str], None, list[str]]:
         """Handle key transformations for right joins.
 
         For right joins:
@@ -553,7 +611,8 @@ class FlowGraphToPolarsConverter:
         """
         join_key_duplication_command = [
             f'pl.col("{ljk.new_name}").alias("__jk_{ljk.new_name}")'
-            for ljk in settings.left_select.join_key_selects if ljk.keep
+            for ljk in settings.left_select.join_key_selects
+            if ljk.keep
         ]
 
         # Update left_on keys
@@ -569,14 +628,15 @@ class FlowGraphToPolarsConverter:
         left_join_keys_keep = {jk.new_name for jk in settings.left_select.join_key_selects if jk.keep}
         after_join_drop_cols_right = [
             jk.new_name if jk.new_name not in left_join_keys_keep else jk.new_name + "_right"
-            for jk in settings.right_select.join_key_selects if not jk.keep
+            for jk in settings.right_select.join_key_selects
+            if not jk.keep
         ]
         after_join_drop_cols = list(set(after_join_drop_cols_right))
         return left_on, right_on, None, after_join_drop_cols
 
-    def _handle_outer_join_keys(self, settings: transform_schema.JoinInputManager, right_df: str,
-                                left_on: List[str],
-                                right_on: List[str]) -> Tuple[List[str], List[str], Dict, List[str]]:
+    def _handle_outer_join_keys(
+        self, settings: transform_schema.JoinInputManager, right_df: str, left_on: list[str], right_on: list[str]
+    ) -> tuple[list[str], list[str], dict, list[str]]:
         """Handle key transformations for outer joins.
 
         For outer joins:
@@ -600,14 +660,10 @@ class FlowGraphToPolarsConverter:
         left_join_keys = {jk.new_name for jk in settings.left_select.join_key_selects}
 
         join_keys_to_keep_and_rename = [
-            rjk for rjk in settings.right_select.join_key_selects
-            if rjk.keep and rjk.new_name in left_join_keys
+            rjk for rjk in settings.right_select.join_key_selects if rjk.keep and rjk.new_name in left_join_keys
         ]
 
-        join_key_rename_command = {
-            rjk.new_name: f"__jk_{rjk.new_name}"
-            for rjk in join_keys_to_keep_and_rename
-        }
+        join_key_rename_command = {rjk.new_name: f"__jk_{rjk.new_name}" for rjk in join_keys_to_keep_and_rename}
 
         # Update right_on keys
         for position, right_on_key in enumerate(right_on):
@@ -621,20 +677,27 @@ class FlowGraphToPolarsConverter:
         reverse_action = {f"__jk_{rjk.new_name}": rjk.new_name for rjk in join_keys_to_keep_and_rename}
 
         # Calculate columns to drop after join
-        after_join_drop_cols_left = [
-            jk.new_name for jk in settings.left_select.join_key_selects if not jk.keep
-        ]
+        after_join_drop_cols_left = [jk.new_name for jk in settings.left_select.join_key_selects if not jk.keep]
         after_join_drop_cols_right = [
             jk.new_name if jk.new_name not in left_join_keys else jk.new_name + "_right"
-            for jk in settings.right_select.join_key_selects if not jk.keep
+            for jk in settings.right_select.join_key_selects
+            if not jk.keep
         ]
         after_join_drop_cols = after_join_drop_cols_left + after_join_drop_cols_right
 
         return left_on, right_on, reverse_action, after_join_drop_cols
 
-    def _execute_join_with_post_processing(self, settings: input_schema.NodeJoin, var_name: str,
-                                           left_df: str, right_df: str, left_on: List[str], right_on: List[str],
-                                           after_join_drop_cols: List[str], reverse_action: Optional[Dict]) -> None:
+    def _execute_join_with_post_processing(
+        self,
+        settings: input_schema.NodeJoin,
+        var_name: str,
+        left_df: str,
+        right_df: str,
+        left_on: list[str],
+        right_on: list[str],
+        after_join_drop_cols: list[str],
+        reverse_action: dict | None,
+    ) -> None:
         """Execute the join operation and apply post-processing steps.
 
         Generates the actual join code with any necessary post-processing:
@@ -665,7 +728,7 @@ class FlowGraphToPolarsConverter:
         self._add_code("    )")
 
         # Handle right join special case
-        if settings.join_input.how == 'right':
+        if settings.join_input.how == "right":
             self._add_code(".collect()")  # Right join needs to be collected first cause of issue with rename
 
         # Apply post-join transformations
@@ -676,21 +739,21 @@ class FlowGraphToPolarsConverter:
             self._add_code(f".rename({reverse_action})")
 
         # Convert back to lazy for right joins
-        if settings.join_input.how == 'right':
-            self._add_code(f".lazy()")
+        if settings.join_input.how == "right":
+            self._add_code(".lazy()")
 
         self._add_code(")")
 
-    def _handle_group_by(self, settings: input_schema.NodeGroupBy, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_group_by(self, settings: input_schema.NodeGroupBy, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle group by nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
 
         # Separate groupby columns from aggregation columns
         group_cols = []
         agg_exprs = []
 
         for agg_col in settings.groupby_input.agg_cols:
-            if agg_col.agg == 'groupby':
+            if agg_col.agg == "groupby":
                 group_cols.append(agg_col.old_name)
             else:
                 agg_func = self._get_agg_function(agg_col.agg)
@@ -703,9 +766,9 @@ class FlowGraphToPolarsConverter:
         self._add_code("])")
         self._add_code("")
 
-    def _handle_formula(self, settings: input_schema.NodeFormula, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_formula(self, settings: input_schema.NodeFormula, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle formula/expression nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         self.imports.add("from polars_expr_transformer.process.polars_expr_transformer import simple_function_to_expr")
 
         # Convert SQL-like formula to Polars expression
@@ -717,7 +780,7 @@ class FlowGraphToPolarsConverter:
             output_type = convert_pl_type_to_string(cast_str_to_polars_type(settings.function.field.data_type))
             if output_type[:3] != "pl.":
                 output_type = "pl." + output_type
-            self._add_code(f'    .cast({output_type})')
+            self._add_code(f"    .cast({output_type})")
 
         self._add_code("])")
         self._add_code("")
@@ -725,11 +788,11 @@ class FlowGraphToPolarsConverter:
     def _handle_pivot_no_index(self, settings: input_schema.NodePivot, var_name: str, input_df: str, agg_func: str):
         pivot_input = settings.pivot_input
 
-        self._add_code(f'{var_name} = ({input_df}.collect()')
+        self._add_code(f"{var_name} = ({input_df}.collect()")
         self._add_code('    .with_columns(pl.lit(1).alias("__temp_index__"))')
-        self._add_code('    .pivot(')
+        self._add_code("    .pivot(")
         self._add_code(f'        values="{pivot_input.value_col}",')
-        self._add_code(f'        index=["__temp_index__"],')
+        self._add_code('        index=["__temp_index__"],')
         self._add_code(f'        columns="{pivot_input.pivot_column}",')
         self._add_code(f'        aggregate_function="{agg_func}"')
         self._add_code("    )")
@@ -737,17 +800,16 @@ class FlowGraphToPolarsConverter:
         self._add_code(").lazy()")
         self._add_code("")
 
-    def _handle_pivot(self, settings: input_schema.NodePivot, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_pivot(self, settings: input_schema.NodePivot, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle pivot nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         pivot_input = settings.pivot_input
         if len(pivot_input.aggregations) > 1:
-            logger.error("Multiple aggregations are not convertable to polars code. "
-                         "Taking the first value")
+            logger.error("Multiple aggregations are not convertable to polars code. " "Taking the first value")
         if len(pivot_input.aggregations) > 0:
             agg_func = pivot_input.aggregations[0]
         else:
-            agg_func = 'first'
+            agg_func = "first"
         if len(settings.pivot_input.index_columns) == 0:
             self._handle_pivot_no_index(settings, var_name, input_df, agg_func)
         else:
@@ -761,9 +823,9 @@ class FlowGraphToPolarsConverter:
             self._add_code(").lazy()")
             self._add_code("")
 
-    def _handle_unpivot(self, settings: input_schema.NodeUnpivot, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_unpivot(self, settings: input_schema.NodeUnpivot, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle unpivot nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         unpivot_input = settings.unpivot_input
 
         self._add_code(f"{var_name} = {input_df}.unpivot(")
@@ -779,22 +841,22 @@ class FlowGraphToPolarsConverter:
         self._add_code(")")
         self._add_code("")
 
-    def _handle_union(self, settings: input_schema.NodeUnion, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_union(self, settings: input_schema.NodeUnion, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle union nodes."""
         # Get all input LazyFrame
         dfs = []
-        if 'main' in input_vars:
-            dfs.append(input_vars['main'])
+        if "main" in input_vars:
+            dfs.append(input_vars["main"])
         else:
             # Multiple main inputs
             for key, df_var in input_vars.items():
-                if key.startswith('main'):
+                if key.startswith("main"):
                     dfs.append(df_var)
 
-        if settings.union_input.mode == 'relaxed':
-            how = 'diagonal_relaxed'
+        if settings.union_input.mode == "relaxed":
+            how = "diagonal_relaxed"
         else:
-            how = 'diagonal'
+            how = "diagonal"
 
         self._add_code(f"{var_name} = pl.concat([")
         for df in dfs:
@@ -802,76 +864,88 @@ class FlowGraphToPolarsConverter:
         self._add_code(f"], how='{how}')")
         self._add_code("")
 
-    def _handle_sort(self, settings: input_schema.NodeSort, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_sort(self, settings: input_schema.NodeSort, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle sort nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
 
         sort_cols = []
         descending = []
 
         for sort_input in settings.sort_input:
             sort_cols.append(f'"{sort_input.column}"')
-            descending.append(sort_input.how == 'desc')
+            descending.append(sort_input.how == "desc")
 
         self._add_code(f"{var_name} = {input_df}.sort([{', '.join(sort_cols)}], descending={descending})")
         self._add_code("")
 
-    def _handle_sample(self, settings: input_schema.NodeSample, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_sample(self, settings: input_schema.NodeSample, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle sample nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         self._add_code(f"{var_name} = {input_df}.head(n={settings.sample_size})")
         self._add_code("")
 
     @staticmethod
-    def _transform_fuzzy_mappings_to_string(fuzzy_mappings: List[FuzzyMapping]) -> str:
-
+    def _transform_fuzzy_mappings_to_string(fuzzy_mappings: list[FuzzyMapping]) -> str:
         output_str = "["
         for i, fuzzy_mapping in enumerate(fuzzy_mappings):
-
-            output_str += (f"FuzzyMapping(left_col='{fuzzy_mapping.left_col}',"
-                           f" right_col='{fuzzy_mapping.right_col}', "
-                           f"threshold_score={fuzzy_mapping.threshold_score}, "
-                           f"fuzzy_type='{fuzzy_mapping.fuzzy_type}')")
+            output_str += (
+                f"FuzzyMapping(left_col='{fuzzy_mapping.left_col}',"
+                f" right_col='{fuzzy_mapping.right_col}', "
+                f"threshold_score={fuzzy_mapping.threshold_score}, "
+                f"fuzzy_type='{fuzzy_mapping.fuzzy_type}')"
+            )
             if i < len(fuzzy_mappings) - 1:
                 output_str += ",\n"
         output_str += "]"
         return output_str
 
-    def _handle_fuzzy_match(self, settings: input_schema.NodeFuzzyMatch, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_fuzzy_match(
+        self, settings: input_schema.NodeFuzzyMatch, var_name: str, input_vars: dict[str, str]
+    ) -> None:
         """Handle fuzzy match nodes."""
         self.imports.add("from pl_fuzzy_frame_match import FuzzyMapping, fuzzy_match_dfs")
         fuzzy_match_handler = transform_schema.FuzzyMatchInputManager(settings.join_input)
-        left_df = input_vars.get('main', input_vars.get('main_0', 'df_left'))
-        right_df = input_vars.get('right', input_vars.get('main_1', 'df_right'))
+        left_df = input_vars.get("main", input_vars.get("main_0", "df_left"))
+        right_df = input_vars.get("right", input_vars.get("main_1", "df_right"))
 
         if left_df == right_df:
             right_df = "df_right"
             self._add_code(f"{right_df} = {left_df}")
 
         if fuzzy_match_handler.left_select.has_drop_cols():
-            self._add_code(f"{left_df} = {left_df}.drop({[c.old_name for c in fuzzy_match_handler.left_select.non_jk_drop_columns]})")
+            self._add_code(
+                f"{left_df} = {left_df}.drop({[c.old_name for c in fuzzy_match_handler.left_select.non_jk_drop_columns]})"
+            )
         if fuzzy_match_handler.right_select.has_drop_cols():
-            self._add_code(f"{right_df} = {right_df}.drop({[c.old_name for c in fuzzy_match_handler.right_select.non_jk_drop_columns]})")
+            self._add_code(
+                f"{right_df} = {right_df}.drop({[c.old_name for c in fuzzy_match_handler.right_select.non_jk_drop_columns]})"
+            )
 
         fuzzy_join_mapping_settings = self._transform_fuzzy_mappings_to_string(fuzzy_match_handler.join_mapping)
-        self._add_code(f"{var_name} = fuzzy_match_dfs(\n"
-                       f"       left_df={left_df}, right_df={right_df},\n"
-                       f"       fuzzy_maps={fuzzy_join_mapping_settings}\n"
-                       f"       ).lazy()")
+        self._add_code(
+            f"{var_name} = fuzzy_match_dfs(\n"
+            f"       left_df={left_df}, right_df={right_df},\n"
+            f"       fuzzy_maps={fuzzy_join_mapping_settings}\n"
+            f"       ).lazy()"
+        )
 
-    def _handle_unique(self, settings: input_schema.NodeUnique, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_unique(self, settings: input_schema.NodeUnique, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle unique/distinct nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
 
         if settings.unique_input.columns:
-            self._add_code(f"{var_name} = {input_df}.unique(subset={settings.unique_input.columns}, keep='{settings.unique_input.strategy}')")
+            self._add_code(
+                f"{var_name} = {input_df}.unique(subset={settings.unique_input.columns}, keep='{settings.unique_input.strategy}')"
+            )
         else:
             self._add_code(f"{var_name} = {input_df}.unique(keep='{settings.unique_input.strategy}')")
         self._add_code("")
 
-    def _handle_text_to_rows(self, settings: input_schema.NodeTextToRows, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_text_to_rows(
+        self, settings: input_schema.NodeTextToRows, var_name: str, input_vars: dict[str, str]
+    ) -> None:
         """Handle text to rows (explode) nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         text_input = settings.text_to_rows_input
 
         # First split the column
@@ -884,96 +958,108 @@ class FlowGraphToPolarsConverter:
 
         self._add_code(f"{var_name} = {input_df}.with_columns({split_expr}).explode('{explode_col}')")
         self._add_code("")
+
     # .with_columns(
     #     (pl.cum_count(record_id_settings.output_column_name)
     #      .over(record_id_settings.group_by_columns) + record_id_settings.offset - 1)
     #     .alias(record_id_settings.output_column_name)
     # )
-    def _handle_record_id(self, settings: input_schema.NodeRecordId, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_record_id(self, settings: input_schema.NodeRecordId, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle record ID nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         record_input = settings.record_id_input
         if record_input.group_by and record_input.group_by_columns:
-
             # Row number within groups
             self._add_code(f"{var_name} = ({input_df}")
             self._add_code(f"    .with_columns(pl.lit(1).alias('{record_input.output_column_name}'))")
-            self._add_code(f"    .with_columns([")
-            self._add_code(f"    (pl.cum_count('{record_input.output_column_name}').over({record_input.group_by_columns}) + {record_input.offset} - 1)")
+            self._add_code("    .with_columns([")
+            self._add_code(
+                f"    (pl.cum_count('{record_input.output_column_name}').over({record_input.group_by_columns}) + {record_input.offset} - 1)"
+            )
             self._add_code(f"    .alias('{record_input.output_column_name}')")
             self._add_code("])")
-            self._add_code(f".select(['{record_input.output_column_name}'] + [col for col in {input_df}.columns if col != '{record_input.output_column_name}'])")
+            self._add_code(
+                f".select(['{record_input.output_column_name}'] + [col for col in {input_df}.columns if col != '{record_input.output_column_name}'])"
+            )
             self._add_code(")")
         else:
             # Simple row number
-            self._add_code(f"{var_name} = {input_df}.with_row_count(name='{record_input.output_column_name}', offset={record_input.offset})")
+            self._add_code(
+                f"{var_name} = {input_df}.with_row_count(name='{record_input.output_column_name}', offset={record_input.offset})"
+            )
         self._add_code("")
 
-    def _handle_cross_join(self, settings: input_schema.NodeCrossJoin, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_cross_join(
+        self, settings: input_schema.NodeCrossJoin, var_name: str, input_vars: dict[str, str]
+    ) -> None:
         """Handle cross join nodes."""
-        left_df = input_vars.get('main', input_vars.get('main_0', 'df_left'))
-        right_df = input_vars.get('right', input_vars.get('main_1', 'df_right'))
+        left_df = input_vars.get("main", input_vars.get("main_0", "df_left"))
+        right_df = input_vars.get("right", input_vars.get("main_1", "df_right"))
 
         self._add_code(f"{var_name} = {left_df}.join({right_df}, how='cross')")
         self._add_code("")
 
-    def _handle_cloud_storage_writer(self, settings: input_schema.NodeCloudStorageWriter, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_cloud_storage_writer(
+        self, settings: input_schema.NodeCloudStorageWriter, var_name: str, input_vars: dict[str, str]
+    ) -> None:
         """Handle cloud storage writer nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         # def write_csv_to_cloud_storage(self, path: str, connection_name: typing.Optional[str] = None, delimiter: str = ';', encoding: typing.Literal['utf8', 'utf8-lossy'] = 'utf8', description: Optional[str] = None) -> 'FlowFrame': ...
 
         output_settings = settings.cloud_storage_settings
         self.imports.add("import flowfile as ff")
         self._add_code(f"(ff.FlowFrame({input_df})")
         if output_settings.file_format == "csv":
-            self._add_code(f'    .write_csv_to_cloud_storage(')
+            self._add_code("    .write_csv_to_cloud_storage(")
             self._add_code(f'        path="{output_settings.resource_path}",')
             self._add_code(f'        connection_name="{output_settings.connection_name}",')
             self._add_code(f'        delimiter="{output_settings.csv_delimiter}",')
             self._add_code(f'        encoding="{output_settings.csv_encoding}",')
             self._add_code(f'        description="{settings.description}"')
         elif output_settings.file_format == "parquet":
-            self._add_code(f'    .write_parquet_to_cloud_storage(')
+            self._add_code("    .write_parquet_to_cloud_storage(")
             self._add_code(f'        path="{output_settings.resource_path}",')
             self._add_code(f'        connection_name="{output_settings.connection_name}",')
             self._add_code(f'        description="{settings.description}"')
         elif output_settings.file_format == "json":
-            self._add_code(f'    .write_json_to_cloud_storage(')
+            self._add_code("    .write_json_to_cloud_storage(")
             self._add_code(f'        path="{output_settings.resource_path}",')
             self._add_code(f'        connection_name="{output_settings.connection_name}",')
             self._add_code(f'        description="{settings.description}"')
         elif output_settings.file_format == "delta":
-            self._add_code(f'    .write_delta(')
+            self._add_code("    .write_delta(")
             self._add_code(f'        path="{output_settings.resource_path}",')
             self._add_code(f'        write_mode="{output_settings.write_mode}",')
             self._add_code(f'        connection_name="{output_settings.connection_name}",')
             self._add_code(f'        description="{settings.description}"')
-        self._add_code('    )')
-        self._add_code(')')
+        self._add_code("    )")
+        self._add_code(")")
 
-    def _handle_output(self, settings: input_schema.NodeOutput, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_output(self, settings: input_schema.NodeOutput, var_name: str, input_vars: dict[str, str]) -> None:
         """Handle output nodes."""
-        input_df = input_vars.get('main', 'df')
+        input_df = input_vars.get("main", "df")
         output_settings = settings.output_settings
 
-        if output_settings.file_type == 'csv':
-            self._add_code(f'{input_df}.sink_csv(')
+        if output_settings.file_type == "csv":
+            self._add_code(f"{input_df}.sink_csv(")
             self._add_code(f'    "{output_settings.abs_file_path}",')
             self._add_code(f'    separator="{output_settings.table_settings.delimiter}"')
-            self._add_code(')')
+            self._add_code(")")
 
-        elif output_settings.file_type == 'parquet':
+        elif output_settings.file_type == "parquet":
             self._add_code(f'{input_df}.sink_parquet("{output_settings.abs_file_path}")')
 
-        elif output_settings.file_type == 'excel':
-            self._add_code(f'{input_df}.collect().write_excel(')
+        elif output_settings.file_type == "excel":
+            self._add_code(f"{input_df}.collect().write_excel(")
             self._add_code(f'    "{output_settings.abs_file_path}",')
             self._add_code(f'    worksheet="{output_settings.table_settings.sheet_name}"')
-            self._add_code(')')
+            self._add_code(")")
 
         self._add_code("")
 
-    def _handle_polars_code(self, settings: input_schema.NodePolarsCode, var_name: str, input_vars: Dict[str, str]) -> None:
+    def _handle_polars_code(
+        self, settings: input_schema.NodePolarsCode, var_name: str, input_vars: dict[str, str]
+    ) -> None:
         """Handle custom Polars code nodes."""
         code = settings.polars_code_input.polars_code.strip()
         # Determine function parameters based on number of inputs
@@ -990,7 +1076,7 @@ class FlowGraphToPolarsConverter:
             arg_list = []
             i = 1
             for key in sorted(input_vars.keys()):
-                if key.startswith('main'):
+                if key.startswith("main"):
                     param_list.append(f"input_df_{i}: pl.LazyFrame")
                     arg_list.append(input_vars[key])
                     i += 1
@@ -1001,7 +1087,7 @@ class FlowGraphToPolarsConverter:
         is_expression = "output_df" not in code
 
         # Wrap the code in a function
-        self._add_code(f"# Custom Polars code")
+        self._add_code("# Custom Polars code")
         self._add_code(f"def _polars_code_{var_name.replace('df_', '')}({params}):")
 
         # Handle the code based on its structure
@@ -1010,18 +1096,18 @@ class FlowGraphToPolarsConverter:
             self._add_code(f"    return {code}")
         else:
             # It contains assignments
-            for line in code.split('\n'):
+            for line in code.split("\n"):
                 if line.strip():
                     self._add_code(f"    {line}")
 
             # If no explicit return, try to detect what to return
-            if 'return' not in code:
+            if "return" not in code:
                 # Try to find the last assignment
-                lines = [l.strip() for l in code.split('\n') if l.strip() and '=' in l]
+                lines = [l.strip() for l in code.split("\n") if l.strip() and "=" in l]
                 if lines:
                     last_assignment = lines[-1]
-                    if '=' in last_assignment:
-                        output_var = last_assignment.split('=')[0].strip()
+                    if "=" in last_assignment:
+                        output_var = last_assignment.split("=")[0].strip()
                         self._add_code(f"    return {output_var}")
 
         self._add_code("")
@@ -1030,6 +1116,291 @@ class FlowGraphToPolarsConverter:
         self._add_code(f"{var_name} = _polars_code_{var_name.replace('df_', '')}({args})")
         self._add_code("")
 
+    # Handlers for unsupported node types - these add nodes to the unsupported list
+
+    def _handle_explore_data(
+        self, settings: input_schema.NodeExploreData, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle explore_data nodes - these are skipped as they are interactive visualization only."""
+        # explore_data is just for visualization in the UI, it doesn't transform data
+        # So we skip it in code generation but don't fail - just add a comment
+        input_df = input_vars.get("main", "df")
+        self._add_comment(f"# Node {settings.node_id}: Explore Data (skipped - interactive visualization only)")
+        self._add_code(f"{var_name} = {input_df}  # Pass through unchanged")
+        self._add_code("")
+
+    def _handle_database_reader(
+        self, settings: input_schema.NodeDatabaseReader, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle database_reader nodes by generating code to read from database using a named connection."""
+        db_settings = settings.database_settings
+
+        # Only reference mode is supported for code generation
+        if db_settings.connection_mode != "reference":
+            self.unsupported_nodes.append((
+                settings.node_id,
+                "database_reader",
+                "Database Reader nodes with inline connections cannot be exported. "
+                "Please use a named connection (reference mode) instead."
+            ))
+            self._add_comment(f"# Node {settings.node_id}: Database Reader - Inline connections not supported")
+            return
+
+        if not db_settings.database_connection_name:
+            self.unsupported_nodes.append((
+                settings.node_id,
+                "database_reader",
+                "Database Reader node is missing a connection name"
+            ))
+            return
+
+        self.imports.add("import flowfile as ff")
+
+        connection_name = db_settings.database_connection_name
+        self._add_code(f"# Read from database using connection: {connection_name}")
+
+        if db_settings.query_mode == "query" and db_settings.query:
+            # Query mode - use triple quotes to preserve query formatting
+            self._add_code(f'{var_name} = ff.read_database(')
+            self._add_code(f'    "{connection_name}",')
+            self._add_code(f'    query="""')
+            # Add each line of the query with proper indentation
+            for line in db_settings.query.split("\n"):
+                self._add_code(f"        {line}")
+            self._add_code('    """,')
+            self._add_code(")")
+        else:
+            # Table mode
+            self._add_code(f'{var_name} = ff.read_database(')
+            self._add_code(f'    "{connection_name}",')
+            if db_settings.table_name:
+                self._add_code(f'    table_name="{db_settings.table_name}",')
+            if db_settings.schema_name:
+                self._add_code(f'    schema_name="{db_settings.schema_name}",')
+            self._add_code(")")
+
+        self._add_code("")
+
+    def _handle_database_writer(
+        self, settings: input_schema.NodeDatabaseWriter, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle database_writer nodes by generating code to write to database using a named connection."""
+        db_settings = settings.database_write_settings
+
+        # Only reference mode is supported for code generation
+        if db_settings.connection_mode != "reference":
+            self.unsupported_nodes.append((
+                settings.node_id,
+                "database_writer",
+                "Database Writer nodes with inline connections cannot be exported. "
+                "Please use a named connection (reference mode) instead."
+            ))
+            self._add_comment(f"# Node {settings.node_id}: Database Writer - Inline connections not supported")
+            return
+
+        if not db_settings.database_connection_name:
+            self.unsupported_nodes.append((
+                settings.node_id,
+                "database_writer",
+                "Database Writer node is missing a connection name"
+            ))
+            return
+
+        self.imports.add("import flowfile as ff")
+
+        connection_name = db_settings.database_connection_name
+        input_df = input_vars.get("main", "df")
+
+        self._add_code(f"# Write to database using connection: {connection_name}")
+        self._add_code(f"ff.write_database(")
+        self._add_code(f"    {input_df}.collect(),")
+        self._add_code(f'    "{connection_name}",')
+        self._add_code(f'    "{db_settings.table_name}",')
+        if db_settings.schema_name:
+            self._add_code(f'    schema_name="{db_settings.schema_name}",')
+        if db_settings.if_exists:
+            self._add_code(f'    if_exists="{db_settings.if_exists}",')
+        self._add_code(")")
+        self._add_code(f"{var_name} = {input_df}  # Pass through the input DataFrame")
+        self._add_code("")
+
+    def _handle_external_source(
+        self, settings: input_schema.NodeExternalSource, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle external_source nodes - these are not supported for code generation."""
+        self.unsupported_nodes.append((
+            settings.node_id,
+            "external_source",
+            "External Source nodes use dynamic data sources that cannot be included in generated code"
+        ))
+        self._add_comment(f"# Node {settings.node_id}: External Source - Not supported for code export")
+        self._add_comment("# (External data sources require runtime configuration)")
+
+    def _check_process_method_signature(self, custom_node_class: type) -> tuple[bool, bool]:
+        """
+        Check the process method signature to determine if collect/lazy is needed.
+
+        Returns:
+            Tuple of (needs_collect, needs_lazy):
+            - needs_collect: True if inputs need to be collected to DataFrame before passing to process()
+            - needs_lazy: True if output needs to be converted to LazyFrame after process()
+        """
+        needs_collect = True  # Default: assume needs DataFrame input
+        needs_lazy = True  # Default: assume returns DataFrame
+
+        process_method = getattr(custom_node_class, 'process', None)
+        if process_method is None:
+            return needs_collect, needs_lazy
+
+        try:
+            # Try to get type hints from the process method
+            type_hints = typing.get_type_hints(process_method)
+
+            # Check return type
+            return_type = type_hints.get('return')
+            if return_type is not None:
+                return_type_str = str(return_type)
+                if 'LazyFrame' in return_type_str:
+                    needs_lazy = False
+
+            # Check input parameter types (look for *inputs parameter or first param after self)
+            sig = inspect.signature(process_method)
+            params = list(sig.parameters.values())
+            for param in params[1:]:  # Skip 'self'
+                if param.annotation != inspect.Parameter.empty:
+                    param_type_str = str(param.annotation)
+                    if 'LazyFrame' in param_type_str:
+                        needs_collect = False
+                        break
+                # Also check the type_hints dict for this param
+                if param.name in type_hints:
+                    hint_str = str(type_hints[param.name])
+                    if 'LazyFrame' in hint_str:
+                        needs_collect = False
+                        break
+        except Exception as e:
+            # If we can't determine types, use defaults (collect + lazy)
+            logger.debug(f"Could not determine process method signature: {e}")
+
+        return needs_collect, needs_lazy
+
+    def _read_custom_node_source_file(self, custom_node_class: type) -> str | None:
+        """
+        Read the entire source file where a custom node class is defined.
+        This includes all class definitions in that file (settings schemas, etc.).
+
+        Returns:
+            The complete source code from the file, or None if not readable.
+        """
+        try:
+            source_file = inspect.getfile(custom_node_class)
+            with open(source_file, 'r') as f:
+                return f.read()
+        except (OSError, TypeError):
+            return None
+
+    def _handle_user_defined(
+        self, node: FlowNode, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle user-defined custom nodes by including their class definition and calling process()."""
+        node_type = node.node_type
+        settings = node.setting_input
+
+        # Get the custom node class from the registry
+        custom_node_class = CUSTOM_NODE_STORE.get(node_type)
+        if custom_node_class is None:
+            self.unsupported_nodes.append((
+                node.node_id,
+                node_type,
+                f"User-defined node type '{node_type}' not found in the custom node registry"
+            ))
+            self._add_comment(f"# Node {node.node_id}: User-defined node '{node_type}' - Not found in registry")
+            return
+
+        # Store the entire source file if we haven't already
+        class_name = custom_node_class.__name__
+        if class_name not in self.custom_node_classes:
+            # Read the entire source file - it contains everything we need
+            file_source = self._read_custom_node_source_file(custom_node_class)
+            if file_source:
+                # Remove import lines from the file since we handle imports separately
+                lines = file_source.split('\n')
+                non_import_lines = []
+                in_multiline_import = False
+                for line in lines:
+                    stripped = line.strip()
+                    # Track multi-line imports (using parentheses)
+                    if stripped.startswith('import ') or stripped.startswith('from '):
+                        if '(' in stripped and ')' not in stripped:
+                            in_multiline_import = True
+                        continue
+                    if in_multiline_import:
+                        if ')' in stripped:
+                            in_multiline_import = False
+                        continue
+                    # Skip comments at the very start (like "# Auto-generated custom node")
+                    if stripped.startswith('#') and not non_import_lines:
+                        continue
+                    non_import_lines.append(line)
+                # Remove leading empty lines
+                while non_import_lines and not non_import_lines[0].strip():
+                    non_import_lines.pop(0)
+                self.custom_node_classes[class_name] = '\n'.join(non_import_lines)
+            else:
+                # Fallback to just the class source
+                try:
+                    self.custom_node_classes[class_name] = inspect.getsource(custom_node_class)
+                except (OSError, TypeError) as e:
+                    self.unsupported_nodes.append((
+                        node.node_id,
+                        node_type,
+                        f"Could not retrieve source code for user-defined node: {e}"
+                    ))
+                    self._add_comment(f"# Node {node.node_id}: User-defined node '{node_type}' - Source code unavailable")
+                    return
+
+            # Add necessary imports
+            self.imports.add("from flowfile_core.flowfile.node_designer import CustomNodeBase, Section, NodeSettings, SingleSelect, MultiSelect, IncomingColumns, ColumnSelector, NumericInput, TextInput, DropdownSelector, TextArea, Toggle")
+
+        # Get settings values to initialize the node
+        settings_dict = getattr(settings, "settings", {}) or {}
+
+        # Check process method signature to determine if collect/lazy is needed
+        needs_collect, needs_lazy = self._check_process_method_signature(custom_node_class)
+
+        # Generate the code to instantiate and run the custom node
+        self._add_code(f"# User-defined node: {custom_node_class.model_fields.get('node_name', type('', (), {'default': node_type})).default}")
+        self._add_code(f"_custom_node_{node.node_id} = {class_name}()")
+
+        # If there are settings, apply them
+        if settings_dict:
+            self._add_code(f"_custom_node_{node.node_id}_settings = {repr(settings_dict)}")
+            self._add_code(f"if _custom_node_{node.node_id}.settings_schema:")
+            self._add_code(f"    _custom_node_{node.node_id}.settings_schema.populate_values(_custom_node_{node.node_id}_settings)")
+
+        # Prepare input arguments based on whether we need to collect
+        if len(input_vars) == 0:
+            input_args = ""
+        elif len(input_vars) == 1:
+            input_df = list(input_vars.values())[0]
+            input_args = f"{input_df}.collect()" if needs_collect else input_df
+        else:
+            arg_list = []
+            for key in sorted(input_vars.keys()):
+                if key.startswith("main"):
+                    if needs_collect:
+                        arg_list.append(f"{input_vars[key]}.collect()")
+                    else:
+                        arg_list.append(input_vars[key])
+            input_args = ", ".join(arg_list)
+
+        # Call the process method, adding .lazy() only if needed
+        if needs_lazy:
+            self._add_code(f"{var_name} = _custom_node_{node.node_id}.process({input_args}).lazy()")
+        else:
+            self._add_code(f"{var_name} = _custom_node_{node.node_id}.process({input_args})")
+        self._add_code("")
+
     # Helper methods
 
     def _add_code(self, line: str) -> None:
@@ -1054,14 +1425,7 @@ class FlowGraphToPolarsConverter:
             col, op, val = match.groups()
 
             # Map operators
-            op_map = {
-                '=': '==',
-                '!=': '!=',
-                '>': '>',
-                '<': '<',
-                '>=': '>=',
-                '<=': '<='
-            }
+            op_map = {"=": "==", "!=": "!=", ">": ">", "<": "<", ">=": ">=", "<=": "<="}
 
             polars_op = op_map.get(op, op)
 
@@ -1075,45 +1439,129 @@ class FlowGraphToPolarsConverter:
         return re.sub(pattern, replace_expr, expr)
 
     def _create_basic_filter_expr(self, basic: transform_schema.BasicFilter) -> str:
-        """Create Polars expression from basic filter."""
+        """Create Polars expression from basic filter.
+
+        Generates proper Polars code for all supported filter operators.
+
+        Args:
+            basic: The BasicFilter configuration.
+
+        Returns:
+            A string containing valid Polars filter expression code.
+        """
+        from flowfile_core.schemas.transform_schema import FilterOperator
+
         col = f'pl.col("{basic.field}")'
+        value = basic.value
+        value2 = basic.value2
+
+        # Determine if value is numeric (for proper quoting)
+        is_numeric = value.replace(".", "", 1).replace("-", "", 1).isnumeric() if value else False
+
+        # Get the operator
+        try:
+            operator = basic.get_operator()
+        except (ValueError, AttributeError):
+            operator = FilterOperator.from_symbol(str(basic.operator))
+
+        # Generate expression based on operator
+        if operator == FilterOperator.EQUALS:
+            if is_numeric:
+                return f"{col} == {value}"
+            return f'{col} == "{value}"'
+
+        elif operator == FilterOperator.NOT_EQUALS:
+            if is_numeric:
+                return f"{col} != {value}"
+            return f'{col} != "{value}"'
+
+        elif operator == FilterOperator.GREATER_THAN:
+            if is_numeric:
+                return f"{col} > {value}"
+            return f'{col} > "{value}"'
+
+        elif operator == FilterOperator.GREATER_THAN_OR_EQUALS:
+            if is_numeric:
+                return f"{col} >= {value}"
+            return f'{col} >= "{value}"'
+
+        elif operator == FilterOperator.LESS_THAN:
+            if is_numeric:
+                return f"{col} < {value}"
+            return f'{col} < "{value}"'
+
+        elif operator == FilterOperator.LESS_THAN_OR_EQUALS:
+            if is_numeric:
+                return f"{col} <= {value}"
+            return f'{col} <= "{value}"'
+
+        elif operator == FilterOperator.CONTAINS:
+            return f'{col}.str.contains("{value}")'
+
+        elif operator == FilterOperator.NOT_CONTAINS:
+            return f'{col}.str.contains("{value}").not_()'
+
+        elif operator == FilterOperator.STARTS_WITH:
+            return f'{col}.str.starts_with("{value}")'
+
+        elif operator == FilterOperator.ENDS_WITH:
+            return f'{col}.str.ends_with("{value}")'
+
+        elif operator == FilterOperator.IS_NULL:
+            return f"{col}.is_null()"
+
+        elif operator == FilterOperator.IS_NOT_NULL:
+            return f"{col}.is_not_null()"
+
+        elif operator == FilterOperator.IN:
+            values = [v.strip() for v in value.split(",")]
+            if all(v.replace(".", "", 1).replace("-", "", 1).isnumeric() for v in values):
+                values_str = ", ".join(values)
+            else:
+                values_str = ", ".join(f'"{v}"' for v in values)
+            return f"{col}.is_in([{values_str}])"
+
+        elif operator == FilterOperator.NOT_IN:
+            values = [v.strip() for v in value.split(",")]
+            if all(v.replace(".", "", 1).replace("-", "", 1).isnumeric() for v in values):
+                values_str = ", ".join(values)
+            else:
+                values_str = ", ".join(f'"{v}"' for v in values)
+            return f"{col}.is_in([{values_str}]).not_()"
 
-        if basic.filter_type == 'equals':
-            return f'{col} == "{basic.filter_value}"'
-        elif basic.filter_type == 'not_equals':
-            return f'{col} != "{basic.filter_value}"'
-        elif basic.filter_type == 'greater':
-            return f'{col} > {basic.filter_value}'
-        elif basic.filter_type == 'less':
-            return f'{col} < {basic.filter_value}'
-        elif basic.filter_type == 'in':
-            values = basic.filter_value.split(',')
-            return f"pl.col('{col}').is_in({values})"
+        elif operator == FilterOperator.BETWEEN:
+            if value2 is None:
+                return f"{col}  # BETWEEN requires two values"
+            if is_numeric and value2.replace(".", "", 1).replace("-", "", 1).isnumeric():
+                return f"({col} >= {value}) & ({col} <= {value2})"
+            return f'({col} >= "{value}") & ({col} <= "{value2}")'
+
+        # Fallback
         return col
 
     def _get_polars_dtype(self, dtype_str: str) -> str:
         """Convert Flowfile dtype string to Polars dtype."""
         dtype_map = {
-            'String': 'pl.Utf8',
-            'Integer': 'pl.Int64',
-            'Double': 'pl.Float64',
-            'Boolean': 'pl.Boolean',
-            'Date': 'pl.Date',
-            'Datetime': 'pl.Datetime',
-            'Float32': 'pl.Float32',
-            'Float64': 'pl.Float64',
-            'Int32': 'pl.Int32',
-            'Int64': 'pl.Int64',
-            'Utf8': 'pl.Utf8',
+            "String": "pl.Utf8",
+            "Integer": "pl.Int64",
+            "Double": "pl.Float64",
+            "Boolean": "pl.Boolean",
+            "Date": "pl.Date",
+            "Datetime": "pl.Datetime",
+            "Float32": "pl.Float32",
+            "Float64": "pl.Float64",
+            "Int32": "pl.Int32",
+            "Int64": "pl.Int64",
+            "Utf8": "pl.Utf8",
         }
-        return dtype_map.get(dtype_str, 'pl.Utf8')
+        return dtype_map.get(dtype_str, "pl.Utf8")
 
     def _get_agg_function(self, agg: str) -> str:
         """Get Polars aggregation function name."""
         agg_map = {
-            'avg': 'mean',
-            'average': 'mean',
-            'concat': 'str.concat',
+            "avg": "mean",
+            "average": "mean",
+            "concat": "str.concat",
         }
         return agg_map.get(agg, agg)
 
@@ -1126,12 +1574,12 @@ class FlowGraphToPolarsConverter:
         import re
 
         # Pattern for column names (simplified)
-        col_pattern = r'\b([a-zA-Z_][a-zA-Z0-9_]*)\b'
+        col_pattern = r"\b([a-zA-Z_][a-zA-Z0-9_]*)\b"
 
         def replace_col(match):
             col_name = match.group(1)
             # Skip SQL keywords
-            keywords = {'CASE', 'WHEN', 'THEN', 'ELSE', 'END', 'AND', 'OR', 'NOT', 'IN', 'AS'}
+            keywords = {"CASE", "WHEN", "THEN", "ELSE", "END", "AND", "OR", "NOT", "IN", "AS"}
             if col_name.upper() in keywords:
                 return col_name
             return f'pl.col("{col_name}")'
@@ -1139,13 +1587,13 @@ class FlowGraphToPolarsConverter:
         result = re.sub(col_pattern, replace_col, sql_expr)
 
         # Handle CASE WHEN
-        if 'CASE' in result:
+        if "CASE" in result:
             # This would need proper parsing
             result = "pl.when(...).then(...).otherwise(...)"
 
         return result
 
-    def add_return_code(self, lines: List[str]) -> None:
+    def add_return_code(self, lines: list[str]) -> None:
         if self.output_nodes:
             # Return marked output nodes
             if len(self.output_nodes) == 1:
@@ -1172,11 +1620,22 @@ class FlowGraphToPolarsConverter:
         lines.append("")
         lines.append("")
 
+        # Add custom node class definitions if any
+        if self.custom_node_classes:
+            lines.append("# Custom Node Class Definitions")
+            lines.append("# These classes are user-defined nodes that were included in the flow")
+            lines.append("")
+            for class_name, source_code in self.custom_node_classes.items():
+                for source_line in source_code.split("\n"):
+                    lines.append(source_line)
+                lines.append("")
+            lines.append("")
+
         # Add main function
         lines.append("def run_etl_pipeline():")
         lines.append('    """')
-        lines.append(f'    ETL Pipeline: {self.flow_graph.__name__}')
-        lines.append('    Generated from Flowfile')
+        lines.append(f"    ETL Pipeline: {self.flow_graph.__name__}")
+        lines.append("    Generated from Flowfile")
         lines.append('    """')
         lines.append("    ")