Flowfile 0.3.9__py3-none-any.whl → 0.4.0__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py

@@ -39,7 +39,7 @@ def trigger_df_operation(flow_id: int, node_id: int | str, lf: pl.LazyFrame, fil
              'flowfile_flow_id': flow_id, 'flowfile_node_id': node_id}
     v = requests.post(url=f'{WORKER_URL}/submit_query/', json=_json)
     if not v.ok:
-        raise Exception(f'Could not cache the data, {v.text}')
+        raise Exception(f'trigger_df_operation: Could not cache the data, {v.text}')
     return Status(**v.json())
 
 
@@ -49,7 +49,7 @@ def trigger_sample_operation(lf: pl.LazyFrame, file_ref: str, flow_id: int, node
              'sample_size': sample_size, 'flowfile_flow_id': flow_id, 'flowfile_node_id': node_id}
     v = requests.post(url=f'{WORKER_URL}/store_sample/', json=_json)
     if not v.ok:
-        raise Exception(f'Could not cache the data, {v.text}')
+        raise Exception(f'trigger_sample_operation: Could not cache the data, {v.text}')
     return Status(**v.json())
 
 
@@ -67,9 +67,10 @@ def trigger_fuzzy_match_operation(left_df: pl.LazyFrame, right_df: pl.LazyFrame,
                                       flowfile_flow_id=flow_id,
                                       flowfile_node_id=node_id
                                       )
+    print("fuzzy join input", fuzzy_join_input)
     v = requests.post(f'{WORKER_URL}/add_fuzzy_join', data=fuzzy_join_input.model_dump_json())
     if not v.ok:
-        raise Exception(f'Could not cache the data, {v.text}')
+        raise Exception(f'trigger_fuzzy_match_operation: Could not cache the data, {v.text}')
     return Status(**v.json())
 
 
@@ -78,7 +79,7 @@ def trigger_create_operation(flow_id: int, node_id: int | str, received_table: R
     f = requests.post(url=f'{WORKER_URL}/create_table/{file_type}', data=received_table.model_dump_json(),
                       params={'flowfile_flow_id': flow_id, 'flowfile_node_id': node_id})
     if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
+        raise Exception(f'trigger_create_operation: Could not cache the data, {f.text}')
     return Status(**f.json())
 
 
@@ -86,7 +87,7 @@ def trigger_database_read_collector(database_external_read_settings: DatabaseExt
     f = requests.post(url=f'{WORKER_URL}/store_database_read_result',
                       data=database_external_read_settings.model_dump_json())
     if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
+        raise Exception(f'trigger_database_read_collector: Could not cache the data, {f.text}')
     return Status(**f.json())
 
 
@@ -94,7 +95,7 @@ def trigger_database_write(database_external_write_settings: DatabaseExternalWri
     f = requests.post(url=f'{WORKER_URL}/store_database_write_result',
                       data=database_external_write_settings.model_dump_json())
     if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
+        raise Exception(f'trigger_database_write: Could not cache the data, {f.text}')
     return Status(**f.json())
 
 
@@ -102,7 +103,7 @@ def trigger_cloud_storage_write(database_external_write_settings: CloudStorageWr
     f = requests.post(url=f'{WORKER_URL}/write_data_to_cloud',
                       data=database_external_write_settings.model_dump_json())
     if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
+        raise Exception(f'trigger_cloud_storage_write: Could not cache the data, {f.text}')
     return Status(**f.json())
 
 
@@ -111,7 +112,7 @@ def get_results(file_ref: str) -> Status | None:
     if f.status_code == 200:
         return Status(**f.json())
     else:
-        raise Exception(f'Could not fetch the data, {f.text}')
+        raise Exception(f'get_results: Could not fetch the data, {f.text}')
 
 
 def results_exists(file_ref: str):
@@ -128,6 +129,25 @@ def results_exists(file_ref: str):
         return False
 
 
+def clear_task_from_worker(file_ref: str) -> bool:
+    """
+    Clears a task from the worker service by making a DELETE request. It also removes associated cached files.
+    Args:
+        file_ref (str): The unique identifier of the task to clear.
+
+    Returns:
+        bool: True if the task was successfully cleared, False otherwise.
+    """
+    try:
+        f = requests.delete(f'{WORKER_URL}/clear_task/{file_ref}')
+        if f.status_code == 200:
+            return True
+        return False
+    except requests.RequestException as e:
+        logger.error(f"Failed to remove results: {str(e)}")
+        return False
+
+
 def get_df_result(encoded_df: str) -> pl.LazyFrame:
     r = decodebytes(encoded_df.encode())
     return pl.LazyFrame.deserialize(io.BytesIO(r))

flowfile_core/flowfile/flow_graph.py

@@ -1,18 +1,22 @@
 import datetime
 import pickle
+
+import os
+
 import polars as pl
+
 import fastexcel
 from fastapi.exceptions import HTTPException
 from time import time
 from functools import partial
-from typing import List, Dict, Union, Callable, Any, Optional, Tuple
+from typing import List, Dict, Union, Callable, Any, Optional, Tuple, Literal
 from uuid import uuid1
 from copy import deepcopy
 from pyarrow.parquet import ParquetFile
 from flowfile_core.configs import logger
 from flowfile_core.configs.flow_logger import FlowLogger
 from flowfile_core.flowfile.sources.external_sources.factory import data_source_factory
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import cast_str_to_polars_type, FlowfileColumn
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, cast_str_to_polars_type
 
 from flowfile_core.flowfile.flow_data_engine.cloud_storage_reader import CloudStorageReader
 from flowfile_core.utils.arrow_reader import get_read_top_n
@@ -47,6 +51,7 @@ from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source impor
 from flowfile_core.flowfile.database_connection_manager.db_connections import (get_local_database_connection,
                                                                                get_local_cloud_connection)
 from flowfile_core.flowfile.util.calculate_layout import calculate_layered_layout
+from flowfile_core.flowfile.node_designer.custom_node import CustomNodeBase
 
 
 def get_xlsx_schema(engine: str, file_path: str, sheet_name: str, start_row: int, start_column: int,
@@ -175,11 +180,9 @@ class FlowGraph:
     schema: Optional[List[FlowfileColumn]] = None
     has_over_row_function: bool = False
     _flow_starts: List[Union[int, str]] = None
-    node_results: List[NodeResult] = None
     latest_run_info: Optional[RunInformation] = None
     start_datetime: datetime = None
     end_datetime: datetime = None
-    nodes_completed: int = 0
     _flow_settings: schemas.FlowSettings = None
     flow_logger: FlowLogger
 
@@ -206,11 +209,9 @@ class FlowGraph:
 
         self._flow_settings = flow_settings
         self.uuid = str(uuid1())
-        self.nodes_completed = 0
         self.start_datetime = None
         self.end_datetime = None
         self.latest_run_info = None
-        self.node_results = []
         self._flow_id = flow_settings.flow_id
         self.flow_logger = FlowLogger(flow_settings.flow_id)
         self._flow_starts: List[FlowNode] = []
@@ -436,6 +437,24 @@ class FlowGraph:
         node = self._node_db.get(node_id)
         if node is not None:
             return node
+        
+    def add_user_defined_node(self, *,
+                              custom_node: CustomNodeBase,
+                              user_defined_node_settings: input_schema.UserDefinedNode
+                              ):
+       
+        def _func(*fdes: FlowDataEngine) -> FlowDataEngine | None:
+            output = custom_node.process(*(fde.data_frame for fde in fdes))
+            if isinstance(output, pl.LazyFrame | pl.DataFrame):
+                return FlowDataEngine(output)
+            return None
+        
+        self.add_node_step(node_id=user_defined_node_settings.node_id,
+                           function=_func,
+                           setting_input=user_defined_node_settings,
+                           input_node_ids=user_defined_node_settings.depending_on_ids,
+                           node_type=custom_node.item,
+                           )
 
     def add_pivot(self, pivot_settings: input_schema.NodePivot):
         """Adds a pivot node to the graph.
@@ -814,11 +833,11 @@ class FlowGraph:
         def _func(main: FlowDataEngine, right: FlowDataEngine) -> FlowDataEngine:
             node = self.get_node(node_id=fuzzy_settings.node_id)
             if self.execution_location == "local":
-                return main.fuzzy_join(fuzzy_match_input=fuzzy_settings.join_input,
+                return main.fuzzy_join(fuzzy_match_input=deepcopy(fuzzy_settings.join_input),
                                        other=right,
                                        node_logger=self.flow_logger.get_node_logger(fuzzy_settings.node_id))
 
-            f = main.start_fuzzy_join(fuzzy_match_input=fuzzy_settings.join_input, other=right, file_ref=node.hash,
+            f = main.start_fuzzy_join(fuzzy_match_input=deepcopy(fuzzy_settings.join_input), other=right, file_ref=node.hash,
                                       flow_id=self.flow_id, node_id=fuzzy_settings.node_id)
             logger.info("Started the fuzzy match action")
             node._fetch_cached_df = f  # Add to the node so it can be cancelled and fetch later if needed
@@ -1599,6 +1618,66 @@ class FlowGraph:
             self.reset()
         self.flow_settings.execution_location = execution_location
 
+    def validate_if_node_can_be_fetched(self, node_id: int) -> None:
+        flow_node = self._node_db.get(node_id)
+        if not flow_node:
+            raise Exception("Node not found found")
+        skip_nodes, execution_order = compute_execution_plan(
+            nodes=self.nodes, flow_starts=self._flow_starts+self.get_implicit_starter_nodes()
+        )
+        if flow_node.node_id in [skip_node.node_id for skip_node in skip_nodes]:
+            raise Exception("Node can not be executed because it does not have it's inputs")
+
+    def create_initial_run_information(self, number_of_nodes: int,
+                                       run_type: Literal["fetch_one", "full_run"]):
+        return RunInformation(
+            flow_id=self.flow_id, start_time=datetime.datetime.now(), end_time=None,
+            success=None, number_of_nodes=number_of_nodes, node_step_result=[],
+            run_type=run_type
+        )
+
+    def trigger_fetch_node(self, node_id: int) -> RunInformation | None:
+        """Executes a specific node in the graph by its ID."""
+        if self.flow_settings.is_running:
+            raise Exception("Flow is already running")
+        flow_node = self.get_node(node_id)
+        self.flow_settings.is_running = True
+        self.flow_settings.is_canceled = False
+        self.flow_logger.clear_log_file()
+        self.latest_run_info = self.create_initial_run_information(1, "fetch_one")
+        node_logger = self.flow_logger.get_node_logger(flow_node.node_id)
+        node_result = NodeResult(node_id=flow_node.node_id, node_name=flow_node.name)
+        logger.info(f'Starting to run: node {flow_node.node_id}, start time: {node_result.start_timestamp}')
+        try:
+            self.latest_run_info.node_step_result.append(node_result)
+            flow_node.execute_node(run_location=self.flow_settings.execution_location,
+                                   performance_mode=False,
+                                   node_logger=node_logger,
+                                   optimize_for_downstream=False,
+                                   reset_cache=True)
+            node_result.error = str(flow_node.results.errors)
+            if self.flow_settings.is_canceled:
+                node_result.success = None
+                node_result.success = None
+                node_result.is_running = False
+            node_result.success = flow_node.results.errors is None
+            node_result.end_timestamp = time()
+            node_result.run_time = int(node_result.end_timestamp - node_result.start_timestamp)
+            node_result.is_running = False
+            self.latest_run_info.nodes_completed += 1
+            self.latest_run_info.end_time = datetime.datetime.now()
+            self.flow_settings.is_running = False
+            return self.get_run_info()
+        except Exception as e:
+            node_result.error = 'Node did not run'
+            node_result.success = False
+            node_result.end_timestamp = time()
+            node_result.run_time = int(node_result.end_timestamp - node_result.start_timestamp)
+            node_result.is_running = False
+            node_logger.error(f'Error in node {flow_node.node_id}: {e}')
+        finally:
+            self.flow_settings.is_running = False
+
     def run_graph(self) -> RunInformation | None:
         """Executes the entire data flow graph from start to finish.
 
@@ -1614,20 +1693,23 @@ class FlowGraph:
         if self.flow_settings.is_running:
             raise Exception('Flow is already running')
         try:
+
             self.flow_settings.is_running = True
             self.flow_settings.is_canceled = False
             self.flow_logger.clear_log_file()
-            self.nodes_completed = 0
-            self.node_results = []
-            self.start_datetime = datetime.datetime.now()
-            self.end_datetime = None
-            self.latest_run_info = None
             self.flow_logger.info('Starting to run flowfile flow...')
-            skip_nodes, execution_order = compute_execution_plan(nodes=self.nodes, flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
+
+            skip_nodes, execution_order = compute_execution_plan(
+                nodes=self.nodes,
+                flow_starts=self._flow_starts+self.get_implicit_starter_nodes()
+            )
+
+            self.latest_run_info = self.create_initial_run_information(len(execution_order), "full_run")
 
             skip_node_message(self.flow_logger, skip_nodes)
             execution_order_message(self.flow_logger, execution_order)
             performance_mode = self.flow_settings.execution_mode == 'Performance'
+
             for node in execution_order:
                 node_logger = self.flow_logger.get_node_logger(node.node_id)
                 if self.flow_settings.is_canceled:
@@ -1637,7 +1719,7 @@ class FlowGraph:
                     node_logger.info(f'Skipping node {node.node_id}')
                     continue
                 node_result = NodeResult(node_id=node.node_id, node_name=node.name)
-                self.node_results.append(node_result)
+                self.latest_run_info.node_step_result.append(node_result)
                 logger.info(f'Starting to run: node {node.node_id}, start time: {node_result.start_timestamp}')
                 node.execute_node(run_location=self.flow_settings.execution_location,
                                   performance_mode=performance_mode,
@@ -1663,7 +1745,7 @@ class FlowGraph:
                 if not node_result.success:
                     skip_nodes.extend(list(node.get_all_dependent_nodes()))
                 node_logger.info(f'Completed node with success: {node_result.success}')
-                self.nodes_completed += 1
+                self.latest_run_info.nodes_completed += 1
             self.flow_logger.info('Flow completed!')
             self.end_datetime = datetime.datetime.now()
             self.flow_settings.is_running = False
@@ -1675,28 +1757,23 @@ class FlowGraph:
         finally:
             self.flow_settings.is_running = False
 
-    def get_run_info(self) -> RunInformation:
+    def get_run_info(self) -> RunInformation | None:
         """Gets a summary of the most recent graph execution.
 
         Returns:
             A RunInformation object with details about the last run.
         """
+        is_running = self.flow_settings.is_running
         if self.latest_run_info is None:
-            node_results = self.node_results
-            success = all(nr.success for nr in node_results)
-            self.latest_run_info = RunInformation(start_time=self.start_datetime, end_time=self.end_datetime,
-                                                  success=success,
-                                                  node_step_result=node_results, flow_id=self.flow_id,
-                                                  nodes_completed=self.nodes_completed,
-                                                  number_of_nodes=len(self.nodes))
-        elif self.latest_run_info.nodes_completed != self.nodes_completed:
-            node_results = self.node_results
-            self.latest_run_info = RunInformation(start_time=self.start_datetime, end_time=self.end_datetime,
-                                                  success=all(nr.success for nr in node_results),
-                                                  node_step_result=node_results, flow_id=self.flow_id,
-                                                  nodes_completed=self.nodes_completed,
-                                                  number_of_nodes=len(self.nodes))
-        return self.latest_run_info
+            return
+
+        elif not is_running and self.latest_run_info.success is not None:
+            return self.latest_run_info
+
+        run_info = self.latest_run_info
+        if not is_running:
+            run_info.success = all(nr.success for nr in run_info.node_step_result)
+        return run_info
 
     @property
     def node_connections(self) -> List[Tuple[int, int]]:
@@ -1767,8 +1844,14 @@ class FlowGraph:
         Args:
             flow_path: The path where the flow file will be saved.
         """
-        with open(flow_path, 'wb') as f:
-            pickle.dump(self.get_node_storage(), f)
+        logger.info("Saving flow to %s", flow_path)
+        os.makedirs(os.path.dirname(flow_path), exist_ok=True)
+        try:
+            with open(flow_path, 'wb') as f:
+                pickle.dump(self.get_node_storage(), f)
+        except Exception as e:
+            logger.error(f"Error saving flow: {e}")
+
         self.flow_settings.path = flow_path
 
     def get_frontend_data(self) -> dict:

flowfile_core/flowfile/flow_node/flow_node.py

@@ -8,11 +8,11 @@ from flowfile_core.configs.flow_logger import NodeLogger
 
 from flowfile_core.schemas.output_model import TableExample, FileColumn, NodeData
 from flowfile_core.flowfile.utils import get_hash
-from flowfile_core.configs.node_store import nodes as node_interface
+from flowfile_core.configs import node_store
 from flowfile_core.flowfile.setting_generator import setting_generator, setting_updator
 from time import sleep
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations import (
-    ExternalDfFetcher, ExternalSampler, results_exists, get_external_df_result,
+    ExternalDfFetcher, ExternalSampler, clear_task_from_worker, results_exists, get_external_df_result,
     ExternalDatabaseFetcher, ExternalDatabaseWriter, ExternalCloudWriter)
 from flowfile_core.flowfile.flow_node.models import (NodeStepSettings, NodeStepInputs, NodeSchemaInformation,
                                                      NodeStepStats, NodeResults)
@@ -27,7 +27,7 @@ class FlowNode:
     """
     parent_uuid: str
     node_type: str
-    node_template: node_interface.NodeTemplate
+    node_template: node_store.NodeTemplate
     node_default: schemas.NodeDefault
     node_schema: NodeSchemaInformation
     node_inputs: NodeStepInputs
@@ -251,10 +251,10 @@ class FlowNode:
         self.results.errors = None
         self.add_lead_to_in_depend_source()
         _ = self.hash
-        self.node_template = node_interface.node_dict.get(self.node_type)
+        self.node_template = node_store.node_dict.get(self.node_type)
         if self.node_template is None:
             raise Exception(f'Node template {self.node_type} not found')
-        self.node_default = node_interface.node_defaults.get(self.node_type)
+        self.node_default = node_store.node_defaults.get(self.node_type)
         self.setting_input = setting_input  # wait until the end so that the hash is calculated correctly
 
     @property
@@ -678,9 +678,10 @@ class FlowNode:
 
         if results_exists(self.hash):
             logger.warning('Not implemented')
+            clear_task_from_worker(self.hash)
 
     def needs_run(self, performance_mode: bool, node_logger: NodeLogger = None,
-                  execution_location: schemas.ExecutionLocationsLiteral = "worker") -> bool:
+                  execution_location: schemas.ExecutionLocationsLiteral = "remote") -> bool:
         """Determines if the node needs to be executed.
 
         The decision is based on its run state, caching settings, and execution mode.
@@ -723,6 +724,8 @@ class FlowNode:
         Raises:
             Exception: Propagates exceptions from the execution.
         """
+        self.clear_table_example()
+
         def example_data_generator():
             example_data = None
 
@@ -735,6 +738,7 @@ class FlowNode:
         resulting_data = self.get_resulting_data()
 
         if not performance_mode:
+            self.node_stats.has_run_with_current_setup = True
             self.results.example_data_generator = example_data_generator()
             self.node_schema.result_schema = self.results.resulting_data.schema
             self.node_stats.has_completed_last_run = True
@@ -854,8 +858,12 @@ class FlowNode:
             logger.warning('No external process to cancel')
         self.node_stats.is_canceled = True
 
-    def execute_node(self, run_location: schemas.ExecutionLocationsLiteral, reset_cache: bool = False,
-                     performance_mode: bool = False, retry: bool = True, node_logger: NodeLogger = None):
+    def execute_node(self, run_location: schemas.ExecutionLocationsLiteral,
+                     reset_cache: bool = False,
+                     performance_mode: bool = False,
+                     retry: bool = True,
+                     node_logger: NodeLogger = None,
+                     optimize_for_downstream: bool = True):
         """Orchestrates the execution, handling location, caching, and retries.
 
         Args:
@@ -864,25 +872,33 @@ class FlowNode:
             performance_mode: If True, optimizes for speed over diagnostics.
             retry: If True, allows retrying execution on recoverable errors.
             node_logger: The logger for this node execution.
+            optimize_for_downstream: If true, operations that shuffle the order of rows are fully cached and provided as
+            input to downstream steps
 
         Raises:
             Exception: If the node_logger is not defined.
         """
         if node_logger is None:
             raise Exception('Flow logger is not defined')
-        # node_logger = flow_logger.get_node_logger(self.node_id)
+        #  TODO: Simplify which route is being picked there are many duplicate checks
+
         if reset_cache:
             self.remove_cache()
             self.node_stats.has_run_with_current_setup = False
             self.node_stats.has_completed_last_run = False
+
         if self.is_setup:
             node_logger.info(f'Starting to run {self.__name__}')
             if (self.needs_run(performance_mode, node_logger, run_location) or self.node_template.node_group == "output"
                     and not (run_location == 'local')):
+                self.clear_table_example()
                 self.prepare_before_run()
+                self.reset()
                 try:
-                    if ((run_location == 'remote' or (self.node_default.transform_type == 'wide')
-                            and not run_location == 'local')) or self.node_settings.cache_results:
+                    if (((run_location == 'remote' or
+                         (self.node_default.transform_type == 'wide' and optimize_for_downstream) and
+                         not run_location == 'local'))
+                            or self.node_settings.cache_results):
                         node_logger.info('Running the node remotely')
                         if self.node_settings.cache_results:
                             performance_mode = False
@@ -924,7 +940,7 @@ class FlowNode:
                     node_logger.error(f'Error with running the node: {e}')
                     self.node_stats.error = str(e)
                     self.node_stats.has_completed_last_run = False
-                self.node_stats.has_run_with_current_setup = True
+
             else:
                 node_logger.info('Node has already run, not running the node')
         else:
@@ -1108,6 +1124,17 @@ class FlowNode:
         if self.singular_input:
             return self.all_inputs[0]
 
+    def clear_table_example(self) -> None:
+        """
+        Clear the table example in the results so that it clears the existing results
+        Returns:
+            None
+        """
+
+        self.results.example_data = None
+        self.results.example_data_generator = None
+        self.results.example_data_path = None
+
     def get_table_example(self, include_data: bool = False) -> TableExample | None:
         """Generates a `TableExample` model summarizing the node's output.
 
@@ -1136,10 +1163,15 @@ class FlowNode:
                 data = []
             schema = [FileColumn.model_validate(c.get_column_repr()) for c in self.schema]
             fl = self.get_resulting_data()
+            has_example_data = self.results.example_data_generator is not None
+
             return TableExample(node_id=self.node_id,
                                 name=str(self.node_id), number_of_records=999,
                                 number_of_columns=fl.number_of_fields,
-                                table_schema=schema, columns=fl.columns, data=data)
+                                table_schema=schema, columns=fl.columns, data=data,
+                                has_example_data=has_example_data,
+                                has_run_with_current_setup=self.node_stats.has_run_with_current_setup
+                                )
         else:
             logger.warning('getting the table example but the node has not run')
             try:

flowfile_core/flowfile/handler.py

@@ -3,11 +3,25 @@ from dataclasses import dataclass
 from typing import Dict, List
 import os
 from pathlib import Path
+from datetime import datetime
 
 from flowfile_core.flowfile.manage.open_flowfile import open_flow
 from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.schemas.schemas import FlowSettings
 from flowfile_core.flowfile.utils import create_unique_id
+from shared.storage_config import storage
+
+
+def get_flow_save_location(flow_name: str) -> Path:
+    """Gets the initial save location for flow files"""
+    if ".flowfile" not in flow_name:
+        flow_name += ".flowfile"
+    return storage.temp_directory_for_flows / flow_name
+
+
+def create_flow_name() -> str:
+    """Creates a unique flow name"""
+    return datetime.now().strftime("%Y%m%d_%H_%M_%S")+"_flow.flowfile"
 
 
 @dataclass
@@ -57,7 +71,7 @@ class FlowfileHandler:
         else:
             raise Exception('Flow not found')
 
-    def add_flow(self, name: str, flow_path: str) -> int:
+    def add_flow(self, name: str = None, flow_path: str = None) -> int:
         """
         Creates a new flow with a reference to the flow path
         Args:
@@ -69,8 +83,13 @@ class FlowfileHandler:
 
         """
         next_id = create_unique_id()
-        flow_info = FlowSettings(name=name, flow_id=next_id, save_location='', path=flow_path)
-        _ = self.register_flow(flow_info)
+        if not name:
+            name = create_flow_name()
+        if not flow_path:
+            flow_path = get_flow_save_location(name)
+        flow_info = FlowSettings(name=name, flow_id=next_id, save_location=str(flow_path), path=str(flow_path))
+        flow = self.register_flow(flow_info)
+        flow.save_flow(flow.flow_settings.path)
         return next_id
 
     def get_flow_info(self, flow_id: int) -> FlowSettings:

flowfile_core/flowfile/manage/open_flowfile.py

@@ -4,6 +4,7 @@ from flowfile_core.flowfile.manage.compatibility_enhancements import ensure_comp
 import pickle
 from flowfile_core.flowfile.flow_graph import FlowGraph
 from pathlib import Path
+from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
 
 
 def determine_insertion_order(node_storage: schemas.FlowInformation):
@@ -81,7 +82,14 @@ def open_flow(flow_path: Path) -> FlowGraph:
         new_flow.add_node_promise(node_promise)
     for node_id in ingestion_order:
         node_info: schemas.NodeInformation = flow_storage_obj.data[node_id]
-        getattr(new_flow, 'add_' + node_info.type)(node_info.setting_input)
+        if hasattr(node_info.setting_input, "is_user_defined") and node_info.setting_input.is_user_defined:
+            if node_info.type not in CUSTOM_NODE_STORE:
+                continue
+            user_defined_node_class = CUSTOM_NODE_STORE[node_info.type]
+            new_flow.add_user_defined_node(custom_node=user_defined_node_class.from_settings(node_info.setting_input.settings),
+                                           user_defined_node_settings=node_info.setting_input)
+        else:
+            getattr(new_flow, 'add_' + node_info.type)(node_info.setting_input)
         from_node = new_flow.get_node(node_id)
         for output_node_id in node_info.outputs:
             to_node = new_flow.get_node(output_node_id)

flowfile_core/flowfile/node_designer/__init__.py

@@ -0,0 +1,47 @@
+# flowfile_core/flowfile/node_designer/__init__.py
+
+"""
+Tools for creating custom Flowfile nodes.
+
+This package provides all the necessary components for developers to build their own
+custom nodes, define their UI, and implement their data processing logic.
+"""
+
+# Import the core base class for creating a new node
+from .custom_node import CustomNodeBase, NodeSettings
+
+# Import all UI components so they can be used directly
+from .ui_components import (
+    Section,
+    TextInput,
+    NumericInput,
+    ToggleSwitch,
+    SingleSelect,
+    MultiSelect,
+    ColumnSelector,
+    IncomingColumns,  # Important marker class for dynamic dropdowns
+)
+
+# Import the main `Types` object for filtering in ColumnSelector
+from .data_types import Types
+
+
+# Define the public API of this package
+__all__ = [
+    # Core Node Class
+    "CustomNodeBase",
+
+    # UI Components & Layout
+    "Section",
+    "TextInput",
+    "NumericInput",
+    "ToggleSwitch",
+    "SingleSelect",
+    "MultiSelect",
+    "NodeSettings",
+    "ColumnSelector",
+    "IncomingColumns",
+
+    # Data Type Filtering
+    "Types",
+]