Flowfile 0.3.9__py3-none-any.whl → 0.5.1__py3-none-any.whl

flowfile_core/flowfile/flow_node/flow_node.py

@@ -8,11 +8,11 @@ from flowfile_core.configs.flow_logger import NodeLogger
 
 from flowfile_core.schemas.output_model import TableExample, FileColumn, NodeData
 from flowfile_core.flowfile.utils import get_hash
-from flowfile_core.configs.node_store import nodes as node_interface
+from flowfile_core.configs import node_store
 from flowfile_core.flowfile.setting_generator import setting_generator, setting_updator
 from time import sleep
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations import (
-    ExternalDfFetcher, ExternalSampler, results_exists, get_external_df_result,
+    ExternalDfFetcher, ExternalSampler, clear_task_from_worker, results_exists, get_external_df_result,
     ExternalDatabaseFetcher, ExternalDatabaseWriter, ExternalCloudWriter)
 from flowfile_core.flowfile.flow_node.models import (NodeStepSettings, NodeStepInputs, NodeSchemaInformation,
                                                      NodeStepStats, NodeResults)
@@ -27,7 +27,7 @@ class FlowNode:
     """
     parent_uuid: str
     node_type: str
-    node_template: node_interface.NodeTemplate
+    node_template: node_store.NodeTemplate
     node_default: schemas.NodeDefault
     node_schema: NodeSchemaInformation
     node_inputs: NodeStepInputs
@@ -251,10 +251,10 @@ class FlowNode:
         self.results.errors = None
         self.add_lead_to_in_depend_source()
         _ = self.hash
-        self.node_template = node_interface.node_dict.get(self.node_type)
+        self.node_template = node_store.node_dict.get(self.node_type)
         if self.node_template is None:
             raise Exception(f'Node template {self.node_type} not found')
-        self.node_default = node_interface.node_defaults.get(self.node_type)
+        self.node_default = node_store.node_defaults.get(self.node_type)
         self.setting_input = setting_input  # wait until the end so that the hash is calculated correctly
 
     @property
@@ -370,6 +370,7 @@ class FlowNode:
                                       self.node_inputs.main_inputs] if self.node_inputs.main_inputs is not None else None
         node_information.setting_input = self.setting_input
         node_information.outputs = [n.node_id for n in self.leads_to_nodes]
+        node_information.description = self.setting_input.description if hasattr(self.setting_input, 'description') else ''
         node_information.is_setup = self.is_setup
         node_information.x_position = self.setting_input.pos_x
         node_information.y_position = self.setting_input.pos_y
@@ -498,6 +499,7 @@ class FlowNode:
         Returns:
             A list of FlowfileColumn objects representing the predicted schema.
         """
+
         if self.node_schema.predicted_schema and not force:
             return self.node_schema.predicted_schema
         if self.schema_callback is not None and (self.node_schema.predicted_schema is None or force):
@@ -514,6 +516,7 @@ class FlowNode:
         if predicted_data is not None and predicted_data.schema is not None:
             self.print('Calculating the schema based on the predicted resulting data')
             self.node_schema.predicted_schema = self._predicted_data_getter().schema
+
         return self.node_schema.predicted_schema
 
     @property
@@ -678,9 +681,10 @@ class FlowNode:
 
         if results_exists(self.hash):
             logger.warning('Not implemented')
+            clear_task_from_worker(self.hash)
 
     def needs_run(self, performance_mode: bool, node_logger: NodeLogger = None,
-                  execution_location: schemas.ExecutionLocationsLiteral = "worker") -> bool:
+                  execution_location: schemas.ExecutionLocationsLiteral = "remote") -> bool:
         """Determines if the node needs to be executed.
 
         The decision is based on its run state, caching settings, and execution mode.
@@ -723,6 +727,8 @@ class FlowNode:
         Raises:
             Exception: Propagates exceptions from the execution.
         """
+        self.clear_table_example()
+
         def example_data_generator():
             example_data = None
 
@@ -735,6 +741,7 @@ class FlowNode:
         resulting_data = self.get_resulting_data()
 
         if not performance_mode:
+            self.node_stats.has_run_with_current_setup = True
             self.results.example_data_generator = example_data_generator()
             self.node_schema.result_schema = self.results.resulting_data.schema
             self.node_stats.has_completed_last_run = True
@@ -854,8 +861,12 @@ class FlowNode:
             logger.warning('No external process to cancel')
         self.node_stats.is_canceled = True
 
-    def execute_node(self, run_location: schemas.ExecutionLocationsLiteral, reset_cache: bool = False,
-                     performance_mode: bool = False, retry: bool = True, node_logger: NodeLogger = None):
+    def execute_node(self, run_location: schemas.ExecutionLocationsLiteral,
+                     reset_cache: bool = False,
+                     performance_mode: bool = False,
+                     retry: bool = True,
+                     node_logger: NodeLogger = None,
+                     optimize_for_downstream: bool = True):
         """Orchestrates the execution, handling location, caching, and retries.
 
         Args:
@@ -864,25 +875,33 @@ class FlowNode:
             performance_mode: If True, optimizes for speed over diagnostics.
             retry: If True, allows retrying execution on recoverable errors.
             node_logger: The logger for this node execution.
+            optimize_for_downstream: If true, operations that shuffle the order of rows are fully cached and provided as
+            input to downstream steps
 
         Raises:
             Exception: If the node_logger is not defined.
         """
         if node_logger is None:
             raise Exception('Flow logger is not defined')
-        # node_logger = flow_logger.get_node_logger(self.node_id)
+        #  TODO: Simplify which route is being picked there are many duplicate checks
+
         if reset_cache:
             self.remove_cache()
             self.node_stats.has_run_with_current_setup = False
             self.node_stats.has_completed_last_run = False
+
         if self.is_setup:
             node_logger.info(f'Starting to run {self.__name__}')
             if (self.needs_run(performance_mode, node_logger, run_location) or self.node_template.node_group == "output"
                     and not (run_location == 'local')):
+                self.clear_table_example()
                 self.prepare_before_run()
+                self.reset()
                 try:
-                    if ((run_location == 'remote' or (self.node_default.transform_type == 'wide')
-                            and not run_location == 'local')) or self.node_settings.cache_results:
+                    if (((run_location == 'remote' or
+                         (self.node_default.transform_type == 'wide' and optimize_for_downstream) and
+                         not run_location == 'local'))
+                            or self.node_settings.cache_results):
                         node_logger.info('Running the node remotely')
                         if self.node_settings.cache_results:
                             performance_mode = False
@@ -924,7 +943,7 @@ class FlowNode:
                     node_logger.error(f'Error with running the node: {e}')
                     self.node_stats.error = str(e)
                     self.node_stats.has_completed_last_run = False
-                self.node_stats.has_run_with_current_setup = True
+
             else:
                 node_logger.info('Node has already run, not running the node')
         else:
@@ -1108,6 +1127,17 @@ class FlowNode:
         if self.singular_input:
             return self.all_inputs[0]
 
+    def clear_table_example(self) -> None:
+        """
+        Clear the table example in the results so that it clears the existing results
+        Returns:
+            None
+        """
+
+        self.results.example_data = None
+        self.results.example_data_generator = None
+        self.results.example_data_path = None
+
     def get_table_example(self, include_data: bool = False) -> TableExample | None:
         """Generates a `TableExample` model summarizing the node's output.
 
@@ -1136,10 +1166,15 @@ class FlowNode:
                 data = []
             schema = [FileColumn.model_validate(c.get_column_repr()) for c in self.schema]
             fl = self.get_resulting_data()
+            has_example_data = self.results.example_data_generator is not None
+
             return TableExample(node_id=self.node_id,
                                 name=str(self.node_id), number_of_records=999,
                                 number_of_columns=fl.number_of_fields,
-                                table_schema=schema, columns=fl.columns, data=data)
+                                table_schema=schema, columns=fl.columns, data=data,
+                                has_example_data=has_example_data,
+                                has_run_with_current_setup=self.node_stats.has_run_with_current_setup
+                                )
         else:
             logger.warning('getting the table example but the node has not run')
             try:

flowfile_core/flowfile/flow_node/models.py

@@ -130,13 +130,14 @@ class NodeStepInputs:
     main_inputs: List["FlowNode"] = None
 
     @property
-    def input_ids(self) -> List[int] | None:
+    def input_ids(self) -> List[int]:
         """
         Gets the IDs of all connected input nodes.
         :return: A list of integer node IDs.
         """
         if self.main_inputs is not None:
             return [node_input.node_information.id for node_input in self.get_all_inputs()]
+        return []
 
     def get_all_inputs(self) -> List["FlowNode"]:
         """

flowfile_core/flowfile/handler.py

@@ -3,11 +3,25 @@ from dataclasses import dataclass
 from typing import Dict, List
 import os
 from pathlib import Path
+from datetime import datetime
 
-from flowfile_core.flowfile.manage.open_flowfile import open_flow
+from flowfile_core.flowfile.manage.io_flowfile import open_flow
 from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.schemas.schemas import FlowSettings
 from flowfile_core.flowfile.utils import create_unique_id
+from shared.storage_config import storage
+
+
+def get_flow_save_location(flow_name: str) -> Path:
+    """Gets the initial save location for flow files"""
+    if ".yaml" not in flow_name and ".yml" not in flow_name:
+        flow_name += ".yaml"
+    return storage.temp_directory_for_flows / flow_name
+
+
+def create_flow_name() -> str:
+    """Creates a unique flow name"""
+    return datetime.now().strftime("%Y%m%d_%H_%M_%S")+"_flow.yaml"
 
 
 @dataclass
@@ -25,7 +39,7 @@ class FlowfileHandler:
         self._flows[other.flow_id] = other
         return other.flow_id
 
-    def import_flow(self, flow_path: Path|str) -> int:
+    def import_flow(self, flow_path: Path | str) -> int:
         if isinstance(flow_path, str):
             flow_path = Path(flow_path)
         imported_flow = open_flow(flow_path)
@@ -57,7 +71,7 @@ class FlowfileHandler:
         else:
             raise Exception('Flow not found')
 
-    def add_flow(self, name: str, flow_path: str) -> int:
+    def add_flow(self, name: str = None, flow_path: str = None) -> int:
         """
         Creates a new flow with a reference to the flow path
         Args:
@@ -69,8 +83,13 @@ class FlowfileHandler:
 
         """
         next_id = create_unique_id()
-        flow_info = FlowSettings(name=name, flow_id=next_id, save_location='', path=flow_path)
-        _ = self.register_flow(flow_info)
+        if not name:
+            name = create_flow_name()
+        if not flow_path:
+            flow_path = get_flow_save_location(name)
+        flow_info = FlowSettings(name=name, flow_id=next_id, save_location=str(flow_path), path=str(flow_path))
+        flow = self.register_flow(flow_info)
+        flow.save_flow(flow.flow_settings.path)
         return next_id
 
     def get_flow_info(self, flow_id: int) -> FlowSettings: