Flowfile 0.2.2__py3-none-any.whl

flowfile_core/flowfile/flow_node/models.py

@@ -0,0 +1,111 @@
+
+import pyarrow as pa
+from typing import List, Union, Callable, Optional, Literal
+
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+from flowfile_core.schemas import schemas
+from dataclasses import dataclass
+
+
+@dataclass
+class NodeStepPromise:
+    node_id: Union[str, int]
+    name: str
+    is_start: bool
+    leads_to_id: Optional[List[Union[str, int]]] = None
+    left_input: Optional[Union[str, int]] = None
+    right_input: Optional[Union[str, int]] = None
+    depends_on: Optional[List[Union[str, int]]] = None
+
+
+class NodeStepStats:
+    error: str = None
+    has_run: bool = False
+    active: bool = True
+    is_canceled: bool = False
+
+
+class NodeStepSettings:
+    cache_results: bool = False
+    renew_schema: bool = True
+    streamable: bool = True
+    setup_errors: bool = False
+    breaking_setup_errors: bool = False
+    execute_location: schemas.ExecutionLocationsLiteral = 'auto'
+
+
+class NodeStepInputs:
+    left_input: "FlowNode" = None
+    right_input: "FlowNode" = None
+    main_inputs: List["FlowNode"] = None
+
+    @property
+    def input_ids(self) -> List[int]:
+        if self.main_inputs is not None:
+            return [node_input.node_information.id for node_input in self.get_all_inputs()]
+
+    def get_all_inputs(self) -> List["FlowNode"]:
+        main_inputs = self.main_inputs or []
+        return [v for v in main_inputs + [self.left_input, self.right_input] if v is not None]
+
+    def __repr__(self) -> str:
+        left_repr = f"Left Input: {self.left_input}" if self.left_input else "Left Input: None"
+        right_repr = f"Right Input: {self.right_input}" if self.right_input else "Right Input: None"
+        main_inputs_repr = f"Main Inputs: {self.main_inputs}" if self.main_inputs else "Main Inputs: None"
+        return f"{self.__class__.__name__}({left_repr}, {right_repr}, {main_inputs_repr})"
+
+    def validate_if_input_connection_exists(self, node_input_id: int,
+                                            connection_name: Literal['main', 'left', 'right']) -> bool:
+        if connection_name == 'main':
+            return any((node_input.node_information.id == node_input_id for node_input in self.main_inputs))
+        if connection_name == 'left':
+            return self.left_input.node_information.id == node_input_id
+        if connection_name == 'right':
+            return self.right_input.node_information.id == node_input_id
+
+
+class NodeSchemaInformation:
+    result_schema: Optional[List[FlowfileColumn]] = []  # resulting schema of the function
+    predicted_schema: Optional[List[FlowfileColumn]] = []  # predicted resulting schema of the function
+    input_columns: List[str] = []  # columns that are needed for the function
+    drop_columns: List[str] = []  # columns that will not be available after the function
+    output_columns: List[FlowfileColumn] = []  # columns that will be added with the function
+
+
+class NodeResults:
+    _resulting_data: Optional[FlowDataEngine] = None  # after successful execution this will contain the Flowfile
+    example_data: Optional[
+        FlowDataEngine] = None  # after success this will contain a sample of the data (to provide frontend data)
+    example_data_path: Optional[str] = None  # Path to the arrow table file
+    example_data_generator: Optional[Callable[[], pa.Table]] = None
+    run_time: int = -1
+    errors: Optional[str] = None
+    warnings: Optional[str] = None
+    analysis_data_generator: Optional[Callable[[], pa.Table]] = None
+
+    def __init__(self):
+        self._resulting_data = None
+        self.example_data = None
+        self.run_time = -1
+        self.errors = None
+        self.warnings = None
+        self.example_data_generator = None
+        self.analysis_data_generator = None
+
+    def get_example_data(self) -> pa.Table | None:
+        if self.example_data_generator:
+            return self.example_data_generator()
+
+    @property
+    def resulting_data(self) -> FlowDataEngine:
+        return self._resulting_data
+
+    @resulting_data.setter
+    def resulting_data(self, d: FlowDataEngine):
+        self._resulting_data = d
+
+    def reset(self):
+        self._resulting_data = None
+        self.run_time = -1
+

flowfile_core/flowfile/flow_node/schema_callback.py

@@ -0,0 +1,70 @@
+
+from typing import Callable, Any, Optional, Generic, TypeVar
+from concurrent.futures import ThreadPoolExecutor, Future
+from flowfile_core.configs import logger
+
+
+T = TypeVar('T')
+
+
+class SingleExecutionFuture(Generic[T]):
+    """Single execution of a function in a separate thread with caching of the result."""
+    executor: ThreadPoolExecutor
+    future: Optional[Future[T]]
+    func: Callable[[], T]
+    on_error: Optional[Callable[[Exception], Any]]
+    result_value: Optional[T]
+
+    def __init__(
+        self,
+        func: Callable[[], T],
+        on_error: Optional[Callable[[Exception], Any]] = None
+    ) -> None:
+        """Initialize with function and optional error handler."""
+        self.executor = ThreadPoolExecutor(max_workers=1)
+        self.future = None
+        self.func = func
+        self.on_error = on_error
+        self.result_value = None
+
+    def start(self) -> None:
+        """Start the function execution if not already started."""
+        if not self.future:
+            logger.info("single executor function started")
+            self.future = self.executor.submit(self.func)
+
+    def cleanup(self) -> None:
+        """Clean up resources by clearing the future and shutting down the executor."""
+        # if self.future:
+        #     self.future = None
+        self.executor.shutdown(wait=False)
+
+    def __call__(self) -> Optional[T]:
+        """Execute function if not running and return its result."""
+        if self.result_value:
+            return self.result_value
+        if not self.future:
+            self.start()
+        else:
+            logger.info("Function already running or did complete")
+        try:
+            self.result_value = self.future.result()
+            logger.info("Done with the function")
+            return self.result_value
+        except Exception as e:
+            if self.on_error:
+                return self.on_error(e)
+            else:
+                raise e
+        finally:
+            self.cleanup()
+
+    def reset(self):
+        """Reset the future and result value."""
+        logger.info("Resetting the future and result value")
+        self.result_value = None
+        self.future = None
+
+    def __del__(self) -> None:
+        """Ensure executor is shut down on deletion."""
+        self.cleanup()

flowfile_core/flowfile/handler.py

@@ -0,0 +1,123 @@
+import time
+import random
+import uuid
+import socket
+import hashlib
+
+from dataclasses import dataclass
+from typing import Dict, List
+import os
+from pathlib import Path
+
+from flowfile_core.flowfile.manage.open_flowfile import open_flow
+from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from flowfile_core.schemas.schemas import FlowSettings
+from flowfile_core.configs import logger
+
+
+def create_unique_id() -> int:
+    """
+    Create a unique id for the flowfile with increased randomness while maintaining 32-bit compatibility
+    Returns:
+        int: unique id within 32 bits (4 bytes)
+    """
+    # Get various entropy sources
+    time_ms = int(time.time() * 1000)
+    pid = os.getpid()
+    random_bytes = random.getrandbits(32)
+    mac_addr = uuid.getnode()  # MAC address as integer
+    hostname = socket.gethostname()
+
+    # Combine all sources into a string
+    seed = f"{time_ms}-{pid}-{random_bytes}-{mac_addr}-{hostname}-{uuid.uuid4()}"
+
+    # Create a hash of all entropy sources
+    hash_obj = hashlib.md5(seed.encode())
+    hash_int = int(hash_obj.hexdigest(), 16)
+
+    # Ensure the result fits within 32 bits (4 bytes)
+    unique_id = hash_int & 0xFFFFFFFF
+
+    return unique_id
+
+
+@dataclass
+class FlowfileHandler:
+    _flows: Dict[int, FlowGraph]
+
+    def __init__(self):
+        self._flows = {}
+
+    @property
+    def flowfile_flows(self) -> List[FlowGraph]:
+        return list(self._flows.values())
+
+    def __add__(self, other: FlowGraph) -> int:
+        self._flows[other.flow_id] = other
+        return other.flow_id
+
+    def import_flow(self, flow_path: Path|str) -> int:
+        if isinstance(flow_path, str):
+            flow_path = Path(flow_path)
+        imported_flow = open_flow(flow_path)
+        self._flows[imported_flow.flow_id] = imported_flow
+        imported_flow.flow_settings = self.get_flow_info(imported_flow.flow_id)
+        imported_flow.flow_settings.is_running = False
+        return imported_flow.flow_id
+
+    def register_flow(self, flow_settings: FlowSettings):
+        if flow_settings.flow_id in self._flows:
+            self.delete_flow(flow_settings.flow_id)
+            raise 'flow already registered'
+        else:
+            name = flow_settings.name if flow_settings.name else flow_settings.flow_id
+            self._flows[flow_settings.flow_id] = FlowGraph(name=name, flow_id=flow_settings.flow_id, flow_settings=flow_settings)
+        return self.get_flow(flow_settings.flow_id)
+
+    def get_flow(self, flow_id: int) -> FlowGraph | None:
+        return self._flows.get(flow_id, None)
+
+    def delete_flow(self, flow_id: int):
+        flow = self._flows.pop(flow_id)
+        del flow
+
+    def save_flow(self, flow_id: int, flow_path: str):
+        flow = self.get_flow(flow_id)
+        if flow:
+            flow.save_flow(flow_path)
+        else:
+            raise Exception('Flow not found')
+
+    def add_flow(self, name: str, flow_path: str) -> int:
+        """
+        Creates a new flow with a reference to the flow path
+        Args:
+            name (str): The name of the flow
+            flow_path (str): The path to the flow file
+
+        Returns:
+            int: The flow id
+
+        """
+        next_id = create_unique_id()
+        flow_info = FlowSettings(name=name, flow_id=next_id, save_location='', path=flow_path)
+        _ = self.register_flow(flow_info)
+        return next_id
+
+    def get_flow_info(self, flow_id: int) -> FlowSettings:
+        flow = self.get_flow(flow_id)
+        if not flow:
+            raise Exception(f'Flow {flow_id} not found')
+        flow_exists = os.path.exists(flow.flow_settings.path)
+        last_modified_ts = os.path.getmtime(flow.flow_settings.path) if flow_exists else -1
+        flow.flow_settings.modified_on = last_modified_ts
+        return flow.flow_settings
+
+    def get_node(self, flow_id: int, node_id: int):
+        flow = self.get_flow(flow_id)
+        if not flow:
+            raise Exception(f'Flow {flow_id} not found')
+        node = flow.get_node(node_id)
+        if not node:
+            raise Exception(f'Node {node_id} not found in flow {flow_id}')
+        return node

flowfile_core/flowfile/manage/compatibility_enhancements.py

@@ -0,0 +1,70 @@
+from flowfile_core.schemas import schemas, input_schema
+
+
+def ensure_compatibility_node_read(node_read: input_schema.NodeRead):
+    if hasattr(node_read, 'received_file'):
+        if not hasattr(node_read.received_file, 'fields'):
+            print('setting fields')
+            setattr(node_read.received_file, 'fields', [])
+
+
+def ensure_compatibility_node_output(node_output: input_schema.NodeOutput):
+    if hasattr(node_output, 'output_settings'):
+        if not hasattr(node_output.output_settings, 'abs_file_path'):
+            new_output_settings = input_schema.OutputSettings.model_validate(node_output.output_settings.model_dump())
+            setattr(node_output, 'output_settings', new_output_settings)
+
+
+def ensure_compatibility_node_select(node_select: input_schema.NodeSelect):
+    if hasattr(node_select, 'select_input'):
+        if any(not hasattr(select_input, 'position') for select_input in node_select.select_input):
+            for _index, select_input in enumerate(node_select.select_input):
+                setattr(select_input, 'position', _index)
+        if not hasattr(node_select, 'sorted_by'):
+            setattr(node_select, 'sorted_by', 'none')
+
+
+def ensure_compatibility_node_joins(node_settings: input_schema.NodeFuzzyMatch | input_schema.NodeJoin):
+    if any(not hasattr(r, 'position') for r in node_settings.join_input.right_select.renames):
+        for _index, select_input in enumerate(node_settings.join_input.right_select.renames +
+                                              node_settings.join_input.left_select.renames):
+            setattr(select_input, 'position', _index)
+
+
+def ensure_description(node: input_schema.NodeBase):
+    if not hasattr(node, 'description'):
+        setattr(node, 'description', '')
+
+
+def ensure_compatibility_node_polars(node_polars: input_schema.NodePolarsCode):
+    if hasattr(node_polars, 'depending_on_id'):
+        setattr(node_polars, 'depending_on_ids', [getattr(node_polars, 'depending_on_id')])
+
+
+def ensure_compatibility(flow_storage_obj: schemas.FlowInformation, flow_path: str):
+    if not hasattr(flow_storage_obj, 'flow_settings'):
+        flow_settings = schemas.FlowSettings(flow_id=flow_storage_obj.flow_id, path=flow_path,
+                                             name=flow_storage_obj.flow_name)
+        setattr(flow_storage_obj, 'flow_settings', flow_settings)
+        flow_storage_obj = schemas.FlowInformation.model_validate(flow_storage_obj)
+    elif not hasattr(getattr(flow_storage_obj, 'flow_settings'), 'execution_location'):
+        setattr(getattr(flow_storage_obj, 'flow_settings'), 'execution_location', 'auto')
+    elif not hasattr(flow_storage_obj.flow_settings, 'is_running'):
+        setattr(flow_storage_obj.flow_settings, 'is_running', False)
+        setattr(flow_storage_obj.flow_settings, 'is_canceled', False)
+    if not hasattr(flow_storage_obj.flow_settings, 'show_detailed_progress'):
+        setattr(flow_storage_obj.flow_settings, 'show_detailed_progress', True)
+    for _id, node_information in flow_storage_obj.data.items():
+        if not hasattr(node_information, 'setting_input'):
+            continue
+        if node_information.setting_input.__class__.__name__ == 'NodeRead':
+            ensure_compatibility_node_read(node_information.setting_input)
+        elif node_information.setting_input.__class__.__name__ == 'NodeSelect':
+            ensure_compatibility_node_select(node_information.setting_input)
+        elif node_information.setting_input.__class__.__name__ == 'NodeOutput':
+            ensure_compatibility_node_output(node_information.setting_input)
+        elif node_information.setting_input.__class__.__name__ in ('NodeJoin', 'NodeFuzzyMatch'):
+            ensure_compatibility_node_joins(node_information.setting_input)
+        elif node_information.setting_input.__class__.__name__ == 'NodePolarsCode':
+            ensure_compatibility_node_polars(node_information.setting_input)
+        ensure_description(node_information.setting_input)

flowfile_core/flowfile/manage/open_flowfile.py

@@ -0,0 +1,136 @@
+from flowfile_core.schemas import schemas, input_schema
+from typing import List, Tuple
+from flowfile_core.flowfile.manage.compatibility_enhancements import ensure_compatibility
+import pickle
+from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from pathlib import Path
+
+
+def determine_insertion_order(node_storage: schemas.FlowInformation):
+    ingest_order: List[int] = []
+    ingest_order_set: set[int] = set()
+    all_nodes = set(node_storage.data.keys())
+
+    def assure_output_id(input_node: schemas.NodeInformation, output_node: schemas.NodeInformation):
+        # assure the output id is in the list with outputs of the input node this is a quick fix
+        if output_node.id not in input_node.outputs:
+            input_node.outputs.append(output_node.id)
+
+    def determine_order(node_id: int):
+        current_node = node_storage.data.get(node_id)
+        if current_node is None:
+            return
+        output_ids = current_node.outputs
+        main_input_ids = current_node.input_ids if current_node.input_ids else []
+        input_ids = [n for n in [current_node.left_input_id,
+                                 current_node.right_input_id] + main_input_ids if (n is not None
+                                                                                   and n not in
+                                                                                   ingest_order_set)]
+        if len(input_ids) > 0:
+            for input_id in input_ids:
+                new_node = node_storage.data.get(input_id)
+                if new_node is None:
+                    ingest_order.append(current_node.id)
+                    ingest_order_set.add(current_node.id)
+                    continue
+                assure_output_id(new_node, current_node)
+                if new_node.id not in ingest_order_set:
+                    determine_order(input_id)
+        elif current_node.id not in ingest_order_set:
+            ingest_order.append(current_node.id)
+            ingest_order_set.add(current_node.id)
+
+        for output_id in output_ids:
+            if output_id not in ingest_order_set:
+                determine_order(output_id)
+
+    if len(node_storage.node_starts) > 0:
+        determine_order(node_storage.node_starts[0])
+    # add the random not connected nodes
+    else:
+        for node_id in all_nodes:
+            determine_order(node_id)
+    ingest_order += list(all_nodes - ingest_order_set)
+    return ingest_order
+
+
+def open_flow(flow_path: Path) -> FlowGraph:
+    """
+    Open a flowfile from a given path
+    Args:
+        flow_path (Path): The absolute or relative path to the flowfile
+
+    Returns:
+        FlowGraph: The flowfile object
+    """
+    with open(str(flow_path), 'rb') as f:
+        flow_storage_obj: schemas.FlowInformation = pickle.load(f)
+    flow_storage_obj.flow_settings.path = str(flow_path)
+    flow_storage_obj.flow_settings.name = str(flow_path.stem)
+    flow_storage_obj.flow_name = str(flow_path.stem)
+    ensure_compatibility(flow_storage_obj, str(flow_path))
+    ingestion_order = determine_insertion_order(flow_storage_obj)
+    new_flow = FlowGraph(flow_id=flow_storage_obj.flow_id, name=flow_storage_obj.flow_name,
+                        flow_settings=flow_storage_obj.flow_settings)
+    for node_id in ingestion_order:
+        node_info: schemas.NodeInformation = flow_storage_obj.data[node_id]
+        node_promise = input_schema.NodePromise(flow_id=new_flow.flow_id, node_id=node_info.id,
+                                                pos_x=node_info.x_position, pos_y=node_info.y_position,
+                                                node_type=node_info.type)
+        if hasattr(node_info.setting_input, 'cache_results'):
+            node_promise.cache_results = node_info.setting_input.cache_results
+        new_flow.add_node_promise(node_promise)
+    for node_id in ingestion_order:
+        node_info: schemas.NodeInformation = flow_storage_obj.data[node_id]
+        getattr(new_flow, 'add_' + node_info.type)(node_info.setting_input)
+        from_node = new_flow.get_node(node_id)
+        for output_node_id in node_info.outputs:
+            to_node = new_flow.get_node(output_node_id)
+            if to_node is not None:
+                output_node_obj = flow_storage_obj.data[output_node_id]
+                is_left_input = (output_node_obj.left_input_id == node_id) and (to_node.left_input.node_id != node_id
+                                                                                if to_node.left_input is not None
+                                                                                else True)
+                is_right_input = (output_node_obj.right_input_id == node_id) and (to_node.right_input.node_id != node_id
+                                                                                  if to_node.right_input is not None
+                                                                                  else True)
+                is_main_input = node_id in output_node_obj.input_ids
+                if is_left_input:
+                    insert_type = 'left'
+                elif is_right_input:
+                    insert_type = 'right'
+                elif is_main_input:
+                    insert_type = 'main'
+                else:
+                    continue
+                to_node.add_node_connection(from_node, insert_type)
+            else:
+                from_node.delete_lead_to_node(output_node_id)
+                if not (from_node.node_id, output_node_id) in flow_storage_obj.node_connections:
+                    continue
+                flow_storage_obj.node_connections.pop(
+                    flow_storage_obj.node_connections.index((from_node.node_id, output_node_id)))
+
+    for missing_connection in set(flow_storage_obj.node_connections) - set(new_flow.node_connections):
+        to_node = new_flow.get_node(missing_connection[1])
+        if not to_node.has_input:
+            test_if_circular_connection(missing_connection, new_flow)
+            from_node = new_flow.get_node(missing_connection[0])
+            if from_node:
+                to_node.add_node_connection(from_node)
+    return new_flow
+
+
+def test_if_circular_connection(connection: Tuple[int, int], flow: FlowGraph):
+    to_node = flow.get_node(connection[1])
+    leads_to_nodes_queue = [n for n in to_node.leads_to_nodes]
+    circular_connection: bool = False
+    while len(leads_to_nodes_queue) > 0:
+        leads_to_node = leads_to_nodes_queue.pop(0)
+        if leads_to_node.node_id == connection[0]:
+            circular_connection = True
+            break
+        for leads_to_node_leads_to in leads_to_node.leads_to_nodes:
+            leads_to_nodes_queue.append(leads_to_node_leads_to)
+    return circular_connection
+

flowfile_core/flowfile/setting_generator/__init__.py

@@ -0,0 +1,2 @@
+
+from flowfile_core.flowfile.setting_generator.settings import setting_generator, setting_updator

flowfile_core/flowfile/setting_generator/setting_generator.py

@@ -0,0 +1,41 @@
+from flowfile_core.configs import logger
+from typing import Callable
+
+
+class SettingGenerator:
+    setting_generator_set = set()
+
+    def __init__(self):
+        self.setting_generator_set = set()
+
+    def add_setting_generator_func(self, f: Callable):
+        self.setting_generator_set.update([f.__name__])
+        setattr(self, f.__name__, f)
+
+    def get_setting_generator(self, node_type: str) -> Callable:
+        logger.info('getting setting generator for ' + node_type)
+
+        if node_type in self.setting_generator_set:
+            logger.info('setting generator found')
+            return getattr(self, node_type)
+        else:
+            return lambda x: x
+
+
+class SettingUpdator:
+    setting_updator_set = set()
+
+    def __init__(self):
+        self.setting_updator_set = set()
+
+    def add_setting_updator_func(self, f: Callable):
+        self.setting_updator_set.update([f.__name__])
+        setattr(self, f.__name__, f)
+
+    def get_setting_updator(self, node_type: str) -> Callable:
+        logger.info('getting setting updator for ' + node_type)
+        if node_type in self.setting_updator_set:
+            logger.info('setting updator found')
+            return getattr(self, node_type)
+        else:
+            return lambda x: x