Flowfile 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

flowfile_core/flowfile/flow_node/models.py

@@ -1,15 +1,32 @@
 
 import pyarrow as pa
 from typing import List, Union, Callable, Optional, Literal
+from dataclasses import dataclass
+
+# Forward declaration for type hints to avoid circular imports
+if False:
+    from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
 from flowfile_core.schemas import schemas
-from dataclasses import dataclass
 
 
 @dataclass
 class NodeStepPromise:
+    """
+    A lightweight, temporary representation of a node during the initial
+    graph construction phase, before full `FlowNode` objects are created.
+
+    Attributes:
+        node_id: The unique identifier for the node.
+        name: The display name of the node.
+        is_start: A boolean indicating if this is a starting node with no inputs.
+        leads_to_id: A list of node IDs that this node connects to.
+        left_input: The ID of the node connected to the left input port.
+        right_input: The ID of the node connected to the right input port.
+        depends_on: A list of node IDs that this node depends on for main inputs.
+    """
     node_id: Union[str, int]
     name: str
     is_start: bool
@@ -20,6 +37,9 @@ class NodeStepPromise:
 
 
 class NodeStepStats:
+    """
+    Tracks the execution status and statistics of a `FlowNode`.
+    """
     error: str = None
     _has_run_with_current_setup: bool = False
     has_completed_last_run: bool = False
@@ -31,23 +51,46 @@ class NodeStepStats:
                  has_completed_last_run: bool = False,
                  active: bool = True,
                  is_canceled: bool = False):
+        """
+        Initializes the node's statistics.
+
+        :param error: Any error message from the last run.
+        :param has_run_with_current_setup: Flag indicating if the node has run successfully with its current configuration.
+        :param has_completed_last_run: Flag indicating if the last triggered run finished (successfully or not).
+        :param active: Flag indicating if the node is active in the flow.
+        :param is_canceled: Flag indicating if the last run was canceled.
+        """
         self.error = error
         self._has_run_with_current_setup = has_run_with_current_setup
         self.has_completed_last_run = has_completed_last_run
         self.active = active
         self.is_canceled = is_canceled
 
-    def __repr__(self):
+    def __repr__(self) -> str:
+        """
+        Provides a string representation of the node's stats.
+        :return: A string detailing the current stats.
+        """
         return (f"NodeStepStats(error={self.error}, has_run_with_current_setup={self.has_run_with_current_setup}, "
                 f"has_completed_last_run={self.has_completed_last_run}, "
                 f"active={self.active}, is_canceled={self.is_canceled})")
 
     @property
     def has_run_with_current_setup(self) -> bool:
+        """
+        Checks if the node has run successfully with its current settings and inputs.
+        This is the primary flag for caching.
+        :return: True if the node is considered up-to-date, False otherwise.
+        """
         return self._has_run_with_current_setup
 
     @has_run_with_current_setup.setter
     def has_run_with_current_setup(self, value: bool):
+        """
+        Sets the run status of the node.
+        If set to True, it implies the last run was completed successfully.
+        :param value: The new boolean status.
+        """
         if value:
             self._has_run_with_current_setup = True
             self.has_completed_last_run = True
@@ -56,6 +99,17 @@ class NodeStepStats:
 
 
 class NodeStepSettings:
+    """
+    Holds the configuration settings that control a node's execution behavior.
+
+    Attributes:
+        cache_results: If True, the node will cache its results to avoid re-computation.
+        renew_schema: If True, the schema will be re-evaluated on changes.
+        streamable: If True, the node can process data in a streaming fashion.
+        setup_errors: If True, indicates a non-blocking error occurred during setup.
+        breaking_setup_errors: If True, indicates an error occurred that prevents execution.
+        execute_location: The preferred location for execution ('auto', 'local', 'remote').
+    """
     cache_results: bool = False
     renew_schema: bool = True
     streamable: bool = True
@@ -65,20 +119,40 @@ class NodeStepSettings:
 
 
 class NodeStepInputs:
+    """
+    Manages the input connections for a `FlowNode`.
+
+    Attributes:
+        left_input: The `FlowNode` connected to the left input port.
+        right_input: The `FlowNode` connected to the right input port.
+        main_inputs: A list of `FlowNode` objects connected to the main input port(s).
+    """
     left_input: "FlowNode" = None
     right_input: "FlowNode" = None
     main_inputs: List["FlowNode"] = None
 
     @property
-    def input_ids(self) -> List[int]:
+    def input_ids(self) -> List[int] | None:
+        """
+        Gets the IDs of all connected input nodes.
+        :return: A list of integer node IDs.
+        """
         if self.main_inputs is not None:
             return [node_input.node_information.id for node_input in self.get_all_inputs()]
 
     def get_all_inputs(self) -> List["FlowNode"]:
+        """
+        Retrieves a single list containing all input nodes (main, left, and right).
+        :return: A list of all connected `FlowNode` objects.
+        """
         main_inputs = self.main_inputs or []
         return [v for v in main_inputs + [self.left_input, self.right_input] if v is not None]
 
     def __repr__(self) -> str:
+        """
+        Provides a string representation of the node's inputs.
+        :return: A string detailing the connected inputs.
+        """
         left_repr = f"Left Input: {self.left_input}" if self.left_input else "Left Input: None"
         right_repr = f"Right Input: {self.right_input}" if self.right_input else "Right Input: None"
         main_inputs_repr = f"Main Inputs: {self.main_inputs}" if self.main_inputs else "Main Inputs: None"
@@ -86,27 +160,46 @@ class NodeStepInputs:
 
     def validate_if_input_connection_exists(self, node_input_id: int,
                                             connection_name: Literal['main', 'left', 'right']) -> bool:
-        if connection_name == 'main':
-            return any((node_input.node_information.id == node_input_id for node_input in self.main_inputs))
-        if connection_name == 'left':
+        """
+        Checks if a connection from a specific node ID exists on a given port.
+
+        :param node_input_id: The ID of the source node to check for.
+        :param connection_name: The name of the input port ('main', 'left', 'right').
+        :return: True if the connection exists, False otherwise.
+        """
+        if connection_name == 'main' and self.main_inputs:
+            return any(node_input.node_information.id == node_input_id for node_input in self.main_inputs)
+        if connection_name == 'left' and self.left_input:
             return self.left_input.node_information.id == node_input_id
         if connection_name == 'right':
             return self.right_input.node_information.id == node_input_id
 
 
 class NodeSchemaInformation:
-    result_schema: Optional[List[FlowfileColumn]] = None  # resulting schema of the function
-    predicted_schema: Optional[List[FlowfileColumn]] = None  # predicted resulting schema of the function
-    input_columns: List[str] = []  # columns that are needed for the function
-    drop_columns: List[str] = []  # columns that will not be available after the function
-    output_columns: List[FlowfileColumn] = []  # columns that will be added with the function
+    """
+    Stores all schema-related information for a `FlowNode`.
+
+    Attributes:
+        result_schema: The actual output schema after a successful execution.
+        predicted_schema: The predicted output schema, calculated without full execution.
+        input_columns: A list of column names the node requires from its inputs.
+        drop_columns: A list of column names that will be dropped by the node.
+        output_columns: A list of `FlowfileColumn` objects that will be added by the node.
+    """
+    result_schema: Optional[List[FlowfileColumn]] = None
+    predicted_schema: Optional[List[FlowfileColumn]] = None
+    input_columns: List[str] = []
+    drop_columns: List[str] = []
+    output_columns: List[FlowfileColumn] = []
 
 
 class NodeResults:
-    _resulting_data: Optional[FlowDataEngine] = None  # after successful execution this will contain the Flowfile
-    example_data: Optional[
-        FlowDataEngine] = None  # after success this will contain a sample of the data (to provide frontend data)
-    example_data_path: Optional[str] = None  # Path to the arrow table file
+    """
+    Stores the outputs of a `FlowNode`'s execution, including data, errors, and metadata.
+    """
+    _resulting_data: Optional[FlowDataEngine] = None
+    example_data: Optional[FlowDataEngine] = None
+    example_data_path: Optional[str] = None
     example_data_generator: Optional[Callable[[], pa.Table]] = None
     run_time: int = -1
     errors: Optional[str] = None
@@ -122,19 +215,31 @@ class NodeResults:
         self.example_data_generator = None
         self.analysis_data_generator = None
 
-    def get_example_data(self) -> pa.Table | None:
+    def get_example_data(self) -> Optional[pa.Table]:
+        """
+        Executes the generator to fetch a sample of the resulting data.
+        :return: A PyArrow Table containing a sample of the data, or None.
+        """
         if self.example_data_generator:
             return self.example_data_generator()
 
     @property
-    def resulting_data(self) -> FlowDataEngine:
+    def resulting_data(self) -> Optional[FlowDataEngine]:
+        """
+        Gets the full resulting data from the node's execution.
+        :return: A `FlowDataEngine` instance containing the result, or None.
+        """
         return self._resulting_data
 
     @resulting_data.setter
-    def resulting_data(self, d: FlowDataEngine):
+    def resulting_data(self, d: Optional[FlowDataEngine]):
+        """
+        Sets the resulting data.
+        :param d: The `FlowDataEngine` instance to store.
+        """
         self._resulting_data = d
 
     def reset(self):
+        """Resets all result attributes to their default, empty state."""
         self._resulting_data = None
-        self.run_time = -1
-
+        self.run_time = -1

flowfile_core/flowfile/handler.py

@@ -1,8 +1,3 @@
-import time
-import random
-import uuid
-import socket
-import hashlib
 
 from dataclasses import dataclass
 from typing import Dict, List
@@ -12,33 +7,7 @@ from pathlib import Path
 from flowfile_core.flowfile.manage.open_flowfile import open_flow
 from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.schemas.schemas import FlowSettings
-from flowfile_core.configs import logger
-
-
-def create_unique_id() -> int:
-    """
-    Create a unique id for the flowfile with increased randomness while maintaining 32-bit compatibility
-    Returns:
-        int: unique id within 32 bits (4 bytes)
-    """
-    # Get various entropy sources
-    time_ms = int(time.time() * 1000)
-    pid = os.getpid()
-    random_bytes = random.getrandbits(32)
-    mac_addr = uuid.getnode()  # MAC address as integer
-    hostname = socket.gethostname()
-
-    # Combine all sources into a string
-    seed = f"{time_ms}-{pid}-{random_bytes}-{mac_addr}-{hostname}-{uuid.uuid4()}"
-
-    # Create a hash of all entropy sources
-    hash_obj = hashlib.md5(seed.encode())
-    hash_int = int(hash_obj.hexdigest(), 16)
-
-    # Ensure the result fits within 32 bits (4 bytes)
-    unique_id = hash_int & 0xFFFFFFFF
-
-    return unique_id
+from flowfile_core.flowfile.utils import create_unique_id
 
 
 @dataclass
@@ -71,7 +40,7 @@ class FlowfileHandler:
             raise 'flow already registered'
         else:
             name = flow_settings.name if flow_settings.name else flow_settings.flow_id
-            self._flows[flow_settings.flow_id] = FlowGraph(name=name, flow_id=flow_settings.flow_id, flow_settings=flow_settings)
+            self._flows[flow_settings.flow_id] = FlowGraph(name=name, flow_settings=flow_settings)
         return self.get_flow(flow_settings.flow_id)
 
     def get_flow(self, flow_id: int) -> FlowGraph | None:

flowfile_core/flowfile/manage/open_flowfile.py

@@ -70,8 +70,7 @@ def open_flow(flow_path: Path) -> FlowGraph:
     flow_storage_obj.flow_name = str(flow_path.stem)
     ensure_compatibility(flow_storage_obj, str(flow_path))
     ingestion_order = determine_insertion_order(flow_storage_obj)
-    new_flow = FlowGraph(flow_id=flow_storage_obj.flow_id, name=flow_storage_obj.flow_name,
-                        flow_settings=flow_storage_obj.flow_settings)
+    new_flow = FlowGraph(name=flow_storage_obj.flow_name, flow_settings=flow_storage_obj.flow_settings)
     for node_id in ingestion_order:
         node_info: schemas.NodeInformation = flow_storage_obj.data[node_id]
         node_promise = input_schema.NodePromise(flow_id=new_flow.flow_id, node_id=node_info.id,

flowfile_core/flowfile/util/calculate_layout.py

@@ -4,7 +4,6 @@ from collections import defaultdict, deque
 from typing import List, Dict, Set, Tuple, TYPE_CHECKING
 
 if TYPE_CHECKING:
-    # Make sure this import path is correct for your project structure
     from flowfile_core.flowfile.flow_graph import FlowGraph
 
 
@@ -64,7 +63,6 @@ def calculate_layered_layout(
                 if child_node.node_id in node_ids:
                     if child_node.node_id not in adj[node.node_id]:
                         adj[node.node_id].append(child_node.node_id)
-                    # Assuming primary method works or in_degree is handled elsewhere
                     in_degree[child_node.node_id] += 1
 
     stages: Dict[int, List[int]] = defaultdict(list)

flowfile_core/flowfile/utils.py

@@ -1,11 +1,15 @@
 import os
-import hashlib
 import json
 import shutil
 
-from datetime import datetime, date, time
+import datetime
 from typing import List
 from decimal import Decimal
+import time
+import random
+import uuid
+import socket
+import hashlib
 
 
 def generate_sha256_hash(data: bytes):
@@ -26,11 +30,11 @@ def snake_case_to_camel_case(text: str) -> str:
 
 
 def json_default(val):
-    if isinstance(val, datetime):
+    if isinstance(val, datetime.datetime):
         return val.isoformat(timespec='microseconds')
-    elif isinstance(val, date):
+    elif isinstance(val, datetime.date):
         return val.isoformat()
-    elif isinstance(val, time):
+    elif isinstance(val, datetime.time):
         return val.isoformat()
     elif hasattr(val, '__dict__'):
         return val.__dict__
@@ -113,3 +117,30 @@ def batch_generator(input_list: List, batch_size: int = 10000):
             yield input_list
             input_list = []
             run = False
+
+
+def create_unique_id() -> int:
+    """
+    Create a unique id for the flowfile with increased randomness while maintaining 32-bit compatibility
+    Returns:
+        int: unique id within 32 bits (4 bytes)
+    """
+    # Get various entropy sources
+    time_ms = int(time.time() * 1000)
+    pid = os.getpid()
+    random_bytes = random.getrandbits(32)
+    mac_addr = uuid.getnode()  # MAC address as integer
+    hostname = socket.gethostname()
+
+    # Combine all sources into a string
+    seed = f"{time_ms}-{pid}-{random_bytes}-{mac_addr}-{hostname}-{uuid.uuid4()}"
+
+    # Create a hash of all entropy sources
+
+    hash_obj = hashlib.sha256(seed.encode())
+    hash_int = int(hash_obj.hexdigest(), 16)
+
+    # Ensure the result fits within 32 bits (4 bytes)
+    unique_id = hash_int & 0xFFFFFFFF
+
+    return unique_id

flowfile_core/main.py

@@ -27,7 +27,11 @@ server_instance = None
 
 @asynccontextmanager
 async def shutdown_handler(app: FastAPI):
-    """Handle graceful shutdown of the application."""
+    """Handles the graceful startup and shutdown of the FastAPI application.
+
+    This context manager ensures that resources, such as log files, are cleaned
+    up properly when the application is terminated.
+    """
     print('Starting core application...')
     try:
         yield
@@ -77,31 +81,46 @@ app.include_router(cloud_connections_router, prefix="/cloud_connections", tags=[
 
 @app.post("/shutdown")
 async def shutdown():
-    """Endpoint to handle graceful shutdown"""
-    ServerRun.exit = True
-    print(f"ServerRun.exit = {ServerRun.exit}")
-    if server_instance:
-        # Schedule the shutdown
-        await asyncio.create_task(trigger_shutdown())
-    return {"message": "Shutting down"}
+    """An API endpoint to gracefully shut down the server.
+
+    This endpoint sets a flag that the Uvicorn server checks, allowing it
+    to terminate cleanly. A background task is used to trigger the shutdown
+    after the HTTP response has been sent.
+    """
+    # Use a background task to trigger the shutdown after the response is sent
+    background_tasks = ServerRun()
+    background_tasks.add_task(trigger_shutdown)
+    return {"message": "Server is shutting down"}
 
 
 async def trigger_shutdown():
-    """Trigger the actual shutdown after responding to the client"""
-    await asyncio.sleep(1)  # Give time for the response to be sent
+    """(Internal) Triggers the actual server shutdown.
+
+    Waits for a moment to allow the `/shutdown` response to be sent before
+    telling the Uvicorn server instance to exit.
+    """
+    await asyncio.sleep(1)
     if server_instance:
         server_instance.should_exit = True
 
 
 def signal_handler(signum, frame):
-    """Handle shutdown signals"""
+    """Handles OS signals like SIGINT (Ctrl+C) and SIGTERM for graceful shutdown."""
     print(f"Received signal {signum}")
     if server_instance:
         server_instance.should_exit = True
 
 
 def run(host: str = None, port: int = None):
-    """Run the FastAPI app with graceful shutdown"""
+    """Runs the FastAPI application using Uvicorn.
+
+    This function configures and starts the Uvicorn server, setting up
+    signal handlers to ensure a graceful shutdown.
+
+    Args:
+        host: The host to bind the server to. Defaults to `SERVER_HOST` from settings.
+        port: The port to bind the server to. Defaults to `SERVER_PORT` from settings.
+    """
     global server_instance
 
     # Use values from settings if not explicitly provided
@@ -134,7 +153,7 @@ def run(host: str = None, port: int = None):
         print("Received interrupt signal, shutting down...")
     finally:
         server_instance = None
-        print("Core server shutdown complete")
+        print("Server has shut down.")
 
 
 if __name__ == "__main__":

flowfile_core/routes/cloud_connections.py

@@ -27,13 +27,11 @@ def create_cloud_storage_connection(input_connection: FullCloudStorageConnection
     """
     Create a new cloud storage connection.
     Parameters
-    ----------
-    input_connection: FullCloudStorageConnection schema containing connection details
-    current_user: User obtained from Depends(get_current_active_user)
-    db: Session obtained from Depends(get_db)
+        input_connection: FullCloudStorageConnection schema containing connection details
+        current_user: User obtained from Depends(get_current_active_user)
+        db: Session obtained from Depends(get_db)
     Returns
-    -------
-    Dict with a success message
+        Dict with a success message
     """
     logger.info(f'Create cloud connection {input_connection.connection_name}')
     try:
@@ -70,12 +68,10 @@ def get_cloud_connections(
     """
     Get all cloud storage connections for the current user.
     Parameters
-    ----------
-    db: Session obtained from Depends(get_db)
-    current_user: User obtained from Depends(get_current_active_user)
+        db: Session obtained from Depends(get_db)
+        current_user: User obtained from Depends(get_current_active_user)
 
     Returns
-    -------
-    List[FullCloudStorageConnectionInterface]
+        List[FullCloudStorageConnectionInterface]
     """
     return get_all_cloud_connections_interface(db, current_user.id)

flowfile_core/routes/logs.py

@@ -33,9 +33,7 @@ async def format_sse_message(data: str) -> str:
 
 @router.post("/logs/{flow_id}", tags=['flow_logging'])
 async def add_log(flow_id: int, log_message: str):
-    """
-    Adds a log message to the log file for a given flow_id.
-    """
+    """Adds a log message to the log file for a given flow_id."""
     flow = flow_file_handler.get_flow(flow_id)
     if not flow:
         raise HTTPException(status_code=404, detail="Flow not found")
@@ -45,9 +43,7 @@ async def add_log(flow_id: int, log_message: str):
 
 @router.post("/raw_logs", tags=['flow_logging'])
 async def add_raw_log(raw_log_input: schemas.RawLogInput):
-    """
-    Adds a log message to the log file for a given flow_id.
-    """
+    """Adds a log message to the log file for a given flow_id."""
     logger.info('Adding raw logs')
     flow = flow_file_handler.get_flow(raw_log_input.flowfile_flow_id)
     if not flow:

flowfile_core/routes/public.py

@@ -7,4 +7,5 @@ router = APIRouter()
 
 @router.get("/", tags=['admin'])
 async def docs_redirect():
+    """ Redirects to the documentation page."""
     return RedirectResponse(url='/docs')