Flowfile 0.2.2__py3-none-any.whl

flowfile_core/flowfile/util/calculate_layout.py

@@ -0,0 +1,137 @@
+import logging
+import math
+from collections import defaultdict, deque
+from typing import List, Dict, Set, Tuple, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    # Make sure this import path is correct for your project structure
+    from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+
+
+def calculate_layered_layout(
+    graph: 'FlowGraph',
+    x_spacing: int = 250,
+    y_spacing: int = 100,
+    initial_y: int = 50
+) -> Dict[int, Tuple[int, int]]:
+    """
+    Calculates node positions using a simplified layered approach for a
+    LEFT-TO-RIGHT flow. Stages determine horizontal position (X), and
+    nodes within a stage are spread vertically (Y).
+
+    Args:
+        graph: The FlowGraph instance.
+        x_spacing: Horizontal distance between stage centers (X spacing).
+        y_spacing: Vertical distance between node centers within a stage (Y spacing).
+        initial_y: Reference Y position for vertically centering stages.
+
+    Returns:
+        A dictionary mapping node_id to calculated (pos_x, pos_y).
+    """
+    if not graph._node_db:
+        return {}
+
+    nodes = graph.nodes
+    node_ids = {node.node_id for node in nodes}
+    adj: Dict[int, List[int]] = defaultdict(list)
+    rev_adj: Dict[int, List[int]] = defaultdict(list)
+    in_degree: Dict[int, int] = defaultdict(int)
+
+    # --- Graph Building Stage ---
+    try:
+        connections = graph.node_connections
+        for u, v in connections:
+            if u in node_ids and v in node_ids:
+                if v not in adj[u]:
+                    adj[u].append(v)
+                if u not in rev_adj[v]:
+                    rev_adj[v].append(u)
+        for node_id in node_ids:
+            in_degree[node_id] = len(rev_adj.get(node_id, []))
+
+    except Exception as e:
+        logging.warning(e)
+        # Fallback graph building
+        adj.clear()
+        rev_adj.clear()
+        in_degree.clear()
+        for node in nodes:
+            if node.node_id not in in_degree:
+                in_degree[node.node_id] = 0
+        for node in nodes:
+            children = node.leads_to_nodes
+            for child_node in children:
+                if child_node.node_id in node_ids:
+                    if child_node.node_id not in adj[node.node_id]:
+                        adj[node.node_id].append(child_node.node_id)
+                    # Assuming primary method works or in_degree is handled elsewhere
+                    in_degree[child_node.node_id] += 1
+
+    stages: Dict[int, List[int]] = defaultdict(list)
+    node_stage: Dict[int, int] = {}
+    initial_sources = sorted([node_id for node_id in node_ids if in_degree.get(node_id, 0) == 0])
+    queue = deque(initial_sources)
+    current_stage = 0
+    processed_nodes_count = 0
+    current_in_degree = defaultdict(int, in_degree)
+
+    while queue:
+        stage_size = len(queue)
+        processing_order = sorted(list(queue))
+        queue.clear()
+        nodes_in_current_stage = []
+        for u in processing_order:
+            nodes_in_current_stage.append(u)
+            node_stage[u] = current_stage
+            processed_nodes_count += 1
+            for v in sorted(adj.get(u, [])):
+                current_in_degree[v] -= 1
+                if current_in_degree[v] == 0:
+                    queue.append(v)
+                elif current_in_degree[v] < 0:
+                    # Basic warning for potential issues
+                    print(f"[Layout Warning] Node {v} negative in-degree.")
+        if nodes_in_current_stage:
+            stages[current_stage] = nodes_in_current_stage
+            current_stage += 1
+
+    # Handle unprocessed nodes (cycles/disconnected)
+    if processed_nodes_count != len(node_ids):
+        print(f"[Layout Warning] Cycles or disconnected? Processed {processed_nodes_count}/{len(node_ids)}.")
+        unprocessed_nodes = sorted(list(node_ids - set(node_stage.keys())))
+        for node_id in unprocessed_nodes:
+            node_stage[node_id] = current_stage
+            stages[current_stage].append(node_id)
+        if unprocessed_nodes:
+            stages[current_stage].sort()
+            current_stage += 1
+
+    # --- Coordinate Assignment Stage ---
+    positions: Dict[int, Tuple[int, int]] = {}
+    max_stage_height = 0
+
+    for stage_index, node_ids_in_stage in stages.items():
+        if not node_ids_in_stage:
+            continue
+        sorted_nodes = sorted(node_ids_in_stage)
+        stages[stage_index] = sorted_nodes
+        stage_height = (len(sorted_nodes) - 1) * y_spacing
+        max_stage_height = max(max_stage_height, stage_height)
+
+    center_offset_y = max_stage_height / 2
+
+    for stage_index, node_ids_in_stage in stages.items():
+        if not node_ids_in_stage:
+            continue
+
+        pos_x = stage_index * x_spacing
+
+        stage_height = (len(node_ids_in_stage) - 1) * y_spacing
+        current_center_y = initial_y + center_offset_y
+        start_y = current_center_y - (stage_height / 2)
+
+        for i, node_id in enumerate(node_ids_in_stage):
+            pos_y = start_y + i * y_spacing
+            positions[node_id] = (int(pos_x), int(pos_y))
+
+    return positions

flowfile_core/flowfile/util/execution_orderer.py

@@ -0,0 +1,141 @@
+from typing import List, Dict, Set
+from flowfile_core.flowfile.flow_node.flow_node import FlowNode
+from flowfile_core.configs import logger
+from collections import deque, defaultdict
+
+
+def determine_execution_order(all_nodes: List[FlowNode], flow_starts: List[FlowNode] = None) -> List[FlowNode]:
+    """
+    Determines the execution order of nodes using topological sorting based on node dependencies.
+
+    Args:
+        all_nodes (List[FlowNode]): A list of all nodes (steps) in the flow.
+        flow_starts (List[FlowNode], optional): A list of starting nodes for the flow. If not provided, the function starts with nodes having zero in-degree.
+
+    Returns:
+        List[FlowNode]: A list of nodes in the order they should be executed.
+
+    Raises:
+        Exception: If a cycle is detected, meaning that a valid execution order cannot be determined.
+    """
+    node_map = build_node_map(all_nodes)
+    in_degree, adjacency_list = compute_in_degrees_and_adjacency_list(all_nodes, node_map)
+
+    queue, visited_nodes = initialize_queue(flow_starts, all_nodes, in_degree)
+
+    execution_order = perform_topological_sort(queue, node_map, in_degree, adjacency_list, visited_nodes)
+    if len(execution_order) != len(node_map):
+        raise Exception("Cycle detected in the graph. Execution order cannot be determined.")
+
+    logger.info(f"execution order: \n {[node for node in execution_order if node.is_correct]}")
+
+    return execution_order
+
+
+def build_node_map(all_nodes: List[FlowNode]) -> Dict[str, FlowNode]:
+    """
+    Creates a mapping from node ID to node object.
+
+    Args:
+        all_nodes (List[FlowNode]): A list of all nodes (steps) in the flow.
+
+    Returns:
+        Dict[str, FlowNode]: A dictionary mapping node IDs to FlowNode objects.
+    """
+    return {node.node_id: node for node in all_nodes}
+
+
+def compute_in_degrees_and_adjacency_list(all_nodes: List[FlowNode],
+                                          node_map: Dict[str, FlowNode]) -> (Dict[str, int], Dict[str, List[str]]):
+    """
+    Computes the in-degree and adjacency list for all nodes.
+
+    Args:
+        all_nodes (List[FlowNode]): A list of all nodes (steps) in the flow.
+        node_map (Dict[str, FlowNode]): A dictionary mapping node IDs to FlowNode objects.
+
+    Returns:
+        (Dict[str, int], Dict[str, List[str]]): A tuple containing:
+            - in_degree: A dictionary mapping node IDs to their in-degree count.
+            - adjacency_list: A dictionary mapping node IDs to a list of their connected nodes (outgoing edges).
+    """
+    in_degree = defaultdict(int)
+    adjacency_list = defaultdict(list)
+
+    for node in all_nodes:
+        for next_node in node.leads_to_nodes:
+            adjacency_list[node.node_id].append(next_node.node_id)
+            in_degree[next_node.node_id] += 1
+            if next_node.node_id not in node_map:
+                node_map[next_node.node_id] = next_node
+
+    return in_degree, adjacency_list
+
+
+def initialize_queue(flow_starts: List[FlowNode], all_nodes: List[FlowNode], in_degree: Dict[str, int]) -> (
+deque, Set[str]):
+    """
+    Initializes the queue with nodes that have zero in-degree or based on specified flow start nodes.
+
+    Args:
+        flow_starts (List[FlowNode]): A list of starting nodes for the flow.
+        all_nodes (List[FlowNode]): A list of all nodes (steps) in the flow.
+        in_degree (Dict[str, int]): A dictionary mapping node IDs to their in-degree count.
+
+    Returns:
+        (deque, Set[str]): A tuple containing:
+            - queue: A deque containing nodes with zero in-degree or specified start nodes.
+            - visited_nodes: A set of visited node IDs to track processing state.
+    """
+    queue = deque()
+    visited_nodes = set()
+
+    if flow_starts and len(flow_starts) > 0:
+        for node in flow_starts:
+            if in_degree[node.node_id] == 0:
+                queue.append(node.node_id)
+                visited_nodes.add(node.node_id)
+            else:
+                logger.warning(f"Flow start node {node.node_id} has non-zero in-degree.")
+    else:
+        for node in all_nodes:
+            if in_degree[node.node_id] == 0:
+                queue.append(node.node_id)
+                visited_nodes.add(node.node_id)
+
+    return queue, visited_nodes
+
+
+def perform_topological_sort(queue: deque, node_map: Dict[str, FlowNode], in_degree: Dict[str, int],
+                             adjacency_list: Dict[str, List[str]], visited_nodes: Set[str]) -> List[FlowNode]:
+    """
+    Performs topological sorting to determine the execution order of nodes.
+
+    Args:
+        queue (deque): A deque containing nodes with zero in-degree.
+        node_map (Dict[str, FlowNode]): A dictionary mapping node IDs to FlowNode objects.
+        in_degree (Dict[str, int]): A dictionary mapping node IDs to their in-degree count.
+        adjacency_list (Dict[str, List[str]]): A dictionary mapping node IDs to a list of their connected nodes (outgoing edges).
+        visited_nodes (Set[str]): A set of visited node IDs.
+
+    Returns:
+        List[FlowNode]: A list of nodes in the order they should be executed.
+    """
+    execution_order = []
+    logger.info('Starting topological sort to determine execution order')
+
+    while queue:
+        current_node_id = queue.popleft()
+        current_node = node_map.get(current_node_id)
+        if current_node is None:
+            logger.warning(f"Node with ID {current_node_id} not found in the node map.")
+            continue
+        execution_order.append(current_node)
+
+        for next_node_id in adjacency_list.get(current_node_id, []):
+            in_degree[next_node_id] -= 1
+            if in_degree[next_node_id] == 0 and next_node_id not in visited_nodes:
+                queue.append(next_node_id)
+                visited_nodes.add(next_node_id)
+
+    return execution_order

flowfile_core/flowfile/utils.py

@@ -0,0 +1,106 @@
+from typing import List
+import os
+import shutil
+import hashlib
+from datetime import datetime
+import json
+
+
+def generate_sha256_hash(data: bytes):
+    sha256 = hashlib.sha256()
+    sha256.update(data)
+    return sha256.hexdigest()
+
+
+def create_directory_if_not_exists(directory: str):
+    if not os.path.exists(directory):
+        os.mkdir(directory)
+
+
+def snake_case_to_camel_case(text: str) -> str:
+    # Split the text by underscores, capitalize each piece, and join them together
+    transformed_text = ''.join(word.capitalize() for word in text.split('_'))
+    return transformed_text
+
+
+def json_default(val):
+    if isinstance(val, datetime):
+        return val.isoformat(timespec='microseconds')
+    elif hasattr(val, '__dict__'):
+        return val.__dict__
+    else:
+        raise Exception('Value is not serializable')
+
+
+def json_dumps(thing) -> str:
+    return json.dumps(
+        thing,
+        default=json_default,
+        ensure_ascii=False,
+        sort_keys=True,
+        indent=None,
+        separators=(',', ':'),
+    )
+
+
+def get_hash(val):
+    if hasattr(val, 'overridden_hash') and val.overridden_hash():
+        val = hash(val)
+    elif hasattr(val, '__dict__'):
+        val = {k: v for k, v in val.__dict__.items() if k not in {'pos_x', 'pos_y'}}
+    elif hasattr(val, 'json'):
+        pass
+    return generate_sha256_hash(json_dumps(val).encode('utf-8'))
+
+
+def cleanup(start_location: str = 'temp_storage'):
+    def get_all_files_and_folders(_start_location) -> List[str]:
+        inspect_items = [_start_location]
+        output = []
+        while len(inspect_items) > 0:
+            attributes = []
+            for inspect_item in inspect_items:
+                output.append(inspect_item)
+                if os.path.isdir(inspect_item):
+                    dir_attributes = [os.path.join(inspect_item, _item) for _item in os.listdir(inspect_item)]
+                    if len(dir_attributes) > 0:
+                        attributes += dir_attributes
+            inspect_items = attributes
+        return output
+
+    output = get_all_files_and_folders(start_location)
+
+    # get level of dept of folder and sort based on that
+    actions = [(_path.count(os.sep), _path) for _path in output]
+    actions = [action for action in actions if action[0] > 0]
+    files_to_delete = {action[0]: [] for action in actions}
+    directories_to_delete = {action[0]: [] for action in actions}
+    for action in actions:
+        if os.path.isfile(action[1]):
+            files_to_delete[action[0]].append(action[1])
+        else:
+            directories_to_delete[action[0]].append(action[1])
+
+    files_to_delete = list(files_to_delete.items())
+    directories_to_delete = list(directories_to_delete.items())
+    files_to_delete.sort(key=lambda x: x[0], reverse=True)
+    directories_to_delete.sort(key=lambda x: x[0], reverse=True)
+    for action in files_to_delete:
+        for _f in action[1]:
+            os.remove(_f)
+    for action in directories_to_delete:
+        for _f in action[1]:
+            shutil.rmtree(_f)
+
+
+def batch_generator(input_list: List, batch_size: int = 10000):
+    run: bool = True
+    while run:
+        if len(input_list) > batch_size:
+            yield input_list[:batch_size]
+            input_list = input_list[batch_size:]
+        else:
+            yield input_list
+            input_list = []
+            run = False
+

flowfile_core/main.py

@@ -0,0 +1,138 @@
+import asyncio
+import os
+import signal
+from contextlib import asynccontextmanager
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from flowfile_core import ServerRun
+from flowfile_core.configs.settings import SERVER_HOST, SERVER_PORT, WORKER_HOST, WORKER_PORT, WORKER_URL
+
+from flowfile_core.routes.auth import router as auth_router
+from flowfile_core.routes.secrets import router as secrets_router
+from flowfile_core.routes.routes import router
+from flowfile_core.routes.public import router as public_router
+from flowfile_core.routes.logs import router as logs_router
+
+from flowfile_core.configs.flow_logger import clear_all_flow_logs
+
+os.environ["FLOWFILE_MODE"] = "electron"
+
+should_exit = False
+server_instance = None
+
+
+@asynccontextmanager
+async def shutdown_handler(app: FastAPI):
+    """Handle graceful shutdown of the application."""
+    print('Starting core application...')
+    try:
+        yield
+    finally:
+        print('Shutting down core application...')
+        print("Cleaning up core service resources...")
+        clear_all_flow_logs()
+        await asyncio.sleep(0.1)  # Give a moment for cleanup
+
+
+# Initialize FastAPI with metadata
+app = FastAPI(
+    title='Flowfile Backend',
+    version='0.1',
+    description='Backend for the Flowfile application',
+    lifespan=shutdown_handler
+)
+
+# Configure CORS
+origins = [
+    "http://localhost",
+    "http://localhost:5173",
+    "http://localhost:3000",
+    "http://localhost:8080",
+    "http://localhost:8081",
+    "http://localhost:4173",
+    "http://localhost:4174",
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(public_router)
+app.include_router(router)
+app.include_router(logs_router, tags=["logs"])
+app.include_router(auth_router, prefix="/auth", tags=["auth"])
+app.include_router(secrets_router, prefix="/secrets", tags=["secrets"])
+
+
+@app.post("/shutdown")
+async def shutdown():
+    """Endpoint to handle graceful shutdown"""
+    ServerRun.exit = True
+    print(f"ServerRun.exit = {ServerRun.exit}")
+    if server_instance:
+        # Schedule the shutdown
+        await asyncio.create_task(trigger_shutdown())
+    return {"message": "Shutting down"}
+
+
+async def trigger_shutdown():
+    """Trigger the actual shutdown after responding to the client"""
+    await asyncio.sleep(1)  # Give time for the response to be sent
+    if server_instance:
+        server_instance.should_exit = True
+
+
+def signal_handler(signum, frame):
+    """Handle shutdown signals"""
+    print(f"Received signal {signum}")
+    if server_instance:
+        server_instance.should_exit = True
+
+
+def run(host: str = None, port: int = None):
+    """Run the FastAPI app with graceful shutdown"""
+    global server_instance
+
+    # Use values from settings if not explicitly provided
+    if host is None:
+        host = SERVER_HOST
+    if port is None:
+        port = SERVER_PORT
+
+    print(f"Starting server on {host}:{port}")
+    print(f"Worker configured at {WORKER_URL} (host: {WORKER_HOST}, port: {WORKER_PORT})")
+
+    # Setup signal handlers
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    config = uvicorn.Config(
+        app,
+        host=host,
+        port=port,
+        loop="asyncio"
+    )
+    server = uvicorn.Server(config)
+    server_instance = server  # Store server instance globally
+
+    print('Starting core server...')
+    print('Core server started')
+
+    try:
+        server.run()
+    except KeyboardInterrupt:
+        print("Received interrupt signal, shutting down...")
+    finally:
+        server_instance = None
+        print("Core server shutdown complete")
+
+
+if __name__ == "__main__":
+    run()

flowfile_core/routes/auth.py

@@ -0,0 +1,34 @@
+# app_routes/auth.py
+
+import os
+
+from fastapi import APIRouter, Depends, HTTPException, status, Request
+from sqlalchemy.orm import Session
+
+from flowfile_core.auth.jwt import get_current_active_user, create_access_token
+from flowfile_core.auth.models import Token, User
+from flowfile_core.database.connection import get_db
+
+router = APIRouter()
+
+
+@router.post("/token", response_model=Token)
+async def login_for_access_token(request: Request, db: Session = Depends(get_db)):
+    # In Electron mode, auto-authenticate without requiring form data
+    if os.environ.get("FLOWFILE_MODE") == "electron" or 1 == 1:
+        access_token = create_access_token(data={"sub": "local_user"})
+        return {"access_token": access_token, "token_type": "bearer"}
+    else:
+        # In Docker mode, authenticate against database
+        # Would typically process form data here
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Docker mode authentication not implemented yet",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+
+# Get current user endpoint
+@router.get("/users/me", response_model=User)
+async def read_users_me(current_user=Depends(get_current_active_user)):
+    return current_user