Flowfile 0.3.9__py3-none-any.whl → 0.4.0__py3-none-any.whl

flowfile_core/routes/user_defined_components.py

@@ -0,0 +1,55 @@
+
+from typing import Dict, Any
+
+from fastapi import APIRouter, HTTPException, Depends
+
+from flowfile_core import flow_file_handler
+# Core modules
+from flowfile_core.auth.jwt import get_current_active_user
+from flowfile_core.configs import logger
+from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
+# File handling
+from flowfile_core.schemas import input_schema
+from flowfile_core.utils.utils import camel_case_to_snake_case
+
+# External dependencies
+
+
+router = APIRouter()
+
+
+@router.get("/custom-node-schema", summary="Get a simple UI schema")
+def get_simple_custom_object(flow_id: int, node_id: int):
+    """
+    This endpoint returns a hardcoded JSON object that represents the UI
+    for our SimpleFilterNode.
+    """
+    try:
+        node = flow_file_handler.get_node(flow_id=flow_id, node_id=node_id)
+    except Exception as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    user_defined_node = CUSTOM_NODE_STORE.get(node.node_type)
+
+    if not user_defined_node:
+        raise HTTPException(status_code=404, detail=f"Node type '{node.node_type}' not found")
+    if node.is_setup:
+        settings = node.setting_input.settings
+        return user_defined_node.from_settings(settings).get_frontend_schema()
+    return user_defined_node().get_frontend_schema()
+
+
+@router.post("/update_user_defined_node", tags=["transform"])
+def update_user_defined_node(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)):
+    input_data['user_id'] = current_user.id
+    node_type = camel_case_to_snake_case(node_type)
+    flow_id = int(input_data.get('flow_id'))
+    logger.info(f'Updating the data for flow: {flow_id}, node {input_data["node_id"]}')
+    flow = flow_file_handler.get_flow(flow_id)
+    user_defined_model = CUSTOM_NODE_STORE.get(node_type)
+    if not user_defined_model:
+        raise HTTPException(status_code=404, detail=f"Node type '{node_type}' not found")
+
+    user_defined_node_settings = input_schema.UserDefinedNode.model_validate(input_data)
+    initialized_model = user_defined_model.from_settings(user_defined_node_settings.settings)
+
+    flow.add_user_defined_node(custom_node=initialized_model, user_defined_node_settings=user_defined_node_settings)

flowfile_core/schemas/input_schema.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Literal, Iterator
+from typing import List, Optional, Literal, Iterator, Any
 from flowfile_core.schemas import transform_schema
 from pathlib import Path
 import os
@@ -195,6 +195,7 @@ class NodeBase(BaseModel):
     description: Optional[str] = ''
     user_id: Optional[int] = None
     is_flow_output: Optional[bool] = False
+    is_user_defined: Optional[bool] = False  # Indicator if the node is a user defined node
 
 
 class NodeSingleInput(NodeBase):
@@ -517,3 +518,8 @@ class NodeRecordCount(NodeSingleInput):
 class NodePolarsCode(NodeMultiInput):
     """Settings for a node that executes arbitrary user-provided Polars code."""
     polars_code_input: transform_schema.PolarsCodeInput
+
+
+class UserDefinedNode(NodeMultiInput):
+    """Settings for a node that contains the user defined node information"""
+    settings: Any

flowfile_core/schemas/output_model.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional, Any
+from typing import List, Dict, Optional, Any, Literal
 from pydantic import BaseModel, Field
 from datetime import datetime
 import time
@@ -21,10 +21,11 @@ class RunInformation(BaseModel):
     flow_id: int
     start_time: Optional[datetime] = Field(default_factory=datetime.now)
     end_time: Optional[datetime] = None
-    success: bool
+    success: Optional[bool] = None
     nodes_completed: int = 0
     number_of_nodes: int = 0
     node_step_result: List[NodeResult]
+    run_type: Literal["fetch_one", "full_run"]
 
 
 class BaseItem(BaseModel):
@@ -61,6 +62,8 @@ class TableExample(BaseModel):
     table_schema: List[FileColumn]
     columns: List[str]
     data: Optional[List[Dict]] = {}
+    has_example_data: bool = False
+    has_run_with_current_setup: bool = False
 
 
 class NodeData(BaseModel):

flowfile_core/schemas/schemas.py

@@ -5,6 +5,9 @@ from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
 ExecutionModeLiteral = Literal['Development', 'Performance']
 ExecutionLocationsLiteral = Literal['local', 'remote']
 
+# Type literals for classifying nodes.
+NodeTypeLiteral = Literal['input', 'output', 'process']
+TransformTypeLiteral = Literal['narrow', 'wide', 'other']
 
 def get_global_execution_location() -> ExecutionLocationsLiteral:
     """
@@ -135,9 +138,14 @@ class NodeTemplate(BaseModel):
     output: int
     image: str
     multi: bool = False
+    node_type: NodeTypeLiteral
+    transform_type: TransformTypeLiteral
     node_group: str
     prod_ready: bool = True
     can_be_start: bool = False
+    drawer_title: str = "Node title"
+    drawer_intro: str = "Drawer into"
+    custom_node: Optional[bool] = False
 
 
 class NodeInformation(BaseModel):
@@ -261,9 +269,6 @@ class VueFlowInput(BaseModel):
     node_inputs: List[NodeInput]
 
 
-# Type literals for classifying nodes.
-NodeTypeLiteral = Literal['input', 'output', 'process']
-TransformTypeLiteral = Literal['narrow', 'wide', 'other']
 
 
 class NodeDefault(BaseModel):

flowfile_core/schemas/transform_schema.py

@@ -8,6 +8,7 @@ from typing import NamedTuple
 
 from pl_fuzzy_frame_match.models import FuzzyMapping
 
+FuzzyMap = FuzzyMapping  # For backwards compatibility
 
 def get_func_type_mapping(func: str):
     """Infers the output data type of common aggregation functions."""

flowfile_core/utils/validate_setup.py

@@ -3,7 +3,7 @@ as have a component in flowfile_frontend"""
 
 from flowfile_core.schemas import input_schema
 from flowfile_core.flowfile.flow_graph import FlowGraph
-from flowfile_core.configs.node_store.nodes import nodes_list, NodeTemplate
+from flowfile_core.configs.node_store import nodes_list, NodeTemplate
 import inspect
 
 
@@ -31,6 +31,8 @@ def validate_setup():
     Raises ValueError if any node is missing either.
     """
     for node in nodes_list:
+        if node.custom_node:
+            continue
         check_if_node_has_add_function_in_flow_graph(node)
         check_if_node_has_input_schema_definition(node)
 

flowfile_worker/__init__.py

@@ -1,15 +1,15 @@
 from typing import Dict
-import tempfile
 import threading
 import multiprocessing
-import os
-import shutil
-multiprocessing.set_start_method('spawn', force=True)
+from shared.storage_config import storage
 
+multiprocessing.set_start_method('spawn', force=True)
 
 from multiprocessing import get_context
 from flowfile_worker.models import Status
+
 mp_context = get_context("spawn")
+
 status_dict: Dict[str, Status] = dict()
 process_dict = dict()
 
@@ -17,39 +17,10 @@ status_dict_lock = threading.Lock()
 process_dict_lock = threading.Lock()
 
 
-class SharedTempDirectory:
-    """A class that mimics tempfile.TemporaryDirectory but uses a fixed directory"""
-    def __init__(self, dir_path):
-        self._path = dir_path
-        os.makedirs(self._path, exist_ok=True)
-
-    @property
-    def name(self):
-        return self._path
-
-    def cleanup(self):
-        """Remove all contents of the temp directory"""
-        try:
-            shutil.rmtree(self._path)
-            os.makedirs(self._path, exist_ok=True)
-            print(f"Cleaned up temporary directory: {self._path}")
-        except Exception as e:
-            print(f"Error during cleanup: {e}")
-
-    def __enter__(self):
-        return self.name
-
-    def __exit__(self, exc, value, tb):
-        self.cleanup()
-
-
 CACHE_EXPIRATION_TIME = 24 * 60 * 60
 
 
-TEMP_DIR = os.getenv('TEMP_DIR')
-if TEMP_DIR:
-    CACHE_DIR = SharedTempDirectory(TEMP_DIR)
-else:
-    CACHE_DIR = tempfile.TemporaryDirectory()
+CACHE_DIR = storage.cache_directory
+
 
 PROCESS_MEMORY_USAGE: Dict[str, float] = dict()

flowfile_worker/main.py

@@ -4,8 +4,11 @@ import signal
 
 from contextlib import asynccontextmanager
 from fastapi import FastAPI
+
+from shared.storage_config import storage
+
 from flowfile_worker.routes import router
-from flowfile_worker import mp_context, CACHE_DIR
+from flowfile_worker import mp_context
 from flowfile_worker.configs import logger, FLOWFILE_CORE_URI, SERVICE_HOST, SERVICE_PORT
 
 
@@ -30,7 +33,7 @@ async def shutdown_handler(app: FastAPI):
                 logger.error(f"Error cleaning up process: {e}")
 
         try:
-            CACHE_DIR.cleanup()
+            storage.cleanup_directories()
         except Exception as e:
             print(f"Error cleaning up cache directory: {e}")
 

flowfile_worker/routes.py

@@ -17,13 +17,21 @@ from flowfile_worker.external_sources.sql_source.main import read_sql_source, wr
 router = APIRouter()
 
 
+def create_and_get_default_cache_dir(flowfile_flow_id: int) -> str:
+    default_cache_dir = CACHE_DIR / str(flowfile_flow_id)
+    default_cache_dir.mkdir(parents=True, exist_ok=True)
+    return str(default_cache_dir)
+
+
 @router.post("/submit_query/")
 def submit_query(polars_script: models.PolarsScript, background_tasks: BackgroundTasks) -> models.Status:
     logger.info(f"Processing query with operation: {polars_script.operation_type}")
 
     try:
         polars_script.task_id = str(uuid.uuid4()) if polars_script.task_id is None else polars_script.task_id
-        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else CACHE_DIR.name
+        default_cache_dir = create_and_get_default_cache_dir(polars_script.flowfile_flow_id)
+
+        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else default_cache_dir
         polars_serializable_object = polars_script.polars_serializable_object()
         file_path = os.path.join(polars_script.cache_dir, f"{polars_script.task_id}.arrow")
         result_type = "polars" if polars_script.operation_type == "store" else "other"
@@ -49,8 +57,9 @@ def store_sample(polars_script: models.PolarsScriptSample, background_tasks: Bac
     logger.info(f"Processing sample storage with size: {polars_script.sample_size}")
 
     try:
+        default_cache_dir = create_and_get_default_cache_dir(polars_script.flowfile_flow_id)
         polars_script.task_id = str(uuid.uuid4()) if polars_script.task_id is None else polars_script.task_id
-        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else CACHE_DIR.name
+        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else default_cache_dir
         polars_serializable_object = polars_script.polars_serializable_object()
 
         file_path = os.path.join(polars_script.cache_dir, f"{polars_script.task_id}.arrow")
@@ -210,7 +219,8 @@ def store_sql_db_result(database_read_settings: DatabaseReadSettings, background
 
     try:
         task_id = str(uuid.uuid4())
-        file_path = os.path.join(CACHE_DIR.name, f"{task_id}.arrow")
+        file_path = os.path.join(create_and_get_default_cache_dir(database_read_settings.flowfile_flow_id),
+                                 f"{task_id}.arrow")
         status = models.Status(background_task_id=task_id, status="Starting", file_ref=file_path,
                                result_type="polars")
         status_dict[task_id] = status
@@ -246,7 +256,7 @@ def create_table(file_type: FileType, received_table: Dict, background_tasks: Ba
 
     try:
         task_id = str(uuid.uuid4())
-        file_ref = os.path.join(CACHE_DIR.name, f"{task_id}.arrow")
+        file_ref = os.path.join(create_and_get_default_cache_dir(flowfile_flow_id), f"{task_id}.arrow")
 
         status = models.Status(background_task_id=task_id, status="Starting", file_ref=file_ref,
                                result_type="polars")
@@ -382,8 +392,9 @@ async def add_fuzzy_join(polars_script: models.FuzzyJoinInput, background_tasks:
     """
     logger.info("Starting fuzzy join operation")
     try:
+        default_cache_dir = create_and_get_default_cache_dir(polars_script.flowfile_flow_id)
         polars_script.task_id = str(uuid.uuid4()) if polars_script.task_id is None else polars_script.task_id
-        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else CACHE_DIR.name
+        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else default_cache_dir
         left_serializable_object = polars_script.left_df_operation.polars_serializable_object()
         right_serializable_object = polars_script.right_df_operation.polars_serializable_object()
 
@@ -405,6 +416,37 @@ async def add_fuzzy_join(polars_script: models.FuzzyJoinInput, background_tasks:
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@router.delete("/clear_task/{task_id}")
+def clear_task(task_id: str):
+    """
+    Clear task data and status by ID.
+
+    Args:
+        task_id: Unique identifier of the task to clear
+    Returns:
+        dict: Success message
+    Raises:
+        HTTPException: If task not found
+    """
+
+    logger.info(f"Clearing task: {task_id}")
+    status = status_dict.get(task_id)
+    if not status:
+        logger.warning(f"Task not found for clearing: {task_id}")
+        raise HTTPException(status_code=404, detail="Task not found")
+    try:
+        if os.path.exists(status.file_ref):
+            os.remove(status.file_ref)
+            logger.debug(f"Removed file: {status.file_ref}")
+    except Exception as e:
+        logger.error(f"Error removing file {status.file_ref}: {str(e)}", exc_info=True)
+    with status_dict_lock:
+        status_dict.pop(task_id, None)
+        PROCESS_MEMORY_USAGE.pop(task_id, None)
+        logger.info(f"Successfully cleared task: {task_id}")
+    return {"message": f"Task {task_id} has been cleared."}
+
+
 @router.post("/cancel_task/{task_id}")
 def cancel_task(task_id: str):
     """Cancel a running task by ID.

shared/__init__.py

@@ -0,0 +1,15 @@
+"""
+Shared utilities for Flowfile services.
+This package contains common functionality that can be used across
+flowfile_core, flowfile_worker, and other components without creating
+circular dependencies.
+"""
+
+from .storage_config import storage, get_cache_directory, get_temp_directory, get_flows_directory
+
+__all__ = [
+    'storage',
+    'get_cache_directory', 
+    'get_temp_directory',
+    'get_flows_directory'
+]

shared/storage_config.py

@@ -0,0 +1,258 @@
+# shared/storage_config.py - Updated for Option 3
+"""
+Centralized storage configuration for Flowfile.
+This module can be imported by both core and worker without creating dependencies.
+"""
+import os
+from pathlib import Path
+from typing import Optional, Literal
+
+DirectoryOptions = Literal["temp_directory", "logs_directory",
+                            "system_logs_directory", "database_directory",
+                            "cache_directory", "flows_directory", "user_defined_nodes_directory"]
+
+
+class FlowfileStorage:
+    """Centralized storage manager for Flowfile applications."""
+
+    def __init__(self):
+        self._base_dir: Optional[Path] = None
+        self._user_data_dir: Optional[Path] = None
+        self._ensure_directories()
+
+    @property
+    def base_directory(self) -> Path:
+        """Get the base Flowfile storage directory (for internal container communication)."""
+        if self._base_dir is None:
+            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+                # In Docker, internal storage stays inside /app
+                base_path = os.environ.get("FLOWFILE_STORAGE_DIR", "/app/internal_storage")
+            else:
+                # Local development
+                base_path = os.environ.get("FLOWFILE_STORAGE_DIR")
+                if not base_path:
+                    home_dir = Path.home()
+                    base_path = home_dir / ".flowfile"
+
+            self._base_dir = Path(base_path)
+        return self._base_dir
+
+    @property
+    def user_data_directory(self) -> Path:
+        """Get the user data directory (completely separate from application code)."""
+        if self._user_data_dir is None:
+            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+                # In Docker, user data is at /data/user (completely outside /app)
+                user_data_path = os.environ.get("FLOWFILE_USER_DATA_DIR", "/data/user")
+            else:
+                # Local development - use user's home directory
+                user_data_path = Path.home()
+
+            self._user_data_dir = Path(user_data_path)
+        return self._user_data_dir
+
+    @property
+    def cache_directory(self) -> Path:
+        """Cache directory for worker-core communication (internal)."""
+        return self.base_directory / "cache"
+
+    def get_flow_cache_directory(self, flow_id: int) -> Path:
+        """Get or create a cache directory for a specific flow (internal)."""
+        flow_cache_dir = self.cache_directory / str(flow_id)
+        flow_cache_dir.mkdir(parents=True, exist_ok=True)
+        return flow_cache_dir
+
+    @property
+    def system_logs_directory(self) -> Path:
+        """Directory for system logs (internal)."""
+        return self.base_directory / "system_logs"
+
+    @property
+    def flows_directory(self) -> Path:
+        """Directory for flow storage (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, flows are in separate user data area
+            return self.user_data_directory / "flows"
+        else:
+            # Local development - flows in ~/.flowfile/flows
+            return self.base_directory / "flows"
+
+    @property
+    def uploads_directory(self) -> Path:
+        """Directory for user uploads (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, uploads are in separate user data area
+            return self.user_data_directory / "uploads"
+        else:
+            # Local development - uploads in ~/.flowfile/uploads
+            return self.base_directory / "uploads"
+
+    @property
+    def user_defined_nodes_directory(self) -> Path:
+        """Directory for user-defined custom nodes (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            return self.user_data_directory / "user_defined_nodes"
+        else:
+            return self.base_directory / "user_defined_nodes"
+
+    @property
+    def user_defined_nodes_icons(self) -> Path:
+        """Directory for user-defined custom node icon (user-accessible)."""
+        return self.user_defined_nodes_directory / "icons"
+
+    @property
+    def outputs_directory(self) -> Path:
+        """Directory for user outputs (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, outputs are in separate user data area
+            return self.user_data_directory / "outputs"
+        else:
+            # Local development - outputs in ~/.flowfile/outputs
+            return self.base_directory / "outputs"
+
+    @property
+    def database_directory(self) -> Path:
+        """Directory for local database files (internal)."""
+        return self.base_directory / "database"
+
+    @property
+    def logs_directory(self) -> Path:
+        """Directory for application logs (internal)."""
+        return self.base_directory / "logs"
+
+    @property
+    def temp_directory(self) -> Path:
+        """Directory for temporary files (internal)."""
+        return self.base_directory / "temp"
+
+    @property
+    def temp_directory_for_flows(self) -> Path:
+        """Directory for temporary files specific to flows (internal)."""
+        return self.temp_directory / "flows"
+
+    def _ensure_directories(self) -> None:
+        """Create all necessary directories if they don't exist."""
+        # Internal directories (always created in base_directory)
+        internal_directories = [
+            self.cache_directory,
+            self.database_directory,
+            self.logs_directory,
+            self.temp_directory,
+            self.system_logs_directory,
+            self.temp_directory_for_flows,
+        ]
+
+        # User-accessible directories (location depends on environment)
+        user_directories = [
+            self.flows_directory,
+            self.uploads_directory,
+            self.outputs_directory,
+            self.user_defined_nodes_directory,
+            self.user_defined_nodes_icons,
+        ]
+
+        for directory in internal_directories + user_directories:
+            directory.mkdir(parents=True, exist_ok=True)
+
+    def get_cache_file_path(self, filename: str) -> Path:
+        """Get full path for a cache file (internal)."""
+        return self.cache_directory / filename
+
+    def get_flow_file_path(self, filename: str) -> Path:
+        """Get full path for a flow file (user-accessible)."""
+        return self.flows_directory / filename
+
+    def get_upload_file_path(self, filename: str) -> Path:
+        """Get full path for an uploaded file (user-accessible)."""
+        return self.uploads_directory / filename
+
+    def get_output_file_path(self, filename: str) -> Path:
+        """Get full path for an output file (user-accessible)."""
+        return self.outputs_directory / filename
+
+    def get_log_file_path(self, filename: str) -> Path:
+        """Get full path for an application log file (internal)."""
+        return self.logs_directory / filename
+
+    def get_system_log_file_path(self, filename: str) -> Path:
+        """Get full path for a system log file (internal)."""
+        return self.system_logs_directory / filename
+
+    def get_temp_file_path(self, filename: str) -> Path:
+        """Get full path for a temporary file (internal)."""
+        return self.temp_directory / filename
+
+    def cleanup_directory(self, directory_option: DirectoryOptions, storage_duration_hours: int = 24) -> None:
+        """Clean up any directory of the folder"""
+        import time
+        import shutil
+
+        if not hasattr(self, directory_option):
+            raise Exception(f"Directory does not exist in {self.base_directory}")
+
+        directory = getattr(self, directory_option)
+        if not isinstance(directory, Path):
+            raise Exception(f"Directory attribute {directory_option} is not a Path object")
+
+        if not directory.exists():
+            return
+
+        current_time = time.time()
+        cutoff_time = current_time - (storage_duration_hours * 60 * 60)
+
+        for item in directory.iterdir():
+            try:
+                if item.stat().st_mtime < cutoff_time:
+                    if item.is_file():
+                        item.unlink()
+                    elif item.is_dir():
+                        shutil.rmtree(item)
+            except (OSError, FileNotFoundError):
+                # Handle permission errors or files that disappeared
+                continue
+
+    def cleanup_directories(self) -> None:
+        """Clean up temporary files older than specified hours."""
+        self.cleanup_directory("temp_directory", storage_duration_hours=24)
+        self.cleanup_directory("cache_directory", storage_duration_hours=1)
+        self.cleanup_directory("logs_directory", storage_duration_hours=168)
+        self.cleanup_directory("system_logs_directory", storage_duration_hours=168)
+
+
+storage = FlowfileStorage()
+
+
+# Convenience functions for backward compatibility
+def get_cache_directory() -> str:
+    """Get cache directory path as string."""
+    return str(storage.cache_directory)
+
+
+def get_temp_directory() -> str:
+    """Get temp directory path as string."""
+    return str(storage.temp_directory)
+
+
+def get_flows_directory() -> str:
+    """Get flows directory path as string."""
+    return str(storage.flows_directory)
+
+
+def get_uploads_directory() -> str:
+    """Get uploads directory path as string."""
+    return str(storage.uploads_directory)
+
+
+def get_outputs_directory() -> str:
+    """Get outputs directory path as string."""
+    return str(storage.outputs_directory)
+
+
+def get_logs_directory() -> str:
+    """Get application logs directory path as string."""
+    return str(storage.logs_directory)
+
+
+def get_system_logs_directory() -> str:
+    """Get system logs directory path as string."""
+    return str(storage.system_logs_directory)