Flowfile 0.3.9__py3-none-any.whl → 0.5.1__py3-none-any.whl

flowfile_core/flowfile/schema_callbacks.py

@@ -12,11 +12,12 @@ from flowfile_core.configs.flow_logger import main_logger
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, PlType
 from flowfile_core.schemas import transform_schema
 from flowfile_core.schemas import input_schema
+from flowfile_core.schemas.transform_schema import FuzzyMatchInputManager
 
 
 def _ensure_all_columns_have_select(left_cols: List[str],
                                     right_cols: List[str],
-                                    fuzzy_match_input: transform_schema.FuzzyMatchInput):
+                                    fuzzy_match_input: transform_schema.FuzzyMatchInputManager):
     """
     Ensure that all columns in the left and right FlowDataEngines are included in the fuzzy match input's select
      statements.
@@ -38,7 +39,7 @@ def _ensure_all_columns_have_select(left_cols: List[str],
     )
 
 
-def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: transform_schema.JoinInputs) -> None:
+def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: transform_schema.JoinInputsManager) -> None:
     """
     Ensure that the select columns in the fuzzy match input match the order of the incoming columns.
     This function modifies the join_inputs object in-place.
@@ -46,36 +47,42 @@ def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: tra
     Returns:
         None
     """
-    select_map = {select.new_name: select for select in join_inputs.renames}
+    select_map = {select.old_name: select for select in join_inputs.renames}
     ordered_renames = [select_map[col] for col in col_order if col in select_map]
-    join_inputs.renames = ordered_renames
+    join_inputs.select_inputs.renames = ordered_renames
 
 
-def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInput,
+def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInputManager,
                                  left_schema: List[FlowfileColumn],
                                  right_schema: List[FlowfileColumn]):
     _ensure_all_columns_have_select(left_cols=[col.column_name for col in left_schema],
                                     right_cols=[col.column_name for col in right_schema],
                                     fuzzy_match_input=fm_input)
+
     _order_join_inputs_based_on_col_order(col_order=[col.column_name for col in left_schema],
                                           join_inputs=fm_input.left_select)
     _order_join_inputs_based_on_col_order(col_order=[col.column_name for col in right_schema],
                                           join_inputs=fm_input.right_select)
+    for column in fm_input.left_select.renames:
+        if column.join_key:
+            column.keep = True
+    for column in fm_input.right_select.renames:
+        if column.join_key:
+            column.keep = True
+
     left_schema_dict, right_schema_dict = ({ls.name: ls for ls in left_schema}, {rs.name: rs for rs in right_schema})
     fm_input.auto_rename()
-
     right_renames = {column.old_name: column.new_name for column in fm_input.right_select.renames}
     new_join_mapping = rename_fuzzy_right_mapping(fm_input.join_mapping, right_renames)
-
     output_schema = []
     for column in fm_input.left_select.renames:
         column_schema = left_schema_dict.get(column.old_name)
-        if column_schema and column.keep:
+        if column_schema and (column.keep or column.join_key):
             output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
                                                            example_values=column_schema.example_values))
     for column in fm_input.right_select.renames:
         column_schema = right_schema_dict.get(column.old_name)
-        if column_schema and column.keep:
+        if column_schema and (column.keep or column.join_key):
             output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
                                                            example_values=column_schema.example_values))
     set_name_in_fuzzy_mappings(new_join_mapping)
@@ -117,7 +124,7 @@ def pre_calculate_pivot_schema(node_input_schema: List[FlowfileColumn],
                             pivot_input.index_columns]
     val_column_schema = get_schema_of_column(node_input_schema, pivot_input.value_col)
     if output_fields is not None and len(output_fields) > 0:
-        return index_columns_schema+[FlowfileColumn(PlType(Plcolumn_name=output_field.name,
+        return index_columns_schema+[FlowfileColumn(PlType(column_name=output_field.name,
                                                            pl_datatype=output_field.data_type)) for output_field in
                                      output_fields]
 

flowfile_core/flowfile/setting_generator/settings.py

@@ -39,11 +39,14 @@ def join(node_data: "NodeData") -> NodeData:
             join_key = overlapping_cols[0]
         else:
             join_key = ''
-        ji = transform_schema.JoinInput(join_mapping=join_key,
-                                        left_select=node_data.main_input.columns,
-                                        right_select=node_data.right_input.columns
-                                        )
-        ji.auto_rename()
+        join_input_manager = transform_schema.JoinInputManager(
+            transform_schema.JoinInput(join_mapping=join_key,
+                                       left_select=node_data.main_input.columns,
+                                       right_select=node_data.right_input.columns
+                                       )
+        )
+        join_input_manager.auto_rename()
+        ji = join_input_manager.to_join_input()
         node_data.setting_input = input_schema.NodeJoin(flow_id=node_data.flow_id,
                                                         node_id=node_data.node_id,
                                                         join_input=ji)
@@ -53,13 +56,15 @@ def join(node_data: "NodeData") -> NodeData:
 @setting_generator_method
 def cross_join(node_data: "NodeData") -> NodeData:
     if node_data.right_input and node_data.main_input:
-        ji = transform_schema.CrossJoinInput(left_select=node_data.main_input.columns,
-                                             right_select=node_data.right_input.columns)
-        ji.auto_rename()
-        print(ji)
+        cj_input_manager = transform_schema.CrossJoinInputManager(
+            transform_schema.CrossJoinInput(left_select=node_data.main_input.columns,
+                                            right_select=node_data.right_input.columns)
+        )
+        cj_input_manager.auto_rename()
+        cj = cj_input_manager.to_cross_join_input()
         node_data.setting_input = input_schema.NodeCrossJoin(flow_id=node_data.flow_id,
                                                              node_id=node_data.node_id,
-                                                             cross_join_input=ji)
+                                                             cross_join_input=cj)
     return node_data
 
 

flowfile_core/main.py

@@ -7,6 +7,8 @@ import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from shared.storage_config import storage
+
 from flowfile_core import ServerRun
 from flowfile_core.configs.settings import (SERVER_HOST, SERVER_PORT, WORKER_HOST, WORKER_PORT, WORKER_URL,)
 
@@ -16,15 +18,16 @@ from flowfile_core.routes.routes import router
 from flowfile_core.routes.public import router as public_router
 from flowfile_core.routes.logs import router as logs_router
 from flowfile_core.routes.cloud_connections import router as cloud_connections_router
+from flowfile_core.routes.user_defined_components import router as user_defined_components_router
 
 from flowfile_core.configs.flow_logger import clear_all_flow_logs
+storage.cleanup_directories()
 
 os.environ["FLOWFILE_MODE"] = "electron"
 
 should_exit = False
 server_instance = None
 
-
 @asynccontextmanager
 async def shutdown_handler(app: FastAPI):
     """Handles the graceful startup and shutdown of the FastAPI application.
@@ -77,6 +80,7 @@ app.include_router(logs_router, tags=["logs"])
 app.include_router(auth_router, prefix="/auth", tags=["auth"])
 app.include_router(secrets_router, prefix="/secrets", tags=["secrets"])
 app.include_router(cloud_connections_router, prefix="/cloud_connections", tags=["cloud_connections"])
+app.include_router(user_defined_components_router, prefix="/user_defined_components", tags=["user_defined_components"])
 
 
 @app.post("/shutdown")

flowfile_core/routes/routes.py

@@ -12,49 +12,49 @@ import logging
 import os
 from pathlib import Path
 from typing import List, Dict, Any, Optional
-from sqlalchemy.orm import Session
 
 from fastapi import APIRouter, File, UploadFile, BackgroundTasks, HTTPException, status, Body, Depends
 from fastapi.responses import JSONResponse, Response
 # External dependencies
 from polars_expr_transformer.function_overview import get_all_expressions, get_expression_overview
+from sqlalchemy.orm import Session
 
+from flowfile_core import flow_file_handler
 # Core modules
 from flowfile_core.auth.jwt import get_current_active_user
 from flowfile_core.configs import logger
-from flowfile_core.configs.node_store import nodes
-from flowfile_core.configs.settings import IS_RUNNING_IN_DOCKER
+from flowfile_core.configs.node_store import nodes_list, check_if_has_default_setting
+from flowfile_core.database.connection import get_db
 # File handling
 from flowfile_core.fileExplorer.funcs import (
-    FileExplorer,
+    SecureFileExplorer,
     FileInfo,
     get_files_from_directory
 )
-from flowfile_core.flowfile.flow_graph import add_connection, delete_connection
-from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars
 from flowfile_core.flowfile.analytics.analytics_processor import AnalyticsProcessor
+from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars
+from flowfile_core.flowfile.database_connection_manager.db_connections import (store_database_connection,
+                                                                               get_database_connection,
+                                                                               delete_database_connection,
+                                                                               get_all_database_connections_interface)
 from flowfile_core.flowfile.extensions import get_instant_func_results
-# Flow handling
-
+from flowfile_core.flowfile.flow_graph import add_connection, delete_connection
 from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import create_sql_source_from_db_settings
 from flowfile_core.run_lock import get_flow_run_lock
-# Schema and models
 from flowfile_core.schemas import input_schema, schemas, output_model
 from flowfile_core.utils import excel_file_manager
-from flowfile_core.utils.fileManager import create_dir, remove_paths
+from flowfile_core.utils.fileManager import create_dir
 from flowfile_core.utils.utils import camel_case_to_snake_case
-from flowfile_core import flow_file_handler
-from flowfile_core.flowfile.database_connection_manager.db_connections import (store_database_connection,
-                                                                               get_database_connection,
-                                                                               delete_database_connection,
-                                                                               get_all_database_connections_interface)
-from flowfile_core.database.connection import get_db
+from shared.storage_config import storage
 
 
 router = APIRouter(dependencies=[Depends(get_current_active_user)])
 
 # Initialize services
-file_explorer = FileExplorer('/app/shared' if IS_RUNNING_IN_DOCKER else None)
+file_explorer = SecureFileExplorer(
+    start_path=storage.user_data_directory,
+    sandbox_root=storage.user_data_directory
+)
 
 
 def get_node_model(setting_name_ref: str):
@@ -148,7 +148,7 @@ async def get_directory_contents(directory: str, file_types: List[str] = None,
     Returns:
         A list of `FileInfo` objects representing the directory's contents.
     """
-    directory_explorer = FileExplorer(directory)
+    directory_explorer = SecureFileExplorer(directory, storage.user_data_directory)
     try:
         return directory_explorer.list_contents(show_hidden=include_hidden, file_types=file_types)
     except Exception as e:
@@ -198,6 +198,24 @@ async def get_active_flow_file_sessions() -> List[schemas.FlowSettings]:
     return [flf.flow_settings for flf in flow_file_handler.flowfile_flows]
 
 
+@router.post("/node/trigger_fetch_data", tags=['editor'])
+async def trigger_fetch_node_data(flow_id: int, node_id: int, background_tasks: BackgroundTasks):
+    """Fetches and refreshes the data for a specific node."""
+    flow = flow_file_handler.get_flow(flow_id)
+    lock = get_flow_run_lock(flow_id)
+    async with lock:
+        if flow.flow_settings.is_running:
+            raise HTTPException(422, 'Flow is already running')
+        try:
+            flow.validate_if_node_can_be_fetched(node_id)
+        except Exception as e:
+            raise HTTPException(422, str(e))
+        background_tasks.add_task(flow.trigger_fetch_node, node_id)
+    return JSONResponse(content={"message": "Data started",
+                                 "flow_id": flow_id,
+                                 "node_id": node_id}, status_code=status.HTTP_200_OK)
+
+
 @router.post('/flow/run/', tags=['editor'])
 async def run_flow(flow_id: int, background_tasks: BackgroundTasks) -> JSONResponse:
     """Executes a flow in a background task.
@@ -228,6 +246,16 @@ def cancel_flow(flow_id: int):
     flow.cancel()
 
 
+@router.post("/flow/apply_standard_layout/", tags=["editor"])
+def apply_standard_layout(flow_id: int):
+    flow = flow_file_handler.get_flow(flow_id)
+    if not flow:
+        raise HTTPException(status_code=404, detail="Flow not found")
+    if flow.flow_settings.is_running:
+        raise HTTPException(422, "Flow is running")
+    flow.apply_layout()
+
+
 @router.get('/flow/run_status/', tags=['editor'],
             response_model=output_model.RunInformation)
 def get_run_status(flow_id: int, response: Response):
@@ -240,8 +268,8 @@ def get_run_status(flow_id: int, response: Response):
         raise HTTPException(status_code=404, detail="Flow not found")
     if flow.flow_settings.is_running:
         response.status_code = status.HTTP_202_ACCEPTED
-        return flow.get_run_info()
-    response.status_code = status.HTTP_200_OK
+    else:
+        response.status_code = status.HTTP_200_OK
     return flow.get_run_info()
 
 
@@ -324,7 +352,7 @@ def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y:
         logger.info("Adding node")
         flow.add_node_promise(node_promise)
 
-    if nodes.check_if_has_default_setting(node_type):
+    if check_if_has_default_setting(node_type):
         logger.info(f'Found standard settings for {node_type}, trying to upload them')
         setting_name_ref = 'node' + node_type.replace('_', '')
         node_model = get_node_model(setting_name_ref)
@@ -439,11 +467,24 @@ def get_generated_code(flow_id: int) -> str:
 
 
 @router.post('/editor/create_flow/', tags=['editor'])
-def create_flow(flow_path: str):
+def create_flow(flow_path: str = None, name: str = None):
     """Creates a new, empty flow file at the specified path and registers a session for it."""
-    flow_path = Path(flow_path)
-    logger.info('Creating flow')
-    return flow_file_handler.add_flow(name=flow_path.stem, flow_path=str(flow_path))
+    if flow_path is not None and name is None:
+        name = Path(flow_path).stem
+    elif flow_path is not None and name is not None:
+        if name not in flow_path and (flow_path.endswith(".yaml") or flow_path.endswith(".yml")):
+            raise HTTPException(422, 'The name must be part of the flow path when a full path is provided')
+        elif name in flow_path and not (flow_path.endswith(".yaml") or flow_path.endswith(".yml")):
+            flow_path = str(Path(flow_path) / (name + ".yaml"))
+        elif name not in flow_path and (name.endswith(".yaml") or name.endswith(".yml")):
+            flow_path = str(Path(flow_path) / name)
+        elif name not in flow_path and not (name.endswith(".yaml") or name.endswith(".yml")):
+            flow_path = str(Path(flow_path) / (name + ".yaml"))
+    if flow_path is not None:
+        flow_path_ref = Path(flow_path)
+        if not flow_path_ref.parent.exists():
+            raise HTTPException(422, 'The directory does not exist')
+    return flow_file_handler.add_flow(name=name, flow_path=flow_path)
 
 
 @router.post('/editor/close_flow/', tags=['editor'])
@@ -471,6 +512,7 @@ def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_use
     add_func = getattr(flow, 'add_' + node_type)
     parsed_input = None
     setting_name_ref = 'node' + node_type.replace('_', '')
+
     if add_func is None:
         raise HTTPException(404, 'could not find the function')
     try:
@@ -496,10 +538,11 @@ def get_list_of_saved_flows(path: str):
     except:
         return []
 
-@router.get('/node_list', response_model=List[nodes.NodeTemplate])
-def get_node_list() -> List[nodes.NodeTemplate]:
+
+@router.get('/node_list', response_model=List[schemas.NodeTemplate])
+def get_node_list() -> List[schemas.NodeTemplate]:
     """Retrieves the list of all available node types and their templates."""
-    return nodes.nodes_list
+    return nodes_list
 
 
 @router.get('/node', response_model=output_model.NodeData, tags=['editor'])
@@ -555,7 +598,7 @@ async def get_downstream_node_ids(flow_id: int, node_id: int) -> List[int]:
 
 @router.get('/import_flow/', tags=['editor'], response_model=int)
 def import_saved_flow(flow_path: str) -> int:
-    """Imports a flow from a saved `.flowfile` and registers it as a new session."""
+    """Imports a flow from a saved `.yaml` and registers it as a new session."""
     flow_path = Path(flow_path)
     if not flow_path.exists():
         raise HTTPException(404, 'File not found')
@@ -564,7 +607,7 @@ def import_saved_flow(flow_path: str) -> int:
 
 @router.get('/save_flow', tags=['editor'])
 def save_flow(flow_id: int, flow_path: str = None):
-    """Saves the current state of a flow to a `.flowfile`."""
+    """Saves the current state of a flow to a `.yaml`."""
     flow = flow_file_handler.get_flow(flow_id)
     flow.save_flow(flow_path=flow_path)
 

flowfile_core/routes/user_defined_components.py

@@ -0,0 +1,55 @@
+
+from typing import Dict, Any
+
+from fastapi import APIRouter, HTTPException, Depends
+
+from flowfile_core import flow_file_handler
+# Core modules
+from flowfile_core.auth.jwt import get_current_active_user
+from flowfile_core.configs import logger
+from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
+# File handling
+from flowfile_core.schemas import input_schema
+from flowfile_core.utils.utils import camel_case_to_snake_case
+
+# External dependencies
+
+
+router = APIRouter()
+
+
+@router.get("/custom-node-schema", summary="Get a simple UI schema")
+def get_simple_custom_object(flow_id: int, node_id: int):
+    """
+    This endpoint returns a hardcoded JSON object that represents the UI
+    for our SimpleFilterNode.
+    """
+    try:
+        node = flow_file_handler.get_node(flow_id=flow_id, node_id=node_id)
+    except Exception as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    user_defined_node = CUSTOM_NODE_STORE.get(node.node_type)
+
+    if not user_defined_node:
+        raise HTTPException(status_code=404, detail=f"Node type '{node.node_type}' not found")
+    if node.is_setup:
+        settings = node.setting_input.settings
+        return user_defined_node.from_settings(settings).get_frontend_schema()
+    return user_defined_node().get_frontend_schema()
+
+
+@router.post("/update_user_defined_node", tags=["transform"])
+def update_user_defined_node(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)):
+    input_data['user_id'] = current_user.id
+    node_type = camel_case_to_snake_case(node_type)
+    flow_id = int(input_data.get('flow_id'))
+    logger.info(f'Updating the data for flow: {flow_id}, node {input_data["node_id"]}')
+    flow = flow_file_handler.get_flow(flow_id)
+    user_defined_model = CUSTOM_NODE_STORE.get(node_type)
+    if not user_defined_model:
+        raise HTTPException(status_code=404, detail=f"Node type '{node_type}' not found")
+
+    user_defined_node_settings = input_schema.UserDefinedNode.model_validate(input_data)
+    initialized_model = user_defined_model.from_settings(user_defined_node_settings.settings)
+
+    flow.add_user_defined_node(custom_node=initialized_model, user_defined_node_settings=user_defined_node_settings)

flowfile_core/schemas/cloud_storage_schemas.py

@@ -136,11 +136,9 @@ class CloudStorageReadSettings(CloudStorageSettings):
 
     scan_mode: Literal["single_file", "directory"] = "single_file"
     file_format: Literal["csv", "parquet", "json", "delta", "iceberg"] = "parquet"
-    # CSV specific options
     csv_has_header: Optional[bool] = True
     csv_delimiter: Optional[str] = ","
     csv_encoding: Optional[str] = "utf8"
-    # Deltalake specific settings
     delta_version: Optional[int] = None