Flowfile 0.3.10__py3-none-any.whl → 0.4.1__py3-none-any.whl

flowfile_core/flowfile/node_designer/ui_components.py

@@ -0,0 +1,277 @@
+# ui_components.py - Updated ColumnSelector
+
+from typing import List, Optional, Any, Literal, Union, Type, Tuple, Dict
+
+from pydantic import Field, BaseModel, computed_field
+
+from flowfile_core.flowfile.node_designer._type_registry import normalize_type_spec
+# Public API import
+from flowfile_core.flowfile.node_designer.data_types import DataType, TypeSpec
+
+InputType = Literal["text", "number", "secret", "array", "date", "boolean"]
+
+
+def normalize_input_to_data_types(
+    v: Any
+) -> Union[Literal["ALL"], List[DataType]]:
+    """
+    Normalizes a wide variety of inputs to either 'ALL' or a sorted list of DataType enums.
+    This function is used as a Pydantic BeforeValidator.
+
+    Args:
+        v: The input value to normalize. Can be a string, a list of strings,
+           a DataType, a TypeGroup, or a list of those.
+
+    Returns:
+        Either the string "ALL" or a sorted list of unique DataType enums.
+    """
+    if v == "ALL":
+        return "ALL"
+    if isinstance(v, list) and all(isinstance(item, DataType) for item in v):
+        return v
+
+    normalized_set = normalize_type_spec(v)
+
+    if normalized_set == set(DataType):
+        return "ALL"
+
+    return sorted(list(normalized_set), key=lambda x: x.value)
+
+
+class FlowfileInComponent(BaseModel):
+    """
+    Base class for all UI components in the node settings panel.
+
+    This class provides the common attributes and methods that all UI components share.
+    It's not meant to be used directly, but rather to be inherited by specific
+    component classes.
+    """
+    component_type: str = Field(..., description="Type of the UI component")
+    value: Any = None
+    label: Optional[str] = None
+    input_type: InputType
+
+    def set_value(self, value: Any):
+        """
+        Sets the value of the component, received from the frontend.
+
+        This method is used internally by the framework to populate the component's
+        value when a user interacts with the UI.
+
+        Args:
+            value: The new value for the component.
+
+        Returns:
+            The component instance with the updated value.
+        """
+        self.value = value
+        return self
+
+
+class IncomingColumns:
+    """
+    A marker class used in `SingleSelect` and `MultiSelect` components.
+
+    When `options` is set to this class, the component will be dynamically
+    populated with the column names from the node's input dataframe.
+    This allows users to select from the available columns at runtime.
+
+    Example:
+        class MyNodeSettings(NodeSettings):
+            column_to_process = SingleSelect(
+                label="Select a column",
+                options=IncomingColumns
+            )
+    """
+    pass
+
+
+class ColumnSelector(FlowfileInComponent):
+    """
+    A UI component that allows users to select one or more columns from the
+    input dataframe, with an optional filter based on column data types.
+
+    This is particularly useful when a node operation should only be applied
+    to columns of a specific type (e.g., numeric, string, date).
+    """
+    component_type: Literal["ColumnSelector"] = "ColumnSelector"
+    required: bool = False
+    multiple: bool = False
+    input_type: InputType = "text"
+
+    # Normalized output: either "ALL" or list of DataType enums
+    data_type_filter_input: TypeSpec = Field(
+        default="ALL",
+        alias="data_types",
+        repr=False,
+        exclude=True
+    )
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    @computed_field
+    @property
+    def data_types_filter(self) -> Union[Literal["ALL"], List[DataType]]:
+        """
+        A computed field that normalizes the `data_type_filter_input` into a
+        standardized format for the frontend.
+        """
+        return normalize_input_to_data_types(self.data_type_filter_input)
+
+    def model_dump(self, **kwargs) -> dict:
+        """
+        Overrides the default `model_dump` to ensure `data_types` is in the
+        correct format for the frontend.
+        """
+        data = super().model_dump(**kwargs)
+        if 'data_types_filter' in data and data['data_types_filter'] != "ALL":
+            data['data_types'] = sorted([dt.value for dt in data['data_types_filter']])
+        return data
+
+
+class TextInput(FlowfileInComponent):
+    """A standard text input field for capturing string values."""
+    component_type: Literal["TextInput"] = "TextInput"
+    default: Optional[str] = ""
+    placeholder: Optional[str] = ""
+    input_type: InputType = "text"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None and self.default is not None:
+            self.value = self.default
+
+
+class NumericInput(FlowfileInComponent):
+    """A numeric input field with optional minimum and maximum value validation."""
+    component_type: Literal["NumericInput"] = "NumericInput"
+    default: Optional[float] = None
+    min_value: Optional[float] = None
+    max_value: Optional[float] = None
+    input_type: InputType = "number"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None and self.default is not None:
+            self.value = self.default
+
+
+class ToggleSwitch(FlowfileInComponent):
+    """A boolean toggle switch, typically used for enabling or disabling a feature."""
+    component_type: Literal["ToggleSwitch"] = "ToggleSwitch"
+    default: bool = False
+    description: Optional[str] = None
+    input_type: InputType = "boolean"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None:
+            self.value = self.default
+
+    def __bool__(self):
+        """Allows the component instance to be evaluated as a boolean."""
+        return bool(self.value)
+
+
+class SingleSelect(FlowfileInComponent):
+    """
+    A dropdown menu for selecting a single option from a list.
+
+    The options can be a static list of strings or tuples, or they can be
+    dynamically populated from the input dataframe's columns by using the
+    `IncomingColumns` marker.
+    """
+    component_type: Literal["SingleSelect"] = "SingleSelect"
+    options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
+    default: Optional[Any] = None
+    input_type: InputType = "text"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None and self.default is not None:
+            self.value = self.default
+
+
+class MultiSelect(FlowfileInComponent):
+    """
+    A multi-select dropdown for choosing multiple options from a list.
+
+    Like `SingleSelect`, the options can be static or dynamically populated
+    from the input columns using the `IncomingColumns` marker.
+    """
+    component_type: Literal["MultiSelect"] = "MultiSelect"
+    options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
+    default: List[Any] = Field(default_factory=list)
+    input_type: InputType = "array"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None:
+            self.value = self.default if self.default else []
+
+
+class Section(BaseModel):
+    """
+    A container for grouping related UI components in the node settings panel.
+
+    Sections help organize the UI by grouping components under a common title
+    and description. Components can be added as keyword arguments during
+    initialization or afterward.
+
+    Example:
+        main_section = Section(
+            title="Main Settings",
+            description="Configure the primary behavior of the node.",
+            my_text_input=TextInput(label="Enter a value")
+        )
+    """
+    title: Optional[str] = None
+    description: Optional[str] = None
+    hidden: bool = False
+
+    class Config:
+        extra = 'allow'
+        arbitrary_types_allowed = True
+
+    def __init__(self, **data):
+        """
+        Initialize a Section with components as keyword arguments.
+        """
+        super().__init__(**data)
+
+    def __call__(self, **kwargs) -> 'Section':
+        """
+        Allows adding components to the section after initialization.
+
+        This makes it possible to build up a section dynamically.
+        """
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+        return self
+
+    def get_components(self) -> Dict[str, FlowfileInComponent]:
+        """
+        Get all FlowfileInComponent instances from the section.
+
+        This method collects all the UI components that have been added to the
+        section, whether as defined fields or as extra fields.
+
+        Returns:
+            A dictionary mapping component names to their instances.
+        """
+        components = {}
+
+        # Get from extra fields
+        for key, value in getattr(self, '__pydantic_extra__', {}).items():
+            if isinstance(value, FlowfileInComponent):
+                components[key] = value
+
+        # Get from defined fields (excluding metadata)
+        for field_name in self.model_fields:
+            if field_name not in {'title', 'description', 'hidden'}:
+                value = getattr(self, field_name, None)
+                if isinstance(value, FlowfileInComponent):
+                    components[field_name] = value
+
+        return components

flowfile_core/flowfile/schema_callbacks.py

@@ -121,7 +121,7 @@ def pre_calculate_pivot_schema(node_input_schema: List[FlowfileColumn],
                             pivot_input.index_columns]
     val_column_schema = get_schema_of_column(node_input_schema, pivot_input.value_col)
     if output_fields is not None and len(output_fields) > 0:
-        return index_columns_schema+[FlowfileColumn(PlType(Plcolumn_name=output_field.name,
+        return index_columns_schema+[FlowfileColumn(PlType(column_name=output_field.name,
                                                            pl_datatype=output_field.data_type)) for output_field in
                                      output_fields]
 

flowfile_core/main.py

@@ -18,6 +18,7 @@ from flowfile_core.routes.routes import router
 from flowfile_core.routes.public import router as public_router
 from flowfile_core.routes.logs import router as logs_router
 from flowfile_core.routes.cloud_connections import router as cloud_connections_router
+from flowfile_core.routes.user_defined_components import router as user_defined_components_router
 
 from flowfile_core.configs.flow_logger import clear_all_flow_logs
 storage.cleanup_directories()
@@ -79,7 +80,7 @@ app.include_router(logs_router, tags=["logs"])
 app.include_router(auth_router, prefix="/auth", tags=["auth"])
 app.include_router(secrets_router, prefix="/secrets", tags=["secrets"])
 app.include_router(cloud_connections_router, prefix="/cloud_connections", tags=["cloud_connections"])
-
+app.include_router(user_defined_components_router, prefix="/user_defined_components", tags=["user_defined_components"])
 
 
 @app.post("/shutdown")

flowfile_core/routes/routes.py

@@ -12,45 +12,40 @@ import logging
 import os
 from pathlib import Path
 from typing import List, Dict, Any, Optional
-from sqlalchemy.orm import Session
 
 from fastapi import APIRouter, File, UploadFile, BackgroundTasks, HTTPException, status, Body, Depends
 from fastapi.responses import JSONResponse, Response
 # External dependencies
 from polars_expr_transformer.function_overview import get_all_expressions, get_expression_overview
+from sqlalchemy.orm import Session
 
+from flowfile_core import flow_file_handler
 # Core modules
 from flowfile_core.auth.jwt import get_current_active_user
 from flowfile_core.configs import logger
-from flowfile_core.configs.node_store import nodes
-from flowfile_core.configs.settings import IS_RUNNING_IN_DOCKER
+from flowfile_core.configs.node_store import nodes_list, check_if_has_default_setting
+from flowfile_core.database.connection import get_db
 # File handling
 from flowfile_core.fileExplorer.funcs import (
     SecureFileExplorer,
     FileInfo,
     get_files_from_directory
 )
-from flowfile_core.flowfile.flow_graph import add_connection, delete_connection
-from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars
 from flowfile_core.flowfile.analytics.analytics_processor import AnalyticsProcessor
+from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars
+from flowfile_core.flowfile.database_connection_manager.db_connections import (store_database_connection,
+                                                                               get_database_connection,
+                                                                               delete_database_connection,
+                                                                               get_all_database_connections_interface)
 from flowfile_core.flowfile.extensions import get_instant_func_results
-# Flow handling
-
+from flowfile_core.flowfile.flow_graph import add_connection, delete_connection
 from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import create_sql_source_from_db_settings
 from flowfile_core.run_lock import get_flow_run_lock
-# Schema and models
-
-from shared.storage_config import storage
 from flowfile_core.schemas import input_schema, schemas, output_model
 from flowfile_core.utils import excel_file_manager
 from flowfile_core.utils.fileManager import create_dir
 from flowfile_core.utils.utils import camel_case_to_snake_case
-from flowfile_core import flow_file_handler
-from flowfile_core.flowfile.database_connection_manager.db_connections import (store_database_connection,
-                                                                               get_database_connection,
-                                                                               delete_database_connection,
-                                                                               get_all_database_connections_interface)
-from flowfile_core.database.connection import get_db
+from shared.storage_config import storage
 
 
 router = APIRouter(dependencies=[Depends(get_current_active_user)])
@@ -359,7 +354,7 @@ def add_node(flow_id: int, node_id: int, node_type: str, pos_x: int = 0, pos_y:
         logger.info("Adding node")
         flow.add_node_promise(node_promise)
 
-    if nodes.check_if_has_default_setting(node_type):
+    if check_if_has_default_setting(node_type):
         logger.info(f'Found standard settings for {node_type}, trying to upload them')
         setting_name_ref = 'node' + node_type.replace('_', '')
         node_model = get_node_model(setting_name_ref)
@@ -545,10 +540,11 @@ def get_list_of_saved_flows(path: str):
     except:
         return []
 
-@router.get('/node_list', response_model=List[nodes.NodeTemplate])
-def get_node_list() -> List[nodes.NodeTemplate]:
+
+@router.get('/node_list', response_model=List[schemas.NodeTemplate])
+def get_node_list() -> List[schemas.NodeTemplate]:
     """Retrieves the list of all available node types and their templates."""
-    return nodes.nodes_list
+    return nodes_list
 
 
 @router.get('/node', response_model=output_model.NodeData, tags=['editor'])

flowfile_core/routes/user_defined_components.py

@@ -0,0 +1,55 @@
+
+from typing import Dict, Any
+
+from fastapi import APIRouter, HTTPException, Depends
+
+from flowfile_core import flow_file_handler
+# Core modules
+from flowfile_core.auth.jwt import get_current_active_user
+from flowfile_core.configs import logger
+from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
+# File handling
+from flowfile_core.schemas import input_schema
+from flowfile_core.utils.utils import camel_case_to_snake_case
+
+# External dependencies
+
+
+router = APIRouter()
+
+
+@router.get("/custom-node-schema", summary="Get a simple UI schema")
+def get_simple_custom_object(flow_id: int, node_id: int):
+    """
+    This endpoint returns a hardcoded JSON object that represents the UI
+    for our SimpleFilterNode.
+    """
+    try:
+        node = flow_file_handler.get_node(flow_id=flow_id, node_id=node_id)
+    except Exception as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    user_defined_node = CUSTOM_NODE_STORE.get(node.node_type)
+
+    if not user_defined_node:
+        raise HTTPException(status_code=404, detail=f"Node type '{node.node_type}' not found")
+    if node.is_setup:
+        settings = node.setting_input.settings
+        return user_defined_node.from_settings(settings).get_frontend_schema()
+    return user_defined_node().get_frontend_schema()
+
+
+@router.post("/update_user_defined_node", tags=["transform"])
+def update_user_defined_node(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)):
+    input_data['user_id'] = current_user.id
+    node_type = camel_case_to_snake_case(node_type)
+    flow_id = int(input_data.get('flow_id'))
+    logger.info(f'Updating the data for flow: {flow_id}, node {input_data["node_id"]}')
+    flow = flow_file_handler.get_flow(flow_id)
+    user_defined_model = CUSTOM_NODE_STORE.get(node_type)
+    if not user_defined_model:
+        raise HTTPException(status_code=404, detail=f"Node type '{node_type}' not found")
+
+    user_defined_node_settings = input_schema.UserDefinedNode.model_validate(input_data)
+    initialized_model = user_defined_model.from_settings(user_defined_node_settings.settings)
+
+    flow.add_user_defined_node(custom_node=initialized_model, user_defined_node_settings=user_defined_node_settings)

flowfile_core/schemas/input_schema.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Literal, Iterator
+from typing import List, Optional, Literal, Iterator, Any
 from flowfile_core.schemas import transform_schema
 from pathlib import Path
 import os
@@ -195,6 +195,8 @@ class NodeBase(BaseModel):
     description: Optional[str] = ''
     user_id: Optional[int] = None
     is_flow_output: Optional[bool] = False
+    is_user_defined: Optional[bool] = False  # Indicator if the node is a user defined node
+
 
 class NodeSingleInput(NodeBase):
     """A base model for any node that takes a single data input."""
@@ -516,3 +518,8 @@ class NodeRecordCount(NodeSingleInput):
 class NodePolarsCode(NodeMultiInput):
     """Settings for a node that executes arbitrary user-provided Polars code."""
     polars_code_input: transform_schema.PolarsCodeInput
+
+
+class UserDefinedNode(NodeMultiInput):
+    """Settings for a node that contains the user defined node information"""
+    settings: Any

flowfile_core/schemas/schemas.py

@@ -5,6 +5,9 @@ from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
 ExecutionModeLiteral = Literal['Development', 'Performance']
 ExecutionLocationsLiteral = Literal['local', 'remote']
 
+# Type literals for classifying nodes.
+NodeTypeLiteral = Literal['input', 'output', 'process']
+TransformTypeLiteral = Literal['narrow', 'wide', 'other']
 
 def get_global_execution_location() -> ExecutionLocationsLiteral:
     """
@@ -135,11 +138,14 @@ class NodeTemplate(BaseModel):
     output: int
     image: str
     multi: bool = False
+    node_type: NodeTypeLiteral
+    transform_type: TransformTypeLiteral
     node_group: str
     prod_ready: bool = True
     can_be_start: bool = False
     drawer_title: str = "Node title"
     drawer_intro: str = "Drawer into"
+    custom_node: Optional[bool] = False
 
 
 class NodeInformation(BaseModel):
@@ -263,9 +269,6 @@ class VueFlowInput(BaseModel):
     node_inputs: List[NodeInput]
 
 
-# Type literals for classifying nodes.
-NodeTypeLiteral = Literal['input', 'output', 'process']
-TransformTypeLiteral = Literal['narrow', 'wide', 'other']
 
 
 class NodeDefault(BaseModel):

flowfile_core/utils/validate_setup.py

@@ -3,7 +3,7 @@ as have a component in flowfile_frontend"""
 
 from flowfile_core.schemas import input_schema
 from flowfile_core.flowfile.flow_graph import FlowGraph
-from flowfile_core.configs.node_store.nodes import nodes_list, NodeTemplate
+from flowfile_core.configs.node_store import nodes_list, NodeTemplate
 import inspect
 
 
@@ -31,6 +31,8 @@ def validate_setup():
     Raises ValueError if any node is missing either.
     """
     for node in nodes_list:
+        if node.custom_node:
+            continue
         check_if_node_has_add_function_in_flow_graph(node)
         check_if_node_has_input_schema_definition(node)
 

flowfile_frame/flow_frame.py

@@ -27,6 +27,8 @@ from flowfile_frame.join import _normalize_columns_to_list, _create_join_mapping
 from flowfile_frame.utils import _check_if_convertible_to_code
 from flowfile_frame.config import logger
 from flowfile_frame.cloud_storage.frame_helpers import add_write_ff_to_cloud_storage
+from collections.abc import Mapping
+
 
 
 def can_be_expr(param: inspect.Parameter) -> bool:
@@ -890,13 +892,18 @@ class FlowFrame:
         self.flow_graph.add_record_count(node_number_of_records)
         return self._create_child_frame(new_node_id)
 
-    def select(self, *columns: Union[str, Expr, Selector], description: Optional[str] = None) -> "FlowFrame":
+    def rename(self, mapping: Mapping[str, str], *, strict: bool = True,
+               description: str = None) -> "FlowFrame":
+        """Rename columns based on a mapping or function."""
+        return self.select([col(old_name).alias(new_name) for old_name, new_name in mapping.items()],
+                           description=description, _keep_missing=True)
+
+    def select(self, *columns: Union[str, Expr, Selector], description: Optional[str] = None, _keep_missing: bool = False) -> "FlowFrame":
         """
         Select columns from the frame.
         """
         columns_iterable = list(_parse_inputs_as_iterable(columns))
         new_node_id = generate_node_id()
-
         if (len(columns_iterable) == 1 and isinstance(columns_iterable[0], Expr)
                 and str(columns_iterable[0]) == "pl.Expr(len()).alias('number_of_records')"):
             return self._add_number_of_records(new_node_id, description)
@@ -914,7 +921,6 @@ class FlowFrame:
         for expr_input in effective_columns_iterable:
             current_expr_obj = expr_input
             is_simple_col_for_native = False
-
             if isinstance(expr_input, str):
                 current_expr_obj = col(expr_input)
                 selected_col_names_for_native.append(transform_schema.SelectInput(old_name=expr_input))
@@ -942,14 +948,18 @@ class FlowFrame:
         if can_use_native_node:
             existing_cols = self.columns
             selected_col_names = {select_col.old_name for select_col in selected_col_names_for_native}
-            dropped_columns = [transform_schema.SelectInput(c, keep=False) for c in existing_cols if
+            not_selected_columns = [transform_schema.SelectInput(c, keep=_keep_missing) for c in existing_cols if
                                c not in selected_col_names]
-            selected_col_names_for_native.extend(dropped_columns)
+            selected_col_names_for_native.extend(not_selected_columns)
+            if _keep_missing:
+                lookup_selection = {_col.old_name: _col for _col in selected_col_names_for_native}
+                selected_col_names_for_native = [lookup_selection.get(_col) for
+                                                 _col in existing_cols if _col in lookup_selection]
             select_settings = input_schema.NodeSelect(
                 flow_id=self.flow_graph.flow_id,
                 node_id=new_node_id,
                 select_input=selected_col_names_for_native,
-                keep_missing=False,
+                keep_missing=_keep_missing,
                 pos_x=200,
                 pos_y=100,
                 is_setup=True,

shared/storage_config.py

@@ -9,7 +9,7 @@ from typing import Optional, Literal
 
 DirectoryOptions = Literal["temp_directory", "logs_directory",
                             "system_logs_directory", "database_directory",
-                            "cache_directory", "flows_directory"]
+                            "cache_directory", "flows_directory", "user_defined_nodes_directory"]
 
 
 class FlowfileStorage:
@@ -87,6 +87,19 @@ class FlowfileStorage:
             # Local development - uploads in ~/.flowfile/uploads
             return self.base_directory / "uploads"
 
+    @property
+    def user_defined_nodes_directory(self) -> Path:
+        """Directory for user-defined custom nodes (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            return self.user_data_directory / "user_defined_nodes"
+        else:
+            return self.base_directory / "user_defined_nodes"
+
+    @property
+    def user_defined_nodes_icons(self) -> Path:
+        """Directory for user-defined custom node icon (user-accessible)."""
+        return self.user_defined_nodes_directory / "icons"
+
     @property
     def outputs_directory(self) -> Path:
         """Directory for user outputs (user-accessible)."""
@@ -134,6 +147,8 @@ class FlowfileStorage:
             self.flows_directory,
             self.uploads_directory,
             self.outputs_directory,
+            self.user_defined_nodes_directory,
+            self.user_defined_nodes_icons,
         ]
 
         for directory in internal_directories + user_directories:
@@ -240,4 +255,4 @@ def get_logs_directory() -> str:
 
 def get_system_logs_directory() -> str:
     """Get system logs directory path as string."""
-    return str(storage.system_logs_directory)
+    return str(storage.system_logs_directory)