Flowfile 0.3.9__py3-none-any.whl → 0.5.1__py3-none-any.whl

flowfile_core/flowfile/node_designer/custom_node.py

@@ -0,0 +1,371 @@
+# Fixed custom_node.py with proper type hints
+
+import polars as pl
+from pydantic import BaseModel
+from typing import Any, Dict, Optional, TypeVar
+from flowfile_core.flowfile.node_designer.ui_components import FlowfileInComponent, IncomingColumns, Section
+from flowfile_core.schemas.schemas import NodeTemplate, NodeTypeLiteral, TransformTypeLiteral
+
+
+def to_frontend_schema(model_instance: BaseModel) -> dict:
+    """
+    Recursively converts a Pydantic model instance into a JSON-serializable
+    dictionary suitable for the frontend.
+
+    This function handles special marker classes like `IncomingColumns` and
+    nested `Section` and `FlowfileInComponent` instances.
+
+    Args:
+        model_instance: The Pydantic model instance to convert.
+
+    Returns:
+        A dictionary representation of the model.
+    """
+    result = {}
+    extra_fields = getattr(model_instance, '__pydantic_extra__', {})
+    model_fields = {k: getattr(model_instance, k) for k in model_instance.model_fields.keys()}
+    for key, value in (extra_fields|model_fields).items():
+        result[key] = _convert_value(value)
+    return result
+
+
+def _convert_value(value: Any) -> Any:
+    """
+    Helper function to convert any value to a frontend-ready format.
+    """
+    if isinstance(value, Section):
+        section_data = value.model_dump(
+            include={'title', 'description', 'hidden'},
+            exclude_none=True
+        )
+        section_data["component_type"] = "Section"
+        section_data["components"] = {
+            key: _convert_value(comp)
+            for key, comp in value.get_components().items()
+        }
+        return section_data
+
+    elif isinstance(value, FlowfileInComponent):
+        component_dict = value.model_dump(exclude_none=True)
+        if 'options' in component_dict:
+            if component_dict['options'] is IncomingColumns or (
+                    isinstance(component_dict['options'], type) and
+                    issubclass(component_dict['options'], IncomingColumns)
+            ):
+                component_dict['options'] = {"__type__": "IncomingColumns"}
+        return component_dict
+    elif isinstance(value, BaseModel):
+        return to_frontend_schema(value)
+    elif isinstance(value, list):
+        return [_convert_value(item) for item in value]
+    elif isinstance(value, dict):
+        return {k: _convert_value(v) for k, v in value.items()}
+    elif isinstance(value, tuple):
+        return tuple(_convert_value(item) for item in value)
+    else:
+        return value
+
+
+# Type variable for the Section factory
+T = TypeVar('T', bound=Section)
+
+
+def create_section(**components: FlowfileInComponent) -> Section:
+    """
+    Factory function to create a Section with proper type hints.
+
+    This is a convenience function that makes it easier to create `Section`
+    objects with autocomplete and type checking in modern editors.
+
+    Usage:
+        advanced_config_section = create_section(
+            case_sensitive=case_sensitive_toggle
+        )
+
+    Args:
+        **components: Keyword arguments where each key is the component name
+                      and the value is a `FlowfileInComponent` instance.
+
+    Returns:
+        A new `Section` instance containing the provided components.
+    """
+    return Section(**components)
+
+
+class NodeSettings(BaseModel):
+    """
+    The top-level container for all sections in a node's UI.
+
+    This class holds all the `Section` objects that make up the settings panel
+    for a custom node.
+
+    Example:
+        class MyNodeSettings(NodeSettings):
+            main_config = main_config_section
+            advanced_options = advanced_config_section
+    """
+    class Config:
+        extra = 'allow'
+        arbitrary_types_allowed = True
+
+    def __init__(self, **sections):
+        """
+        Initialize NodeSettings with sections as keyword arguments.
+        """
+        super().__init__(**sections)
+
+    def populate_values(self, values: Dict[str, Any]) -> 'NodeSettings':
+        """
+        Populates the settings with values received from the frontend.
+
+        This method is used internally to update the node's state based on
+        user input in the UI.
+
+        Args:
+            values: A dictionary of values from the frontend, where keys are
+                    section names and values are dictionaries of component
+                    values.
+
+        Returns:
+            The `NodeSettings` instance with updated component values.
+        """
+        # Handle both extra fields and defined fields
+        all_sections = {}
+
+        # Get extra fields
+        extra_fields = getattr(self, '__pydantic_extra__', {})
+        all_sections.update(extra_fields)
+
+        # Get defined fields that are Sections
+        for field_name in self.model_fields:
+            field_value = getattr(self, field_name, None)
+            if isinstance(field_value, Section):
+                all_sections[field_name] = field_value
+
+        for section_name, section in all_sections.items():
+            if section_name in values:
+                section_values = values[section_name]
+                for component_name, component in section.get_components().items():
+                    if component_name in section_values:
+                        component.set_value(section_values[component_name])
+        return self
+
+
+def create_node_settings(**sections: Section) -> NodeSettings:
+    """
+    Factory function to create NodeSettings with proper type hints.
+
+    This is a convenience function for creating `NodeSettings` instances.
+
+    Usage:
+        FilterNodeSchema = create_node_settings(
+            main_config=main_config_section,
+            advanced_options=advanced_config_section
+        )
+
+    Args:
+        **sections: Keyword arguments where each key is the section name
+                    and the value is a `Section` instance.
+
+    Returns:
+        A new `NodeSettings` instance containing the provided sections.
+    """
+    return NodeSettings(**sections)
+
+
+class SectionBuilder:
+    """
+    A builder pattern for creating `Section` objects with proper type hints.
+
+    This provides a more fluent and readable way to construct complex sections,
+    especially when the number of components is large.
+
+    Usage:
+        builder = SectionBuilder(title="Advanced Settings")
+        builder.add_component("timeout", NumericInput(label="Timeout (s)"))
+        builder.add_component("retries", NumericInput(label="Number of Retries"))
+        advanced_section = builder.build()
+    """
+
+    def __init__(self, title: Optional[str] = None, description: Optional[str] = None, hidden: bool = False):
+        self._section = Section(title=title, description=description, hidden=hidden)
+
+    def add_component(self, name: str, component: FlowfileInComponent) -> 'SectionBuilder':
+        """Add a component to the section."""
+        setattr(self._section, name, component)
+        extra = getattr(self._section, '__pydantic_extra__', {})
+        extra[name] = component
+        return self
+
+    def build(self) -> Section:
+        """Build and return the Section."""
+        return self._section
+
+
+class NodeSettingsBuilder:
+    """
+    A builder pattern for creating `NodeSettings` objects.
+
+    Provides a fluent interface for constructing the entire settings schema
+    for a custom node.
+
+    Usage:
+        settings_builder = NodeSettingsBuilder()
+        settings_builder.add_section("main", main_section)
+        settings_builder.add_section("advanced", advanced_section)
+        my_node_settings = settings_builder.build()
+    """
+
+    def __init__(self):
+        self._settings = NodeSettings()
+
+    def add_section(self, name: str, section: Section) -> 'NodeSettingsBuilder':
+        """Add a section to the node settings."""
+        setattr(self._settings, name, section)
+        extra = getattr(self._settings, '__pydantic_extra__', {})
+        extra[name] = section
+        return self
+
+    def build(self) -> NodeSettings:
+        """Build and return the NodeSettings."""
+        return self._settings
+
+
+class CustomNodeBase(BaseModel):
+    """
+    The base class for creating a custom node in Flowfile.
+
+    To create a new node, you should inherit from this class and define its
+    attributes and the `process` method.
+    """
+    # Core node properties
+    node_name: str
+    node_category: str = "Custom"
+    node_icon: str = "user-defined-icon.png"
+    settings_schema: Optional[NodeSettings] = None
+
+    # I/O configuration
+    number_of_inputs: int = 1
+    number_of_outputs: int = 1
+
+    # Display properties in the UI
+    node_group: Optional[str] = "custom"
+    title: Optional[str] = "Custom Node"
+    intro: Optional[str] = "A custom node for data processing"
+
+    # Behavior properties
+    node_type: NodeTypeLiteral = "process"
+    transform_type: TransformTypeLiteral = "wide"
+
+    @property
+    def item(self):
+        """A unique identifier for the node, derived from its name."""
+        return self.node_name.replace(" ", "_").lower()
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def __init__(self, **data):
+        """
+        Initialize the node, optionally populating settings from initial values.
+        """
+        initial_values = data.pop('initial_values', None)
+        super().__init__(**data)
+        if self.settings_schema and initial_values:
+            self.settings_schema.populate_values(initial_values)
+
+    def get_frontend_schema(self) -> dict:
+        """
+        Get the frontend-ready schema with current values.
+
+        This method is called by the backend to send the node's UI definition
+        and current state to the frontend.
+
+        Returns:
+            A dictionary representing the node's schema and values.
+        """
+        schema = {
+            "node_name": self.node_name,
+            "node_category": self.node_category,
+            "node_icon": self.node_icon,
+            "number_of_inputs": self.number_of_inputs,
+            "number_of_outputs": self.number_of_outputs,
+            "node_group": self.node_group,
+            "title": self.title,
+            "intro": self.intro,
+        }
+
+        if self.settings_schema:
+            schema["settings_schema"] = to_frontend_schema(self.settings_schema)
+        else:
+            schema["settings_schema"] = {}
+
+        return schema
+
+    @classmethod
+    def from_frontend_schema(cls, schema: dict) -> 'CustomNodeBase':
+        """
+        Create a node instance from a frontend schema.
+
+        This is used when loading a node from a saved flow.
+        """
+        settings_values = schema.pop('settings_schema', {})
+        node = cls(**schema)
+        if settings_values and node.settings_schema:
+            node.settings_schema.populate_values(settings_values)
+        return node
+
+    @classmethod
+    def from_settings(cls, settings_values: dict) -> 'CustomNodeBase':
+        """
+        Create a node instance with just its settings values.
+
+        Useful for creating a configured node instance programmatically.
+        """
+        node = cls()
+        if settings_values and node.settings_schema:
+            node.settings_schema.populate_values(settings_values)
+        return node
+
+    def update_settings(self, values: Dict[str, Any]) -> 'CustomNodeBase':
+        """
+        Update the settings with new values from the frontend.
+        """
+        if self.settings_schema:
+            self.settings_schema.populate_values(values)
+        return self
+
+    def process(self, *inputs: pl.DataFrame) -> pl.DataFrame:
+        """
+        The main data processing logic for the node.
+
+        This method must be implemented by all subclasses. It receives one or
+        more Polars DataFrames as input and should return a single DataFrame
+        as output.
+
+        Args:
+            *inputs: A variable number of Polars DataFrames, corresponding to
+                     the inputs connected to the node.
+
+        Returns:
+            A Polars DataFrame containing the processed data.
+        """
+        raise NotImplementedError
+
+    def to_node_template(self) -> NodeTemplate:
+        """
+        Convert the node to a `NodeTemplate` for storage or transmission.
+        """
+        return NodeTemplate(
+            name=self.node_name,
+            item=self.item,
+            input=self.number_of_inputs,
+            output=self.number_of_outputs,
+            image=self.node_icon,
+            node_group=self.node_group,
+            drawer_title=self.title,
+            drawer_intro=self.intro,
+            node_type=self.node_type,
+            transform_type=self.transform_type,
+            custom_node=True
+        )

flowfile_core/flowfile/node_designer/ui_components.py

@@ -0,0 +1,277 @@
+# ui_components.py - Updated ColumnSelector
+
+from typing import List, Optional, Any, Literal, Union, Type, Tuple, Dict
+
+from pydantic import Field, BaseModel, computed_field
+
+from flowfile_core.flowfile.node_designer._type_registry import normalize_type_spec
+# Public API import
+from flowfile_core.types import DataType, TypeSpec
+
+InputType = Literal["text", "number", "secret", "array", "date", "boolean"]
+
+
+def normalize_input_to_data_types(
+    v: Any
+) -> Union[Literal["ALL"], List[DataType]]:
+    """
+    Normalizes a wide variety of inputs to either 'ALL' or a sorted list of DataType enums.
+    This function is used as a Pydantic BeforeValidator.
+
+    Args:
+        v: The input value to normalize. Can be a string, a list of strings,
+           a DataType, a TypeGroup, or a list of those.
+
+    Returns:
+        Either the string "ALL" or a sorted list of unique DataType enums.
+    """
+    if v == "ALL":
+        return "ALL"
+    if isinstance(v, list) and all(isinstance(item, DataType) for item in v):
+        return v
+
+    normalized_set = normalize_type_spec(v)
+
+    if normalized_set == set(DataType):
+        return "ALL"
+
+    return sorted(list(normalized_set), key=lambda x: x.value)
+
+
+class FlowfileInComponent(BaseModel):
+    """
+    Base class for all UI components in the node settings panel.
+
+    This class provides the common attributes and methods that all UI components share.
+    It's not meant to be used directly, but rather to be inherited by specific
+    component classes.
+    """
+    component_type: str = Field(..., description="Type of the UI component")
+    value: Any = None
+    label: Optional[str] = None
+    input_type: InputType
+
+    def set_value(self, value: Any):
+        """
+        Sets the value of the component, received from the frontend.
+
+        This method is used internally by the framework to populate the component's
+        value when a user interacts with the UI.
+
+        Args:
+            value: The new value for the component.
+
+        Returns:
+            The component instance with the updated value.
+        """
+        self.value = value
+        return self
+
+
+class IncomingColumns:
+    """
+    A marker class used in `SingleSelect` and `MultiSelect` components.
+
+    When `options` is set to this class, the component will be dynamically
+    populated with the column names from the node's input dataframe.
+    This allows users to select from the available columns at runtime.
+
+    Example:
+        class MyNodeSettings(NodeSettings):
+            column_to_process = SingleSelect(
+                label="Select a column",
+                options=IncomingColumns
+            )
+    """
+    pass
+
+
+class ColumnSelector(FlowfileInComponent):
+    """
+    A UI component that allows users to select one or more columns from the
+    input dataframe, with an optional filter based on column data types.
+
+    This is particularly useful when a node operation should only be applied
+    to columns of a specific type (e.g., numeric, string, date).
+    """
+    component_type: Literal["ColumnSelector"] = "ColumnSelector"
+    required: bool = False
+    multiple: bool = False
+    input_type: InputType = "text"
+
+    # Normalized output: either "ALL" or list of DataType enums
+    data_type_filter_input: TypeSpec = Field(
+        default="ALL",
+        alias="data_types",
+        repr=False,
+        exclude=True
+    )
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    @computed_field
+    @property
+    def data_types_filter(self) -> Union[Literal["ALL"], List[DataType]]:
+        """
+        A computed field that normalizes the `data_type_filter_input` into a
+        standardized format for the frontend.
+        """
+        return normalize_input_to_data_types(self.data_type_filter_input)
+
+    def model_dump(self, **kwargs) -> dict:
+        """
+        Overrides the default `model_dump` to ensure `data_types` is in the
+        correct format for the frontend.
+        """
+        data = super().model_dump(**kwargs)
+        if 'data_types_filter' in data and data['data_types_filter'] != "ALL":
+            data['data_types'] = sorted([dt.value for dt in data['data_types_filter']])
+        return data
+
+
+class TextInput(FlowfileInComponent):
+    """A standard text input field for capturing string values."""
+    component_type: Literal["TextInput"] = "TextInput"
+    default: Optional[str] = ""
+    placeholder: Optional[str] = ""
+    input_type: InputType = "text"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None and self.default is not None:
+            self.value = self.default
+
+
+class NumericInput(FlowfileInComponent):
+    """A numeric input field with optional minimum and maximum value validation."""
+    component_type: Literal["NumericInput"] = "NumericInput"
+    default: Optional[float] = None
+    min_value: Optional[float] = None
+    max_value: Optional[float] = None
+    input_type: InputType = "number"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None and self.default is not None:
+            self.value = self.default
+
+
+class ToggleSwitch(FlowfileInComponent):
+    """A boolean toggle switch, typically used for enabling or disabling a feature."""
+    component_type: Literal["ToggleSwitch"] = "ToggleSwitch"
+    default: bool = False
+    description: Optional[str] = None
+    input_type: InputType = "boolean"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None:
+            self.value = self.default
+
+    def __bool__(self):
+        """Allows the component instance to be evaluated as a boolean."""
+        return bool(self.value)
+
+
+class SingleSelect(FlowfileInComponent):
+    """
+    A dropdown menu for selecting a single option from a list.
+
+    The options can be a static list of strings or tuples, or they can be
+    dynamically populated from the input dataframe's columns by using the
+    `IncomingColumns` marker.
+    """
+    component_type: Literal["SingleSelect"] = "SingleSelect"
+    options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
+    default: Optional[Any] = None
+    input_type: InputType = "text"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None and self.default is not None:
+            self.value = self.default
+
+
+class MultiSelect(FlowfileInComponent):
+    """
+    A multi-select dropdown for choosing multiple options from a list.
+
+    Like `SingleSelect`, the options can be static or dynamically populated
+    from the input columns using the `IncomingColumns` marker.
+    """
+    component_type: Literal["MultiSelect"] = "MultiSelect"
+    options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
+    default: List[Any] = Field(default_factory=list)
+    input_type: InputType = "array"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None:
+            self.value = self.default if self.default else []
+
+
+class Section(BaseModel):
+    """
+    A container for grouping related UI components in the node settings panel.
+
+    Sections help organize the UI by grouping components under a common title
+    and description. Components can be added as keyword arguments during
+    initialization or afterward.
+
+    Example:
+        main_section = Section(
+            title="Main Settings",
+            description="Configure the primary behavior of the node.",
+            my_text_input=TextInput(label="Enter a value")
+        )
+    """
+    title: Optional[str] = None
+    description: Optional[str] = None
+    hidden: bool = False
+
+    class Config:
+        extra = 'allow'
+        arbitrary_types_allowed = True
+
+    def __init__(self, **data):
+        """
+        Initialize a Section with components as keyword arguments.
+        """
+        super().__init__(**data)
+
+    def __call__(self, **kwargs) -> 'Section':
+        """
+        Allows adding components to the section after initialization.
+
+        This makes it possible to build up a section dynamically.
+        """
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+        return self
+
+    def get_components(self) -> Dict[str, FlowfileInComponent]:
+        """
+        Get all FlowfileInComponent instances from the section.
+
+        This method collects all the UI components that have been added to the
+        section, whether as defined fields or as extra fields.
+
+        Returns:
+            A dictionary mapping component names to their instances.
+        """
+        components = {}
+
+        # Get from extra fields
+        for key, value in getattr(self, '__pydantic_extra__', {}).items():
+            if isinstance(value, FlowfileInComponent):
+                components[key] = value
+
+        # Get from defined fields (excluding metadata)
+        for field_name in self.model_fields:
+            if field_name not in {'title', 'description', 'hidden'}:
+                value = getattr(self, field_name, None)
+                if isinstance(value, FlowfileInComponent):
+                    components[field_name] = value
+
+        return components