Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_core/flowfile/node_designer/custom_node.py

@@ -1,9 +1,16 @@
 # Fixed custom_node.py with proper type hints
 
+from typing import Any, TypeVar
+
 import polars as pl
 from pydantic import BaseModel
-from typing import Any, Dict, Optional, TypeVar, Callable
-from flowfile_core.flowfile.node_designer.ui_components import FlowfileInComponent, IncomingColumns, Section
+
+from flowfile_core.flowfile.node_designer.ui_components import (
+    FlowfileInComponent,
+    IncomingColumns,
+    SecretSelector,
+    Section,
+)
 from flowfile_core.schemas.schemas import NodeTemplate, NodeTypeLiteral, TransformTypeLiteral
 
 
@@ -22,9 +29,9 @@ def to_frontend_schema(model_instance: BaseModel) -> dict:
         A dictionary representation of the model.
     """
     result = {}
-    extra_fields = getattr(model_instance, '__pydantic_extra__', {})
+    extra_fields = getattr(model_instance, "__pydantic_extra__", {})
     model_fields = {k: getattr(model_instance, k) for k in model_instance.model_fields.keys()}
-    for key, value in (extra_fields|model_fields).items():
+    for key, value in (extra_fields | model_fields).items():
         result[key] = _convert_value(value)
     return result
 
@@ -34,25 +41,18 @@ def _convert_value(value: Any) -> Any:
     Helper function to convert any value to a frontend-ready format.
     """
     if isinstance(value, Section):
-        section_data = value.model_dump(
-            include={'title', 'description', 'hidden'},
-            exclude_none=True
-        )
+        section_data = value.model_dump(include={"title", "description", "hidden"}, exclude_none=True)
         section_data["component_type"] = "Section"
-        section_data["components"] = {
-            key: _convert_value(comp)
-            for key, comp in value.get_components().items()
-        }
+        section_data["components"] = {key: _convert_value(comp) for key, comp in value.get_components().items()}
         return section_data
 
     elif isinstance(value, FlowfileInComponent):
         component_dict = value.model_dump(exclude_none=True)
-        if 'options' in component_dict:
-            if component_dict['options'] is IncomingColumns or (
-                    isinstance(component_dict['options'], type) and
-                    issubclass(component_dict['options'], IncomingColumns)
+        if "options" in component_dict:
+            if component_dict["options"] is IncomingColumns or (
+                isinstance(component_dict["options"], type) and issubclass(component_dict["options"], IncomingColumns)
             ):
-                component_dict['options'] = {"__type__": "IncomingColumns"}
+                component_dict["options"] = {"__type__": "IncomingColumns"}
         return component_dict
     elif isinstance(value, BaseModel):
         return to_frontend_schema(value)
@@ -67,7 +67,7 @@ def _convert_value(value: Any) -> Any:
 
 
 # Type variable for the Section factory
-T = TypeVar('T', bound=Section)
+T = TypeVar("T", bound=Section)
 
 
 def create_section(**components: FlowfileInComponent) -> Section:
@@ -104,17 +104,30 @@ class NodeSettings(BaseModel):
             main_config = main_config_section
             advanced_options = advanced_config_section
     """
+
     class Config:
-        extra = 'allow'
+        extra = "allow"
         arbitrary_types_allowed = True
 
+    def has_sections(self) -> bool:
+        """Check if this settings class has any sections defined."""
+        if self.model_fields:
+            return True
+        extra = getattr(self, "__pydantic_extra__", {})
+        return bool(extra)
+
+    def is_empty(self) -> bool:
+        """Check if this is an empty settings class with no configuration."""
+
+        return not self.has_sections()
+
     def __init__(self, **sections):
         """
         Initialize NodeSettings with sections as keyword arguments.
         """
         super().__init__(**sections)
 
-    def populate_values(self, values: Dict[str, Any]) -> 'NodeSettings':
+    def populate_values(self, values: dict[str, Any]) -> "NodeSettings":
         """
         Populates the settings with values received from the frontend.
 
@@ -133,7 +146,7 @@ class NodeSettings(BaseModel):
         all_sections = {}
 
         # Get extra fields
-        extra_fields = getattr(self, '__pydantic_extra__', {})
+        extra_fields = getattr(self, "__pydantic_extra__", {})
         all_sections.update(extra_fields)
 
         # Get defined fields that are Sections
@@ -150,6 +163,80 @@ class NodeSettings(BaseModel):
                         component.set_value(section_values[component_name])
         return self
 
+    def get_value(self, field_name: str) -> Any:
+        """
+        Gets the current value of a field by name.
+
+        Searches through direct fields, extra fields, and sections.
+
+        Args:
+            field_name: The name of the field to retrieve.
+
+        Returns:
+            The current value of the field, or None if not found.
+        """
+        # Check direct model fields
+        if field_name in self.model_fields:
+            component = getattr(self, field_name, None)
+            if component is not None:
+                if isinstance(component, FlowfileInComponent):
+                    return component.value
+                return component
+
+        # Check pydantic extra fields
+        extras = getattr(self, "__pydantic_extra__", {}) or {}
+        if field_name in extras:
+            component = extras[field_name]
+            if isinstance(component, FlowfileInComponent):
+                return component.value
+            return component
+
+        # Check within sections (both in model_fields and extras)
+        all_fields = {**{k: getattr(self, k) for k in self.model_fields}, **extras}
+        for value in all_fields.values():
+            if isinstance(value, Section):
+                components = value.get_components()
+                if field_name in components:
+                    component = components[field_name]
+                    if isinstance(component, FlowfileInComponent):
+                        return component.value
+                    return component
+
+        return None
+
+    def get_all_components(self) -> dict[str, FlowfileInComponent]:
+        """
+        Returns all UI components in the settings, including those nested in sections.
+
+        Returns:
+            Dictionary mapping field names to their FlowfileInComponent instances.
+        """
+        components = {}
+
+        # Get from model fields
+        for field_name in self.model_fields:
+            value = getattr(self, field_name, None)
+            if isinstance(value, FlowfileInComponent):
+                components[field_name] = value
+            elif isinstance(value, Section):
+                components.update(value.get_components())
+
+        # Get from extra fields
+        extras = getattr(self, "__pydantic_extra__", {}) or {}
+        for field_name, value in extras.items():
+            if isinstance(value, FlowfileInComponent):
+                components[field_name] = value
+            elif isinstance(value, Section):
+                components.update(value.get_components())
+
+        return components
+
+    def set_secret_context(self, user_id: int, accessed_secrets: set):
+        """Inject execution context into all SecretSelector components."""
+        for component in self.get_all_components().values():
+            if isinstance(component, SecretSelector):
+                component.set_execution_context(user_id, accessed_secrets)
+
 
 def create_node_settings(**sections: Section) -> NodeSettings:
     """
@@ -187,13 +274,13 @@ class SectionBuilder:
         advanced_section = builder.build()
     """
 
-    def __init__(self, title: Optional[str] = None, description: Optional[str] = None, hidden: bool = False):
+    def __init__(self, title: str | None = None, description: str | None = None, hidden: bool = False):
         self._section = Section(title=title, description=description, hidden=hidden)
 
-    def add_component(self, name: str, component: FlowfileInComponent) -> 'SectionBuilder':
+    def add_component(self, name: str, component: FlowfileInComponent) -> "SectionBuilder":
         """Add a component to the section."""
         setattr(self._section, name, component)
-        extra = getattr(self._section, '__pydantic_extra__', {})
+        extra = getattr(self._section, "__pydantic_extra__", {})
         extra[name] = component
         return self
 
@@ -219,10 +306,10 @@ class NodeSettingsBuilder:
     def __init__(self):
         self._settings = NodeSettings()
 
-    def add_section(self, name: str, section: Section) -> 'NodeSettingsBuilder':
+    def add_section(self, name: str, section: Section) -> "NodeSettingsBuilder":
         """Add a section to the node settings."""
         setattr(self._settings, name, section)
-        extra = getattr(self._settings, '__pydantic_extra__', {})
+        extra = getattr(self._settings, "__pydantic_extra__", {})
         extra[name] = section
         return self
 
@@ -238,25 +325,29 @@ class CustomNodeBase(BaseModel):
     To create a new node, you should inherit from this class and define its
     attributes and the `process` method.
     """
+
     # Core node properties
     node_name: str
     node_category: str = "Custom"
     node_icon: str = "user-defined-icon.png"
-    settings_schema: Optional[NodeSettings] = None
+    settings_schema: NodeSettings | None = None
 
     # I/O configuration
     number_of_inputs: int = 1
     number_of_outputs: int = 1
 
     # Display properties in the UI
-    node_group: Optional[str] = "custom"
-    title: Optional[str] = "Custom Node"
-    intro: Optional[str] = "A custom node for data processing"
+    node_group: str | None = "custom"
+    title: str | None = "Custom Node"
+    intro: str | None = "A custom node for data processing"
 
     # Behavior properties
     node_type: NodeTypeLiteral = "process"
     transform_type: TransformTypeLiteral = "wide"
 
+    _user_id: int | None = None
+    accessed_secrets: set[str] = set()
+
     @property
     def item(self):
         """A unique identifier for the node, derived from its name."""
@@ -269,11 +360,46 @@ class CustomNodeBase(BaseModel):
         """
         Initialize the node, optionally populating settings from initial values.
         """
-        initial_values = data.pop('initial_values', None)
+        initial_values = data.pop("initial_values", None)
         super().__init__(**data)
         if self.settings_schema and initial_values:
             self.settings_schema.populate_values(initial_values)
 
+    def set_execution_context(self, user_id: int):
+        """
+        Sets the execution context for the node.
+        Called by the framework before executing the node.
+
+        Args:
+            user_id: The ID of the user executing this node.
+        """
+        self._user_id = user_id
+        self.accessed_secrets = set()
+
+    def get_accessed_secrets(self) -> set[str]:
+        """
+        Returns the set of secret values accessed during this execution.
+        Used by the output scanner to detect accidental leaks.
+        """
+        return self.accessed_secrets.copy()
+
+    def get_secret_names(self) -> list[str]:
+        """
+        Returns a list of all SecretSelector field names in the settings schema.
+        Useful for validation and debugging.
+        """
+        from flowfile_core.flowfile.node_designer.ui_components import SecretSelector
+
+        if self.settings_schema is None:
+            return []
+
+        secret_fields = []
+        for name, field in self.settings_schema.get_all_components().items():
+            if isinstance(field, SecretSelector):
+                secret_fields.append(name)
+
+        return secret_fields
+
     def get_frontend_schema(self) -> dict:
         """
         Get the frontend-ready schema with current values.
@@ -303,20 +429,20 @@ class CustomNodeBase(BaseModel):
         return schema
 
     @classmethod
-    def from_frontend_schema(cls, schema: dict) -> 'CustomNodeBase':
+    def from_frontend_schema(cls, schema: dict) -> "CustomNodeBase":
         """
         Create a node instance from a frontend schema.
 
         This is used when loading a node from a saved flow.
         """
-        settings_values = schema.pop('settings_schema', {})
+        settings_values = schema.pop("settings_schema", {})
         node = cls(**schema)
         if settings_values and node.settings_schema:
             node.settings_schema.populate_values(settings_values)
         return node
 
     @classmethod
-    def from_settings(cls, settings_values: dict) -> 'CustomNodeBase':
+    def from_settings(cls, settings_values: dict) -> "CustomNodeBase":
         """
         Create a node instance with just its settings values.
 
@@ -327,7 +453,7 @@ class CustomNodeBase(BaseModel):
             node.settings_schema.populate_values(settings_values)
         return node
 
-    def update_settings(self, values: Dict[str, Any]) -> 'CustomNodeBase':
+    def update_settings(self, values: dict[str, Any]) -> "CustomNodeBase":
         """
         Update the settings with new values from the frontend.
         """
@@ -367,5 +493,5 @@ class CustomNodeBase(BaseModel):
             drawer_intro=self.intro,
             node_type=self.node_type,
             transform_type=self.transform_type,
-            custom_node=True
+            custom_node=True,
         )

flowfile_core/flowfile/node_designer/ui_components.py

@@ -1,19 +1,17 @@
-# ui_components.py - Updated ColumnSelector
+from typing import Any, Literal
 
-from typing import List, Optional, Any, Literal, Union, Type, Tuple, Dict
-
-from pydantic import Field, BaseModel, computed_field
+from pydantic import BaseModel, Field, SecretStr, computed_field
 
 from flowfile_core.flowfile.node_designer._type_registry import normalize_type_spec
+from flowfile_core.secret_manager.secret_manager import decrypt_secret, get_encrypted_secret
+
 # Public API import
-from flowfile_core.flowfile.node_designer.data_types import DataType, TypeSpec
+from flowfile_core.types import DataType, TypeSpec
 
 InputType = Literal["text", "number", "secret", "array", "date", "boolean"]
 
 
-def normalize_input_to_data_types(
-    v: Any
-) -> Union[Literal["ALL"], List[DataType]]:
+def normalize_input_to_data_types(v: Any) -> Literal["ALL"] | list[DataType]:
     """
     Normalizes a wide variety of inputs to either 'ALL' or a sorted list of DataType enums.
     This function is used as a Pydantic BeforeValidator.
@@ -46,9 +44,10 @@ class FlowfileInComponent(BaseModel):
     It's not meant to be used directly, but rather to be inherited by specific
     component classes.
     """
+
     component_type: str = Field(..., description="Type of the UI component")
     value: Any = None
-    label: Optional[str] = None
+    label: str | None = None
     input_type: InputType
 
     def set_value(self, value: Any):
@@ -83,6 +82,7 @@ class IncomingColumns:
                 options=IncomingColumns
             )
     """
+
     pass
 
 
@@ -94,25 +94,21 @@ class ColumnSelector(FlowfileInComponent):
     This is particularly useful when a node operation should only be applied
     to columns of a specific type (e.g., numeric, string, date).
     """
+
     component_type: Literal["ColumnSelector"] = "ColumnSelector"
     required: bool = False
     multiple: bool = False
     input_type: InputType = "text"
 
     # Normalized output: either "ALL" or list of DataType enums
-    data_type_filter_input: TypeSpec = Field(
-        default="ALL",
-        alias="data_types",
-        repr=False,
-        exclude=True
-    )
+    data_type_filter_input: TypeSpec = Field(default="ALL", alias="data_types", repr=False, exclude=True)
 
     class Config:
         arbitrary_types_allowed = True
 
     @computed_field
     @property
-    def data_types_filter(self) -> Union[Literal["ALL"], List[DataType]]:
+    def data_types_filter(self) -> Literal["ALL"] | list[DataType]:
         """
         A computed field that normalizes the `data_type_filter_input` into a
         standardized format for the frontend.
@@ -125,16 +121,17 @@ class ColumnSelector(FlowfileInComponent):
         correct format for the frontend.
         """
         data = super().model_dump(**kwargs)
-        if 'data_types_filter' in data and data['data_types_filter'] != "ALL":
-            data['data_types'] = sorted([dt.value for dt in data['data_types_filter']])
+        if "data_types_filter" in data and data["data_types_filter"] != "ALL":
+            data["data_types"] = sorted([dt.value for dt in data["data_types_filter"]])
         return data
 
 
 class TextInput(FlowfileInComponent):
     """A standard text input field for capturing string values."""
+
     component_type: Literal["TextInput"] = "TextInput"
-    default: Optional[str] = ""
-    placeholder: Optional[str] = ""
+    default: str | None = ""
+    placeholder: str | None = ""
     input_type: InputType = "text"
 
     def __init__(self, **data):
@@ -145,23 +142,43 @@ class TextInput(FlowfileInComponent):
 
 class NumericInput(FlowfileInComponent):
     """A numeric input field with optional minimum and maximum value validation."""
+
     component_type: Literal["NumericInput"] = "NumericInput"
-    default: Optional[float] = None
-    min_value: Optional[float] = None
-    max_value: Optional[float] = None
+    default: float | None = None
+    min_value: float | None = None
+    max_value: float | None = None
+    input_type: InputType = "number"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if self.value is None and self.default is not None:
+            self.value = self.default
+
+
+class SliderInput(FlowfileInComponent):
+    """A slider input for selecting a numeric value within a range."""
+
+    component_type: Literal["SliderInput"] = "SliderInput"
+    default: float | None = None
+    min_value: float = 0
+    max_value: float = 100
+    step: float = 1
     input_type: InputType = "number"
 
     def __init__(self, **data):
         super().__init__(**data)
         if self.value is None and self.default is not None:
             self.value = self.default
+        elif self.value is None:
+            self.value = self.min_value
 
 
 class ToggleSwitch(FlowfileInComponent):
     """A boolean toggle switch, typically used for enabling or disabling a feature."""
+
     component_type: Literal["ToggleSwitch"] = "ToggleSwitch"
     default: bool = False
-    description: Optional[str] = None
+    description: str | None = None
     input_type: InputType = "boolean"
 
     def __init__(self, **data):
@@ -182,9 +199,10 @@ class SingleSelect(FlowfileInComponent):
     dynamically populated from the input dataframe's columns by using the
     `IncomingColumns` marker.
     """
+
     component_type: Literal["SingleSelect"] = "SingleSelect"
-    options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
-    default: Optional[Any] = None
+    options: list[str | tuple[str, Any]] | type[IncomingColumns]
+    default: Any | None = None
     input_type: InputType = "text"
 
     def __init__(self, **data):
@@ -200,9 +218,10 @@ class MultiSelect(FlowfileInComponent):
     Like `SingleSelect`, the options can be static or dynamically populated
     from the input columns using the `IncomingColumns` marker.
     """
+
     component_type: Literal["MultiSelect"] = "MultiSelect"
-    options: Union[List[Union[str, Tuple[str, Any]]], Type[IncomingColumns]]
-    default: List[Any] = Field(default_factory=list)
+    options: list[str | tuple[str, Any]] | type[IncomingColumns]
+    default: list[Any] = Field(default_factory=list)
     input_type: InputType = "array"
 
     def __init__(self, **data):
@@ -226,12 +245,13 @@ class Section(BaseModel):
             my_text_input=TextInput(label="Enter a value")
         )
     """
-    title: Optional[str] = None
-    description: Optional[str] = None
+
+    title: str | None = None
+    description: str | None = None
     hidden: bool = False
 
     class Config:
-        extra = 'allow'
+        extra = "allow"
         arbitrary_types_allowed = True
 
     def __init__(self, **data):
@@ -240,7 +260,7 @@ class Section(BaseModel):
         """
         super().__init__(**data)
 
-    def __call__(self, **kwargs) -> 'Section':
+    def __call__(self, **kwargs) -> "Section":
         """
         Allows adding components to the section after initialization.
 
@@ -250,7 +270,7 @@ class Section(BaseModel):
             setattr(self, key, value)
         return self
 
-    def get_components(self) -> Dict[str, FlowfileInComponent]:
+    def get_components(self) -> dict[str, FlowfileInComponent]:
         """
         Get all FlowfileInComponent instances from the section.
 
@@ -263,15 +283,96 @@ class Section(BaseModel):
         components = {}
 
         # Get from extra fields
-        for key, value in getattr(self, '__pydantic_extra__', {}).items():
+        for key, value in getattr(self, "__pydantic_extra__", {}).items():
             if isinstance(value, FlowfileInComponent):
                 components[key] = value
 
         # Get from defined fields (excluding metadata)
         for field_name in self.model_fields:
-            if field_name not in {'title', 'description', 'hidden'}:
+            if field_name not in {"title", "description", "hidden"}:
                 value = getattr(self, field_name, None)
                 if isinstance(value, FlowfileInComponent):
                     components[field_name] = value
 
         return components
+
+
+class AvailableSecrets:
+    """
+    A marker class used in `SecretSelector` components.
+
+    When `options` is set to this class, the component will be dynamically
+    populated with the secret names available to the current user.
+    This allows users to select from available secrets at runtime.
+
+    Example:
+        class MyNodeSettings(NodeSettings):
+            api_key = SecretSelector(
+                label="Select an API Key",
+                options=AvailableSecrets
+            )
+    """
+
+    pass
+
+
+class SecretSelector(FlowfileInComponent):
+    component_type: Literal["SecretSelector"] = "SecretSelector"
+    options: type[AvailableSecrets] = AvailableSecrets
+    required: bool = False
+    description: str | None = None
+    input_type: InputType = "secret"
+    name_prefix: str | None = None
+
+    # Private fields for runtime context
+    _user_id: int | None = None
+    _accessed_secrets: set | None = None  # Reference to node's tracking set
+
+    def set_execution_context(self, user_id: int, accessed_secrets: set):
+        """Called by framework before process() runs."""
+        self._user_id = user_id
+        self._accessed_secrets = accessed_secrets
+
+    @property
+    def secret_value(self) -> SecretStr | None:
+        """
+        Get the decrypted secret value.
+
+        Can only be called during node execution (after context is set).
+        Returns None if no secret is selected.
+        """
+        if self.value is None:
+            return None
+
+        if self._user_id is None:
+            raise ValueError(
+                "Secret can only be accessed during node execution. "
+                "Ensure you're calling this from within the process() method."
+            )
+
+        encrypted = get_encrypted_secret(current_user_id=self._user_id, secret_name=self.value)
+
+        if encrypted is None:
+            raise ValueError(
+                f"Secret '{self.value}' not found for user. " f"Please ensure the secret exists in your secrets store."
+            )
+
+        decrypted = decrypt_secret(encrypted)
+
+        if self._accessed_secrets is not None:
+            self._accessed_secrets.add(decrypted.get_secret_value())
+        else:
+            self._accessed_secrets = {decrypted.get_secret_value()}
+        return decrypted
+
+    def model_dump(self, **kwargs) -> dict:
+        """
+        Overrides the default `model_dump` to signal to the frontend
+        that this needs dynamic population from available secrets.
+        """
+        data = super().model_dump(**kwargs)
+        # Signal to frontend that options should be fetched from /secrets endpoint
+        data["options"] = {"__type__": "AvailableSecrets"}
+        if self.name_prefix:
+            data["name_prefix"] = self.name_prefix
+        return data