Flowfile 0.5.3__py3-none-any.whl → 0.5.4__py3-none-any.whl

flowfile_core/flowfile/node_designer/__init__.py

@@ -15,7 +15,9 @@ from .custom_node import CustomNodeBase, NodeSettings
 
 # Import all UI components so they can be used directly
 from .ui_components import (
+    ActionOption,
     AvailableSecrets,
+    ColumnActionInput,
     ColumnSelector,
     IncomingColumns,
     MultiSelect,
@@ -42,6 +44,8 @@ __all__ = [
     "MultiSelect",
     "NodeSettings",
     "ColumnSelector",
+    "ColumnActionInput",
+    "ActionOption",
     "IncomingColumns",
     "AvailableSecrets",
     "SecretSelector",

flowfile_core/flowfile/node_designer/ui_components.py

@@ -1,4 +1,4 @@
-from typing import Any, Literal
+from typing import Any, Literal, NamedTuple
 
 from pydantic import BaseModel, Field, SecretStr, computed_field
 
@@ -11,6 +11,29 @@ from flowfile_core.types import DataType, TypeSpec
 InputType = Literal["text", "number", "secret", "array", "date", "boolean"]
 
 
+class ActionOption(NamedTuple):
+    """
+    A named tuple representing an action option with a value and display label.
+
+    Use this to define actions with custom labels in ColumnActionInput:
+        actions=[
+            ActionOption("sum", "Sum"),
+            ActionOption("avg", "Average"),
+            "count"  # plain strings also work
+        ]
+    """
+
+    value: str
+    """The internal value used in the data."""
+
+    label: str
+    """The display label shown in the UI."""
+
+
+# Type alias for action specifications - accepts strings or ActionOption tuples
+ActionSpec = str | ActionOption
+
+
 def normalize_input_to_data_types(v: Any) -> Literal["ALL"] | list[DataType]:
     """
     Normalizes a wide variety of inputs to either 'ALL' or a sorted list of DataType enums.
@@ -376,3 +399,123 @@ class SecretSelector(FlowfileInComponent):
         if self.name_prefix:
             data["name_prefix"] = self.name_prefix
         return data
+
+
+class ColumnActionInput(FlowfileInComponent):
+    """
+    A generic UI component for configuring column-based transformations.
+
+    This component allows users to select columns, choose an action/transformation,
+    and optionally rename the output. It can be configured for many use cases:
+    rolling windows, aggregations, string transformations, type conversions, etc.
+
+    The component displays:
+    - A list of available columns (filterable by data type)
+    - A table of configured operations with: Column, Action, Output Name
+    - Optional group by and order by selectors
+
+    Example - Rolling Window:
+        ColumnActionInput(
+            label="Rolling Calculations",
+            actions=["sum", "mean", "min", "max"],
+            output_name_template="{column}_rolling_{action}",
+            show_group_by=True,
+            show_order_by=True,
+            data_types="Numeric"
+        )
+
+    Example - String Transformations:
+        ColumnActionInput(
+            label="String Operations",
+            actions=["upper", "lower", "trim", "reverse"],
+            output_name_template="{column}_{action}",
+            data_types="String"
+        )
+
+    Example - Aggregations with ActionOption:
+        ColumnActionInput(
+            label="Aggregations",
+            actions=[
+                ActionOption("sum", "Sum"),
+                ActionOption("count", "Count"),
+                ActionOption("mean", "Average"),
+                "min",  # plain strings also work
+            ],
+            output_name_template="{column}_{action}",
+            show_group_by=True
+        )
+    """
+
+    component_type: Literal["ColumnActionInput"] = "ColumnActionInput"
+    input_type: InputType = "array"
+
+    # Configurable actions - list of action names or ActionOption tuples
+    actions: list[ActionSpec] = Field(default_factory=list)
+    """Actions available for selection. Can be strings or ActionOption(value, label) tuples."""
+
+    # Template for auto-generating output names
+    # Supports placeholders: {column}, {action}
+    output_name_template: str = "{column}_{action}"
+    """Template for generating default output names. Use {column} and {action} placeholders."""
+
+    # Optional grouping/ordering support
+    show_group_by: bool = False
+    """Whether to show the group by column selector."""
+
+    show_order_by: bool = False
+    """Whether to show the order by column selector."""
+
+    # Type filtering for column selection
+    data_type_filter_input: TypeSpec = Field(default="ALL", alias="data_types", repr=False, exclude=True)
+    """Filter columns by data type. Defaults to ALL."""
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        # Initialize value if not set
+        if self.value is None:
+            self.value = {
+                "rows": [],
+                "group_by_columns": [],
+                "order_by_column": None,
+            }
+
+    def set_value(self, value: Any):
+        """
+        Sets the value from frontend.
+        """
+        self.value = value
+        return self
+
+    @computed_field
+    @property
+    def data_types_filter(self) -> Literal["ALL"] | list[DataType]:
+        """
+        A computed field that normalizes the `data_type_filter_input` into a
+        standardized format for the frontend.
+        """
+        return normalize_input_to_data_types(self.data_type_filter_input)
+
+    def model_dump(self, **kwargs) -> dict:
+        """
+        Serializes the component for the frontend.
+        """
+        data = super().model_dump(**kwargs)
+        # Normalize actions to list of {value, label} objects for frontend
+        normalized_actions = []
+        for action in self.actions:
+            if isinstance(action, tuple):
+                normalized_actions.append({"value": action[0], "label": action[1]})
+            else:
+                normalized_actions.append({"value": action, "label": action})
+        data["actions"] = normalized_actions
+        data["output_name_template"] = self.output_name_template
+        data["show_group_by"] = self.show_group_by
+        data["show_order_by"] = self.show_order_by
+        if "data_types_filter" in data and data["data_types_filter"] != "ALL":
+            data["data_types"] = sorted([dt.value for dt in data["data_types_filter"]])
+        else:
+            data["data_types"] = "ALL"
+        return data

flowfile_core/routes/public.py

@@ -1,11 +1,53 @@
+import os
+
 from fastapi import APIRouter
 from fastapi.responses import RedirectResponse
+from pydantic import BaseModel
+
+from flowfile_core.auth.secrets import generate_master_key, is_master_key_configured
 
-# Router setup
 router = APIRouter()
 
 
+class SetupStatus(BaseModel):
+    """Response model for the setup status endpoint."""
+
+    setup_required: bool
+    master_key_configured: bool
+    mode: str
+
+
+class GeneratedKey(BaseModel):
+    """Response model for the generate key endpoint."""
+
+    key: str
+    instructions: str
+
+
 @router.get("/", tags=["admin"])
 async def docs_redirect():
     """Redirects to the documentation page."""
     return RedirectResponse(url="/docs")
+
+
+@router.get("/health/status", response_model=SetupStatus, tags=["health"])
+async def get_setup_status():
+    """Get the current setup status of the application."""
+    mode = os.environ.get("FLOWFILE_MODE", "electron")
+    master_key_ok = is_master_key_configured()
+    return SetupStatus(
+        setup_required=not master_key_ok,
+        master_key_configured=master_key_ok,
+        mode=mode,
+    )
+
+
+@router.post("/setup/generate-key", response_model=GeneratedKey, tags=["setup"])
+async def generate_key():
+    """Generate a new master encryption key."""
+    key = generate_master_key()
+    instructions = (
+        f'Add to your .env file:\n  FLOWFILE_MASTER_KEY="{key}"\n\n'
+        "Then restart: docker-compose down && docker-compose up"
+    )
+    return GeneratedKey(key=key, instructions=instructions)

flowfile_core/schemas/cloud_storage_schemas.py

@@ -14,14 +14,20 @@ AuthMethod = Literal[
 ]
 
 
-def encrypt_for_worker(secret_value: SecretStr | None) -> str | None:
+def encrypt_for_worker(secret_value: SecretStr | None, user_id: int) -> str | None:
     """
-    Encrypts a secret value for use in worker contexts.
-    This is a placeholder function that simulates encryption.
-    In practice, you would use a secure encryption method.
+    Encrypts a secret value for use in worker contexts using per-user key derivation.
+
+    Args:
+        secret_value: The secret value to encrypt
+        user_id: The user ID for key derivation
+
+    Returns:
+        Encrypted secret with embedded user_id, or None if secret_value is None
     """
     if secret_value is not None:
-        return encrypt_secret(secret_value.get_secret_value())
+        return encrypt_secret(secret_value.get_secret_value(), user_id)
+    return None
 
 
 class AuthSettingsInput(BaseModel):
@@ -80,25 +86,31 @@ class FullCloudStorageConnection(AuthSettingsInput):
     endpoint_url: str | None = None
     verify_ssl: bool = True
 
-    def get_worker_interface(self) -> "FullCloudStorageConnectionWorkerInterface":
+    def get_worker_interface(self, user_id: int) -> "FullCloudStorageConnectionWorkerInterface":
         """
-        Convert to a public interface model without secrets.
+        Convert to a worker interface model with encrypted secrets.
+
+        Args:
+            user_id: The user ID for per-user key derivation
+
+        Returns:
+            FullCloudStorageConnectionWorkerInterface with encrypted secrets
         """
         return FullCloudStorageConnectionWorkerInterface(
             storage_type=self.storage_type,
             auth_method=self.auth_method,
             connection_name=self.connection_name,
             aws_allow_unsafe_html=self.aws_allow_unsafe_html,
-            aws_secret_access_key=encrypt_for_worker(self.aws_secret_access_key),
+            aws_secret_access_key=encrypt_for_worker(self.aws_secret_access_key, user_id),
             aws_region=self.aws_region,
             aws_access_key_id=self.aws_access_key_id,
             aws_role_arn=self.aws_role_arn,
-            aws_session_token=encrypt_for_worker(self.aws_session_token),
+            aws_session_token=encrypt_for_worker(self.aws_session_token, user_id),
             azure_account_name=self.azure_account_name,
             azure_tenant_id=self.azure_tenant_id,
-            azure_account_key=encrypt_for_worker(self.azure_account_key),
+            azure_account_key=encrypt_for_worker(self.azure_account_key, user_id),
             azure_client_id=self.azure_client_id,
-            azure_client_secret=encrypt_for_worker(self.azure_client_secret),
+            azure_client_secret=encrypt_for_worker(self.azure_client_secret, user_id),
             endpoint_url=self.endpoint_url,
             verify_ssl=self.verify_ssl,
         )
@@ -202,18 +214,30 @@ def get_cloud_storage_write_settings_worker_interface(
     write_settings: CloudStorageWriteSettings,
     connection: FullCloudStorageConnection,
     lf: pl.LazyFrame,
+    user_id: int,
     flowfile_flow_id: int = 1,
     flowfile_node_id: int | str = -1,
 ) -> CloudStorageWriteSettingsWorkerInterface:
     """
-    Convert to a worker interface model with hashed secrets.
+    Convert to a worker interface model with encrypted secrets.
+
+    Args:
+        write_settings: Cloud storage write settings
+        connection: Full cloud storage connection with secrets
+        lf: LazyFrame to serialize
+        user_id: User ID for per-user key derivation
+        flowfile_flow_id: Flow ID for tracking
+        flowfile_node_id: Node ID for tracking
+
+    Returns:
+        CloudStorageWriteSettingsWorkerInterface ready for worker
     """
     operation = base64.b64encode(lf.serialize()).decode()
 
     return CloudStorageWriteSettingsWorkerInterface(
         operation=operation,
         write_settings=write_settings.get_write_setting_worker_interface(),
-        connection=connection.get_worker_interface(),
-        flowfile_flow_id=flowfile_flow_id,  # Default value, can be overridden
-        flowfile_node_id=flowfile_node_id,  # Default value, can be overridden
+        connection=connection.get_worker_interface(user_id),
+        flowfile_flow_id=flowfile_flow_id,
+        flowfile_node_id=flowfile_node_id,
     )

flowfile_core/secret_manager/secret_manager.py

@@ -1,4 +1,8 @@
+import base64
+
 from cryptography.fernet import Fernet
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
 from fastapi.exceptions import HTTPException
 from pydantic import SecretStr
 from sqlalchemy import and_
@@ -9,21 +13,118 @@ from flowfile_core.auth.secrets import get_master_key
 from flowfile_core.database import models as db_models
 from flowfile_core.database.connection import get_db_context
 
+# Version identifier for key derivation scheme (allows future migrations)
+KEY_DERIVATION_VERSION = b"flowfile-secrets-v1"
+
+# Encrypted secret format: $ffsec$1${user_id}${fernet_token}
+# This embeds the user_id so the worker can derive the correct key
+SECRET_FORMAT_PREFIX = "$ffsec$1$"
+
+
+def derive_user_key(user_id: int) -> bytes:
+    """
+    Derive a user-specific encryption key from the master key using HKDF.
+
+    This provides cryptographic isolation between users - each user's secrets
+    are encrypted with a unique key derived from the master key.
+
+    Args:
+        user_id: The unique identifier for the user
+
+    Returns:
+        bytes: A 32-byte URL-safe base64-encoded key suitable for Fernet
+    """
+    master_key = get_master_key().encode()
 
-def encrypt_secret(secret_value):
-    """Encrypt a secret value using the master key."""
+    # Use HKDF to derive a user-specific key
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=32,  # Fernet requires 32 bytes
+        salt=KEY_DERIVATION_VERSION,  # Static salt is fine for key derivation
+        info=f"user-{user_id}".encode(),  # User-specific context
+    )
+
+    # Derive raw key material and encode for Fernet
+    derived_key = hkdf.derive(master_key)
+    return base64.urlsafe_b64encode(derived_key)
+
+
+def _encrypt_with_master_key(secret_value: str) -> str:
+    """Legacy encryption using master key directly (for backward compatibility)."""
     key = get_master_key().encode()
     f = Fernet(key)
     return f.encrypt(secret_value.encode()).decode()
 
 
-def decrypt_secret(encrypted_value) -> SecretStr:
-    """Decrypt an encrypted value using the master key."""
+def _decrypt_with_master_key(encrypted_value: str) -> SecretStr:
+    """Legacy decryption using master key directly (for backward compatibility)."""
     key = get_master_key().encode()
     f = Fernet(key)
     return SecretStr(f.decrypt(encrypted_value.encode()).decode())
 
 
+def encrypt_secret(secret_value: str, user_id: int) -> str:
+    """
+    Encrypt a secret value using a user-specific derived key.
+
+    The encrypted format embeds the user_id so it can be decrypted
+    without knowing the user context (e.g., by the worker service).
+
+    Format: $ffsec$1${user_id}${fernet_token}
+
+    Args:
+        secret_value: The plaintext secret to encrypt
+        user_id: The user ID to derive the encryption key for
+
+    Returns:
+        str: The encrypted value with embedded user_id
+    """
+    key = derive_user_key(user_id)
+    f = Fernet(key)
+    fernet_token = f.encrypt(secret_value.encode()).decode()
+    return f"{SECRET_FORMAT_PREFIX}{user_id}${fernet_token}"
+
+
+def decrypt_secret(encrypted_value: str, user_id: int | None = None) -> SecretStr:
+    """
+    Decrypt an encrypted value.
+
+    Supports both new format (with embedded user_id) and legacy format.
+    - New format: $ffsec$1${user_id}${fernet_token} - user_id extracted automatically
+    - Legacy format: raw Fernet token - requires user_id parameter or uses master key
+
+    Args:
+        encrypted_value: The encrypted secret
+        user_id: Optional user ID (required for legacy secrets, ignored for new format)
+
+    Returns:
+        SecretStr: The decrypted value wrapped in SecretStr
+    """
+    # Check for new versioned format with embedded user_id
+    if encrypted_value.startswith(SECRET_FORMAT_PREFIX):
+        # Parse: $ffsec$1${user_id}${fernet_token}
+        remainder = encrypted_value[len(SECRET_FORMAT_PREFIX):]
+        parts = remainder.split("$", 1)
+        if len(parts) != 2:
+            raise ValueError("Invalid encrypted secret format")
+
+        embedded_user_id = int(parts[0])
+        fernet_token = parts[1]
+
+        key = derive_user_key(embedded_user_id)
+        f = Fernet(key)
+        return SecretStr(f.decrypt(fernet_token.encode()).decode())
+
+    # Legacy format - use provided user_id or fall back to master key
+    if user_id is not None:
+        key = derive_user_key(user_id)
+        f = Fernet(key)
+        return SecretStr(f.decrypt(encrypted_value.encode()).decode())
+
+    # Fall back to master key for legacy secrets without user context
+    return _decrypt_with_master_key(encrypted_value)
+
+
 def get_encrypted_secret(current_user_id: int, secret_name: str) -> str | None:
     with get_db_context() as db:
         user_id = current_user_id
@@ -39,13 +140,13 @@ def get_encrypted_secret(current_user_id: int, secret_name: str) -> str | None:
 
 
 def store_secret(db: Session, secret: SecretInput, user_id: int) -> db_models.Secret:
-    encrypted_value = encrypt_secret(secret.value.get_secret_value())
+    encrypted_value = encrypt_secret(secret.value.get_secret_value(), user_id)
 
     # Store in database
     db_secret = db_models.Secret(
         name=secret.name,
         encrypted_value=encrypted_value,
-        iv="",  # Not used with Fernet
+        iv="",  # Legacy field, not used with current encryption
         user_id=user_id,
     )
     db.add(db_secret)

flowfile_frame/__init__.py

@@ -54,6 +54,17 @@ from flowfile_frame.cloud_storage.secret_manager import (
     get_all_available_cloud_storage_connections,
 )
 
+# Database I/O
+from flowfile_frame.database import (
+    create_database_connection,
+    create_database_connection_if_not_exists,
+    del_database_connection,
+    get_all_available_database_connections,
+    get_database_connection_by_name,
+    read_database,
+    write_database,
+)
+
 # Commonly used functions
 from flowfile_frame.expr import (  # noqa: F401
     col,

flowfile_frame/database/__init__.py

@@ -0,0 +1,36 @@
+"""Database module for flowfile_frame.
+
+This module provides functions for:
+- Managing database connections (create, list, delete)
+- Reading from databases
+- Writing to databases
+"""
+
+from flowfile_frame.database.connection_manager import (
+    create_database_connection,
+    create_database_connection_if_not_exists,
+    del_database_connection,
+    get_all_available_database_connections,
+    get_database_connection_by_name,
+)
+from flowfile_frame.database.frame_helpers import (
+    add_read_from_database,
+    add_write_to_database,
+    read_database,
+    write_database,
+)
+
+__all__ = [
+    # Connection management
+    "create_database_connection",
+    "create_database_connection_if_not_exists",
+    "del_database_connection",
+    "get_all_available_database_connections",
+    "get_database_connection_by_name",
+    # FlowGraph helpers
+    "add_read_from_database",
+    "add_write_to_database",
+    # Direct read/write
+    "read_database",
+    "write_database",
+]