Flowfile 0.5.3__py3-none-any.whl → 0.5.6__py3-none-any.whl

flowfile_core/flowfile/flow_node/flow_node.py

@@ -1,3 +1,4 @@
+import threading
 from collections.abc import Callable, Generator
 from time import sleep
 from typing import Any, Literal, Optional
@@ -128,6 +129,7 @@ class FlowNode:
         self._cache_progress = None
         self._schema_callback = None
         self._state_needs_reset = False
+        self._execution_lock = threading.RLock()  # Protects concurrent access to get_resulting_data
 
     @property
     def state_needs_reset(self) -> bool:
@@ -147,10 +149,11 @@ class FlowNode:
         """
         self._state_needs_reset = v
 
-    @staticmethod
-    def create_schema_callback_from_function(f: Callable) -> Callable[[], list[FlowfileColumn]]:
+    def create_schema_callback_from_function(self, f: Callable) -> Callable[[], list[FlowfileColumn]]:
         """Wraps a node's function to create a schema callback that extracts the schema.
 
+        Thread-safe: uses _execution_lock to prevent concurrent execution with get_resulting_data.
+
         Args:
             f: The node's core function that returns a FlowDataEngine instance.
 
@@ -161,7 +164,8 @@ class FlowNode:
         def schema_callback() -> list[FlowfileColumn]:
             try:
                 logger.info("Executing the schema callback function based on the node function")
-                return f().schema
+                with self._execution_lock:
+                    return f().schema
             except Exception as e:
                 logger.warning(f"Error with the schema callback: {e}")
                 return []
@@ -575,6 +579,7 @@ class FlowNode:
         """Executes the node's function to produce the actual output data.
 
         Handles both regular functions and external data sources.
+        Thread-safe: uses _execution_lock to prevent concurrent execution.
 
         Returns:
             A FlowDataEngine instance containing the result, or None on error.
@@ -583,30 +588,40 @@ class FlowNode:
             Exception: Propagates exceptions from the node's function execution.
         """
         if self.is_setup:
-            if self.results.resulting_data is None and self.results.errors is None:
-                self.print("getting resulting data")
-                try:
-                    if isinstance(self.function, FlowDataEngine):
-                        fl: FlowDataEngine = self.function
-                    elif self.node_type == "external_source":
-                        fl: FlowDataEngine = self.function()
-                        fl.collect_external()
-                        self.node_settings.streamable = False
-                    else:
-                        try:
-                            fl = self._function(*[v.get_resulting_data() for v in self.all_inputs])
-                        except Exception as e:
-                            raise e
-                    fl.set_streamable(self.node_settings.streamable)
-                    self.results.resulting_data = fl
-                    self.node_schema.result_schema = fl.schema
-                except Exception as e:
-                    self.results.resulting_data = FlowDataEngine()
-                    self.results.errors = str(e)
-                    self.node_stats.has_run_with_current_setup = False
-                    self.node_stats.has_completed_last_run = False
-                    raise e
-            return self.results.resulting_data
+            with self._execution_lock:
+                if self.results.resulting_data is None and self.results.errors is None:
+                    self.print("getting resulting data")
+                    try:
+                        if isinstance(self.function, FlowDataEngine):
+                            fl: FlowDataEngine = self.function
+                        elif self.node_type == "external_source":
+                            fl: FlowDataEngine = self.function()
+                            fl.collect_external()
+                            self.node_settings.streamable = False
+                        else:
+                            try:
+                                self.print("Collecting input data from all inputs")
+                                input_data = []
+                                for i, v in enumerate(self.all_inputs):
+                                    self.print(f"Getting resulting data from input {i} (node {v.node_id})")
+                                    input_result = v.get_resulting_data()
+                                    self.print(f"Input {i} data type: {type(input_result)}, dataframe type: {type(input_result.data_frame) if input_result else 'None'}")
+                                    input_data.append(input_result)
+                                self.print(f"All {len(input_data)} inputs collected, calling node function")
+                                fl = self._function(*input_data)
+                                self.print(f"Node function returned, result type: {type(fl)}")
+                            except Exception as e:
+                                raise e
+                        fl.set_streamable(self.node_settings.streamable)
+                        self.results.resulting_data = fl
+                        self.node_schema.result_schema = fl.schema
+                    except Exception as e:
+                        self.results.resulting_data = FlowDataEngine()
+                        self.results.errors = str(e)
+                        self.node_stats.has_run_with_current_setup = False
+                        self.node_stats.has_completed_last_run = False
+                        raise e
+                return self.results.resulting_data
 
     def _predicted_data_getter(self) -> FlowDataEngine | None:
         """Internal helper to get a predicted data result.
@@ -844,11 +859,18 @@ class FlowNode:
             self.results.resulting_data = self.get_resulting_data()
             self.node_stats.has_run_with_current_setup = True
             return
+
         try:
-            self.get_resulting_data()
+            result_data = self.get_resulting_data()
+            # Use 'is not None' instead of truthiness check to avoid triggering __len__()
+            # which calls .collect() on the LazyFrame and can cause issues
+            if result_data is None:
+                self.results.errors = "Error with creating the lazy frame, most likely due to invalid graph"
+                raise Exception("get_resulting_data returned None")
         except Exception as e:
             self.results.errors = "Error with creating the lazy frame, most likely due to invalid graph"
             raise e
+
         if not performance_mode:
             external_df_fetcher = ExternalDfFetcher(
                 lf=self.get_resulting_data().data_frame,
@@ -858,6 +880,7 @@ class FlowNode:
                 node_id=self.node_id,
             )
             self._fetch_cached_df = external_df_fetcher
+
             try:
                 lf = external_df_fetcher.get_result()
                 self.results.resulting_data = FlowDataEngine(
@@ -869,6 +892,7 @@ class FlowNode:
                         node_id=self.node_id,
                     ).result,
                 )
+
                 if not performance_mode:
                     self.store_example_data_generator(external_df_fetcher)
                     self.node_stats.has_run_with_current_setup = True

flowfile_core/flowfile/node_designer/__init__.py

@@ -15,7 +15,9 @@ from .custom_node import CustomNodeBase, NodeSettings
 
 # Import all UI components so they can be used directly
 from .ui_components import (
+    ActionOption,
     AvailableSecrets,
+    ColumnActionInput,
     ColumnSelector,
     IncomingColumns,
     MultiSelect,
@@ -42,6 +44,8 @@ __all__ = [
     "MultiSelect",
     "NodeSettings",
     "ColumnSelector",
+    "ColumnActionInput",
+    "ActionOption",
     "IncomingColumns",
     "AvailableSecrets",
     "SecretSelector",

flowfile_core/flowfile/node_designer/ui_components.py

@@ -1,4 +1,4 @@
-from typing import Any, Literal
+from typing import Any, Literal, NamedTuple
 
 from pydantic import BaseModel, Field, SecretStr, computed_field
 
@@ -11,6 +11,29 @@ from flowfile_core.types import DataType, TypeSpec
 InputType = Literal["text", "number", "secret", "array", "date", "boolean"]
 
 
+class ActionOption(NamedTuple):
+    """
+    A named tuple representing an action option with a value and display label.
+
+    Use this to define actions with custom labels in ColumnActionInput:
+        actions=[
+            ActionOption("sum", "Sum"),
+            ActionOption("avg", "Average"),
+            "count"  # plain strings also work
+        ]
+    """
+
+    value: str
+    """The internal value used in the data."""
+
+    label: str
+    """The display label shown in the UI."""
+
+
+# Type alias for action specifications - accepts strings or ActionOption tuples
+ActionSpec = str | ActionOption
+
+
 def normalize_input_to_data_types(v: Any) -> Literal["ALL"] | list[DataType]:
     """
     Normalizes a wide variety of inputs to either 'ALL' or a sorted list of DataType enums.
@@ -376,3 +399,123 @@ class SecretSelector(FlowfileInComponent):
         if self.name_prefix:
             data["name_prefix"] = self.name_prefix
         return data
+
+
+class ColumnActionInput(FlowfileInComponent):
+    """
+    A generic UI component for configuring column-based transformations.
+
+    This component allows users to select columns, choose an action/transformation,
+    and optionally rename the output. It can be configured for many use cases:
+    rolling windows, aggregations, string transformations, type conversions, etc.
+
+    The component displays:
+    - A list of available columns (filterable by data type)
+    - A table of configured operations with: Column, Action, Output Name
+    - Optional group by and order by selectors
+
+    Example - Rolling Window:
+        ColumnActionInput(
+            label="Rolling Calculations",
+            actions=["sum", "mean", "min", "max"],
+            output_name_template="{column}_rolling_{action}",
+            show_group_by=True,
+            show_order_by=True,
+            data_types="Numeric"
+        )
+
+    Example - String Transformations:
+        ColumnActionInput(
+            label="String Operations",
+            actions=["upper", "lower", "trim", "reverse"],
+            output_name_template="{column}_{action}",
+            data_types="String"
+        )
+
+    Example - Aggregations with ActionOption:
+        ColumnActionInput(
+            label="Aggregations",
+            actions=[
+                ActionOption("sum", "Sum"),
+                ActionOption("count", "Count"),
+                ActionOption("mean", "Average"),
+                "min",  # plain strings also work
+            ],
+            output_name_template="{column}_{action}",
+            show_group_by=True
+        )
+    """
+
+    component_type: Literal["ColumnActionInput"] = "ColumnActionInput"
+    input_type: InputType = "array"
+
+    # Configurable actions - list of action names or ActionOption tuples
+    actions: list[ActionSpec] = Field(default_factory=list)
+    """Actions available for selection. Can be strings or ActionOption(value, label) tuples."""
+
+    # Template for auto-generating output names
+    # Supports placeholders: {column}, {action}
+    output_name_template: str = "{column}_{action}"
+    """Template for generating default output names. Use {column} and {action} placeholders."""
+
+    # Optional grouping/ordering support
+    show_group_by: bool = False
+    """Whether to show the group by column selector."""
+
+    show_order_by: bool = False
+    """Whether to show the order by column selector."""
+
+    # Type filtering for column selection
+    data_type_filter_input: TypeSpec = Field(default="ALL", alias="data_types", repr=False, exclude=True)
+    """Filter columns by data type. Defaults to ALL."""
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        # Initialize value if not set
+        if self.value is None:
+            self.value = {
+                "rows": [],
+                "group_by_columns": [],
+                "order_by_column": None,
+            }
+
+    def set_value(self, value: Any):
+        """
+        Sets the value from frontend.
+        """
+        self.value = value
+        return self
+
+    @computed_field
+    @property
+    def data_types_filter(self) -> Literal["ALL"] | list[DataType]:
+        """
+        A computed field that normalizes the `data_type_filter_input` into a
+        standardized format for the frontend.
+        """
+        return normalize_input_to_data_types(self.data_type_filter_input)
+
+    def model_dump(self, **kwargs) -> dict:
+        """
+        Serializes the component for the frontend.
+        """
+        data = super().model_dump(**kwargs)
+        # Normalize actions to list of {value, label} objects for frontend
+        normalized_actions = []
+        for action in self.actions:
+            if isinstance(action, tuple):
+                normalized_actions.append({"value": action[0], "label": action[1]})
+            else:
+                normalized_actions.append({"value": action, "label": action})
+        data["actions"] = normalized_actions
+        data["output_name_template"] = self.output_name_template
+        data["show_group_by"] = self.show_group_by
+        data["show_order_by"] = self.show_order_by
+        if "data_types_filter" in data and data["data_types_filter"] != "ALL":
+            data["data_types"] = sorted([dt.value for dt in data["data_types_filter"]])
+        else:
+            data["data_types"] = "ALL"
+        return data

flowfile_core/main.py

@@ -4,10 +4,9 @@ import signal
 from contextlib import asynccontextmanager
 
 import uvicorn
-from fastapi import FastAPI
+from fastapi import BackgroundTasks, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
-from flowfile_core import ServerRun
 from flowfile_core.configs.flow_logger import clear_all_flow_logs
 from flowfile_core.configs.settings import (
     SERVER_HOST,
@@ -91,7 +90,7 @@ app.include_router(user_defined_components_router, prefix="/user_defined_compone
 
 
 @app.post("/shutdown")
-async def shutdown():
+async def shutdown(background_tasks: BackgroundTasks):
     """An API endpoint to gracefully shut down the server.
 
     This endpoint sets a flag that the Uvicorn server checks, allowing it
@@ -99,7 +98,6 @@ async def shutdown():
     after the HTTP response has been sent.
     """
     # Use a background task to trigger the shutdown after the response is sent
-    background_tasks = ServerRun()
     background_tasks.add_task(trigger_shutdown)
     return {"message": "Server is shutting down"}
 

flowfile_core/routes/public.py

@@ -1,11 +1,53 @@
+import os
+
 from fastapi import APIRouter
 from fastapi.responses import RedirectResponse
+from pydantic import BaseModel
+
+from flowfile_core.auth.secrets import generate_master_key, is_master_key_configured
 
-# Router setup
 router = APIRouter()
 
 
+class SetupStatus(BaseModel):
+    """Response model for the setup status endpoint."""
+
+    setup_required: bool
+    master_key_configured: bool
+    mode: str
+
+
+class GeneratedKey(BaseModel):
+    """Response model for the generate key endpoint."""
+
+    key: str
+    instructions: str
+
+
 @router.get("/", tags=["admin"])
 async def docs_redirect():
     """Redirects to the documentation page."""
     return RedirectResponse(url="/docs")
+
+
+@router.get("/health/status", response_model=SetupStatus, tags=["health"])
+async def get_setup_status():
+    """Get the current setup status of the application."""
+    mode = os.environ.get("FLOWFILE_MODE", "electron")
+    master_key_ok = is_master_key_configured()
+    return SetupStatus(
+        setup_required=not master_key_ok,
+        master_key_configured=master_key_ok,
+        mode=mode,
+    )
+
+
+@router.post("/setup/generate-key", response_model=GeneratedKey, tags=["setup"])
+async def generate_key():
+    """Generate a new master encryption key."""
+    key = generate_master_key()
+    instructions = (
+        f'Add to your .env file:\n  FLOWFILE_MASTER_KEY="{key}"\n\n'
+        "Then restart: docker-compose down && docker-compose up"
+    )
+    return GeneratedKey(key=key, instructions=instructions)

flowfile_core/schemas/cloud_storage_schemas.py

@@ -14,14 +14,20 @@ AuthMethod = Literal[
 ]
 
 
-def encrypt_for_worker(secret_value: SecretStr | None) -> str | None:
+def encrypt_for_worker(secret_value: SecretStr | None, user_id: int) -> str | None:
     """
-    Encrypts a secret value for use in worker contexts.
-    This is a placeholder function that simulates encryption.
-    In practice, you would use a secure encryption method.
+    Encrypts a secret value for use in worker contexts using per-user key derivation.
+
+    Args:
+        secret_value: The secret value to encrypt
+        user_id: The user ID for key derivation
+
+    Returns:
+        Encrypted secret with embedded user_id, or None if secret_value is None
     """
     if secret_value is not None:
-        return encrypt_secret(secret_value.get_secret_value())
+        return encrypt_secret(secret_value.get_secret_value(), user_id)
+    return None
 
 
 class AuthSettingsInput(BaseModel):
@@ -80,25 +86,31 @@ class FullCloudStorageConnection(AuthSettingsInput):
     endpoint_url: str | None = None
     verify_ssl: bool = True
 
-    def get_worker_interface(self) -> "FullCloudStorageConnectionWorkerInterface":
+    def get_worker_interface(self, user_id: int) -> "FullCloudStorageConnectionWorkerInterface":
         """
-        Convert to a public interface model without secrets.
+        Convert to a worker interface model with encrypted secrets.
+
+        Args:
+            user_id: The user ID for per-user key derivation
+
+        Returns:
+            FullCloudStorageConnectionWorkerInterface with encrypted secrets
         """
         return FullCloudStorageConnectionWorkerInterface(
             storage_type=self.storage_type,
             auth_method=self.auth_method,
             connection_name=self.connection_name,
             aws_allow_unsafe_html=self.aws_allow_unsafe_html,
-            aws_secret_access_key=encrypt_for_worker(self.aws_secret_access_key),
+            aws_secret_access_key=encrypt_for_worker(self.aws_secret_access_key, user_id),
             aws_region=self.aws_region,
             aws_access_key_id=self.aws_access_key_id,
             aws_role_arn=self.aws_role_arn,
-            aws_session_token=encrypt_for_worker(self.aws_session_token),
+            aws_session_token=encrypt_for_worker(self.aws_session_token, user_id),
             azure_account_name=self.azure_account_name,
             azure_tenant_id=self.azure_tenant_id,
-            azure_account_key=encrypt_for_worker(self.azure_account_key),
+            azure_account_key=encrypt_for_worker(self.azure_account_key, user_id),
             azure_client_id=self.azure_client_id,
-            azure_client_secret=encrypt_for_worker(self.azure_client_secret),
+            azure_client_secret=encrypt_for_worker(self.azure_client_secret, user_id),
             endpoint_url=self.endpoint_url,
             verify_ssl=self.verify_ssl,
         )
@@ -202,18 +214,30 @@ def get_cloud_storage_write_settings_worker_interface(
     write_settings: CloudStorageWriteSettings,
     connection: FullCloudStorageConnection,
     lf: pl.LazyFrame,
+    user_id: int,
     flowfile_flow_id: int = 1,
     flowfile_node_id: int | str = -1,
 ) -> CloudStorageWriteSettingsWorkerInterface:
     """
-    Convert to a worker interface model with hashed secrets.
+    Convert to a worker interface model with encrypted secrets.
+
+    Args:
+        write_settings: Cloud storage write settings
+        connection: Full cloud storage connection with secrets
+        lf: LazyFrame to serialize
+        user_id: User ID for per-user key derivation
+        flowfile_flow_id: Flow ID for tracking
+        flowfile_node_id: Node ID for tracking
+
+    Returns:
+        CloudStorageWriteSettingsWorkerInterface ready for worker
     """
     operation = base64.b64encode(lf.serialize()).decode()
 
     return CloudStorageWriteSettingsWorkerInterface(
         operation=operation,
         write_settings=write_settings.get_write_setting_worker_interface(),
-        connection=connection.get_worker_interface(),
-        flowfile_flow_id=flowfile_flow_id,  # Default value, can be overridden
-        flowfile_node_id=flowfile_node_id,  # Default value, can be overridden
+        connection=connection.get_worker_interface(user_id),
+        flowfile_flow_id=flowfile_flow_id,
+        flowfile_node_id=flowfile_node_id,
     )

flowfile_core/secret_manager/secret_manager.py

@@ -1,4 +1,8 @@
+import base64
+
 from cryptography.fernet import Fernet
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
 from fastapi.exceptions import HTTPException
 from pydantic import SecretStr
 from sqlalchemy import and_
@@ -9,21 +13,118 @@ from flowfile_core.auth.secrets import get_master_key
 from flowfile_core.database import models as db_models
 from flowfile_core.database.connection import get_db_context
 
+# Version identifier for key derivation scheme (allows future migrations)
+KEY_DERIVATION_VERSION = b"flowfile-secrets-v1"
+
+# Encrypted secret format: $ffsec$1${user_id}${fernet_token}
+# This embeds the user_id so the worker can derive the correct key
+SECRET_FORMAT_PREFIX = "$ffsec$1$"
+
+
+def derive_user_key(user_id: int) -> bytes:
+    """
+    Derive a user-specific encryption key from the master key using HKDF.
+
+    This provides cryptographic isolation between users - each user's secrets
+    are encrypted with a unique key derived from the master key.
+
+    Args:
+        user_id: The unique identifier for the user
+
+    Returns:
+        bytes: A 32-byte URL-safe base64-encoded key suitable for Fernet
+    """
+    master_key = get_master_key().encode()
 
-def encrypt_secret(secret_value):
-    """Encrypt a secret value using the master key."""
+    # Use HKDF to derive a user-specific key
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=32,  # Fernet requires 32 bytes
+        salt=KEY_DERIVATION_VERSION,  # Static salt is fine for key derivation
+        info=f"user-{user_id}".encode(),  # User-specific context
+    )
+
+    # Derive raw key material and encode for Fernet
+    derived_key = hkdf.derive(master_key)
+    return base64.urlsafe_b64encode(derived_key)
+
+
+def _encrypt_with_master_key(secret_value: str) -> str:
+    """Legacy encryption using master key directly (for backward compatibility)."""
     key = get_master_key().encode()
     f = Fernet(key)
     return f.encrypt(secret_value.encode()).decode()
 
 
-def decrypt_secret(encrypted_value) -> SecretStr:
-    """Decrypt an encrypted value using the master key."""
+def _decrypt_with_master_key(encrypted_value: str) -> SecretStr:
+    """Legacy decryption using master key directly (for backward compatibility)."""
     key = get_master_key().encode()
     f = Fernet(key)
     return SecretStr(f.decrypt(encrypted_value.encode()).decode())
 
 
+def encrypt_secret(secret_value: str, user_id: int) -> str:
+    """
+    Encrypt a secret value using a user-specific derived key.
+
+    The encrypted format embeds the user_id so it can be decrypted
+    without knowing the user context (e.g., by the worker service).
+
+    Format: $ffsec$1${user_id}${fernet_token}
+
+    Args:
+        secret_value: The plaintext secret to encrypt
+        user_id: The user ID to derive the encryption key for
+
+    Returns:
+        str: The encrypted value with embedded user_id
+    """
+    key = derive_user_key(user_id)
+    f = Fernet(key)
+    fernet_token = f.encrypt(secret_value.encode()).decode()
+    return f"{SECRET_FORMAT_PREFIX}{user_id}${fernet_token}"
+
+
+def decrypt_secret(encrypted_value: str, user_id: int | None = None) -> SecretStr:
+    """
+    Decrypt an encrypted value.
+
+    Supports both new format (with embedded user_id) and legacy format.
+    - New format: $ffsec$1${user_id}${fernet_token} - user_id extracted automatically
+    - Legacy format: raw Fernet token - requires user_id parameter or uses master key
+
+    Args:
+        encrypted_value: The encrypted secret
+        user_id: Optional user ID (required for legacy secrets, ignored for new format)
+
+    Returns:
+        SecretStr: The decrypted value wrapped in SecretStr
+    """
+    # Check for new versioned format with embedded user_id
+    if encrypted_value.startswith(SECRET_FORMAT_PREFIX):
+        # Parse: $ffsec$1${user_id}${fernet_token}
+        remainder = encrypted_value[len(SECRET_FORMAT_PREFIX):]
+        parts = remainder.split("$", 1)
+        if len(parts) != 2:
+            raise ValueError("Invalid encrypted secret format")
+
+        embedded_user_id = int(parts[0])
+        fernet_token = parts[1]
+
+        key = derive_user_key(embedded_user_id)
+        f = Fernet(key)
+        return SecretStr(f.decrypt(fernet_token.encode()).decode())
+
+    # Legacy format - use provided user_id or fall back to master key
+    if user_id is not None:
+        key = derive_user_key(user_id)
+        f = Fernet(key)
+        return SecretStr(f.decrypt(encrypted_value.encode()).decode())
+
+    # Fall back to master key for legacy secrets without user context
+    return _decrypt_with_master_key(encrypted_value)
+
+
 def get_encrypted_secret(current_user_id: int, secret_name: str) -> str | None:
     with get_db_context() as db:
         user_id = current_user_id
@@ -39,13 +140,13 @@ def get_encrypted_secret(current_user_id: int, secret_name: str) -> str | None:
 
 
 def store_secret(db: Session, secret: SecretInput, user_id: int) -> db_models.Secret:
-    encrypted_value = encrypt_secret(secret.value.get_secret_value())
+    encrypted_value = encrypt_secret(secret.value.get_secret_value(), user_id)
 
     # Store in database
     db_secret = db_models.Secret(
         name=secret.name,
         encrypted_value=encrypted_value,
-        iv="",  # Not used with Fernet
+        iv="",  # Legacy field, not used with current encryption
         user_id=user_id,
     )
     db.add(db_secret)

flowfile_frame/__init__.py

@@ -54,6 +54,17 @@ from flowfile_frame.cloud_storage.secret_manager import (
     get_all_available_cloud_storage_connections,
 )
 
+# Database I/O
+from flowfile_frame.database import (
+    create_database_connection,
+    create_database_connection_if_not_exists,
+    del_database_connection,
+    get_all_available_database_connections,
+    get_database_connection_by_name,
+    read_database,
+    write_database,
+)
+
 # Commonly used functions
 from flowfile_frame.expr import (  # noqa: F401
     col,