Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

flowfile_worker/secrets.py

@@ -1,17 +1,29 @@
 """
 Simplified secure storage module for FlowFile worker to read credentials and secrets.
 """
-from cryptography.fernet import Fernet
-import os
-from pathlib import Path
+
+import base64
 import json
 import logging
+import os
+from pathlib import Path
+
+from cryptography.fernet import Fernet
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
 from pydantic import SecretStr
+
 from flowfile_worker.configs import TEST_MODE
 
 # Set up logging
 logger = logging.getLogger(__name__)
 
+# Version identifier for key derivation scheme (must match flowfile_core)
+KEY_DERIVATION_VERSION = b"flowfile-secrets-v1"
+
+# Encrypted secret format: $ffsec$1${user_id}${fernet_token}
+SECRET_FORMAT_PREFIX = "$ffsec$1$"
+
 
 class SecureStorage:
     """A secure local storage mechanism for reading secrets using Fernet encryption."""
@@ -66,27 +78,42 @@ def get_password(service_name, username):
     return _storage.get_password(service_name, username)
 
 
-def get_docker_secret_key():
+def get_docker_secret_key() -> str | None:
     """
-    Get the master key from Docker secret.
+    Get the master key from Docker secret or environment variable.
 
     Returns:
-        str: The master key if successfully read from Docker secret.
+        str: The master key if successfully read, None if not configured.
 
     Raises:
-        RuntimeError: If running in Docker but unable to access the secret.
+        RuntimeError: If the secret file exists but cannot be read, or key is invalid.
     """
+    # First, check for environment variable (allows runtime configuration)
+    env_key = os.environ.get("FLOWFILE_MASTER_KEY")
+    if env_key:
+        # Validate it's a proper Fernet key
+        try:
+            Fernet(env_key.encode())
+            return env_key
+        except Exception:
+            logger.error("FLOWFILE_MASTER_KEY environment variable is not a valid Fernet key")
+            raise RuntimeError("FLOWFILE_MASTER_KEY is not a valid Fernet key")
+
+    # Then, check for Docker secret file
     secret_path = "/run/secrets/flowfile_master_key"
     if os.path.exists(secret_path):
         try:
-            with open(secret_path, "r") as f:
-                return f.read().strip()
+            with open(secret_path) as f:
+                key = f.read().strip()
+                # Validate the key
+                Fernet(key.encode())
+                return key
         except Exception as e:
-            logger.error(f"Failed to read master key from Docker secret: {e}")
+            logger.error(f"Failed to read or validate master key from Docker secret: {e}")
             raise RuntimeError("Failed to read master key from Docker secret")
-    else:
-        logger.critical("Running in Docker but flowfile_master_key secret is not mounted!")
-        raise RuntimeError("Docker secret 'flowfile_master_key' is not mounted")
+
+    # No key configured
+    return None
 
 
 def get_master_key() -> str:
@@ -94,22 +121,29 @@ def get_master_key() -> str:
     Get the master encryption key.
 
     If in TEST_MODE, returns a test key.
-    If running in Docker, retrieves the key from Docker secrets.
+    If running in Docker, retrieves the key from Docker secrets or environment.
     Otherwise, retrieves the key from secure storage.
 
     Returns:
         str: The master encryption key
 
     Raises:
+        RuntimeError: If in Docker mode and no key is configured.
         ValueError: If the master key is not found in storage.
     """
     # First check for test mode
     if TEST_MODE:
-        return b'06t640eu3AG2FmglZS0n0zrEdqadoT7lYDwgSmKyxE4='.decode()
+        return b"06t640eu3AG2FmglZS0n0zrEdqadoT7lYDwgSmKyxE4=".decode()
 
     # Next check if running in Docker
-    if os.environ.get("RUNNING_IN_DOCKER") == "true":
-        return get_docker_secret_key()
+    if os.environ.get("FLOWFILE_MODE") == "docker":
+        key = get_docker_secret_key()
+        if key is None:
+            raise RuntimeError(
+                "Master key not configured. Set FLOWFILE_MASTER_KEY environment variable "
+                "or mount the flowfile_master_key Docker secret."
+            )
+        return key
 
     # Otherwise read from local storage
     key = get_password("flowfile", "master_key")
@@ -118,9 +152,41 @@ def get_master_key() -> str:
     return key
 
 
-def decrypt_secret(encrypted_value) -> SecretStr:
+def derive_user_key(user_id: int) -> bytes:
     """
-    Decrypt an encrypted value using the master key.
+    Derive a user-specific encryption key from the master key using HKDF.
+
+    This provides cryptographic isolation between users - each user's secrets
+    are encrypted with a unique key derived from the master key.
+
+    Args:
+        user_id: The unique identifier for the user
+
+    Returns:
+        bytes: A 32-byte URL-safe base64-encoded key suitable for Fernet
+    """
+    master_key = get_master_key().encode()
+
+    # Use HKDF to derive a user-specific key
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=32,  # Fernet requires 32 bytes
+        salt=KEY_DERIVATION_VERSION,  # Static salt is fine for key derivation
+        info=f"user-{user_id}".encode(),  # User-specific context
+    )
+
+    # Derive raw key material and encode for Fernet
+    derived_key = hkdf.derive(master_key)
+    return base64.urlsafe_b64encode(derived_key)
+
+
+def decrypt_secret(encrypted_value: str) -> SecretStr:
+    """
+    Decrypt an encrypted value.
+
+    Supports both new format (with embedded user_id) and legacy format.
+    - New format: $ffsec$1${user_id}${fernet_token} - user_id extracted automatically
+    - Legacy format: raw Fernet token - uses master key directly
 
     Args:
         encrypted_value: The encrypted value as a string
@@ -128,21 +194,48 @@ def decrypt_secret(encrypted_value) -> SecretStr:
     Returns:
         SecretStr: The decrypted value as a SecretStr
     """
+    # Check for new versioned format with embedded user_id
+    if encrypted_value.startswith(SECRET_FORMAT_PREFIX):
+        # Parse: $ffsec$1${user_id}${fernet_token}
+        remainder = encrypted_value[len(SECRET_FORMAT_PREFIX):]
+        parts = remainder.split("$", 1)
+        if len(parts) != 2:
+            raise ValueError("Invalid encrypted secret format")
+
+        embedded_user_id = int(parts[0])
+        fernet_token = parts[1]
+
+        key = derive_user_key(embedded_user_id)
+        f = Fernet(key)
+        return SecretStr(f.decrypt(fernet_token.encode()).decode())
+
+    # Legacy format - use master key directly
     key = get_master_key().encode()
     f = Fernet(key)
     return SecretStr(f.decrypt(encrypted_value.encode()).decode())
 
 
-def encrypt_secret(secret_value):
+def encrypt_secret(secret_value: str, user_id: int | None = None) -> str:
     """
-    Encrypt a secret value using the master key.
+    Encrypt a secret value.
+
+    If user_id is provided, uses per-user key derivation with embedded user_id format.
+    Otherwise, uses legacy master key encryption (for backward compatibility in tests).
 
     Args:
         secret_value: The secret value to encrypt
+        user_id: Optional user ID for per-user key derivation
 
     Returns:
         str: The encrypted value as a string
     """
+    if user_id is not None:
+        key = derive_user_key(user_id)
+        f = Fernet(key)
+        fernet_token = f.encrypt(secret_value.encode()).decode()
+        return f"{SECRET_FORMAT_PREFIX}{user_id}${fernet_token}"
+
+    # Legacy format for backward compatibility
     key = get_master_key().encode()
     f = Fernet(key)
     return f.encrypt(secret_value.encode()).decode()

flowfile_worker/spawner.py

@@ -1,15 +1,15 @@
-from flowfile_worker import status_dict
-from time import sleep
 import gc
-from typing import List, Tuple
-from multiprocessing import Process, Queue
+from multiprocessing import Process
+from multiprocessing.queues import Queue
+from time import sleep
+
+from flowfile_worker import funcs, models, mp_context, status_dict, status_dict_lock
 from flowfile_worker.process_manager import ProcessManager
-from flowfile_worker import models, mp_context, funcs, status_dict_lock
 
 # Initialize ProcessManager
 process_manager = ProcessManager()
 
-flowfile_node_id_type = int|str
+flowfile_node_id_type = int | str
 
 
 def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_message: mp_context.Array, q: Queue):
@@ -48,7 +48,7 @@ def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_mess
                 with status_dict_lock:
                     status_dict[task_id].status = "Error"
                     with error_message.get_lock():
-                        status_dict[task_id].error_message = error_message.value.decode().rstrip('\x00')
+                        status_dict[task_id].error_message = error_message.value.decode().rstrip("\x00")
                 break
 
         p.join()
@@ -56,11 +56,14 @@ def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_mess
         with status_dict_lock:
             status = status_dict[task_id]
             if status.status != "Cancelled":
-                if progress.value == 100:
+                # Read progress value with lock to ensure consistency
+                with progress.get_lock():
+                    final_progress = progress.value
+                if final_progress == 100:
                     status.status = "Completed"
                     if not q.empty():
                         status.results = q.get()
-                elif progress.value != -1:
+                elif final_progress != -1:
                     status_dict[task_id].status = "Unknown Error"
 
     finally:
@@ -72,11 +75,15 @@ def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_mess
         gc.collect()
 
 
-def start_process(polars_serializable_object: bytes, task_id: str,
-                  operation: models.OperationType,
-                  file_ref: str, flowfile_flow_id: int,
-                  flowfile_node_id: flowfile_node_id_type,
-                  kwargs: dict = None) -> None:
+def start_process(
+    polars_serializable_object: bytes,
+    task_id: str,
+    operation: models.OperationType,
+    file_ref: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+    kwargs: dict = None,
+) -> None:
     """
     Starts a new process for handling Polars dataframe operations.
 
@@ -97,24 +104,31 @@ def start_process(polars_serializable_object: bytes, task_id: str,
     if kwargs is None:
         kwargs = {}
     process_task = getattr(funcs, operation)
-    kwargs['polars_serializable_object'] = polars_serializable_object
-    kwargs['progress'] = mp_context.Value('i', 0)
-    kwargs['error_message'] = mp_context.Array('c', 1024)
-    kwargs['queue'] = Queue(maxsize=1)
-    kwargs['file_path'] = file_ref
-    kwargs['flowfile_flow_id'] = flowfile_flow_id
-    kwargs['flowfile_node_id'] = flowfile_node_id
+    kwargs["polars_serializable_object"] = polars_serializable_object
+    kwargs["progress"] = mp_context.Value("i", 0)
+    kwargs["error_message"] = mp_context.Array("c", 1024)
+    kwargs["queue"] = mp_context.Queue(maxsize=1)
+    kwargs["file_path"] = file_ref
+    kwargs["flowfile_flow_id"] = flowfile_flow_id
+    kwargs["flowfile_node_id"] = flowfile_node_id
 
     p: Process = mp_context.Process(target=process_task, kwargs=kwargs)
     p.start()
 
     process_manager.add_process(task_id, p)
-    handle_task(task_id=task_id, p=p, progress=kwargs['progress'], error_message=kwargs['error_message'], q=kwargs['queue'])
-
-
-def start_generic_process(func_ref: callable, task_id: str,
-                          file_ref: str, flowfile_flow_id: int,
-                          flowfile_node_id: flowfile_node_id_type, kwargs: dict = None) -> None:
+    handle_task(
+        task_id=task_id, p=p, progress=kwargs["progress"], error_message=kwargs["error_message"], q=kwargs["queue"]
+    )
+
+
+def start_generic_process(
+    func_ref: callable,
+    task_id: str,
+    file_ref: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+    kwargs: dict = None,
+) -> None:
     """
     Starts a new process for handling generic function execution.
 
@@ -132,30 +146,33 @@ def start_generic_process(func_ref: callable, task_id: str,
         - Delegates to handle_task for process monitoring
     """
     kwargs = {} if kwargs is None else kwargs
-    kwargs['func'] = func_ref
-    kwargs['progress'] = mp_context.Value('i', 0)
-    kwargs['error_message'] = mp_context.Array('c', 1024)
-    kwargs['queue'] = Queue(maxsize=1)
-    kwargs['file_path'] = file_ref
-    kwargs['flowfile_flow_id'] = flowfile_flow_id
-    kwargs['flowfile_node_id'] = flowfile_node_id
-
-    process_task = getattr(funcs, 'generic_task')
+    kwargs["func"] = func_ref
+    kwargs["progress"] = mp_context.Value("i", 0)
+    kwargs["error_message"] = mp_context.Array("c", 1024)
+    kwargs["queue"] = mp_context.Queue(maxsize=1)
+    kwargs["file_path"] = file_ref
+    kwargs["flowfile_flow_id"] = flowfile_flow_id
+    kwargs["flowfile_node_id"] = flowfile_node_id
+
+    process_task = funcs.generic_task
     p: Process = mp_context.Process(target=process_task, kwargs=kwargs)
     p.start()
 
     process_manager.add_process(task_id, p)  # Add process to process manager
-    handle_task(task_id=task_id, p=p, progress=kwargs['progress'],
-                error_message=kwargs['error_message'], q=kwargs['queue'])
-
-
-def start_fuzzy_process(left_serializable_object: bytes,
-                        right_serializable_object: bytes,
-                        file_ref: str,
-                        fuzzy_maps: List[models.FuzzyMapping],
-                        task_id: str,
-                        flowfile_flow_id: int,
-                        flowfile_node_id: flowfile_node_id_type) -> None:
+    handle_task(
+        task_id=task_id, p=p, progress=kwargs["progress"], error_message=kwargs["error_message"], q=kwargs["queue"]
+    )
+
+
+def start_fuzzy_process(
+    left_serializable_object: bytes,
+    right_serializable_object: bytes,
+    file_ref: str,
+    fuzzy_maps: list[models.FuzzyMapping],
+    task_id: str,
+    flowfile_flow_id: int,
+    flowfile_node_id: flowfile_node_id_type,
+) -> None:
     """
     Starts a new process for performing fuzzy joining operations on two datasets.
 
@@ -172,13 +189,31 @@ def start_fuzzy_process(left_serializable_object: bytes,
         - Initializes and starts a new process for fuzzy joining operation
         - Delegates to handle_task for process monitoring
     """
-    progress = mp_context.Value('i', 0)
-    error_message = mp_context.Array('c', 1024)
-    q = Queue(maxsize=1)
-
-    args: Tuple[bytes, bytes, List[models.FuzzyMapping], mp_context.Array, str, mp_context.Value, Queue, int, flowfile_node_id_type] = \
-        (left_serializable_object, right_serializable_object, fuzzy_maps, error_message, file_ref, progress, q,
-         flowfile_flow_id, flowfile_node_id)
+    progress = mp_context.Value("i", 0)
+    error_message = mp_context.Array("c", 1024)
+    q = mp_context.Queue(maxsize=1)
+
+    args: tuple[
+        bytes,
+        bytes,
+        list[models.FuzzyMapping],
+        mp_context.Array,
+        str,
+        mp_context.Value,
+        Queue,
+        int,
+        flowfile_node_id_type,
+    ] = (
+        left_serializable_object,
+        right_serializable_object,
+        fuzzy_maps,
+        error_message,
+        file_ref,
+        progress,
+        q,
+        flowfile_flow_id,
+        flowfile_node_id,
+    )
 
     p: Process = mp_context.Process(target=funcs.fuzzy_join_task, args=args)
     p.start()

flowfile_worker/utils.py

@@ -1,6 +1,7 @@
+from dataclasses import dataclass
+
 import polars as pl
 from polars.exceptions import PanicException
-from dataclasses import dataclass
 
 
 def collect_lazy_frame(lf: pl.LazyFrame) -> pl.DataFrame:
@@ -12,7 +13,7 @@ def collect_lazy_frame(lf: pl.LazyFrame) -> pl.DataFrame:
 
 @dataclass
 class CollectStreamingInfo:
-    __slots__ = 'df', 'streaming_collect_available'
+    __slots__ = "df", "streaming_collect_available"
     df: pl.DataFrame
     streaming_collect_available: bool
 

shared/__init__.py

@@ -5,11 +5,6 @@ flowfile_core, flowfile_worker, and other components without creating
 circular dependencies.
 """
 
-from .storage_config import storage, get_cache_directory, get_temp_directory, get_flows_directory
+from .storage_config import get_cache_directory, get_flows_directory, get_temp_directory, storage
 
-__all__ = [
-    'storage',
-    'get_cache_directory', 
-    'get_temp_directory',
-    'get_flows_directory'
-]
+__all__ = ["storage", "get_cache_directory", "get_temp_directory", "get_flows_directory"]

shared/storage_config.py

@@ -3,28 +3,40 @@
 Centralized storage configuration for Flowfile.
 This module can be imported by both core and worker without creating dependencies.
 """
+
 import os
 from pathlib import Path
-from typing import Optional, Literal
+from typing import Literal
+
+DirectoryOptions = Literal[
+    "temp_directory",
+    "logs_directory",
+    "system_logs_directory",
+    "database_directory",
+    "cache_directory",
+    "flows_directory",
+    "user_defined_nodes_directory",
+]
+
 
-DirectoryOptions = Literal["temp_directory", "logs_directory",
-                            "system_logs_directory", "database_directory",
-                            "cache_directory", "flows_directory", "user_defined_nodes_directory"]
+def _is_docker_mode() -> bool:
+    """Check if running in Docker mode based on FLOWFILE_MODE."""
+    return os.environ.get("FLOWFILE_MODE") == "docker"
 
 
 class FlowfileStorage:
     """Centralized storage manager for Flowfile applications."""
 
     def __init__(self):
-        self._base_dir: Optional[Path] = None
-        self._user_data_dir: Optional[Path] = None
+        self._base_dir: Path | None = None
+        self._user_data_dir: Path | None = None
         self._ensure_directories()
 
     @property
     def base_directory(self) -> Path:
         """Get the base Flowfile storage directory (for internal container communication)."""
         if self._base_dir is None:
-            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            if _is_docker_mode():
                 # In Docker, internal storage stays inside /app
                 base_path = os.environ.get("FLOWFILE_STORAGE_DIR", "/app/internal_storage")
             else:
@@ -41,7 +53,7 @@ class FlowfileStorage:
     def user_data_directory(self) -> Path:
         """Get the user data directory (completely separate from application code)."""
         if self._user_data_dir is None:
-            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            if _is_docker_mode():
                 # In Docker, user data is at /data/user (completely outside /app)
                 user_data_path = os.environ.get("FLOWFILE_USER_DATA_DIR", "/data/user")
             else:
@@ -70,7 +82,7 @@ class FlowfileStorage:
     @property
     def flows_directory(self) -> Path:
         """Directory for flow storage (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, flows are in separate user data area
             return self.user_data_directory / "flows"
         else:
@@ -80,7 +92,7 @@ class FlowfileStorage:
     @property
     def uploads_directory(self) -> Path:
         """Directory for user uploads (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, uploads are in separate user data area
             return self.user_data_directory / "uploads"
         else:
@@ -90,7 +102,7 @@ class FlowfileStorage:
     @property
     def user_defined_nodes_directory(self) -> Path:
         """Directory for user-defined custom nodes (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             return self.user_data_directory / "user_defined_nodes"
         else:
             return self.base_directory / "user_defined_nodes"
@@ -103,7 +115,7 @@ class FlowfileStorage:
     @property
     def outputs_directory(self) -> Path:
         """Directory for user outputs (user-accessible)."""
-        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+        if _is_docker_mode():
             # In Docker, outputs are in separate user data area
             return self.user_data_directory / "outputs"
         else:
@@ -184,8 +196,8 @@ class FlowfileStorage:
 
     def cleanup_directory(self, directory_option: DirectoryOptions, storage_duration_hours: int = 24) -> None:
         """Clean up any directory of the folder"""
-        import time
         import shutil
+        import time
 
         if not hasattr(self, directory_option):
             raise Exception(f"Directory does not exist in {self.base_directory}")

test_utils/postgres/commands.py

@@ -6,8 +6,9 @@ to start and stop PostgreSQL containers with sample data.
 """
 
 import argparse
-import sys
 import logging
+import sys
+
 from . import fixtures
 
 # Set up logging
@@ -106,4 +107,4 @@ if __name__ == "__main__":
         sys.exit(stop_postgres())
     else:
         print("Usage: python -m test_utils.postgres.commands [start|stop] [options]")
-        sys.exit(1)
+        sys.exit(1)

test_utils/postgres/fixtures.py

@@ -5,13 +5,13 @@ This module provides utilities to set up, manage, and tear down PostgreSQL
 containers with sample data for testing.
 """
 
-import os
-import time
 import logging
-import subprocess
+import os
 import shutil
+import subprocess
+import time
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Dict, Generator, Optional, Tuple
 
 # Configure logging
 logging.basicConfig(
@@ -208,7 +208,7 @@ def start_postgres_container(
         container_name: str = POSTGRES_CONTAINER_NAME,
         port: int = POSTGRES_PORT,
         image_tag: str = POSTGRES_IMAGE_TAG
-) -> Tuple[Optional[subprocess.Popen], bool]:
+) -> tuple[subprocess.Popen | None, bool]:
     """
     Start the PostgreSQL container with sample data.
 
@@ -225,7 +225,7 @@ def start_postgres_container(
         logger.warning("Docker not available, skipping PostgreSQL container start")
         return None, False
 
-    logger.info(f"Starting PostgreSQL container with sample data...")
+    logger.info("Starting PostgreSQL container with sample data...")
 
     # Check if container is already running
     if is_container_running(container_name):
@@ -345,12 +345,12 @@ def print_connection_info(
     print(f"Connection string: postgresql://{user}:{password}@{host}:{port}/{db}")
     print("=" * 50)
     print("\nTo stop the container, run:")
-    print(f"poetry run stop_postgres")
+    print("poetry run stop_postgres")
     print("=" * 50 + "\n")
 
 
 @contextmanager
-def managed_postgres() -> Generator[Dict[str, any], None, None]:
+def managed_postgres() -> Generator[dict[str, any], None, None]:
     """
     Context manager for PostgreSQL container management.
     Ensures proper cleanup even when tests fail.
@@ -414,4 +414,4 @@ def get_db_engine():
         return engine
     except ImportError:
         logger.error("SQLAlchemy not installed. Run: pip install sqlalchemy")
-        raise
+        raise

test_utils/s3/commands.py

@@ -43,4 +43,4 @@ def stop_minio():
     if fixtures.stop_minio_container():
         print("MinIO stopped successfully")
         return 0
-    return 1
+    return 1

test_utils/s3/data_generator.py

@@ -1,17 +1,16 @@
 
-import logging
 import io
+import logging
 import os
 
 # Third-party libraries
 import boto3
-from botocore.client import Config
 import polars as pl
 import pyarrow as pa
+from botocore.client import Config
 from deltalake import write_deltalake
 from pyiceberg.catalog import load_catalog
 
-
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -289,4 +288,4 @@ if __name__ == '__main__':
     populate_test_data(endpoint_url=MINIO_ENDPOINT_URL,
                        access_key=MINIO_ACCESS_KEY,
                        secret_key=MINIO_SECRET_KEY,
-                       bucket_name="test-bucket")
+                       bucket_name="test-bucket")