Flowfile 0.5.3__py3-none-any.whl → 0.5.6__py3-none-any.whl

flowfile_worker/secrets.py

@@ -2,12 +2,15 @@
 Simplified secure storage module for FlowFile worker to read credentials and secrets.
 """
 
+import base64
 import json
 import logging
 import os
 from pathlib import Path
 
 from cryptography.fernet import Fernet
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
 from pydantic import SecretStr
 
 from flowfile_worker.configs import TEST_MODE
@@ -15,6 +18,12 @@ from flowfile_worker.configs import TEST_MODE
 # Set up logging
 logger = logging.getLogger(__name__)
 
+# Version identifier for key derivation scheme (must match flowfile_core)
+KEY_DERIVATION_VERSION = b"flowfile-secrets-v1"
+
+# Encrypted secret format: $ffsec$1${user_id}${fernet_token}
+SECRET_FORMAT_PREFIX = "$ffsec$1$"
+
 
 class SecureStorage:
     """A secure local storage mechanism for reading secrets using Fernet encryption."""
@@ -69,27 +78,42 @@ def get_password(service_name, username):
     return _storage.get_password(service_name, username)
 
 
-def get_docker_secret_key():
+def get_docker_secret_key() -> str | None:
     """
-    Get the master key from Docker secret.
+    Get the master key from Docker secret or environment variable.
 
     Returns:
-        str: The master key if successfully read from Docker secret.
+        str: The master key if successfully read, None if not configured.
 
     Raises:
-        RuntimeError: If running in Docker but unable to access the secret.
+        RuntimeError: If the secret file exists but cannot be read, or key is invalid.
     """
+    # First, check for environment variable (allows runtime configuration)
+    env_key = os.environ.get("FLOWFILE_MASTER_KEY")
+    if env_key:
+        # Validate it's a proper Fernet key
+        try:
+            Fernet(env_key.encode())
+            return env_key
+        except Exception:
+            logger.error("FLOWFILE_MASTER_KEY environment variable is not a valid Fernet key")
+            raise RuntimeError("FLOWFILE_MASTER_KEY is not a valid Fernet key")
+
+    # Then, check for Docker secret file
     secret_path = "/run/secrets/flowfile_master_key"
     if os.path.exists(secret_path):
         try:
             with open(secret_path) as f:
-                return f.read().strip()
+                key = f.read().strip()
+                # Validate the key
+                Fernet(key.encode())
+                return key
         except Exception as e:
-            logger.error(f"Failed to read master key from Docker secret: {e}")
+            logger.error(f"Failed to read or validate master key from Docker secret: {e}")
             raise RuntimeError("Failed to read master key from Docker secret")
-    else:
-        logger.critical("Running in Docker but flowfile_master_key secret is not mounted!")
-        raise RuntimeError("Docker secret 'flowfile_master_key' is not mounted")
+
+    # No key configured
+    return None
 
 
 def get_master_key() -> str:
@@ -97,13 +121,14 @@ def get_master_key() -> str:
     Get the master encryption key.
 
     If in TEST_MODE, returns a test key.
-    If running in Docker, retrieves the key from Docker secrets.
+    If running in Docker, retrieves the key from Docker secrets or environment.
     Otherwise, retrieves the key from secure storage.
 
     Returns:
         str: The master encryption key
 
     Raises:
+        RuntimeError: If in Docker mode and no key is configured.
         ValueError: If the master key is not found in storage.
     """
     # First check for test mode
@@ -112,7 +137,13 @@ def get_master_key() -> str:
 
     # Next check if running in Docker
     if os.environ.get("FLOWFILE_MODE") == "docker":
-        return get_docker_secret_key()
+        key = get_docker_secret_key()
+        if key is None:
+            raise RuntimeError(
+                "Master key not configured. Set FLOWFILE_MASTER_KEY environment variable "
+                "or mount the flowfile_master_key Docker secret."
+            )
+        return key
 
     # Otherwise read from local storage
     key = get_password("flowfile", "master_key")
@@ -121,9 +152,41 @@ def get_master_key() -> str:
     return key
 
 
-def decrypt_secret(encrypted_value) -> SecretStr:
+def derive_user_key(user_id: int) -> bytes:
     """
-    Decrypt an encrypted value using the master key.
+    Derive a user-specific encryption key from the master key using HKDF.
+
+    This provides cryptographic isolation between users - each user's secrets
+    are encrypted with a unique key derived from the master key.
+
+    Args:
+        user_id: The unique identifier for the user
+
+    Returns:
+        bytes: A 32-byte URL-safe base64-encoded key suitable for Fernet
+    """
+    master_key = get_master_key().encode()
+
+    # Use HKDF to derive a user-specific key
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=32,  # Fernet requires 32 bytes
+        salt=KEY_DERIVATION_VERSION,  # Static salt is fine for key derivation
+        info=f"user-{user_id}".encode(),  # User-specific context
+    )
+
+    # Derive raw key material and encode for Fernet
+    derived_key = hkdf.derive(master_key)
+    return base64.urlsafe_b64encode(derived_key)
+
+
+def decrypt_secret(encrypted_value: str) -> SecretStr:
+    """
+    Decrypt an encrypted value.
+
+    Supports both new format (with embedded user_id) and legacy format.
+    - New format: $ffsec$1${user_id}${fernet_token} - user_id extracted automatically
+    - Legacy format: raw Fernet token - uses master key directly
 
     Args:
         encrypted_value: The encrypted value as a string
@@ -131,21 +194,48 @@ def decrypt_secret(encrypted_value) -> SecretStr:
     Returns:
         SecretStr: The decrypted value as a SecretStr
     """
+    # Check for new versioned format with embedded user_id
+    if encrypted_value.startswith(SECRET_FORMAT_PREFIX):
+        # Parse: $ffsec$1${user_id}${fernet_token}
+        remainder = encrypted_value[len(SECRET_FORMAT_PREFIX):]
+        parts = remainder.split("$", 1)
+        if len(parts) != 2:
+            raise ValueError("Invalid encrypted secret format")
+
+        embedded_user_id = int(parts[0])
+        fernet_token = parts[1]
+
+        key = derive_user_key(embedded_user_id)
+        f = Fernet(key)
+        return SecretStr(f.decrypt(fernet_token.encode()).decode())
+
+    # Legacy format - use master key directly
     key = get_master_key().encode()
     f = Fernet(key)
     return SecretStr(f.decrypt(encrypted_value.encode()).decode())
 
 
-def encrypt_secret(secret_value):
+def encrypt_secret(secret_value: str, user_id: int | None = None) -> str:
     """
-    Encrypt a secret value using the master key.
+    Encrypt a secret value.
+
+    If user_id is provided, uses per-user key derivation with embedded user_id format.
+    Otherwise, uses legacy master key encryption (for backward compatibility in tests).
 
     Args:
         secret_value: The secret value to encrypt
+        user_id: Optional user ID for per-user key derivation
 
     Returns:
         str: The encrypted value as a string
     """
+    if user_id is not None:
+        key = derive_user_key(user_id)
+        f = Fernet(key)
+        fernet_token = f.encrypt(secret_value.encode()).decode()
+        return f"{SECRET_FORMAT_PREFIX}{user_id}${fernet_token}"
+
+    # Legacy format for backward compatibility
     key = get_master_key().encode()
     f = Fernet(key)
     return f.encrypt(secret_value.encode()).decode()

flowfile_worker/spawner.py

@@ -1,5 +1,6 @@
 import gc
-from multiprocessing import Process, Queue
+from multiprocessing import Process
+from multiprocessing.queues import Queue
 from time import sleep
 
 from flowfile_worker import funcs, models, mp_context, status_dict, status_dict_lock
@@ -55,11 +56,14 @@ def handle_task(task_id: str, p: Process, progress: mp_context.Value, error_mess
         with status_dict_lock:
             status = status_dict[task_id]
             if status.status != "Cancelled":
-                if progress.value == 100:
+                # Read progress value with lock to ensure consistency
+                with progress.get_lock():
+                    final_progress = progress.value
+                if final_progress == 100:
                     status.status = "Completed"
                     if not q.empty():
                         status.results = q.get()
-                elif progress.value != -1:
+                elif final_progress != -1:
                     status_dict[task_id].status = "Unknown Error"
 
     finally:
@@ -103,7 +107,7 @@ def start_process(
     kwargs["polars_serializable_object"] = polars_serializable_object
     kwargs["progress"] = mp_context.Value("i", 0)
     kwargs["error_message"] = mp_context.Array("c", 1024)
-    kwargs["queue"] = Queue(maxsize=1)
+    kwargs["queue"] = mp_context.Queue(maxsize=1)
     kwargs["file_path"] = file_ref
     kwargs["flowfile_flow_id"] = flowfile_flow_id
     kwargs["flowfile_node_id"] = flowfile_node_id
@@ -145,7 +149,7 @@ def start_generic_process(
     kwargs["func"] = func_ref
     kwargs["progress"] = mp_context.Value("i", 0)
     kwargs["error_message"] = mp_context.Array("c", 1024)
-    kwargs["queue"] = Queue(maxsize=1)
+    kwargs["queue"] = mp_context.Queue(maxsize=1)
     kwargs["file_path"] = file_ref
     kwargs["flowfile_flow_id"] = flowfile_flow_id
     kwargs["flowfile_node_id"] = flowfile_node_id
@@ -187,7 +191,7 @@ def start_fuzzy_process(
     """
     progress = mp_context.Value("i", 0)
     error_message = mp_context.Array("c", 1024)
-    q = Queue(maxsize=1)
+    q = mp_context.Queue(maxsize=1)
 
     args: tuple[
         bytes,

flowfile/web/static/assets/ContextMenu-26d4dd27.css

@@ -1,26 +0,0 @@
-
-.context-menu[data-v-f6c94f99] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: var(--color-background-primary);
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-  user-select: none;
-}
-.context-menu ul[data-v-f6c94f99] {
-  list-style: none;
-  padding: 0;
-  margin: 0;
-}
-.context-menu li[data-v-f6c94f99] {
-  padding: 8px 16px;
-  cursor: pointer;
-}
-.context-menu li.disabled[data-v-f6c94f99] {
-  color: #ccc;
-  cursor: not-allowed;
-}
-.context-menu li[data-v-f6c94f99]:hover:not(.disabled) {
-  background-color: #f0f0f0;
-}

flowfile/web/static/assets/ContextMenu-31ee57f0.js

@@ -1,41 +0,0 @@
-import { d as defineComponent, o as openBlock, c as createElementBlock, b as createBaseVNode, F as Fragment, D as renderList, n as normalizeClass, t as toDisplayString, Z as normalizeStyle, i as _export_sfc } from "./index-fb6493ae.js";
-const _hoisted_1 = ["onClick"];
-const _sfc_main = /* @__PURE__ */ defineComponent({
-  __name: "ContextMenu",
-  props: {
-    position: { type: Object, required: true },
-    options: {
-      type: Array,
-      required: true
-    }
-  },
-  emits: ["select", "close"],
-  setup(__props, { emit: __emit }) {
-    const emit = __emit;
-    const selectOption = (action) => {
-      emit("select", action);
-      emit("close");
-    };
-    return (_ctx, _cache) => {
-      return openBlock(), createElementBlock("div", {
-        class: "context-menu",
-        style: normalizeStyle({ top: __props.position.y + "px", left: __props.position.x + "px" })
-      }, [
-        createBaseVNode("ul", null, [
-          (openBlock(true), createElementBlock(Fragment, null, renderList(__props.options, (option) => {
-            return openBlock(), createElementBlock("li", {
-              key: option.action,
-              class: normalizeClass({ disabled: option.disabled }),
-              onClick: ($event) => !option.disabled && selectOption(option.action)
-            }, toDisplayString(option.label), 11, _hoisted_1);
-          }), 128))
-        ])
-      ], 4);
-    };
-  }
-});
-const ContextMenu_vue_vue_type_style_index_0_scoped_f6c94f99_lang = "";
-const ContextMenu = /* @__PURE__ */ _export_sfc(_sfc_main, [["__scopeId", "data-v-f6c94f99"]]);
-export {
-  ContextMenu as default
-};

flowfile/web/static/assets/ContextMenu-69a74055.js

@@ -1,41 +0,0 @@
-import { d as defineComponent, o as openBlock, c as createElementBlock, b as createBaseVNode, F as Fragment, D as renderList, n as normalizeClass, t as toDisplayString, Z as normalizeStyle, i as _export_sfc } from "./index-fb6493ae.js";
-const _hoisted_1 = ["onClick"];
-const _sfc_main = /* @__PURE__ */ defineComponent({
-  __name: "ContextMenu",
-  props: {
-    position: { type: Object, required: true },
-    options: {
-      type: Array,
-      required: true
-    }
-  },
-  emits: ["select", "close"],
-  setup(__props, { emit: __emit }) {
-    const emit = __emit;
-    const selectOption = (action) => {
-      emit("select", action);
-      emit("close");
-    };
-    return (_ctx, _cache) => {
-      return openBlock(), createElementBlock("div", {
-        class: "context-menu",
-        style: normalizeStyle({ top: __props.position.y + "px", left: __props.position.x + "px" })
-      }, [
-        createBaseVNode("ul", null, [
-          (openBlock(true), createElementBlock(Fragment, null, renderList(__props.options, (option) => {
-            return openBlock(), createElementBlock("li", {
-              key: option.action,
-              class: normalizeClass({ disabled: option.disabled }),
-              onClick: ($event) => !option.disabled && selectOption(option.action)
-            }, toDisplayString(option.label), 11, _hoisted_1);
-          }), 128))
-        ])
-      ], 4);
-    };
-  }
-});
-const ContextMenu_vue_vue_type_style_index_0_scoped_6b361c01_lang = "";
-const ContextMenu = /* @__PURE__ */ _export_sfc(_sfc_main, [["__scopeId", "data-v-6b361c01"]]);
-export {
-  ContextMenu as default
-};

flowfile/web/static/assets/ContextMenu-8e2051c6.js

@@ -1,41 +0,0 @@
-import { d as defineComponent, o as openBlock, c as createElementBlock, b as createBaseVNode, F as Fragment, D as renderList, n as normalizeClass, t as toDisplayString, Z as normalizeStyle, i as _export_sfc } from "./index-fb6493ae.js";
-const _hoisted_1 = ["onClick"];
-const _sfc_main = /* @__PURE__ */ defineComponent({
-  __name: "ContextMenu",
-  props: {
-    position: { type: Object, required: true },
-    options: {
-      type: Array,
-      required: true
-    }
-  },
-  emits: ["select", "close"],
-  setup(__props, { emit: __emit }) {
-    const emit = __emit;
-    const selectOption = (action) => {
-      emit("select", action);
-      emit("close");
-    };
-    return (_ctx, _cache) => {
-      return openBlock(), createElementBlock("div", {
-        class: "context-menu",
-        style: normalizeStyle({ top: __props.position.y + "px", left: __props.position.x + "px" })
-      }, [
-        createBaseVNode("ul", null, [
-          (openBlock(true), createElementBlock(Fragment, null, renderList(__props.options, (option) => {
-            return openBlock(), createElementBlock("li", {
-              key: option.action,
-              class: normalizeClass({ disabled: option.disabled }),
-              onClick: ($event) => !option.disabled && selectOption(option.action)
-            }, toDisplayString(option.label), 11, _hoisted_1);
-          }), 128))
-        ])
-      ], 4);
-    };
-  }
-});
-const ContextMenu_vue_vue_type_style_index_0_scoped_5437ad46_lang = "";
-const ContextMenu = /* @__PURE__ */ _export_sfc(_sfc_main, [["__scopeId", "data-v-5437ad46"]]);
-export {
-  ContextMenu as default
-};

flowfile/web/static/assets/ContextMenu-8ec1729e.css

@@ -1,26 +0,0 @@
-
-.context-menu[data-v-6b361c01] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: var(--color-background-primary);
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-  user-select: none;
-}
-.context-menu ul[data-v-6b361c01] {
-  list-style: none;
-  padding: 0;
-  margin: 0;
-}
-.context-menu li[data-v-6b361c01] {
-  padding: 8px 16px;
-  cursor: pointer;
-}
-.context-menu li.disabled[data-v-6b361c01] {
-  color: #ccc;
-  cursor: not-allowed;
-}
-.context-menu li[data-v-6b361c01]:hover:not(.disabled) {
-  background-color: #f0f0f0;
-}

flowfile/web/static/assets/ContextMenu-9b310c60.css

@@ -1,26 +0,0 @@
-
-.context-menu[data-v-5437ad46] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: var(--color-background-primary);
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-  user-select: none;
-}
-.context-menu ul[data-v-5437ad46] {
-  list-style: none;
-  padding: 0;
-  margin: 0;
-}
-.context-menu li[data-v-5437ad46] {
-  padding: 8px 16px;
-  cursor: pointer;
-}
-.context-menu li.disabled[data-v-5437ad46] {
-  color: #ccc;
-  cursor: not-allowed;
-}
-.context-menu li[data-v-5437ad46]:hover:not(.disabled) {
-  background-color: #f0f0f0;
-}

flowfile/web/static/assets/CustomNode-59e99a86.css

@@ -1,32 +0,0 @@
-
-.custom-node-wrapper[data-v-04422d4f] {
-  padding: 1.5rem;
-  background-color: var(--color-background-primary);
-}
-.node-header[data-v-04422d4f] {
-  padding-bottom: 1rem;
-  border-bottom: 1px solid var(--color-border-primary);
-  margin-bottom: 1.5rem;
-}
-.node-title[data-v-04422d4f] {
-  font-size: 1.25rem;
-  font-weight: 700;
-  color: var(--color-text-primary);
-}
-.node-category[data-v-04422d4f] {
-  font-size: 0.875rem;
-  color: var(--color-text-secondary);
-  margin-top: 0.25rem;
-}
-.section-description[data-v-04422d4f] {
-  font-size: 0.875rem;
-  color: var(--color-text-secondary);
-  margin-top: 0.25rem;
-  margin-bottom: 1.25rem;
-  padding-left: 0.5rem;
-}
-.components-container[data-v-04422d4f] {
-  display: flex;
-  flex-direction: column;
-  gap: 1.25rem;
-}

flowfile/web/static/assets/GroupBy-be7ac0bf.css

@@ -1,51 +0,0 @@
-
-.context-menu[data-v-9d79e03b] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: var(--color-background-primary);
-  padding: 8px;
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-  user-select: none;
-}
-.context-menu button[data-v-9d79e03b] {
-  display: block;
-  background: none;
-  border: none;
-  padding: 4px 8px;
-  text-align: left;
-  width: 100%;
-  cursor: pointer;
-  z-index: 100;
-}
-.context-menu button[data-v-9d79e03b]:hover {
-  background-color: #f0f0f0;
-}
-.table-wrapper[data-v-9d79e03b] {
-  max-height: 300px; /* Adjust this value as needed */
-  box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); /* subtle shadow for depth */
-  border-radius: 8px; /* rounded corners */
-  overflow: auto; /* ensures the rounded corners are applied to the child elements */
-  margin: 5px; /* adds a small margin around the table */
-}
-.context-menu[data-v-9d79e03b] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: var(--color-background-primary);
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-}
-.context-menu ul[data-v-9d79e03b] {
-  list-style: none;
-  padding: 0;
-  margin: 0;
-}
-.context-menu li[data-v-9d79e03b] {
-  padding: 8px 16px;
-  cursor: pointer;
-}
-.context-menu li[data-v-9d79e03b]:hover {
-  background-color: #f0f0f0;
-}