Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

flowfile_core/schemas/yaml_types.py

@@ -1,8 +1,21 @@
-from typing import TypedDict, List
-
+from typing import TypedDict
 
 # === Transform Schema YAML Types ===
 
+
+class BasicFilterYaml(TypedDict, total=False):
+    field: str
+    operator: str
+    value: str
+    value2: str  # For BETWEEN operator
+
+
+class FilterInputYaml(TypedDict, total=False):
+    mode: str  # "basic" or "advanced"
+    basic_filter: BasicFilterYaml
+    advanced_filter: str
+
+
 class SelectInputYaml(TypedDict, total=False):
     old_name: str
     new_name: str
@@ -11,7 +24,7 @@ class SelectInputYaml(TypedDict, total=False):
 
 
 class JoinInputsYaml(TypedDict):
-    select: List[SelectInputYaml]
+    select: list[SelectInputYaml]
 
 
 class JoinMapYaml(TypedDict):
@@ -20,7 +33,7 @@ class JoinMapYaml(TypedDict):
 
 
 class JoinInputYaml(TypedDict):
-    join_mapping: List[JoinMapYaml]
+    join_mapping: list[JoinMapYaml]
     left_select: JoinInputsYaml
     right_select: JoinInputsYaml
     how: str
@@ -42,7 +55,7 @@ class FuzzyMappingYaml(TypedDict, total=False):
 
 
 class FuzzyMatchInputYaml(TypedDict):
-    join_mapping: List[FuzzyMappingYaml]
+    join_mapping: list[FuzzyMappingYaml]
     left_select: JoinInputsYaml
     right_select: JoinInputsYaml
     how: str
@@ -51,20 +64,21 @@ class FuzzyMatchInputYaml(TypedDict):
 
 # === Input Schema YAML Types ===
 
+
 class OutputSettingsYaml(TypedDict, total=False):
     name: str
     directory: str
     file_type: str
     write_mode: str
     abs_file_path: str
-    fields: List[str]
+    fields: list[str]
     table_settings: dict
 
 
 class NodeSelectYaml(TypedDict):
     cache_results: bool
     keep_missing: bool
-    select_input: List[SelectInputYaml]
+    select_input: list[SelectInputYaml]
     sorted_by: str
 
 

flowfile_core/secret_manager/secret_manager.py

@@ -1,34 +1,138 @@
+import base64
 
 from cryptography.fernet import Fernet
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from fastapi.exceptions import HTTPException
+from pydantic import SecretStr
 from sqlalchemy import and_
 from sqlalchemy.orm import Session
+
+from flowfile_core.auth.models import SecretInput
+from flowfile_core.auth.secrets import get_master_key
 from flowfile_core.database import models as db_models
 from flowfile_core.database.connection import get_db_context
-from flowfile_core.auth.secrets import get_master_key
-from pydantic import SecretStr
-from flowfile_core.auth.models import SecretInput
-from fastapi.exceptions import HTTPException
 
+# Version identifier for key derivation scheme (allows future migrations)
+KEY_DERIVATION_VERSION = b"flowfile-secrets-v1"
+
+# Encrypted secret format: $ffsec$1${user_id}${fernet_token}
+# This embeds the user_id so the worker can derive the correct key
+SECRET_FORMAT_PREFIX = "$ffsec$1$"
+
+
+def derive_user_key(user_id: int) -> bytes:
+    """
+    Derive a user-specific encryption key from the master key using HKDF.
+
+    This provides cryptographic isolation between users - each user's secrets
+    are encrypted with a unique key derived from the master key.
+
+    Args:
+        user_id: The unique identifier for the user
+
+    Returns:
+        bytes: A 32-byte URL-safe base64-encoded key suitable for Fernet
+    """
+    master_key = get_master_key().encode()
+
+    # Use HKDF to derive a user-specific key
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=32,  # Fernet requires 32 bytes
+        salt=KEY_DERIVATION_VERSION,  # Static salt is fine for key derivation
+        info=f"user-{user_id}".encode(),  # User-specific context
+    )
+
+    # Derive raw key material and encode for Fernet
+    derived_key = hkdf.derive(master_key)
+    return base64.urlsafe_b64encode(derived_key)
 
-def encrypt_secret(secret_value):
-    """Encrypt a secret value using the master key."""
+
+def _encrypt_with_master_key(secret_value: str) -> str:
+    """Legacy encryption using master key directly (for backward compatibility)."""
     key = get_master_key().encode()
     f = Fernet(key)
     return f.encrypt(secret_value.encode()).decode()
 
 
-def decrypt_secret(encrypted_value) -> SecretStr:
-    """Decrypt an encrypted value using the master key."""
+def _decrypt_with_master_key(encrypted_value: str) -> SecretStr:
+    """Legacy decryption using master key directly (for backward compatibility)."""
     key = get_master_key().encode()
     f = Fernet(key)
     return SecretStr(f.decrypt(encrypted_value.encode()).decode())
 
 
-def get_encrypted_secret(current_user_id: int, secret_name: str) -> str|None:
+def encrypt_secret(secret_value: str, user_id: int) -> str:
+    """
+    Encrypt a secret value using a user-specific derived key.
+
+    The encrypted format embeds the user_id so it can be decrypted
+    without knowing the user context (e.g., by the worker service).
+
+    Format: $ffsec$1${user_id}${fernet_token}
+
+    Args:
+        secret_value: The plaintext secret to encrypt
+        user_id: The user ID to derive the encryption key for
+
+    Returns:
+        str: The encrypted value with embedded user_id
+    """
+    key = derive_user_key(user_id)
+    f = Fernet(key)
+    fernet_token = f.encrypt(secret_value.encode()).decode()
+    return f"{SECRET_FORMAT_PREFIX}{user_id}${fernet_token}"
+
+
+def decrypt_secret(encrypted_value: str, user_id: int | None = None) -> SecretStr:
+    """
+    Decrypt an encrypted value.
+
+    Supports both new format (with embedded user_id) and legacy format.
+    - New format: $ffsec$1${user_id}${fernet_token} - user_id extracted automatically
+    - Legacy format: raw Fernet token - requires user_id parameter or uses master key
+
+    Args:
+        encrypted_value: The encrypted secret
+        user_id: Optional user ID (required for legacy secrets, ignored for new format)
+
+    Returns:
+        SecretStr: The decrypted value wrapped in SecretStr
+    """
+    # Check for new versioned format with embedded user_id
+    if encrypted_value.startswith(SECRET_FORMAT_PREFIX):
+        # Parse: $ffsec$1${user_id}${fernet_token}
+        remainder = encrypted_value[len(SECRET_FORMAT_PREFIX):]
+        parts = remainder.split("$", 1)
+        if len(parts) != 2:
+            raise ValueError("Invalid encrypted secret format")
+
+        embedded_user_id = int(parts[0])
+        fernet_token = parts[1]
+
+        key = derive_user_key(embedded_user_id)
+        f = Fernet(key)
+        return SecretStr(f.decrypt(fernet_token.encode()).decode())
+
+    # Legacy format - use provided user_id or fall back to master key
+    if user_id is not None:
+        key = derive_user_key(user_id)
+        f = Fernet(key)
+        return SecretStr(f.decrypt(encrypted_value.encode()).decode())
+
+    # Fall back to master key for legacy secrets without user context
+    return _decrypt_with_master_key(encrypted_value)
+
+
+def get_encrypted_secret(current_user_id: int, secret_name: str) -> str | None:
     with get_db_context() as db:
         user_id = current_user_id
-        db_secret = db.query(db_models.Secret).filter(and_(db_models.Secret.user_id == user_id,
-                                                      db_models.Secret.name == secret_name)).first()
+        db_secret = (
+            db.query(db_models.Secret)
+            .filter(and_(db_models.Secret.user_id == user_id, db_models.Secret.name == secret_name))
+            .first()
+        )
         if db_secret:
             return db_secret.encrypted_value
         else:
@@ -36,14 +140,14 @@ def get_encrypted_secret(current_user_id: int, secret_name: str) -> str|None:
 
 
 def store_secret(db: Session, secret: SecretInput, user_id: int) -> db_models.Secret:
-    encrypted_value = encrypt_secret(secret.value.get_secret_value())
+    encrypted_value = encrypt_secret(secret.value.get_secret_value(), user_id)
 
     # Store in database
     db_secret = db_models.Secret(
         name=secret.name,
         encrypted_value=encrypted_value,
-        iv="",  # Not used with Fernet
-        user_id=user_id
+        iv="",  # Legacy field, not used with current encryption
+        user_id=user_id,
     )
     db.add(db_secret)
     db.commit()
@@ -52,10 +156,11 @@ def store_secret(db: Session, secret: SecretInput, user_id: int) -> db_models.Se
 
 
 def delete_secret(db: Session, secret_name: str, user_id: int) -> None:
-    db_secret = db.query(db_models.Secret).filter(
-        db_models.Secret.user_id == user_id,
-        db_models.Secret.name == secret_name
-    ).first()
+    db_secret = (
+        db.query(db_models.Secret)
+        .filter(db_models.Secret.user_id == user_id, db_models.Secret.name == secret_name)
+        .first()
+    )
 
     if not db_secret:
         raise HTTPException(status_code=404, detail="Secret not found")

flowfile_core/types.py

@@ -18,22 +18,41 @@ Usage:
 """
 
 from enum import Enum
-from typing import List, Literal, Union
-import polars as pl
+from typing import Literal, Union
 
+import polars as pl
 
 DataTypeStr = Literal[
-    "Int8", "Int16", "Int32", "Int64",
-    "UInt8", "UInt16", "UInt32", "UInt64",
-    "Float32", "Float64", "Decimal",
+    "Int8",
+    "Int16",
+    "Int32",
+    "Int64",
+    "UInt8",
+    "UInt16",
+    "UInt32",
+    "UInt64",
+    "Float32",
+    "Float64",
+    "Decimal",
     "String",
-    "Date", "Datetime", "Time", "Duration",
-    "Boolean", "Binary", "List", "Struct", "Array", "Integer", "Double", "Utf8"
+    "Date",
+    "Datetime",
+    "Time",
+    "Duration",
+    "Boolean",
+    "Binary",
+    "List",
+    "Struct",
+    "Array",
+    "Integer",
+    "Double",
+    "Utf8",
 ]
 
 
 class TypeGroup(str, Enum):
     """High-level type groups for column selection."""
+
     Numeric = "Numeric"
     String = "String"
     Date = "Date"
@@ -51,6 +70,7 @@ class TypeGroup(str, Enum):
 
 class DataType(str, Enum):
     """Specific data types for fine-grained control."""
+
     # Numeric types
     Int8 = "Int8"
     Int16 = "Int16"
@@ -150,7 +170,7 @@ TypeSpec = Union[
     TypeGroup,
     DataType,
     str,
-    List[Union[TypeGroup, DataType, str, type[pl.DataType], pl.DataType]],
+    list[TypeGroup | DataType | str | type[pl.DataType] | pl.DataType],
     type[pl.DataType],
-    pl.DataType
+    pl.DataType,
 ]

flowfile_core/utils/arrow_reader.py

@@ -1,5 +1,7 @@
+from collections.abc import Callable, Iterator
+
 import pyarrow as pa
-from typing import List, Iterator, Tuple, Callable
+
 from flowfile_core.configs import logger
 
 
@@ -36,7 +38,7 @@ def open_validated_file(file_path: str, n: int) -> pa.OSFile:
         logger.error(f"Invalid file_path type: {type(file_path)}")
         raise TypeError("file_path must be a string")
     try:
-        file = pa.OSFile(file_path, 'rb')
+        file = pa.OSFile(file_path, "rb")
         logger.info(f"Successfully opened file: {file_path}")
         return file
     except FileNotFoundError:
@@ -112,7 +114,7 @@ def iter_batches(reader: pa.ipc.RecordBatchFileReader, n: int, rows_collected: i
             break
 
 
-def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> Tuple[List[pa.RecordBatch], int]:
+def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> tuple[list[pa.RecordBatch], int]:
     """
     Collect record batches from a reader up to a specified number of rows.
 
@@ -134,11 +136,10 @@ def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> Tuple[List[
         >>> print(f"Collected {row_count} rows in {len(batches)} batches")
     """
     logger.debug(f"Collecting batches up to {n} rows")
-    batches: List[pa.RecordBatch] = []
+    batches: list[pa.RecordBatch] = []
     rows_collected = 0
 
     for batch in iter_batches(reader, n, rows_collected):
-
         rows_collected += batch.num_rows
         logger.debug(f"Collected batch: total rows now {rows_collected}")
         if rows_collected >= n:
@@ -178,7 +179,7 @@ def read(file_path: str) -> pa.Table:
     logger.info(f"Reading entire file: {file_path}")
     with open_validated_file(file_path, 0) as source:
         reader = create_reader(source)
-        batches, total_rows = collect_batches(reader, float('inf'))
+        batches, total_rows = collect_batches(reader, float("inf"))
         table = pa.Table.from_batches(batches)  # type: ignore
         logger.info(f"Successfully read {total_rows} rows from {file_path}")
         return table

flowfile_core/utils/excel_file_manager.py

@@ -1,13 +1,13 @@
 import os
-import fastexcel
-from typing import List
 from functools import lru_cache
 
+import fastexcel
+
 from flowfile_core.configs import logger
 
 
 @lru_cache(maxsize=32)
-def get_sheet_names(file_path: str) -> List[str] | None:
+def get_sheet_names(file_path: str) -> list[str] | None:
     if not os.path.exists(file_path):
         logger.error(f"File does not exist: {file_path}")
         return

flowfile_core/utils/fileManager.py

@@ -1,13 +1,13 @@
 import os
-from flowfile_core.schemas.input_schema import NewDirectory, RemoveItem, RemoveItemsInput
-from typing import Tuple, Optional
+
 from flowfile_core.configs import logger
+from flowfile_core.schemas.input_schema import NewDirectory, RemoveItem, RemoveItemsInput
 
 local_database_connection = None
 
 
-def create_dir(new_directory: NewDirectory) -> Tuple[bool, Optional[Exception]]:
-    full_path: str = os.path.join(new_directory.source_path,new_directory.dir_name)
+def create_dir(new_directory: NewDirectory) -> tuple[bool, Exception | None]:
+    full_path: str = os.path.join(new_directory.source_path, new_directory.dir_name)
     try:
         os.mkdir(full_path)
         logger.info("Successfully created a new folder")
@@ -16,7 +16,7 @@ def create_dir(new_directory: NewDirectory) -> Tuple[bool, Optional[Exception]]:
         return False, e
 
 
-def remove_path(path: str) -> Tuple[bool, Optional[Exception]]:
+def remove_path(path: str) -> tuple[bool, Exception | None]:
     try:
         os.remove(path)
         logger.info(f"Succesfully removed {path}")
@@ -35,11 +35,11 @@ def remove_item(item_to_remove: RemoveItem):
         os.rmdir(item_to_remove.path)
 
 
-def remove_paths(remove_items: RemoveItemsInput) -> Tuple[bool, Optional[Exception]]:
+def remove_paths(remove_items: RemoveItemsInput) -> tuple[bool, Exception | None]:
     try:
         for path in remove_items.paths:
             remove_item(path)
-        logger.info(f'Successfully removed {remove_items.paths}')
+        logger.info(f"Successfully removed {remove_items.paths}")
         return True, None
     except Exception as e:
         return False, e

flowfile_core/utils/fl_executor.py

@@ -1,9 +1,9 @@
-from flowfile_core.configs import logger
+from functools import wraps
 from inspect import isfunction
+
 from loky import get_reusable_executor
-from functools import wraps
-from concurrent.futures import ThreadPoolExecutor
-import atexit
+
+from flowfile_core.configs import logger
 
 
 # process_executor: Uses loky for process-based parallelism
@@ -13,18 +13,18 @@ def process_executor(wait_on_completion: bool = False, max_workers: int = 12):
     def executor(f):
         @wraps(f)
         def inner(*args, **kwargs):
-            logger.debug(f'Added task {f.__name__} to a process executor')
-            logger.debug(f'max_workers: {max_workers}')
+            logger.debug(f"Added task {f.__name__} to a process executor")
+            logger.debug(f"max_workers: {max_workers}")
 
             # Create a new executor with the required number of workers
             func_executor = get_reusable_executor(max_workers=max_workers, timeout=2, kill_workers=False, reuse=True)
             r = func_executor.submit(f, *args, **kwargs)
             if wait_on_completion:
                 result = r.result()
-                logger.info(f'done executing {f.__name__}')
+                logger.info(f"done executing {f.__name__}")
                 return result
 
-            logger.info(f'done submitting {f.__name__} to a process executor')
+            logger.info(f"done submitting {f.__name__} to a process executor")
             return r
 
         return inner
@@ -34,5 +34,3 @@ def process_executor(wait_on_completion: bool = False, max_workers: int = 12):
         wait_on_completion = False
         return executor(f)
     return executor
-
-

flowfile_core/utils/utils.py

@@ -1,14 +1,14 @@
 import re
 from itertools import chain
-from typing import List, Dict
+
 
 def camel_case_to_snake_case(text: str) -> str:
     # Use a regular expression to find capital letters and replace them with _ followed by the lowercase letter
-    transformed_text = re.sub(r'(?<!^)(?=[A-Z])', '_', text).lower()
+    transformed_text = re.sub(r"(?<!^)(?=[A-Z])", "_", text).lower()
     return transformed_text
 
 
-def ensure_similarity_dicts(datas: List[Dict], respect_order: bool = True):
+def ensure_similarity_dicts(datas: list[dict], respect_order: bool = True):
     all_cols = (data.keys() for data in datas)
     if not respect_order:
         unique_cols = set(chain(*all_cols))
@@ -44,4 +44,4 @@ def standardize_col_dtype(vals):
     elif int in types and float in types:
         return vals
     else:
-        return [convert_to_string(v) for v in vals]
+        return [convert_to_string(v) for v in vals]

flowfile_core/utils/validate_setup.py

@@ -1,11 +1,12 @@
 """This script runs on run time and checks if all the nodes that are created have a function in the flow_graph as well
 as have a component in flowfile_frontend"""
 
-from flowfile_core.schemas import input_schema
-from flowfile_core.flowfile.flow_graph import FlowGraph
-from flowfile_core.configs.node_store import nodes_list, NodeTemplate
 import inspect
 
+from flowfile_core.configs.node_store import NodeTemplate, nodes_list
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.schemas import input_schema
+
 
 def check_if_node_has_add_function_in_flow_graph(node: NodeTemplate):
     func_name = "add_" + node.item
@@ -17,7 +18,7 @@ def check_if_node_has_add_function_in_flow_graph(node: NodeTemplate):
 
 
 def check_if_node_has_input_schema_definition(node: NodeTemplate):
-    if "node"+node.item.replace("_","") not in {k.lower() for k in inspect.getmodule(input_schema).__dict__.keys()}:
+    if "node" + node.item.replace("_", "") not in {k.lower() for k in inspect.getmodule(input_schema).__dict__.keys()}:
         raise ValueError(
             f"Node {node.name} ({node.item}) does not have a corresponding input schema definition in input_schema.py."
             "Check if the schema is implemented or if the node item is correct."