Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

flowfile_core/auth/jwt.py

@@ -3,15 +3,14 @@
 import os
 import secrets
 from datetime import datetime, timedelta
-from typing import Optional
 
-from fastapi import APIRouter, Depends, HTTPException, status, Query
+from fastapi import APIRouter, Depends, HTTPException, Query, status
 from fastapi.security import OAuth2PasswordBearer
 from jose import JWTError, jwt
 from sqlalchemy.orm import Session
-from flowfile_core.auth.secrets import get_password, set_password
 
-from flowfile_core.auth.models import User, TokenData
+from flowfile_core.auth.models import TokenData, User
+from flowfile_core.auth.secrets import get_password, set_password
 from flowfile_core.database import models as db_models
 from flowfile_core.database.connection import get_db
 
@@ -59,10 +58,10 @@ def get_current_user_sync(token: str, db: Session):
     except JWTError:
         raise credentials_exception
 
-    # In Electron mode, if token is valid, return default user
+    # In Electron mode, if token is valid, return default user (always admin in electron mode)
     if os.environ.get("FLOWFILE_MODE") == "electron":
         if token_data.username == "local_user":
-            electron_user = User(username="local_user", id=1, disabled=False)
+            electron_user = User(username="local_user", id=1, disabled=False, is_admin=True, must_change_password=False)
             return electron_user
         else:
             # Invalid username in token
@@ -74,7 +73,16 @@ def get_current_user_sync(token: str, db: Session):
             raise credentials_exception
         if user.disabled:
             raise HTTPException(status_code=400, detail="Inactive user")
-        return user
+        # Convert to Pydantic User model
+        return User(
+            username=user.username,
+            id=user.id,
+            email=user.email,
+            full_name=user.full_name,
+            disabled=user.disabled,
+            is_admin=user.is_admin,
+            must_change_password=user.must_change_password
+        )
 
 
 async def get_current_user(token: str = Depends(oauth2_scheme), db: Session = Depends(get_db)):
@@ -98,10 +106,10 @@ async def get_current_user(token: str = Depends(oauth2_scheme), db: Session = De
     except JWTError:
         raise credentials_exception
 
-    # In Electron mode, if token is valid, return default user
+    # In Electron mode, if token is valid, return default user (always admin in electron mode)
     if os.environ.get("FLOWFILE_MODE") == "electron":
         if token_data.username == "local_user":
-            electron_user = User(username="local_user", id=1, disabled=False)
+            electron_user = User(username="local_user", id=1, disabled=False, is_admin=True, must_change_password=False)
             return electron_user
         else:
             # Invalid username in token
@@ -113,16 +121,25 @@ async def get_current_user(token: str = Depends(oauth2_scheme), db: Session = De
             raise credentials_exception
         if user.disabled:
             raise HTTPException(status_code=400, detail="Inactive user")
-        return user
+        # Convert to Pydantic User model
+        return User(
+            username=user.username,
+            id=user.id,
+            email=user.email,
+            full_name=user.full_name,
+            disabled=user.disabled,
+            is_admin=user.is_admin,
+            must_change_password=user.must_change_password
+        )
 
 
 def get_current_active_user(current_user=Depends(get_current_user)):
-    if hasattr(current_user, 'disabled') and current_user.disabled:
+    if hasattr(current_user, "disabled") and current_user.disabled:
         raise HTTPException(status_code=400, detail="Inactive user")
     return current_user
 
 
-def create_access_token(data: dict, expires_delta: Optional[timedelta] = None):
+def create_access_token(data: dict, expires_delta: timedelta | None = None):
     to_encode = data.copy()
 
     if expires_delta:
@@ -136,8 +153,7 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None):
 
 
 async def get_current_user_from_query(
-    access_token: str = Query(..., description="JWT access token"),
-    db: Session = Depends(get_db)
+    access_token: str = Query(..., description="JWT access token"), db: Session = Depends(get_db)
 ):
     """
     Authenticate user using only the query parameter token.
@@ -165,7 +181,7 @@ async def get_current_user_from_query(
     # Handle authentication based on deployment mode (same as your existing logic)
     if os.environ.get("FLOWFILE_MODE") == "electron":
         if token_data.username == "local_user":
-            electron_user = User(username="local_user", id=1, disabled=False)
+            electron_user = User(username="local_user", id=1, disabled=False, is_admin=True, must_change_password=False)
             return electron_user
         else:
             raise credentials_exception
@@ -176,4 +192,23 @@ async def get_current_user_from_query(
             raise credentials_exception
         if user.disabled:
             raise HTTPException(status_code=400, detail="Inactive user")
-        return user
+        # Convert to Pydantic User model
+        return User(
+            username=user.username,
+            id=user.id,
+            email=user.email,
+            full_name=user.full_name,
+            disabled=user.disabled,
+            is_admin=user.is_admin,
+            must_change_password=user.must_change_password
+        )
+
+
+async def get_current_admin_user(current_user: User = Depends(get_current_user)):
+    """Dependency that requires the current user to be an admin"""
+    if not current_user.is_admin:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Admin privileges required"
+        )
+    return current_user

flowfile_core/auth/models.py

@@ -1,6 +1,4 @@
-
 from pydantic import BaseModel, SecretStr
-from typing import Optional, List
 
 
 class Token(BaseModel):
@@ -9,21 +7,48 @@ class Token(BaseModel):
 
 
 class TokenData(BaseModel):
-    username: Optional[str] = None
+    username: str | None = None
 
 
 class User(BaseModel):
     username: str
-    id: Optional[int] = None
-    email: Optional[str] = None
-    full_name: Optional[str] = None
-    disabled: Optional[bool] = False
+    id: int | None = None
+    email: str | None = None
+    full_name: str | None = None
+    disabled: bool | None = False
+    is_admin: bool | None = False
+    must_change_password: bool | None = False
 
 
 class UserInDB(User):
     hashed_password: str
 
 
+class UserCreate(BaseModel):
+    """Model for creating a new user (admin only)"""
+    username: str
+    password: str
+    email: str | None = None
+    full_name: str | None = None
+    is_admin: bool = False
+
+
+class UserUpdate(BaseModel):
+    """Model for updating a user (admin only)"""
+    email: str | None = None
+    full_name: str | None = None
+    disabled: bool | None = None
+    is_admin: bool | None = None
+    password: str | None = None  # Optional password change
+    must_change_password: bool | None = None
+
+
+class ChangePassword(BaseModel):
+    """Model for user changing their own password"""
+    current_password: str
+    new_password: str
+
+
 class SecretInput(BaseModel):
     name: str
     value: SecretStr

flowfile_core/auth/password.py

@@ -0,0 +1,89 @@
+"""Password hashing and verification utilities."""
+
+import re
+from passlib.context import CryptContext
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+# Password requirements
+PASSWORD_MIN_LENGTH = 8
+PASSWORD_REQUIREMENTS = {
+    "min_length": PASSWORD_MIN_LENGTH,
+    "require_number": True,
+    "require_special": True,
+    "special_chars": "!@#$%^&*()_+-=[]{}|;:,.<>?"
+}
+
+
+class PasswordValidationError(Exception):
+    """Raised when password doesn't meet requirements."""
+    pass
+
+
+def validate_password(password: str) -> tuple[bool, str]:
+    """
+    Validate password against security requirements.
+
+    Requirements:
+    - Minimum 8 characters
+    - At least one number
+    - At least one special character
+
+    Args:
+        password: The plain text password to validate
+
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    if len(password) < PASSWORD_MIN_LENGTH:
+        return False, f"Password must be at least {PASSWORD_MIN_LENGTH} characters long"
+
+    if not re.search(r'\d', password):
+        return False, "Password must contain at least one number"
+
+    if not re.search(r'[!@#$%^&*()_+\-=\[\]{}|;:,.<>?]', password):
+        return False, "Password must contain at least one special character (!@#$%^&*()_+-=[]{}|;:,.<>?)"
+
+    return True, ""
+
+
+def validate_password_or_raise(password: str) -> None:
+    """
+    Validate password and raise exception if invalid.
+
+    Args:
+        password: The plain text password to validate
+
+    Raises:
+        PasswordValidationError: If password doesn't meet requirements
+    """
+    is_valid, error_message = validate_password(password)
+    if not is_valid:
+        raise PasswordValidationError(error_message)
+
+
+def verify_password(plain: str, hashed: str) -> bool:
+    """
+    Verify a plain password against a hashed password.
+
+    Args:
+        plain: The plain text password
+        hashed: The hashed password to verify against
+
+    Returns:
+        True if the password matches, False otherwise
+    """
+    return pwd_context.verify(plain, hashed)
+
+
+def get_password_hash(password: str) -> str:
+    """
+    Hash a plain text password.
+
+    Args:
+        password: The plain text password to hash
+
+    Returns:
+        The hashed password
+    """
+    return pwd_context.hash(password)

flowfile_core/auth/secrets.py

@@ -1,11 +1,13 @@
 """
 Secure storage module for FlowFile credentials and secrets.
 """
-from cryptography.fernet import Fernet
-import os
-from pathlib import Path
+
 import json
 import logging
+import os
+from pathlib import Path
+
+from cryptography.fernet import Fernet
 
 logger = logging.getLogger(__name__)
 
@@ -15,7 +17,7 @@ class SecureStorage:
 
     def __init__(self):
         env = os.environ.get("FLOWFILE_MODE")
-        logger.debug(f'Using secure storage in {env} mode')
+        logger.debug(f"Using secure storage in {env} mode")
         if os.environ.get("FLOWFILE_MODE") == "electron":
             app_data = os.environ.get("APPDATA") or os.path.expanduser("~/.config")
             self.storage_path = Path(app_data) / "flowfile"
@@ -135,41 +137,84 @@ def delete_password(service_name, username):
     _storage.delete_password(service_name, username)
 
 
-def get_docker_secret_key():
+def get_docker_secret_key() -> str | None:
     """
-    Get the master key from Docker secret.
+    Get the master key from Docker secret or environment variable.
 
     Returns:
-        str: The master key if successfully read from Docker secret.
+        str: The master key if successfully read, None if not configured.
 
     Raises:
-        RuntimeError: If running in Docker but unable to access the secret.
+        RuntimeError: If the secret file exists but cannot be read, or key is invalid.
     """
+    env_key = os.environ.get("FLOWFILE_MASTER_KEY")
+    if env_key:
+        env_key = env_key.strip().strip('"').strip("'")
+        try:
+            Fernet(env_key.encode())
+            return env_key
+        except Exception:
+            raise RuntimeError("FLOWFILE_MASTER_KEY is not a valid Fernet key")
+
     secret_path = "/run/secrets/flowfile_master_key"
-    if os.path.exists(secret_path):
+    if os.path.isfile(secret_path):
         try:
-            with open(secret_path, "r") as f:
-                return f.read().strip()
-        except Exception as e:
-            logger.error(f"Failed to read master key from Docker secret: {e}")
+            with open(secret_path) as f:
+                key = f.read().strip()
+                Fernet(key.encode())
+                return key
+        except Exception:
             raise RuntimeError("Failed to read master key from Docker secret")
-    else:
-        logger.critical("Running in Docker but flowfile_master_key secret is not mounted!")
-        raise RuntimeError("Docker secret 'flowfile_master_key' is not mounted")
+
+    return None
+
+
+def is_master_key_configured() -> bool:
+    """
+    Check if the master key is properly configured.
+
+    Returns:
+        bool: True if master key is configured and valid, False otherwise.
+    """
+    try:
+        if os.environ.get("FLOWFILE_MODE") == "docker":
+            return get_docker_secret_key() is not None
+        return True
+    except RuntimeError:
+        return False
+
+
+def generate_master_key() -> str:
+    """
+    Generate a new Fernet master key.
+
+    Returns:
+        str: A new valid Fernet encryption key.
+    """
+    return Fernet.generate_key().decode()
 
 
 def get_master_key():
     """
     Get or generate the master encryption key.
 
-    If running in Docker, retrieves the key from Docker secrets.
+    If running in Docker, retrieves the key from Docker secrets or environment.
     Otherwise, retrieves or generates a key using the secure storage.
 
     Returns:
         str: The master encryption key
+
+    Raises:
+        RuntimeError: If in Docker mode and no key is configured.
     """
-    if os.environ.get("RUNNING_IN_DOCKER") == "true":
-        return get_docker_secret_key()
+    if os.environ.get("FLOWFILE_MODE") == "docker":
+        key = get_docker_secret_key()
+        if key is None:
+            raise RuntimeError(
+                "Master key not configured. Set FLOWFILE_MASTER_KEY environment variable "
+                "or mount the flowfile_master_key Docker secret."
+            )
+        return key
 
     key = get_password("flowfile", "master_key")
     if not key:

flowfile_core/configs/__init__.py

@@ -1,13 +1,14 @@
 # flowfile_core/flowfile_core/configs/__init__.py
 import logging
+import os
 import sys
 from pathlib import Path
-import os
 
-os.environ["FLOWFILE_MODE"] = "electron"
+if "FLOWFILE_MODE" not in os.environ:
+    os.environ["FLOWFILE_MODE"] = "electron"
 
 # Create and configure the logger
-logger = logging.getLogger('PipelineHandler')
+logger = logging.getLogger("PipelineHandler")
 logger.setLevel(logging.INFO)
 logger.propagate = False
 
@@ -17,9 +18,9 @@ if logger.hasHandlers():
 
 # Try to determine the best output stream
 output_stream = None
-if hasattr(sys.stdout, 'isatty') and sys.stdout.isatty():
+if hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
     output_stream = sys.stdout
-elif hasattr(sys.stderr, 'isatty') and sys.stderr.isatty():
+elif hasattr(sys.stderr, "isatty") and sys.stderr.isatty():
     output_stream = sys.stderr
 else:
     # Use __stdout__ for debugger environments (PyDev, PyCharm, etc.)
@@ -29,7 +30,7 @@ console_handler = logging.StreamHandler(output_stream)
 console_handler.setLevel(logging.INFO)
 
 # Create formatter
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 console_handler.setFormatter(formatter)
 
 logger.addHandler(console_handler)
@@ -37,10 +38,11 @@ logger.addHandler(console_handler)
 # Create logs directory in temp at startup
 try:
     from tempfile import gettempdir
+
     log_dir = Path(gettempdir()) / "flowfile_logs"
     log_dir.mkdir(exist_ok=True)
 except Exception as e:
     logger.warning(f"Failed to create logs directory: {e}")
 
 # Initialize vault
-logger.info("Logging system initialized")
+logger.info("Logging system initialized")

flowfile_core/configs/flow_logger.py

@@ -1,14 +1,15 @@
 import logging
-from pathlib import Path
-from datetime import datetime
-import os
 import logging.handlers
+import os
 import queue
 import threading
+from datetime import datetime
+from pathlib import Path
+
 from shared.storage_config import storage
 
 _process_safe_queue = queue.Queue(-1)
-main_logger = logging.getLogger('PipelineHandler')
+main_logger = logging.getLogger("PipelineHandler")
 
 
 class NodeLogger:
@@ -38,6 +39,7 @@ class NodeLogger:
 
 class FlowLogger:
     """Thread-safe logger for flow execution"""
+
     _instances = {}
     _instances_lock = threading.RLock()
     _queue_listener = None
@@ -47,7 +49,7 @@ class FlowLogger:
     def handle_extra_log_info(flow_id: int, extra: dict = None) -> dict:
         if extra is None:
             extra = {}
-        extra['flow_id'] = flow_id
+        extra["flow_id"] = flow_id
         return extra
 
     def __new__(cls, flow_id: int, clear_existing_logs: bool = False):
@@ -75,7 +77,7 @@ class FlowLogger:
 
     def _setup_new_logger(self):
         """Creates a new logger instance with appropriate handlers"""
-        logger_name = f'FlowExecution.{self.flow_id}'
+        logger_name = f"FlowExecution.{self.flow_id}"
         self._logger = logging.getLogger(logger_name)
         self._logger.setLevel(logging.INFO)
         self.setup_logging()
@@ -131,7 +133,7 @@ class FlowLogger:
 
         try:
             # Create an empty file
-            with open(self.log_file_path, 'w') as f:
+            with open(self.log_file_path, "w") as f:
                 pass
 
             # Re-setup the logger
@@ -154,9 +156,7 @@ class FlowLogger:
         """Start the queue listener for asynchronous logging"""
         queue_handler = logging.handlers.QueueHandler(_process_safe_queue)
         cls._queue_listener = logging.handlers.QueueListener(
-            _process_safe_queue,
-            queue_handler,
-            respect_handler_level=True
+            _process_safe_queue, queue_handler, respect_handler_level=True
         )
         cls._queue_listener.start()
 
@@ -193,7 +193,7 @@ class FlowLogger:
 
         # Add file handler
         file_handler = logging.FileHandler(self.log_file_path)
-        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+        formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
         file_handler.setFormatter(formatter)
         self._logger.addHandler(file_handler)
 
@@ -214,7 +214,8 @@ class FlowLogger:
             else:
                 # If still can't get lock, proceed anyway
                 main_logger.warning(
-                    f"Could not acquire lock for flow {self.flow_id}, proceeding with file clearing anyway")
+                    f"Could not acquire lock for flow {self.flow_id}, proceeding with file clearing anyway"
+                )
                 self._clear_log_impl()
 
     def _clear_log_impl(self):
@@ -223,7 +224,7 @@ class FlowLogger:
             # Ensure parent directory exists
             self.refresh_logger_if_needed()
             # Truncate file
-            with open(self.log_file_path, 'w') as f:
+            with open(self.log_file_path, "w") as f:
                 pass
             main_logger.info(f"Log file cleared for flow {self.flow_id}")
         except Exception as e:
@@ -409,7 +410,7 @@ def read_log_from_line(log_file_path: Path, start_line: int = 0):
     """Read log file content starting from a specific line"""
     lines = []
     try:
-        with open(log_file_path, "r") as file:
+        with open(log_file_path) as file:
             # Skip lines efficiently if needed
             if start_line > 0:
                 for _ in range(start_line):

flowfile_core/configs/node_store/__init__.py

@@ -1,10 +1,17 @@
-from flowfile_core.configs.node_store.user_defined_node_registry import get_all_nodes_from_standard_location
+import logging
+
 from flowfile_core.configs.node_store.nodes import get_all_standard_nodes
-from flowfile_core.schemas.schemas import NodeTemplate
+from flowfile_core.configs.node_store.user_defined_node_registry import (
+    get_all_nodes_from_standard_location,
+    load_single_node_from_file,
+    unload_node_by_name,
+)
 from flowfile_core.flowfile.node_designer.custom_node import CustomNodeBase
+from flowfile_core.schemas.schemas import NodeTemplate
 
+logger = logging.getLogger(__name__)
 
-nodes_with_defaults = {'sample', 'sort', 'union', 'select', 'record_count'}
+nodes_with_defaults = {"sample", "sort", "union", "select", "record_count"}
 
 
 def register_custom_node(node: NodeTemplate):
@@ -18,6 +25,68 @@ def add_to_custom_node_store(custom_node: type[CustomNodeBase]):
         register_custom_node(custom_node().to_node_template())
 
 
+def remove_from_custom_node_store(node_key: str, file_stem: str = None) -> bool:
+    """
+    Remove a custom node from both CUSTOM_NODE_STORE and node registries.
+
+    Args:
+        node_key: The key/item name of the node to remove
+        file_stem: Optional file name stem (without .py) to use as fallback for matching
+
+    Returns:
+        True if the node was found and removed, False otherwise
+    """
+    removed = False
+
+    logger.info(f"Attempting to remove node with key: '{node_key}' (file_stem: '{file_stem}')")
+    logger.info(f"Current CUSTOM_NODE_STORE keys: {list(CUSTOM_NODE_STORE.keys())}")
+    logger.info(f"Current nodes_list items: {[n.item for n in nodes_list if hasattr(n, 'item')]}")
+
+    # Try to find the key - use exact match first, then fallback to file_stem
+    actual_key = None
+    if node_key in CUSTOM_NODE_STORE:
+        actual_key = node_key
+    elif file_stem and file_stem in CUSTOM_NODE_STORE:
+        actual_key = file_stem
+        logger.info(f"Using file_stem '{file_stem}' as key instead of '{node_key}'")
+
+    # Remove from CUSTOM_NODE_STORE
+    if actual_key and actual_key in CUSTOM_NODE_STORE:
+        del CUSTOM_NODE_STORE[actual_key]
+        logger.info(f"Removed '{actual_key}' from CUSTOM_NODE_STORE")
+        removed = True
+    else:
+        logger.warning(f"Key '{node_key}' (or file_stem '{file_stem}') not found in CUSTOM_NODE_STORE")
+
+    # Remove from node_dict - try both keys
+    key_to_use = actual_key or node_key
+    if key_to_use in node_dict:
+        del node_dict[key_to_use]
+        logger.info(f"Removed '{key_to_use}' from node_dict")
+    elif file_stem and file_stem in node_dict:
+        del node_dict[file_stem]
+        logger.info(f"Removed '{file_stem}' from node_dict")
+
+    # Remove from nodes_list - try both keys
+    removed_from_list = False
+    for i, node in enumerate(nodes_list):
+        if node.item == key_to_use or (file_stem and node.item == file_stem):
+            nodes_list.pop(i)
+            logger.info(f"Removed '{node.item}' from nodes_list at index {i}")
+            removed_from_list = True
+            break
+
+    if not removed_from_list:
+        logger.warning(f"Key '{node_key}' not found in nodes_list")
+
+    # Clean up module cache
+    unload_node_by_name(node_key)
+    if file_stem and file_stem != node_key:
+        unload_node_by_name(file_stem)
+
+    return removed
+
+
 CUSTOM_NODE_STORE = get_all_nodes_from_standard_location()
 nodes_list, node_dict, node_defaults = get_all_standard_nodes()
 
@@ -26,5 +95,4 @@ for custom_node in CUSTOM_NODE_STORE.values():
 
 
 def check_if_has_default_setting(node_item: str):
-
     return node_item in nodes_with_defaults