Flowfile 0.5.6__py3-none-any.whl → 0.6.1__py3-none-any.whl

flowfile_core/routes/user_defined_components.py

@@ -1,23 +1,25 @@
 
 import ast
 import re
-from typing import Dict, Any, List, Optional
 from pathlib import Path
+from typing import Any
 
-from fastapi import APIRouter, HTTPException, Depends, UploadFile, File
+from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
 from fastapi.responses import FileResponse
 from pydantic import BaseModel
 
 from flowfile_core import flow_file_handler
+
 # Core modules
 from flowfile_core.auth.jwt import get_current_active_user
 from flowfile_core.configs import logger
 from flowfile_core.configs.node_store import (
     CUSTOM_NODE_STORE,
     add_to_custom_node_store,
-    remove_from_custom_node_store,
     load_single_node_from_file,
+    remove_from_custom_node_store,
 )
+
 # File handling
 from flowfile_core.schemas import input_schema
 from flowfile_core.utils.utils import camel_case_to_snake_case
@@ -66,7 +68,7 @@ def get_simple_custom_object(flow_id: int, node_id: int):
 
 
 @router.post("/update_user_defined_node", tags=["transform"])
-def update_user_defined_node(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)):
+def update_user_defined_node(input_data: dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)):
     input_data['user_id'] = current_user.id
     node_type = camel_case_to_snake_case(node_type)
     flow_id = int(input_data.get('flow_id'))
@@ -149,7 +151,7 @@ def _extract_node_info_from_file(file_path: Path) -> CustomNodeInfo:
     info = CustomNodeInfo(file_name=file_path.name)
 
     try:
-        with open(file_path, 'r', encoding='utf-8') as f:
+        with open(file_path, encoding='utf-8') as f:
             content = f.read()
 
         tree = ast.parse(content)
@@ -199,14 +201,14 @@ def _extract_node_info_from_file(file_path: Path) -> CustomNodeInfo:
     return info
 
 
-@router.get("/list-custom-nodes", summary="List all custom nodes", response_model=List[CustomNodeInfo])
-def list_custom_nodes() -> List[CustomNodeInfo]:
+@router.get("/list-custom-nodes", summary="List all custom nodes", response_model=list[CustomNodeInfo])
+def list_custom_nodes() -> list[CustomNodeInfo]:
     """
     List all custom node Python files in the user-defined nodes directory.
     Returns basic metadata extracted from each file.
     """
     nodes_dir = storage.user_defined_nodes_directory
-    nodes: List[CustomNodeInfo] = []
+    nodes: list[CustomNodeInfo] = []
 
     if not nodes_dir.exists():
         return nodes
@@ -223,7 +225,7 @@ def list_custom_nodes() -> List[CustomNodeInfo]:
 
 
 @router.get("/get-custom-node/{file_name}", summary="Get custom node details")
-def get_custom_node(file_name: str) -> Dict[str, Any]:
+def get_custom_node(file_name: str) -> dict[str, Any]:
     """
     Get the full content and parsed metadata of a custom node file.
     This endpoint is used by the Node Designer to load an existing node for editing.
@@ -239,7 +241,7 @@ def get_custom_node(file_name: str) -> Dict[str, Any]:
         raise HTTPException(status_code=404, detail=f"Node file '{safe_name}' not found")
 
     try:
-        with open(file_path, 'r', encoding='utf-8') as f:
+        with open(file_path, encoding='utf-8') as f:
             content = f.read()
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Failed to read file: {str(e)}")
@@ -320,7 +322,7 @@ def get_custom_node(file_name: str) -> Dict[str, Any]:
 
 
 @router.delete("/delete-custom-node/{file_name}", summary="Delete a custom node")
-def delete_custom_node(file_name: str) -> Dict[str, Any]:
+def delete_custom_node(file_name: str) -> dict[str, Any]:
     """
     Delete a custom node Python file from the user-defined nodes directory.
     This also attempts to unregister the node from the node store.
@@ -384,14 +386,14 @@ ALLOWED_ICON_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.svg', '.gif', '.webp'}
 MAX_ICON_SIZE = 5 * 1024 * 1024  # 5MB
 
 
-@router.get("/list-icons", summary="List all available icons", response_model=List[IconInfo])
-def list_icons() -> List[IconInfo]:
+@router.get("/list-icons", summary="List all available icons", response_model=list[IconInfo])
+def list_icons() -> list[IconInfo]:
     """
     List all icon files available for custom nodes.
     Returns icons from the user_defined_nodes/icons directory.
     """
     icons_dir = storage.user_defined_nodes_icons
-    icons: List[IconInfo] = []
+    icons: list[IconInfo] = []
 
     if not icons_dir.exists():
         return icons
@@ -406,7 +408,7 @@ def list_icons() -> List[IconInfo]:
 
 
 @router.post("/upload-icon", summary="Upload a custom icon")
-async def upload_icon(file: UploadFile = File(...)) -> Dict[str, Any]:
+async def upload_icon(file: UploadFile = File(...)) -> dict[str, Any]:
     """
     Upload a new icon file to the user_defined_nodes/icons directory.
 
@@ -507,7 +509,7 @@ def get_icon(file_name: str) -> FileResponse:
 
 
 @router.delete("/delete-icon/{file_name}", summary="Delete a custom icon")
-def delete_icon(file_name: str) -> Dict[str, Any]:
+def delete_icon(file_name: str) -> dict[str, Any]:
     """
     Delete a custom icon file from the icons directory.
     """

flowfile_core/schemas/history_schema.py

@@ -0,0 +1,220 @@
+"""
+Schema definitions for the undo/redo history system.
+
+This module defines the Pydantic models for tracking flow graph history,
+enabling users to undo and redo changes to their flow graphs.
+"""
+
+import pickle
+import zlib
+from enum import Enum
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, Field
+
+if TYPE_CHECKING:
+    pass
+
+
+class HistoryActionType(str, Enum):
+    """Enumeration of action types that can be tracked in history."""
+
+    ADD_NODE = "add_node"
+    DELETE_NODE = "delete_node"
+    MOVE_NODE = "move_node"
+    ADD_CONNECTION = "add_connection"
+    DELETE_CONNECTION = "delete_connection"
+    UPDATE_SETTINGS = "update_settings"
+    COPY_NODE = "copy_node"
+    PASTE_NODES = "paste_nodes"
+    APPLY_LAYOUT = "apply_layout"
+    BATCH = "batch"
+
+
+class HistoryConfig(BaseModel):
+    """Configuration for the history system."""
+
+    enabled: bool = Field(default=True, description="Whether history tracking is enabled")
+    max_stack_size: int = Field(default=50, description="Maximum number of history entries to keep")
+    use_compression: bool = Field(default=True, description="Whether to compress snapshots")
+    compression_level: int = Field(default=6, ge=1, le=9, description="Compression level (1-9)")
+
+
+class CompressedSnapshot:
+    """Efficiently stores a compressed flow state snapshot.
+
+    Uses zlib compression to reduce memory usage by 60-80%.
+    This is not a Pydantic model to avoid serialization overhead.
+    """
+
+    __slots__ = ('_compressed_data', '_hash')
+
+    def __init__(self, snapshot_dict: dict, compression_level: int = 6):
+        """Create a compressed snapshot from a dictionary.
+
+        Args:
+            snapshot_dict: The flow state dictionary to compress.
+            compression_level: Compression level 1-9 (higher = smaller but slower).
+        """
+        # Pickle and compress the snapshot
+        pickled = pickle.dumps(snapshot_dict, protocol=pickle.HIGHEST_PROTOCOL)
+        self._compressed_data = zlib.compress(pickled, level=compression_level)
+
+        # Pre-compute hash for fast comparison
+        self._hash = self._compute_hash(snapshot_dict)
+
+    @staticmethod
+    def _compute_hash(snapshot_dict: dict) -> int:
+        """Compute a fast structural hash of the snapshot."""
+        nodes = snapshot_dict.get("nodes", [])
+
+        # Build tuple of node signatures for hashing
+        node_signatures = []
+        for n in sorted(nodes, key=lambda x: x.get("id", 0)):
+            sig = (
+                n.get("id"),
+                n.get("type"),
+                tuple(n.get("input_ids") or []),
+                n.get("left_input_id"),
+                n.get("right_input_id"),
+                tuple(n.get("outputs") or []),
+                n.get("x_position"),
+                n.get("y_position"),
+                # Include a hash of setting_input for change detection
+                hash(str(n.get("setting_input"))) if n.get("setting_input") else None,
+            )
+            node_signatures.append(sig)
+
+        settings = snapshot_dict.get("flowfile_settings", {})
+        settings_tuple = tuple(sorted(settings.items())) if isinstance(settings, dict) else hash(str(settings))
+
+        return hash((
+            snapshot_dict.get("flowfile_id"),
+            settings_tuple,
+            tuple(node_signatures),
+        ))
+
+    def decompress(self) -> dict:
+        """Decompress and return the original snapshot dictionary."""
+        pickled = zlib.decompress(self._compressed_data)
+        return pickle.loads(pickled)
+
+    @property
+    def hash(self) -> int:
+        """Get the pre-computed hash for fast comparison."""
+        return self._hash
+
+    @property
+    def compressed_size(self) -> int:
+        """Get the size of the compressed data in bytes."""
+        return len(self._compressed_data)
+
+    def __eq__(self, other: "CompressedSnapshot") -> bool:
+        """Fast equality check using pre-computed hashes."""
+        if not isinstance(other, CompressedSnapshot):
+            return False
+        return self._hash == other._hash
+
+
+class HistoryEntry:
+    """A single entry in the history stack.
+
+    Stores a compressed snapshot of the flow state along with metadata
+    about the action that created this entry.
+
+    Uses __slots__ for memory efficiency.
+    """
+
+    __slots__ = ('_snapshot', 'action_type', 'description', 'timestamp', 'node_id')
+
+    def __init__(
+        self,
+        snapshot: CompressedSnapshot,
+        action_type: HistoryActionType,
+        description: str,
+        timestamp: float,
+        node_id: int | None = None,
+    ):
+        self._snapshot = snapshot
+        self.action_type = action_type
+        self.description = description
+        self.timestamp = timestamp
+        self.node_id = node_id
+
+    @classmethod
+    def from_dict(
+        cls,
+        snapshot_dict: dict,
+        action_type: HistoryActionType,
+        description: str,
+        timestamp: float,
+        node_id: int | None = None,
+        compression_level: int = 6,
+    ) -> "HistoryEntry":
+        """Create a HistoryEntry from a snapshot dictionary.
+
+        Args:
+            snapshot_dict: The flow state dictionary.
+            action_type: The type of action.
+            description: Human-readable description.
+            timestamp: Unix timestamp.
+            node_id: Optional affected node ID.
+            compression_level: Compression level 1-9.
+        """
+        compressed = CompressedSnapshot(snapshot_dict, compression_level)
+        return cls(compressed, action_type, description, timestamp, node_id)
+
+    def get_snapshot(self) -> dict:
+        """Decompress and return the snapshot dictionary."""
+        return self._snapshot.decompress()
+
+    @property
+    def snapshot_hash(self) -> int:
+        """Get the hash of the snapshot for comparison."""
+        return self._snapshot.hash
+
+    @property
+    def compressed_size(self) -> int:
+        """Get the compressed size in bytes."""
+        return self._snapshot.compressed_size
+
+
+class HistoryState(BaseModel):
+    """Current state of the history system.
+
+    Provides information about what undo/redo operations are available.
+    """
+
+    can_undo: bool = Field(default=False, description="Whether undo is available")
+    can_redo: bool = Field(default=False, description="Whether redo is available")
+    undo_description: str | None = Field(
+        default=None, description="Description of the action that would be undone"
+    )
+    redo_description: str | None = Field(
+        default=None, description="Description of the action that would be redone"
+    )
+    undo_count: int = Field(default=0, description="Number of available undo steps")
+    redo_count: int = Field(default=0, description="Number of available redo steps")
+
+
+class UndoRedoResult(BaseModel):
+    """Result of an undo or redo operation."""
+
+    success: bool = Field(..., description="Whether the operation succeeded")
+    action_description: str | None = Field(
+        default=None, description="Description of the action that was undone/redone"
+    )
+    error_message: str | None = Field(
+        default=None, description="Error message if the operation failed"
+    )
+
+
+class OperationResponse(BaseModel):
+    """Standard response for operations that modify the flow graph.
+
+    Includes the current history state so the frontend can update its UI.
+    """
+
+    success: bool = Field(default=True, description="Whether the operation succeeded")
+    message: str | None = Field(default=None, description="Optional message")
+    history: HistoryState = Field(..., description="Current history state after the operation")

flowfile_core/schemas/input_schema.py

@@ -25,6 +25,7 @@ from flowfile_core.schemas.yaml_types import (
     NodeSelectYaml,
     OutputSettingsYaml,
 )
+from flowfile_core.types import DataTypeStr
 from flowfile_core.utils.utils import ensure_similarity_dicts, standardize_col_dtype
 
 SecretRef = Annotated[
@@ -80,6 +81,28 @@ class MinimalFieldInfo(BaseModel):
     data_type: str = "String"
 
 
+class OutputFieldInfo(BaseModel):
+    """Field information with optional default value for output field configuration."""
+
+    name: str
+    data_type: DataTypeStr = "String"
+    default_value: str | None = None  # Can be a literal value or expression
+
+
+class OutputFieldConfig(BaseModel):
+    """Configuration for output field validation and transformation behavior."""
+
+    enabled: bool = False
+    validation_mode_behavior: Literal[
+        "add_missing",  # Add missing fields with defaults, remove extra columns
+        "add_missing_keep_extra",  # Add missing fields with defaults, keep all incoming columns
+        "raise_on_missing",  # Raise error if any fields are missing
+        "select_only"  # Select only specified fields, skip missing silently
+    ] = "select_only"
+    fields: list[OutputFieldInfo] = Field(default_factory=list)
+    validate_data_types: bool = False  # Enable data type validation without casting
+
+
 class InputTableBase(BaseModel):
     """Base settings for input file operations."""
 
@@ -334,9 +357,25 @@ class NodeBase(BaseModel):
     pos_y: float | None = 0
     is_setup: bool | None = True
     description: str | None = ""
+    node_reference: str | None = None  # Unique reference identifier for code generation (lowercase, no spaces)
     user_id: int | None = None
     is_flow_output: bool | None = False
     is_user_defined: bool | None = False  # Indicator if the node is a user defined node
+    output_field_config: OutputFieldConfig | None = None
+
+    @field_validator("node_reference", mode="before")
+    @classmethod
+    def validate_node_reference(cls, v):
+        """Validates that node_reference is lowercase and contains no spaces."""
+        if v is None or v == "":
+            return None
+        if not isinstance(v, str):
+            raise ValueError("node_reference must be a string")
+        if " " in v:
+            raise ValueError("node_reference cannot contain spaces")
+        if v != v.lower():
+            raise ValueError("node_reference must be lowercase")
+        return v
 
 
 class NodeSingleInput(NodeBase):
@@ -360,12 +399,27 @@ class NodeSelect(NodeSingleInput):
 
     def to_yaml_dict(self) -> NodeSelectYaml:
         """Converts the select node settings to a dictionary for YAML serialization."""
-        return {
-            "cache_results": self.cache_results,
+        result: NodeSelectYaml = {
+            "cache_results": bool(self.cache_results),
             "keep_missing": self.keep_missing,
             "select_input": [s.to_yaml_dict() for s in self.select_input],
             "sorted_by": self.sorted_by,
         }
+        if self.output_field_config:
+            result["output_field_config"] = {
+                "enabled": self.output_field_config.enabled,
+                "validation_mode_behavior": self.output_field_config.validation_mode_behavior,
+                "validate_data_types": self.output_field_config.validate_data_types,
+                "fields": [
+                    {
+                        "name": f.name,
+                        "data_type": f.data_type,
+                        "default_value": f.default_value,
+                    }
+                    for f in self.output_field_config.fields
+                ],
+            }
+        return result
 
 
 class NodeFilter(NodeSingleInput):
@@ -410,7 +464,7 @@ class NodeJoin(NodeMultiInput):
 
     def to_yaml_dict(self) -> NodeJoinYaml:
         """Converts the join node settings to a dictionary for YAML serialization."""
-        return {
+        result: NodeJoinYaml = {
             "cache_results": self.cache_results,
             "auto_generate_selection": self.auto_generate_selection,
             "verify_integrity": self.verify_integrity,
@@ -419,6 +473,21 @@ class NodeJoin(NodeMultiInput):
             "auto_keep_right": self.auto_keep_right,
             "auto_keep_left": self.auto_keep_left,
         }
+        if self.output_field_config:
+            result["output_field_config"] = {
+                "enabled": self.output_field_config.enabled,
+                "validation_mode_behavior": self.output_field_config.validation_mode_behavior,
+                "validate_data_types": self.output_field_config.validate_data_types,
+                "fields": [
+                    {
+                        "name": f.name,
+                        "data_type": f.data_type,
+                        "default_value": f.default_value,
+                    }
+                    for f in self.output_field_config.fields
+                ],
+            }
+        return result
 
 
 class NodeCrossJoin(NodeMultiInput):
@@ -433,7 +502,7 @@ class NodeCrossJoin(NodeMultiInput):
 
     def to_yaml_dict(self) -> NodeCrossJoinYaml:
         """Converts the cross join node settings to a dictionary for YAML serialization."""
-        return {
+        result: NodeCrossJoinYaml = {
             "cache_results": self.cache_results,
             "auto_generate_selection": self.auto_generate_selection,
             "verify_integrity": self.verify_integrity,
@@ -442,6 +511,21 @@ class NodeCrossJoin(NodeMultiInput):
             "auto_keep_right": self.auto_keep_right,
             "auto_keep_left": self.auto_keep_left,
         }
+        if self.output_field_config:
+            result["output_field_config"] = {
+                "enabled": self.output_field_config.enabled,
+                "validation_mode_behavior": self.output_field_config.validation_mode_behavior,
+                "validate_data_types": self.output_field_config.validate_data_types,
+                "fields": [
+                    {
+                        "name": f.name,
+                        "data_type": f.data_type,
+                        "default_value": f.default_value,
+                    }
+                    for f in self.output_field_config.fields
+                ],
+            }
+        return result
 
 
 class NodeFuzzyMatch(NodeJoin):
@@ -451,7 +535,7 @@ class NodeFuzzyMatch(NodeJoin):
 
     def to_yaml_dict(self) -> NodeFuzzyMatchYaml:
         """Converts the fuzzy match node settings to a dictionary for YAML serialization."""
-        return {
+        result: NodeFuzzyMatchYaml = {
             "cache_results": self.cache_results,
             "auto_generate_selection": self.auto_generate_selection,
             "verify_integrity": self.verify_integrity,
@@ -460,6 +544,21 @@ class NodeFuzzyMatch(NodeJoin):
             "auto_keep_right": self.auto_keep_right,
             "auto_keep_left": self.auto_keep_left,
         }
+        if self.output_field_config:
+            result["output_field_config"] = {
+                "enabled": self.output_field_config.enabled,
+                "validation_mode_behavior": self.output_field_config.validation_mode_behavior,
+                "validate_data_types": self.output_field_config.validate_data_types,
+                "fields": [
+                    {
+                        "name": f.name,
+                        "data_type": f.data_type,
+                        "default_value": f.default_value,
+                    }
+                    for f in self.output_field_config.fields
+                ],
+            }
+        return result
 
 
 class NodeDatasource(NodeBase):
@@ -485,6 +584,16 @@ class RawData(BaseModel):
         columns = [MinimalFieldInfo(name=c, data_type=str(next(data_types))) for c in pylist[0].keys()]
         return cls(columns=columns, data=values)
 
+    @classmethod
+    def from_pydict(cls, pydict: dict[str, list]):
+        """Creates a RawData object from a dictionary of lists."""
+        if len(pydict) == 0:
+            return cls(columns=[], data=[])
+        values = [standardize_col_dtype(column_values) for column_values in pydict.values()]
+        data_types = (pl.DataType.from_python(type(next((v for v in column_values), None))) for column_values in values)
+        columns = [MinimalFieldInfo(name=c, data_type=str(next(data_types))) for c in pydict.keys()]
+        return cls(columns=columns, data=values)
+
     def to_pylist(self) -> list[dict]:
         """Converts the RawData object back into a list of Python dictionaries."""
         return [{c.name: self.data[ci][ri] for ci, c in enumerate(self.columns)} for ri in range(len(self.data[0]))]
@@ -691,10 +800,25 @@ class NodeOutput(NodeSingleInput):
 
     def to_yaml_dict(self) -> NodeOutputYaml:
         """Converts the output node settings to a dictionary for YAML serialization."""
-        return {
+        result: NodeOutputYaml = {
             "cache_results": self.cache_results,
             "output_settings": self.output_settings.to_yaml_dict(),
         }
+        if self.output_field_config:
+            result["output_field_config"] = {
+                "enabled": self.output_field_config.enabled,
+                "validation_mode_behavior": self.output_field_config.validation_mode_behavior,
+                "validate_data_types": self.output_field_config.validate_data_types,
+                "fields": [
+                    {
+                        "name": f.name,
+                        "data_type": f.data_type,
+                        "default_value": f.default_value,
+                    }
+                    for f in self.output_field_config.fields
+                ],
+            }
+        return result
 
 
 class NodeOutputConnection(BaseModel):

flowfile_core/schemas/schemas.py

@@ -106,6 +106,8 @@ class FlowGraphConfig(BaseModel):
         path (str): The file path associated with the flow.
         execution_mode (ExecutionModeLiteral): The mode of execution ('Development' or 'Performance').
         execution_location (ExecutionLocationsLiteral): The location for execution ('local', 'remote').
+        max_parallel_workers (int): Maximum number of threads used for parallel node execution within a
+            stage. Set to 1 to disable parallelism. Defaults to 4.
     """
 
     flow_id: int = Field(default_factory=create_unique_id, description="Unique identifier for the flow.")
@@ -115,6 +117,7 @@ class FlowGraphConfig(BaseModel):
     path: str = ""
     execution_mode: ExecutionModeLiteral = "Performance"
     execution_location: ExecutionLocationsLiteral = Field(default_factory=get_global_execution_location)
+    max_parallel_workers: int = Field(default=4, ge=1, description="Max threads for parallel node execution.")
 
     @field_validator("execution_location", mode="before")
     def validate_and_set_execution_location(cls, v: ExecutionLocationsLiteral | None) -> ExecutionLocationsLiteral:
@@ -143,6 +146,7 @@ class FlowSettings(FlowGraphConfig):
         show_detailed_progress (bool): Flag to show detailed progress during execution.
         is_running (bool): Indicates if the flow is currently running.
         is_canceled (bool): Indicates if the flow execution has been canceled.
+        track_history (bool): Flag to enable or disable undo/redo history tracking.
     """
 
     auto_save: bool = False
@@ -150,6 +154,7 @@ class FlowSettings(FlowGraphConfig):
     show_detailed_progress: bool = True
     is_running: bool = False
     is_canceled: bool = False
+    track_history: bool = True
 
     @classmethod
     def from_flow_settings_input(cls, flow_graph_config: FlowGraphConfig):
@@ -190,6 +195,7 @@ class FlowfileSettings(BaseModel):
     execution_location: ExecutionLocationsLiteral = "local"
     auto_save: bool = False
     show_detailed_progress: bool = True
+    max_parallel_workers: int = Field(default=4, ge=1)
 
 
 class FlowfileNode(BaseModel):
@@ -199,6 +205,7 @@ class FlowfileNode(BaseModel):
     type: str
     is_start_node: bool = False
     description: str | None = ""
+    node_reference: str | None = None  # Unique reference identifier for code generation
     x_position: int | None = 0
     y_position: int | None = 0
     left_input_id: int | None = None
@@ -214,6 +221,7 @@ class FlowfileNode(BaseModel):
         "pos_y",
         "is_setup",
         "description",
+        "node_reference",
         "user_id",
         "is_flow_output",
         "is_user_defined",
@@ -286,6 +294,7 @@ class NodeInformation(BaseModel):
     is_setup: bool | None = None
     is_start_node: bool = False
     description: str | None = ""
+    node_reference: str | None = None  # Unique reference identifier for code generation
     x_position: int | None = 0
     y_position: int | None = 0
     left_input_id: int | None = None