Flowfile 0.5.6__py3-none-any.whl → 0.6.1__py3-none-any.whl

flowfile_core/flowfile/history_manager.py

@@ -0,0 +1,401 @@
+"""
+History Manager for undo/redo functionality in flow graphs.
+
+This module provides the HistoryManager class which manages undo/redo stacks
+and enables users to revert or reapply changes to their flow graphs.
+
+Optimizations:
+- Compressed snapshots using zlib (60-80% memory reduction)
+- Pre-computed hashes for O(1) snapshot comparison
+- __slots__ for memory-efficient entry storage
+"""
+
+from collections import deque
+from time import time
+from typing import TYPE_CHECKING
+
+from flowfile_core.configs import logger
+from flowfile_core.schemas.history_schema import (
+    CompressedSnapshot,
+    HistoryActionType,
+    HistoryConfig,
+    HistoryEntry,
+    HistoryState,
+    UndoRedoResult,
+)
+from flowfile_core.schemas.schemas import FlowfileData
+
+if TYPE_CHECKING:
+    from flowfile_core.flowfile.flow_graph import FlowGraph
+
+
+class HistoryManager:
+    """Manages undo/redo history for a FlowGraph.
+
+    Uses two deques (undo_stack and redo_stack) to track state changes.
+    Snapshots are captured BEFORE changes occur, so undo restores to that state.
+
+    Memory Optimization:
+    - Snapshots are compressed using zlib (typically 60-80% size reduction)
+    - Hashes are pre-computed for O(1) equality checks
+    - HistoryEntry uses __slots__ for reduced memory overhead
+    """
+
+    __slots__ = ('_config', '_undo_stack', '_redo_stack', '_is_restoring', '_last_snapshot_hash')
+
+    def __init__(self, config: HistoryConfig | None = None):
+        """Initialize the HistoryManager.
+
+        Args:
+            config: Optional configuration for history behavior.
+        """
+        self._config = config or HistoryConfig()
+        self._undo_stack: deque[HistoryEntry] = deque(maxlen=self._config.max_stack_size)
+        self._redo_stack: deque[HistoryEntry] = deque(maxlen=self._config.max_stack_size)
+        self._is_restoring: bool = False
+        self._last_snapshot_hash: int | None = None
+
+    @property
+    def config(self) -> HistoryConfig:
+        """Get the history configuration."""
+        return self._config
+
+    @config.setter
+    def config(self, config: HistoryConfig):
+        """Set the history configuration.
+
+        Note: Changing max_stack_size won't resize existing stacks.
+        """
+        self._config = config
+
+    def _create_entry(
+        self,
+        snapshot_dict: dict,
+        action_type: HistoryActionType,
+        description: str,
+        node_id: int | None = None,
+    ) -> HistoryEntry:
+        """Create a history entry with the configured compression settings.
+
+        Args:
+            snapshot_dict: The flow state dictionary.
+            action_type: The type of action.
+            description: Human-readable description.
+            node_id: Optional affected node ID.
+
+        Returns:
+            A new HistoryEntry instance.
+        """
+        return HistoryEntry.from_dict(
+            snapshot_dict=snapshot_dict,
+            action_type=action_type,
+            description=description,
+            timestamp=time(),
+            node_id=node_id,
+            compression_level=self._config.compression_level if self._config.use_compression else 1,
+        )
+
+    def capture_snapshot(
+        self,
+        flow_graph: "FlowGraph",
+        action_type: HistoryActionType,
+        description: str,
+        node_id: int | None = None,
+    ) -> bool:
+        """Capture the current state of the flow graph BEFORE a change.
+
+        This method captures state BEFORE an operation. We detect duplicates by
+        comparing against the last CAPTURED snapshot (top of undo stack), not
+        against _last_snapshot_hash (which tracks the post-operation state).
+
+        Args:
+            flow_graph: The FlowGraph to capture.
+            action_type: The type of action being performed.
+            description: Human-readable description of the action.
+            node_id: Optional ID of the affected node.
+
+        Returns:
+            True if snapshot was captured, False if skipped (disabled or restoring).
+        """
+        logger.info(f"History: capture_snapshot called for '{description}' (enabled={self._config.enabled}, restoring={self._is_restoring})")
+
+        if not self._config.enabled:
+            logger.info(f"History: Skipping '{description}' - history disabled")
+            return False
+
+        if self._is_restoring:
+            logger.info(f"History: Skipping '{description}' - currently restoring")
+            return False
+
+        try:
+            # Get the current state as FlowfileData
+            flowfile_data = flow_graph.get_flowfile_data()
+            snapshot_dict = flowfile_data.model_dump()
+
+            # Compute hash for duplicate detection
+            current_hash = CompressedSnapshot._compute_hash(snapshot_dict)
+
+            # Compare against the LAST CAPTURED snapshot (top of undo stack), not _last_snapshot_hash
+            # This correctly detects if we're capturing the same pre-state twice,
+            # without being confused by post-operation hash updates from capture_if_changed
+            if self._undo_stack:
+                last_entry_hash = self._undo_stack[-1].snapshot_hash
+                if last_entry_hash == current_hash:
+                    logger.info(f"History: Skipping duplicate snapshot for: {description}")
+                    return False
+
+            # Create compressed entry
+            entry = self._create_entry(snapshot_dict, action_type, description, node_id)
+
+            # Add to undo stack
+            self._undo_stack.append(entry)
+
+            # Clear redo stack when new action is performed
+            self._redo_stack.clear()
+
+            logger.info(
+                f"History: Captured '{description}' "
+                f"(undo_stack={len(self._undo_stack)}, redo_stack={len(self._redo_stack)})"
+            )
+            return True
+
+        except Exception as e:
+            logger.error(f"History: Failed to capture snapshot for '{description}': {e}")
+            return False
+
+    def capture_if_changed(
+        self,
+        flow_graph: "FlowGraph",
+        pre_snapshot: FlowfileData,
+        action_type: HistoryActionType,
+        description: str,
+        node_id: int | None = None,
+    ) -> bool:
+        """Capture history only if the flow state actually changed.
+
+        Use this for settings updates where the change might be a no-op.
+        Call this AFTER the change is applied.
+
+        Args:
+            flow_graph: The FlowGraph after the change.
+            pre_snapshot: The FlowfileData captured BEFORE the change.
+            action_type: The type of action that was performed.
+            description: Human-readable description of the action.
+            node_id: Optional ID of the affected node.
+
+        Returns:
+            True if a change was detected and snapshot was captured.
+        """
+        if not self._config.enabled:
+            logger.debug(f"History: Skipping '{description}' (if_changed) - history disabled")
+            return False
+
+        if self._is_restoring:
+            logger.debug(f"History: Skipping '{description}' (if_changed) - currently restoring")
+            return False
+
+        try:
+            # Get the current (post-change) state
+            current_snapshot = flow_graph.get_flowfile_data()
+            current_dict = current_snapshot.model_dump()
+            pre_dict = pre_snapshot.model_dump()
+
+            # Fast hash comparison (no JSON serialization)
+            pre_hash = CompressedSnapshot._compute_hash(pre_dict)
+            current_hash = CompressedSnapshot._compute_hash(current_dict)
+
+            if pre_hash == current_hash:
+                logger.debug(f"History: No change detected for: {description}")
+                return False
+
+            # State changed - capture the BEFORE state (compressed)
+            entry = self._create_entry(pre_dict, action_type, description, node_id)
+
+            # Add to undo stack
+            self._undo_stack.append(entry)
+            self._last_snapshot_hash = current_hash
+
+            # Clear redo stack when new action is performed
+            self._redo_stack.clear()
+
+            logger.info(
+                f"History: Captured '{description}' (after change detection) "
+                f"(undo_stack={len(self._undo_stack)}, redo_stack={len(self._redo_stack)})"
+            )
+            return True
+
+        except Exception as e:
+            logger.error(f"History: Failed to capture snapshot for '{description}': {e}")
+            return False
+
+    def undo(self, flow_graph: "FlowGraph") -> UndoRedoResult:
+        """Undo the last action by restoring to the previous state.
+
+        Args:
+            flow_graph: The FlowGraph to restore.
+
+        Returns:
+            UndoRedoResult indicating success or failure.
+        """
+        if not self._undo_stack:
+            return UndoRedoResult(
+                success=False,
+                error_message="Nothing to undo",
+            )
+
+        try:
+            # Set flag to prevent capturing during restore
+            self._is_restoring = True
+
+            # Get the entry to restore from
+            entry = self._undo_stack.pop()
+
+            # Save current state to redo stack BEFORE restoring
+            current_snapshot = flow_graph.get_flowfile_data()
+            current_dict = current_snapshot.model_dump()
+            redo_entry = self._create_entry(
+                current_dict,
+                entry.action_type,
+                entry.description,
+                entry.node_id,
+            )
+            self._redo_stack.append(redo_entry)
+
+            # Decompress and restore the flow graph from the snapshot
+            snapshot_dict = entry.get_snapshot()
+            snapshot_data = FlowfileData.model_validate(snapshot_dict)
+            flow_graph.restore_from_snapshot(snapshot_data)
+
+            # Update last snapshot hash
+            self._last_snapshot_hash = entry.snapshot_hash
+
+            logger.info(f"Undo successful: {entry.description}")
+            return UndoRedoResult(
+                success=True,
+                action_description=entry.description,
+            )
+
+        except Exception as e:
+            logger.error(f"Undo failed: {e}")
+            return UndoRedoResult(
+                success=False,
+                error_message=str(e),
+            )
+
+        finally:
+            self._is_restoring = False
+
+    def redo(self, flow_graph: "FlowGraph") -> UndoRedoResult:
+        """Redo the last undone action.
+
+        Args:
+            flow_graph: The FlowGraph to restore.
+
+        Returns:
+            UndoRedoResult indicating success or failure.
+        """
+        if not self._redo_stack:
+            return UndoRedoResult(
+                success=False,
+                error_message="Nothing to redo",
+            )
+
+        try:
+            # Set flag to prevent capturing during restore
+            self._is_restoring = True
+
+            # Get the entry to restore from
+            entry = self._redo_stack.pop()
+
+            # Save current state to undo stack BEFORE restoring
+            current_snapshot = flow_graph.get_flowfile_data()
+            current_dict = current_snapshot.model_dump()
+            undo_entry = self._create_entry(
+                current_dict,
+                entry.action_type,
+                entry.description,
+                entry.node_id,
+            )
+            self._undo_stack.append(undo_entry)
+
+            # Decompress and restore the flow graph from the snapshot
+            snapshot_dict = entry.get_snapshot()
+            snapshot_data = FlowfileData.model_validate(snapshot_dict)
+            flow_graph.restore_from_snapshot(snapshot_data)
+
+            # Update last snapshot hash
+            self._last_snapshot_hash = entry.snapshot_hash
+
+            logger.info(f"Redo successful: {entry.description}")
+            return UndoRedoResult(
+                success=True,
+                action_description=entry.description,
+            )
+
+        except Exception as e:
+            logger.error(f"Redo failed: {e}")
+            return UndoRedoResult(
+                success=False,
+                error_message=str(e),
+            )
+
+        finally:
+            self._is_restoring = False
+
+    def get_state(self) -> HistoryState:
+        """Get the current state of the history system.
+
+        Returns:
+            HistoryState with information about available undo/redo operations.
+        """
+        can_undo = len(self._undo_stack) > 0
+        can_redo = len(self._redo_stack) > 0
+
+        undo_description = None
+        if can_undo:
+            undo_description = self._undo_stack[-1].description
+
+        redo_description = None
+        if can_redo:
+            redo_description = self._redo_stack[-1].description
+
+        return HistoryState(
+            can_undo=can_undo,
+            can_redo=can_redo,
+            undo_description=undo_description,
+            redo_description=redo_description,
+            undo_count=len(self._undo_stack),
+            redo_count=len(self._redo_stack),
+        )
+
+    def clear(self) -> None:
+        """Clear all history entries."""
+        self._undo_stack.clear()
+        self._redo_stack.clear()
+        self._last_snapshot_hash = None
+        logger.debug("History cleared")
+
+    def is_restoring(self) -> bool:
+        """Check if a restore operation is currently in progress.
+
+        Returns:
+            True if undo/redo is in progress.
+        """
+        return self._is_restoring
+
+    def get_memory_usage(self) -> dict:
+        """Get memory usage statistics for the history stacks.
+
+        Returns:
+            Dictionary with memory usage information.
+        """
+        undo_size = sum(e.compressed_size for e in self._undo_stack)
+        redo_size = sum(e.compressed_size for e in self._redo_stack)
+
+        return {
+            "undo_stack_entries": len(self._undo_stack),
+            "redo_stack_entries": len(self._redo_stack),
+            "undo_stack_bytes": undo_size,
+            "redo_stack_bytes": redo_size,
+            "total_bytes": undo_size + redo_size,
+        }

flowfile_core/flowfile/manage/compatibility_enhancements.py

@@ -454,6 +454,15 @@ def ensure_flow_settings(flow_storage_obj: schemas.FlowInformation, flow_path: s
     if not hasattr(fs, "show_detailed_progress"):
         fs.show_detailed_progress = True
 
+    # For track_history, we need to handle legacy pickled objects that were
+    # serialized before this field existed. Use object.__setattr__ to bypass
+    # Pydantic validation which would reject adding a new field.
+    if "track_history" not in fs.__dict__:
+        object.__setattr__(fs, '__dict__', {**fs.__dict__, 'track_history': True})
+
+    if "max_parallel_workers" not in fs.__dict__ or fs.max_parallel_workers is None:
+        object.__setattr__(fs, '__dict__', {**fs.__dict__, 'max_parallel_workers': 4})
+
     return flow_storage_obj
 
 

flowfile_core/flowfile/manage/io_flowfile.py

@@ -173,6 +173,7 @@ def _flowfile_data_to_flow_information(flowfile_data: schemas.FlowfileData) -> s
             setting_data["pos_x"] = float(node.x_position or 0)
             setting_data["pos_y"] = float(node.y_position or 0)
             setting_data["description"] = node.description or ""
+            setting_data["node_reference"] = node.node_reference
             setting_data["is_setup"] = True
 
             if is_user_defined:
@@ -209,6 +210,7 @@ def _flowfile_data_to_flow_information(flowfile_data: schemas.FlowfileData) -> s
             type=node.type,
             is_setup=setting_input is not None,
             description=node.description,
+            node_reference=node.node_reference,
             x_position=node.x_position,
             y_position=node.y_position,
             left_input_id=node.left_input_id,
@@ -231,6 +233,7 @@ def _flowfile_data_to_flow_information(flowfile_data: schemas.FlowfileData) -> s
         execution_location=flowfile_data.flowfile_settings.execution_location,
         auto_save=flowfile_data.flowfile_settings.auto_save,
         show_detailed_progress=flowfile_data.flowfile_settings.show_detailed_progress,
+        max_parallel_workers=flowfile_data.flowfile_settings.max_parallel_workers,
     )
 
     return schemas.FlowInformation(
@@ -305,7 +308,6 @@ def open_flow(flow_path: Path) -> FlowGraph:
     ingestion_order = determine_insertion_order(flow_storage_obj)
     new_flow = FlowGraph(name=flow_storage_obj.flow_name, flow_settings=flow_storage_obj.flow_settings)
     # Create new FlowGraph
-
     # First pass: add node promises
     for node_id in ingestion_order:
         node_info: schemas.NodeInformation = flow_storage_obj.data[node_id]

flowfile_core/flowfile/sources/external_sources/sql_source/models.py

@@ -1,8 +1,8 @@
 import base64
-from typing import Literal
+from typing import Annotated, Any, Literal
 
 import polars as pl
-from pydantic import BaseModel
+from pydantic import BaseModel, BeforeValidator, PlainSerializer
 
 from flowfile_core.schemas.input_schema import (
     DatabaseConnection,
@@ -12,6 +12,22 @@ from flowfile_core.schemas.input_schema import (
 )
 
 
+# Custom type for bytes that serializes to/from base64 string in JSON
+def _decode_bytes(v: Any) -> bytes:
+    if isinstance(v, bytes):
+        return v
+    if isinstance(v, str):
+        return base64.b64decode(v)
+    raise ValueError(f"Expected bytes or base64 string, got {type(v)}")
+
+
+Base64Bytes = Annotated[
+    bytes,
+    BeforeValidator(_decode_bytes),
+    PlainSerializer(lambda x: base64.b64encode(x).decode('ascii'), return_type=str),
+]
+
+
 class ExtDatabaseConnection(DatabaseConnection):
     """Database connection configuration with password handling."""
 
@@ -26,7 +42,7 @@ class DatabaseExternalWriteSettings(BaseModel):
     if_exists: Literal["append", "replace", "fail"] | None = "append"
     flowfile_flow_id: int = 1
     flowfile_node_id: int | str = -1
-    operation: str
+    operation: Base64Bytes  # Accepts bytes or base64 string, serializes to base64
 
     @classmethod
     def create_from_from_node_database_writer(
@@ -60,7 +76,7 @@ class DatabaseExternalWriteSettings(BaseModel):
             if_exists=node_database_writer.database_write_settings.if_exists,
             flowfile_flow_id=node_database_writer.flow_id,
             flowfile_node_id=node_database_writer.node_id,
-            operation=base64.b64encode(lf.serialize()).decode(),
+            operation=lf.serialize(),  # Pass raw bytes, Base64Bytes handles encoding
         )
 
 

flowfile_core/flowfile/util/execution_orderer.py

@@ -1,45 +1,90 @@
 from collections import defaultdict, deque
+from dataclasses import dataclass
 
 from flowfile_core.configs import logger
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_core.flowfile.util.node_skipper import determine_nodes_to_skip
 
 
-def compute_execution_plan(nodes: list[FlowNode], flow_starts: list[FlowNode] = None):
-    """Computes the execution order after finding the nodes to skip on the execution step."""
+@dataclass(frozen=True)
+class ExecutionStage:
+    """A group of nodes with no mutual dependencies that can execute in parallel."""
+
+    nodes: list[FlowNode]
+
+    def __len__(self) -> int:
+        return len(self.nodes)
+
+    def __iter__(self):
+        return iter(self.nodes)
+
+
+@dataclass(frozen=True)
+class ExecutionPlan:
+    """Complete execution plan: nodes to skip and ordered stages of parallelizable nodes."""
+
+    skip_nodes: list[FlowNode]
+    stages: list[ExecutionStage]
+
+    @property
+    def all_nodes(self) -> list[FlowNode]:
+        """Flattened list of all nodes across all stages, preserving topological order."""
+        return [node for stage in self.stages for node in stage.nodes]
+
+    @property
+    def node_count(self) -> int:
+        return sum(len(stage) for stage in self.stages)
+
+
+def compute_execution_plan(nodes: list[FlowNode], flow_starts: list[FlowNode] = None) -> ExecutionPlan:
+    """Computes the execution plan: nodes to skip and parallelizable execution stages.
+
+    Args:
+        nodes: All nodes in the flow.
+        flow_starts: Explicit starting nodes for the flow.
+
+    Returns:
+        An ExecutionPlan containing skip_nodes and ordered execution stages.
+    """
     skip_nodes = determine_nodes_to_skip(nodes=nodes)
-    computed_execution_order = determine_execution_order(
+    stages = determine_execution_order(
         all_nodes=[node for node in nodes if node not in skip_nodes], flow_starts=flow_starts
     )
-    return skip_nodes, computed_execution_order
+    return ExecutionPlan(skip_nodes=skip_nodes, stages=stages)
 
 
-def determine_execution_order(all_nodes: list[FlowNode], flow_starts: list[FlowNode] = None) -> list[FlowNode]:
+def determine_execution_order(
+    all_nodes: list[FlowNode], flow_starts: list[FlowNode] = None
+) -> list[ExecutionStage]:
     """
     Determines the execution order of nodes using topological sorting based on node dependencies.
+    Returns stages of nodes where each stage contains nodes that can execute in parallel.
 
     Args:
-        all_nodes (List[FlowNode]): A list of all nodes (steps) in the flow.
-        flow_starts (List[FlowNode], optional): A list of starting nodes for the flow. If not provided, the function starts with nodes having zero in-degree.
+        all_nodes: A list of all nodes in the flow.
+        flow_starts: Starting nodes for the flow. If not provided, starts with zero in-degree nodes.
 
     Returns:
-        List[FlowNode]: A list of nodes in the order they should be executed.
+        A list of ExecutionStage objects in dependency order. Nodes within a stage have no
+        mutual dependencies and can run concurrently.
 
     Raises:
-        Exception: If a cycle is detected, meaning that a valid execution order cannot be determined.
+        Exception: If a cycle is detected in the graph.
     """
     node_map = build_node_map(all_nodes)
     in_degree, adjacency_list = compute_in_degrees_and_adjacency_list(all_nodes, node_map)
 
     queue, visited_nodes = initialize_queue(flow_starts, all_nodes, in_degree)
 
-    execution_order = perform_topological_sort(queue, node_map, in_degree, adjacency_list, visited_nodes)
-    if len(execution_order) != len(node_map):
+    stages = perform_topological_sort(queue, node_map, in_degree, adjacency_list, visited_nodes)
+    total_nodes = sum(len(stage) for stage in stages)
+    if total_nodes != len(node_map):
         raise Exception("Cycle detected in the graph. Execution order cannot be determined.")
 
-    logger.info(f"execution order: \n {[node for node in execution_order if node.is_correct]}")
+    all_nodes_flat = [node for stage in stages for node in stage if node.is_correct]
+    logger.info(f"execution order: \n {all_nodes_flat}")
 
-    return execution_order
+    return stages
 
 
 def build_node_map(all_nodes: list[FlowNode]) -> dict[str, FlowNode]:
@@ -124,35 +169,43 @@ def perform_topological_sort(
     in_degree: dict[str, int],
     adjacency_list: dict[str, list[str]],
     visited_nodes: set[str],
-) -> list[FlowNode]:
+) -> list[ExecutionStage]:
     """
-    Performs topological sorting to determine the execution order of nodes.
+    Performs a level-based topological sort, grouping nodes into execution stages.
+    Nodes within the same stage have no dependencies on each other and can run in parallel.
 
     Args:
-        queue (deque): A deque containing nodes with zero in-degree.
-        node_map (Dict[str, FlowNode]): A dictionary mapping node IDs to FlowNode objects.
-        in_degree (Dict[str, int]): A dictionary mapping node IDs to their in-degree count.
-        adjacency_list (Dict[str, List[str]]): A dictionary mapping node IDs to a list of their connected nodes (outgoing edges).
-        visited_nodes (Set[str]): A set of visited node IDs.
+        queue: A deque containing node IDs with zero in-degree.
+        node_map: A dictionary mapping node IDs to FlowNode objects.
+        in_degree: A dictionary mapping node IDs to their in-degree count.
+        adjacency_list: A dictionary mapping node IDs to a list of their outgoing node IDs.
+        visited_nodes: A set of visited node IDs.
 
     Returns:
-        List[FlowNode]: A list of nodes in the order they should be executed.
+        A list of ExecutionStage objects. Each stage contains nodes that can execute concurrently.
     """
-    execution_order = []
+    stages: list[ExecutionStage] = []
     logger.info("Starting topological sort to determine execution order")
 
     while queue:
-        current_node_id = queue.popleft()
-        current_node = node_map.get(current_node_id)
-        if current_node is None:
-            logger.warning(f"Node with ID {current_node_id} not found in the node map.")
-            continue
-        execution_order.append(current_node)
-
-        for next_node_id in adjacency_list.get(current_node_id, []):
-            in_degree[next_node_id] -= 1
-            if in_degree[next_node_id] == 0 and next_node_id not in visited_nodes:
-                queue.append(next_node_id)
-                visited_nodes.add(next_node_id)
-
-    return execution_order
+        current_stage_ids = list(queue)
+        queue.clear()
+
+        stage_nodes: list[FlowNode] = []
+        for node_id in current_stage_ids:
+            node = node_map.get(node_id)
+            if node is None:
+                logger.warning(f"Node with ID {node_id} not found in the node map.")
+                continue
+            stage_nodes.append(node)
+
+            for next_node_id in adjacency_list.get(node_id, []):
+                in_degree[next_node_id] -= 1
+                if in_degree[next_node_id] == 0 and next_node_id not in visited_nodes:
+                    queue.append(next_node_id)
+                    visited_nodes.add(next_node_id)
+
+        if stage_nodes:
+            stages.append(ExecutionStage(nodes=stage_nodes))
+
+    return stages