Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_core/schemas/output_model.py

@@ -1,47 +1,52 @@
-from typing import List, Dict, Optional, Any, Literal
-from pydantic import BaseModel, Field
-from datetime import datetime
 import time
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
 
 
 class NodeResult(BaseModel):
     """Represents the execution result of a single node in a FlowGraph run."""
+
     node_id: int
-    node_name: Optional[str] = None
+    node_name: str | None = None
     start_timestamp: float = Field(default_factory=time.time)
     end_timestamp: float = 0
-    success: Optional[bool] = None
-    error: str = ''
+    success: bool | None = None
+    error: str = ""
     run_time: int = -1
     is_running: bool = True
 
 
 class RunInformation(BaseModel):
     """Contains summary information about a complete FlowGraph execution."""
+
     flow_id: int
-    start_time: Optional[datetime] = Field(default_factory=datetime.now)
-    end_time: Optional[datetime] = None
-    success: Optional[bool] = None
+    start_time: datetime | None = Field(default_factory=datetime.now)
+    end_time: datetime | None = None
+    success: bool | None = None
     nodes_completed: int = 0
     number_of_nodes: int = 0
-    node_step_result: List[NodeResult]
-    run_type: Literal["fetch_one", "full_run"]
+    node_step_result: list[NodeResult]
+    run_type: Literal["fetch_one", "full_run", "init"]
 
 
 class BaseItem(BaseModel):
     """A base model for any item in a file system, like a file or directory."""
+
     name: str
     path: str
-    size: Optional[int] = None
-    creation_date: Optional[datetime] = None
-    access_date: Optional[datetime] = None
-    modification_date: Optional[datetime] = None
-    source_path: Optional[str] = None
+    size: int | None = None
+    creation_date: datetime | None = None
+    access_date: datetime | None = None
+    modification_date: datetime | None = None
+    source_path: str | None = None
     number_of_items: int = -1
 
 
 class FileColumn(BaseModel):
     """Represents detailed schema and statistics for a single column (field)."""
+
     name: str
     data_type: str
     is_unique: bool
@@ -55,13 +60,14 @@ class FileColumn(BaseModel):
 
 class TableExample(BaseModel):
     """Represents a preview of a table, including schema and sample data."""
+
     node_id: int
     number_of_records: int
     number_of_columns: int
     name: str
-    table_schema: List[FileColumn]
-    columns: List[str]
-    data: Optional[List[Dict]] = {}
+    table_schema: list[FileColumn]
+    columns: list[str]
+    data: list[dict] | None = {}
     has_example_data: bool = False
     has_run_with_current_setup: bool = False
 
@@ -71,15 +77,16 @@ class NodeData(BaseModel):
 
     This includes its input/output data previews, settings, and run status.
     """
+
     flow_id: int
     node_id: int
     flow_type: str
-    left_input: Optional[TableExample] = None
-    right_input: Optional[TableExample] = None
-    main_input: Optional[TableExample] = None
-    main_output: Optional[TableExample] = None
-    left_output: Optional[TableExample] = None
-    right_output: Optional[TableExample] = None
+    left_input: TableExample | None = None
+    right_input: TableExample | None = None
+    main_input: TableExample | None = None
+    main_output: TableExample | None = None
+    left_output: TableExample | None = None
+    right_output: TableExample | None = None
     has_run: bool = False
     is_cached: bool = False
     setting_input: Any = None
@@ -87,22 +94,26 @@ class NodeData(BaseModel):
 
 class OutputFile(BaseItem):
     """Represents a single file in an output directory, extending BaseItem."""
-    ext: Optional[str] = None
-    mimetype: Optional[str] = None
+
+    ext: str | None = None
+    mimetype: str | None = None
 
 
 class OutputFiles(BaseItem):
     """Represents a collection of files, typically within a directory."""
-    files: List[OutputFile] = Field(default_factory=list)
+
+    files: list[OutputFile] = Field(default_factory=list)
 
 
 class OutputTree(OutputFiles):
     """Represents a directory tree, including subdirectories."""
-    directories: List[OutputFiles] = Field(default_factory=list)
+
+    directories: list[OutputFiles] = Field(default_factory=list)
 
 
 class ItemInfo(OutputFile):
     """Provides detailed information about a single item in an output directory."""
+
     id: int = -1
     type: str
     analysis_file_available: bool = False
@@ -112,23 +123,27 @@ class ItemInfo(OutputFile):
 
 class OutputDir(BaseItem):
     """Represents the contents of a single output directory."""
-    all_items: List[str]
-    items: List[ItemInfo]
+
+    all_items: list[str]
+    items: list[ItemInfo]
 
 
 class ExpressionRef(BaseModel):
     """A reference to a single Polars expression, including its name and docstring."""
+
     name: str
-    doc: Optional[str]
+    doc: str | None
 
 
 class ExpressionsOverview(BaseModel):
     """Represents a categorized list of available Polars expressions."""
+
     expression_type: str
-    expressions: List[ExpressionRef]
+    expressions: list[ExpressionRef]
 
 
 class InstantFuncResult(BaseModel):
     """Represents the result of a function that is expected to execute instantly."""
-    success: Optional[bool] = None
-    result: str
+
+    success: bool | None = None
+    result: str

flowfile_core/schemas/schemas.py

@@ -1,13 +1,51 @@
-from typing import Optional, List, Dict, Tuple, Any, Literal, Annotated
-from pydantic import BaseModel, field_validator, ConfigDict, Field, StringConstraints
-from flowfile_core.flowfile.utils import create_unique_id
+from typing import Any, ClassVar, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_serializer, field_validator
+
 from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
-ExecutionModeLiteral = Literal['Development', 'Performance']
-ExecutionLocationsLiteral = Literal['local', 'remote']
+from flowfile_core.flowfile.utils import create_unique_id
+from flowfile_core.schemas import input_schema
+
+ExecutionModeLiteral = Literal["Development", "Performance"]
+ExecutionLocationsLiteral = Literal["local", "remote"]
 
 # Type literals for classifying nodes.
-NodeTypeLiteral = Literal['input', 'output', 'process']
-TransformTypeLiteral = Literal['narrow', 'wide', 'other']
+NodeTypeLiteral = Literal["input", "output", "process"]
+TransformTypeLiteral = Literal["narrow", "wide", "other"]
+_custom_node_store_cache = None
+
+NODE_TYPE_TO_SETTINGS_CLASS = {
+    "manual_input": input_schema.NodeManualInput,
+    "filter": input_schema.NodeFilter,
+    "formula": input_schema.NodeFormula,
+    "select": input_schema.NodeSelect,
+    "sort": input_schema.NodeSort,
+    "record_id": input_schema.NodeRecordId,
+    "sample": input_schema.NodeSample,
+    "unique": input_schema.NodeUnique,
+    "group_by": input_schema.NodeGroupBy,
+    "pivot": input_schema.NodePivot,
+    "unpivot": input_schema.NodeUnpivot,
+    "text_to_rows": input_schema.NodeTextToRows,
+    "graph_solver": input_schema.NodeGraphSolver,
+    "polars_code": input_schema.NodePolarsCode,
+    "join": input_schema.NodeJoin,
+    "cross_join": input_schema.NodeCrossJoin,
+    "fuzzy_match": input_schema.NodeFuzzyMatch,
+    "record_count": input_schema.NodeRecordCount,
+    "explore_data": input_schema.NodeExploreData,
+    "union": input_schema.NodeUnion,
+    "output": input_schema.NodeOutput,
+    "read": input_schema.NodeRead,
+    "database_reader": input_schema.NodeDatabaseReader,
+    "database_writer": input_schema.NodeDatabaseWriter,
+    "cloud_storage_reader": input_schema.NodeCloudStorageReader,
+    "cloud_storage_writer": input_schema.NodeCloudStorageWriter,
+    "external_source": input_schema.NodeExternalSource,
+    "promise": input_schema.NodePromise,
+    "user_defined": input_schema.UserDefinedNode,
+}
+
 
 def get_global_execution_location() -> ExecutionLocationsLiteral:
     """
@@ -21,12 +59,33 @@ def get_global_execution_location() -> ExecutionLocationsLiteral:
     return "local"
 
 
+def _get_custom_node_store():
+    """Lazy load CUSTOM_NODE_STORE once and cache it."""
+    global _custom_node_store_cache
+    if _custom_node_store_cache is None:
+        from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
+
+        _custom_node_store_cache = CUSTOM_NODE_STORE
+    return _custom_node_store_cache
+
+
+def get_settings_class_for_node_type(node_type: str):
+    """Get the settings class for a node type, supporting both standard and user-defined nodes."""
+    model_class = NODE_TYPE_TO_SETTINGS_CLASS.get(node_type)
+    if model_class is None:
+        if node_type in _get_custom_node_store():
+            return input_schema.UserDefinedNode
+        return None
+    return model_class
+
+
 def is_valid_execution_location_in_current_global_settings(execution_location: ExecutionLocationsLiteral) -> bool:
     return not (get_global_execution_location() == "local" and execution_location == "remote")
 
 
-def get_prio_execution_location(local_execution_location: ExecutionLocationsLiteral,
-                                global_execution_location: ExecutionLocationsLiteral) -> ExecutionLocationsLiteral:
+def get_prio_execution_location(
+    local_execution_location: ExecutionLocationsLiteral, global_execution_location: ExecutionLocationsLiteral
+) -> ExecutionLocationsLiteral:
     if local_execution_location == global_execution_location:
         return local_execution_location
     elif global_execution_location == "local" and local_execution_location == "remote":
@@ -48,16 +107,17 @@ class FlowGraphConfig(BaseModel):
         execution_mode (ExecutionModeLiteral): The mode of execution ('Development' or 'Performance').
         execution_location (ExecutionLocationsLiteral): The location for execution ('local', 'remote').
     """
+
     flow_id: int = Field(default_factory=create_unique_id, description="Unique identifier for the flow.")
-    description: Optional[str] = None
-    save_location: Optional[str] = None
-    name: str = ''
-    path: str = ''
-    execution_mode: ExecutionModeLiteral = 'Performance'
+    description: str | None = None
+    save_location: str | None = None
+    name: str = ""
+    path: str = ""
+    execution_mode: ExecutionModeLiteral = "Performance"
     execution_location: ExecutionLocationsLiteral = Field(default_factory=get_global_execution_location)
 
-    @field_validator('execution_location', mode='before')
-    def validate_and_set_execution_location(cls, v: Optional[ExecutionLocationsLiteral]) -> ExecutionLocationsLiteral:
+    @field_validator("execution_location", mode="before")
+    def validate_and_set_execution_location(cls, v: ExecutionLocationsLiteral | None) -> ExecutionLocationsLiteral:
         """
         Validates and sets the execution location.
         1.  **If `None` is provided**: It defaults to the location determined by global settings.
@@ -84,8 +144,9 @@ class FlowSettings(FlowGraphConfig):
         is_running (bool): Indicates if the flow is currently running.
         is_canceled (bool): Indicates if the flow execution has been canceled.
     """
+
     auto_save: bool = False
-    modified_on: Optional[float] = None
+    modified_on: float | None = None
     show_detailed_progress: bool = True
     is_running: bool = False
     is_canceled: bool = False
@@ -111,10 +172,76 @@ class RawLogInput(BaseModel):
         log_type (Literal["INFO", "ERROR"]): The type of log.
         extra (Optional[dict]): Extra context data for the log.
     """
+
     flowfile_flow_id: int
     log_message: str
     log_type: Literal["INFO", "ERROR"]
-    extra: Optional[dict] = None
+    extra: dict | None = None
+
+
+class FlowfileSettings(BaseModel):
+    """Settings for flowfile serialization (YAML/JSON).
+
+    Excludes runtime state fields like is_running, is_canceled, modified_on.
+    """
+
+    description: str | None = None
+    execution_mode: ExecutionModeLiteral = "Performance"
+    execution_location: ExecutionLocationsLiteral = "local"
+    auto_save: bool = False
+    show_detailed_progress: bool = True
+
+
+class FlowfileNode(BaseModel):
+    """Node representation for flowfile serialization (YAML/JSON)."""
+
+    id: int
+    type: str
+    is_start_node: bool = False
+    description: str | None = ""
+    x_position: int | None = 0
+    y_position: int | None = 0
+    left_input_id: int | None = None
+    right_input_id: int | None = None
+    input_ids: list[int] | None = Field(default_factory=list)
+    outputs: list[int] | None = Field(default_factory=list)
+    setting_input: Any | None = None
+
+    _setting_input_exclude: ClassVar[set] = {
+        "flow_id",
+        "node_id",
+        "pos_x",
+        "pos_y",
+        "is_setup",
+        "description",
+        "user_id",
+        "is_flow_output",
+        "is_user_defined",
+        "depending_on_id",
+        "depending_on_ids",
+    }
+
+    @field_serializer("setting_input")
+    def serialize_setting_input(self, value, _info):
+        if value is None:
+            return None
+        if isinstance(value, input_schema.NodePromise):
+            return None
+        if hasattr(value, "to_yaml_dict"):
+            return value.to_yaml_dict()
+        if hasattr(value, "to_yaml_dict"):
+            return value.to_yaml_dict()
+        return value.model_dump(exclude=self._setting_input_exclude)
+
+
+class FlowfileData(BaseModel):
+    """Root model for flowfile serialization (YAML/JSON)."""
+
+    flowfile_version: str
+    flowfile_id: int
+    flowfile_name: str
+    flowfile_settings: FlowfileSettings
+    nodes: list[FlowfileNode]
 
 
 class NodeTemplate(BaseModel):
@@ -132,6 +259,7 @@ class NodeTemplate(BaseModel):
         prod_ready (bool): Whether the node is considered production-ready.
         can_be_start (bool): Whether the node can be a starting point in a flow.
     """
+
     name: str
     item: str
     input: int
@@ -145,54 +273,54 @@ class NodeTemplate(BaseModel):
     can_be_start: bool = False
     drawer_title: str = "Node title"
     drawer_intro: str = "Drawer into"
-    custom_node: Optional[bool] = False
+    custom_node: bool | None = False
 
 
 class NodeInformation(BaseModel):
     """
     Stores the state and configuration of a specific node instance within a flow.
-
-    Attributes:
-        id (Optional[int]): The unique ID of the node instance.
-        type (Optional[str]): The type of the node (e.g., 'join', 'filter').
-        is_setup (Optional[bool]): Whether the node has been configured.
-        description (Optional[str]): A user-provided description.
-        x_position (Optional[int]): The x-coordinate on the canvas.
-        y_position (Optional[int]): The y-coordinate on the canvas.
-        left_input_id (Optional[int]): The ID of the node connected to the left input.
-        right_input_id (Optional[int]): The ID of the node connected to the right input.
-        input_ids (Optional[List[int]]): A list of IDs for main input nodes.
-        outputs (Optional[List[int]]): A list of IDs for nodes this node outputs to.
-        setting_input (Optional[Any]): The specific settings for this node instance.
     """
-    id: Optional[int] = None
-    type: Optional[str] = None
-    is_setup: Optional[bool] = None
-    description: Optional[str] = ''
-    x_position: Optional[int] = 0
-    y_position: Optional[int] = 0
-    left_input_id: Optional[int] = None
-    right_input_id: Optional[int] = None
-    input_ids: Optional[List[int]] = [-1]
-    outputs: Optional[List[int]] = [-1]
-    setting_input: Optional[Any] = None
+
+    id: int | None = None
+    type: str | None = None
+    is_setup: bool | None = None
+    is_start_node: bool = False
+    description: str | None = ""
+    x_position: int | None = 0
+    y_position: int | None = 0
+    left_input_id: int | None = None
+    right_input_id: int | None = None
+    input_ids: list[int] | None = Field(default_factory=list)
+    outputs: list[int] | None = Field(default_factory=list)
+    setting_input: Any | None = None
 
     @property
     def data(self) -> Any:
-        """
-        Property to access the node's specific settings.
-        :return: The settings of the node.
-        """
         return self.setting_input
 
     @property
-    def main_input_ids(self) -> Optional[List[int]]:
-        """
-        Property to access the main input node IDs.
-        :return: A list of main input node IDs.
-        """
+    def main_input_ids(self) -> list[int] | None:
         return self.input_ids
 
+    @field_validator("setting_input", mode="before")
+    @classmethod
+    def validate_setting_input(cls, v, info: ValidationInfo):
+        if v is None:
+            return None
+        if isinstance(v, BaseModel):
+            return v
+
+        node_type = info.data.get("type")
+        model_class = get_settings_class_for_node_type(node_type)
+
+        if model_class is None:
+            raise ValueError(f"Unknown node type: {node_type}")
+
+        if isinstance(v, model_class):
+            return v
+
+        return model_class.model_validate(v)
+
 
 class FlowInformation(BaseModel):
     """
@@ -206,21 +334,36 @@ class FlowInformation(BaseModel):
         node_starts (List[int]): A list of starting node IDs.
         node_connections (List[Tuple[int, int]]): A list of tuples representing connections between nodes.
     """
+
     flow_id: int
-    flow_name: Optional[str] = ''
+    flow_name: str | None = ""
     flow_settings: FlowSettings
-    data: Dict[int, NodeInformation] = {}
-    node_starts: List[int]
-    node_connections: List[Tuple[int, int]] = []
+    data: dict[int, NodeInformation] = {}
+    node_starts: list[int]
+    node_connections: list[tuple[int, int]] = []
 
-    @field_validator('flow_name', mode="before")
+    @field_validator("flow_name", mode="before")
     def ensure_string(cls, v):
         """
         Validator to ensure the flow_name is always a string.
         :param v: The value to validate.
         :return: The value as a string, or an empty string if it's None.
         """
-        return str(v) if v is not None else ''
+        return str(v) if v is not None else ""
+
+
+class NodeConnection(BaseModel):
+    """
+    Represents a connection between two nodes in the flow.
+
+    Attributes:
+        from_node_id (int): The ID of the source node.
+        to_node_id (int): The ID of the target node.
+    """
+
+    model_config = ConfigDict(frozen=True)
+    from_node_id: int
+    to_node_id: int
 
 
 class NodeInput(NodeTemplate):
@@ -232,6 +375,7 @@ class NodeInput(NodeTemplate):
         pos_x (float): The x-coordinate on the canvas.
         pos_y (float): The y-coordinate on the canvas.
     """
+
     id: int
     pos_x: float
     pos_y: float
@@ -248,6 +392,7 @@ class NodeEdge(BaseModel):
         targetHandle (str): The specific input handle on the target node.
         sourceHandle (str): The specific output handle on the source node.
     """
+
     model_config = ConfigDict(coerce_numbers_to_str=True)
     id: str
     source: str
@@ -265,10 +410,9 @@ class VueFlowInput(BaseModel):
         node_edges (List[NodeEdge]): A list of all edges in the graph.
         node_inputs (List[NodeInput]): A list of all nodes in the graph.
     """
-    node_edges: List[NodeEdge]
-    node_inputs: List[NodeInput]
-
 
+    node_edges: list[NodeEdge]
+    node_inputs: list[NodeInput]
 
 
 class NodeDefault(BaseModel):
@@ -281,12 +425,8 @@ class NodeDefault(BaseModel):
         transform_type (TransformTypeLiteral): The data transformation behavior ('narrow', 'wide', 'other').
         has_default_settings (Optional[Any]): Indicates if the node has predefined default settings.
     """
+
     node_name: str
     node_type: NodeTypeLiteral
     transform_type: TransformTypeLiteral
-    has_default_settings: Optional[Any] = None
-
-
-# Define SecretRef here if not in a common location
-SecretRef = Annotated[str, StringConstraints(min_length=1, max_length=100),
-                      Field(description="An ID referencing an encrypted secret.")]
+    has_default_settings: Any | None = None