Flowfile 0.5.6__py3-none-any.whl → 0.6.1__py3-none-any.whl

flowfile_core/flowfile/flow_node/output_field_config_applier.py

@@ -0,0 +1,217 @@
+"""Utility module for applying output field configuration to FlowDataEngine results."""
+
+from typing import List, Set
+import polars as pl
+from flowfile_core.configs import logger
+from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
+from flowfile_core.schemas.input_schema import OutputFieldConfig, OutputFieldInfo
+
+
+def _parse_default_value(field: OutputFieldInfo) -> pl.Expr:
+    """Parse default value from field configuration.
+
+    Args:
+        field: Output field info containing default_value
+
+    Returns:
+        Polars expression for the default value cast to the target data type
+    """
+    # Get target Polars dtype from the field's data_type
+    target_dtype = cast_str_to_polars_type(field.data_type)
+    # Treat as literal value and cast to target type
+    return pl.lit(field.default_value).cast(target_dtype, strict=False, wrap_numerical=True)
+
+
+def _select_columns_in_order(df: pl.DataFrame, fields: list[OutputFieldInfo]) -> pl.DataFrame:
+    """Select columns in the specified field order.
+
+    Args:
+        df: Input dataframe
+        fields: List of fields specifying column order
+
+    Returns:
+        DataFrame with columns selected in specified order
+    """
+    return df.select([field.name for field in fields])
+
+
+def _apply_raise_on_missing(
+    flowfile_engine: FlowDataEngine,
+    fields: list[OutputFieldInfo],
+) -> FlowDataEngine:
+    """Apply raise_on_missing validation mode.
+
+    Raises error if any expected columns are missing, then selects columns in order.
+
+    Args:
+        flowfile_engine: Input flow data engine
+        fields: List of expected output fields
+
+    Returns:
+        Flow Data Engine with columns selected in specified order
+
+    Raises:
+        ValueError: If any expected columns are missing
+    """
+    cols = [f.name for f in fields]
+    missing_columns = set(cols) - set(flowfile_engine.columns)
+    if missing_columns:
+        raise ValueError(f"Missing required columns: {', '.join(sorted(missing_columns))}")
+    if flowfile_engine.columns != cols:
+        return FlowDataEngine(_select_columns_in_order(flowfile_engine.data_frame, fields))
+    else:
+        return flowfile_engine
+
+
+def _apply_add_missing(
+    engine: FlowDataEngine,
+    fields: list[OutputFieldInfo],
+) -> FlowDataEngine:
+    """Apply add_missing validation mode.
+
+    Adds missing columns with default values, then selects columns in order.
+    Extra columns not in the config are removed.
+
+    Args:
+        engine: Input flow data engine
+        fields: List of expected output fields
+
+    Returns:
+        FlowDataEngine with missing columns added and only configured columns in specified order
+    """
+    # Add missing columns with default values
+    current_columns = set(engine.columns)
+    expressions = [_parse_default_value(field).alias(field.name)
+                   for field in fields if field.name not in current_columns]
+    if expressions:
+        new_df = engine.data_frame.with_columns(expressions)
+    else:
+        new_df = engine.data_frame
+    return FlowDataEngine(_select_columns_in_order(new_df, fields))
+
+
+def _apply_add_missing_keep_extra(
+    engine: FlowDataEngine,
+    fields: list[OutputFieldInfo],
+) -> FlowDataEngine:
+    """Apply add_missing_keep_extra validation mode.
+
+    Adds missing columns with default values, but keeps all incoming columns.
+    Configured columns come first in specified order, followed by extra columns.
+
+    Args:
+        engine: Input flow data engine
+        fields: List of expected output fields
+
+    Returns:
+        FlowDataEngine with missing columns added and all columns preserved
+        (configured columns first, then extras)
+    """
+    current_columns = set(engine.columns)
+    configured_names = {field.name for field in fields}
+
+    # Add missing columns with default values
+    expressions = [_parse_default_value(field).alias(field.name)
+                   for field in fields if field.name not in current_columns]
+    if expressions:
+        new_df = engine.data_frame.with_columns(expressions)
+    else:
+        new_df = engine.data_frame
+
+    # Build column order: configured columns first (in order), then extras
+    configured_column_order = [field.name for field in fields]
+    extra_columns = [col for col in engine.columns if col not in configured_names]
+
+    final_column_order = configured_column_order + extra_columns
+    return FlowDataEngine(new_df.select(final_column_order))
+
+
+def _validate_data_types(df: FlowDataEngine, fields: list[OutputFieldInfo]) -> None:
+    """Validate that dataframe column types match expected types.
+
+    Args:
+        df: Input dataframe or lazyframe
+        fields: List of expected output fields with data types
+
+    Raises:
+        ValueError: If any data type mismatches are found
+    """
+    # Get schema (works for both DataFrame and LazyFrame)
+    schema: dict[str, FlowfileColumn] = {column.column_name: column for column in df.schema}
+    mismatches = []
+    for field in fields:
+
+        if field.name not in schema:
+            continue
+        # Use FlowfileColumn infrastructure to convert dtype to string
+        column = schema.get(field.name)
+        column.get_minimal_field_info()
+        if column.data_type != field.data_type:
+            mismatches.append(
+                f"Column '{field.name}': expected {field.data_type}, got {column.data_type}"
+            )
+
+    if mismatches:
+        error_msg = "Data type validation failed:\n" + "\n".join(f"  - {m}" for m in mismatches)
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    logger.info(f"Data type validation passed for {len(fields)} fields")
+
+
+def apply_output_field_config(
+    flow_data_engine: FlowDataEngine, output_field_config: OutputFieldConfig
+) -> FlowDataEngine:
+    """Apply output field configuration to enforce schema requirements.
+
+    Args:
+        flow_data_engine: The FlowDataEngine instance to apply configuration to
+        output_field_config: The output field configuration specifying behavior
+
+    Returns:
+        Modified FlowDataEngine with enforced schema
+
+    Raises:
+        ValueError: If raise_on_missing behavior is set and required columns are missing,
+                   or if data type validation fails
+    """
+    if not output_field_config or not output_field_config.enabled:
+        return flow_data_engine
+
+    if not output_field_config.fields:
+        return flow_data_engine
+    # breakpoint()
+    try:
+        # Get column sets for validation (works for both DataFrame and LazyFrame)
+
+        # Apply validation mode behavior
+        mode = output_field_config.validation_mode_behavior
+        if mode == "raise_on_missing":
+            new_flow_engine = _apply_raise_on_missing(flow_data_engine, output_field_config.fields)
+        elif mode == "add_missing":
+            new_flow_engine = _apply_add_missing(engine=flow_data_engine, fields=output_field_config.fields)
+        elif mode == "add_missing_keep_extra":
+            new_flow_engine = _apply_add_missing_keep_extra(engine=flow_data_engine, fields=output_field_config.fields)
+        elif mode == "select_only":
+            new_flow_engine = flow_data_engine.select_columns(
+                [field.name for field in output_field_config.fields]
+            )
+        else:
+            raise ValueError(f"Unknown validation mode behavior: {mode}")
+        # Validate data types if enabled
+        if output_field_config.validate_data_types:
+            _validate_data_types(new_flow_engine, output_field_config.fields)
+
+        logger.info(
+            f"Applied output field config: behavior={mode}, "
+            f"fields={len(output_field_config.fields)}, "
+            f"validate_data_types={output_field_config.validate_data_types}"
+        )
+
+    except Exception as e:
+        logger.error(f"Error applying output field config: {e}")
+        raise
+
+    return new_flow_engine

flowfile_core/flowfile/flow_node/schema_utils.py

@@ -0,0 +1,78 @@
+"""Schema utilities for output field configuration."""
+
+from flowfile_core.configs import logger
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.schemas.input_schema import OutputFieldConfig
+
+
+def create_schema_from_output_field_config(output_field_config: OutputFieldConfig) -> list[FlowfileColumn]:
+    """Create a FlowfileColumn schema from OutputFieldConfig.
+
+    This is used for schema prediction - instead of running the transformation,
+    we can directly return the configured output schema.
+
+    Note: For 'add_missing_keep_extra' mode, this returns only the configured fields.
+    Extra columns from the incoming data are not predictable without running the flow,
+    so _predicted_data_getter provides more accurate results for that mode.
+
+    Args:
+        output_field_config: The output field configuration
+
+    Returns:
+        List of FlowfileColumn objects representing the expected output schema
+    """
+    if not output_field_config or not output_field_config.enabled or not output_field_config.fields:
+        logger.debug("create_schema_from_output_field_config: config not enabled or no fields, returning None")
+        return None
+
+    schema = [
+        FlowfileColumn.from_input(column_name=field.name, data_type=field.data_type)
+        for field in output_field_config.fields
+    ]
+    logger.info(
+        f"create_schema_from_output_field_config: Created schema with {len(schema)} fields: "
+        f"{[f.name for f in schema]}"
+    )
+    return schema
+
+
+def create_schema_callback_with_output_config(
+    base_schema_callback: callable,
+    output_field_config: OutputFieldConfig | None
+) -> callable:
+    """Wraps a schema callback to use output_field_config when available.
+
+    This allows nodes to use their configured output schema for prediction
+    instead of running through transformation logic.
+
+    Args:
+        base_schema_callback: The original schema callback function
+        output_field_config: The output field configuration, if any
+
+    Returns:
+        A wrapped schema callback that prioritizes output_field_config
+    """
+    logger.debug(
+        f"create_schema_callback_with_output_config called: "
+        f"base_callback={'present' if base_schema_callback else 'None'}, "
+        f"config={'enabled' if (output_field_config and output_field_config.enabled) else 'disabled/None'}"
+    )
+
+    def wrapped_schema_callback():
+        # If output_field_config is enabled, use it directly for schema prediction
+        if output_field_config and output_field_config.enabled and output_field_config.fields:
+            logger.info(
+                f"wrapped_schema_callback: Using output_field_config for schema prediction "
+                f"(validation_mode={output_field_config.validation_mode_behavior}, {len(output_field_config.fields)} fields)"
+            )
+            return create_schema_from_output_field_config(output_field_config)
+
+        # Otherwise fall back to the original schema callback
+        if base_schema_callback:
+            logger.debug("wrapped_schema_callback: Falling back to base_schema_callback")
+            return base_schema_callback()
+        else:
+            logger.warning("wrapped_schema_callback: No base_schema_callback and no output_field_config, returning None")
+            return None
+
+    return wrapped_schema_callback

flowfile_core/flowfile/flow_node/state.py

@@ -0,0 +1,155 @@
+"""
+Node execution state - separable from node definition.
+Can be persisted to database/cache for stateless operation.
+"""
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+import pyarrow as pa
+
+if TYPE_CHECKING:
+    from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+    from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+
+
+@dataclass
+class SourceFileInfo:
+    """
+    Tracks source file state for cache invalidation.
+    Used by read nodes to detect when source files change.
+    """
+    path: str
+    mtime: float
+    size: int
+
+    @classmethod
+    def from_path(cls, path: str) -> SourceFileInfo | None:
+        """Create from file path, returns None if file doesn't exist."""
+        try:
+            stat = os.stat(path)
+            return cls(path=path, mtime=stat.st_mtime, size=stat.st_size)
+        except OSError:
+            return None
+
+    def has_changed(self) -> bool:
+        """Check if file has changed since this info was recorded."""
+        try:
+            stat = os.stat(self.path)
+            return stat.st_mtime != self.mtime or stat.st_size != self.size
+        except OSError:
+            return True  # File missing = changed
+
+    def to_dict(self) -> dict:
+        """Serialize for external storage."""
+        return {"path": self.path, "mtime": self.mtime, "size": self.size}
+
+    @classmethod
+    def from_dict(cls, data: dict) -> SourceFileInfo:
+        """Deserialize from external storage."""
+        return cls(path=data["path"], mtime=data["mtime"], size=data["size"])
+
+
+@dataclass
+class NodeExecutionState:
+    """
+    All mutable state for a node's execution.
+
+    This can be:
+    - Stored in memory (current behavior)
+    - Persisted to database (stateless workers)
+    - Stored in Redis/cache (distributed execution)
+
+    Kept separate from FlowNode to enable stateless execution patterns.
+    """
+    # Execution tracking
+    has_run_with_current_setup: bool = False
+    has_completed_last_run: bool = False
+    is_canceled: bool = False
+    error: str | None = None
+
+    # Results (not serialized - too large)
+    resulting_data: FlowDataEngine | None = field(default=None, repr=False)
+    example_data_path: str | None = None
+    example_data_generator: Callable[[], pa.Table] | None = field(default=None, repr=False)
+    warnings: str | None = None
+
+    # Schema
+    result_schema: list[FlowfileColumn] | None = field(default=None, repr=False)
+    predicted_schema: list[FlowfileColumn] | None = field(default=None, repr=False)
+
+    # Source tracking (for read nodes)
+    source_file_info: SourceFileInfo | None = None
+
+    # Hash for cache lookup
+    execution_hash: str | None = None
+
+    def reset(self) -> None:
+        """Reset to clean state for re-execution."""
+        self.has_run_with_current_setup = False
+        self.has_completed_last_run = False
+        self.is_canceled = False
+        self.error = None
+        self.resulting_data = None
+        self.example_data_path = None
+        self.example_data_generator = None
+        self.warnings = None
+        self.result_schema = None
+        self.predicted_schema = None
+        # Note: source_file_info intentionally NOT reset - needed for change detection
+        self.execution_hash = None
+
+    def reset_results_only(self) -> None:
+        """Reset just the results, keep tracking state."""
+        self.error = None
+        self.resulting_data = None
+        self.example_data_path = None
+        self.example_data_generator = None
+        self.warnings = None
+
+    def mark_successful(self) -> None:
+        """Mark execution as successful."""
+        self.has_run_with_current_setup = True
+        self.has_completed_last_run = True
+        self.error = None
+
+    def mark_failed(self, error: str) -> None:
+        """Mark execution as failed."""
+        self.has_run_with_current_setup = False
+        self.has_completed_last_run = False
+        self.error = error
+
+    def to_dict(self) -> dict:
+        """
+        Serialize for external storage (stateless mode).
+        Note: resulting_data and generators are not serialized.
+        """
+        return {
+            "has_run_with_current_setup": self.has_run_with_current_setup,
+            "has_completed_last_run": self.has_completed_last_run,
+            "is_canceled": self.is_canceled,
+            "error": self.error,
+            "example_data_path": self.example_data_path,
+            "warnings": self.warnings,
+            "execution_hash": self.execution_hash,
+            "source_file_info": self.source_file_info.to_dict() if self.source_file_info else None,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> NodeExecutionState:
+        """Deserialize from external storage."""
+        state = cls(
+            has_run_with_current_setup=data.get("has_run_with_current_setup", False),
+            has_completed_last_run=data.get("has_completed_last_run", False),
+            is_canceled=data.get("is_canceled", False),
+            error=data.get("error"),
+            example_data_path=data.get("example_data_path"),
+            warnings=data.get("warnings"),
+            execution_hash=data.get("execution_hash"),
+        )
+        if data.get("source_file_info"):
+            state.source_file_info = SourceFileInfo.from_dict(data["source_file_info"])
+        return state