Flowfile 0.5.6__py3-none-any.whl → 0.6.1__py3-none-any.whl

flowfile_core/schemas/transform_schema.py

@@ -184,7 +184,9 @@ class SelectInput(BaseModel):
             result["new_name"] = self.new_name
         if not self.keep:
             result["keep"] = self.keep
-        if self.data_type_change and self.data_type:
+        # Always include data_type if it's set, not just when data_type_change is True
+        # This ensures undo/redo snapshots preserve the data_type field
+        if self.data_type:
             result["data_type"] = self.data_type
         return result
 
@@ -193,22 +195,42 @@ class SelectInput(BaseModel):
         """Load from slim YAML format."""
         old_name = data["old_name"]
         new_name = data.get("new_name", old_name)
+        data_type = data.get("data_type")
+        # is_altered should be True if either name was changed OR data_type was explicitly set
+        # This ensures updateNodeSelect in the frontend won't overwrite user-specified data_type
+        is_altered = (old_name != new_name) or (data_type is not None)
         return cls(
             old_name=old_name,
             new_name=new_name,
             keep=data.get("keep", True),
-            data_type=data.get("data_type"),
-            data_type_change=data.get("data_type") is not None,
-            is_altered=old_name != new_name,
+            data_type=data_type,
+            data_type_change=data_type is not None,
+            is_altered=is_altered,
         )
 
+    @model_validator(mode="before")
+    @classmethod
+    def infer_data_type_change(cls, data):
+        """Infer data_type_change when loading from YAML.
+
+        When data_type is present but data_type_change is not explicitly set,
+        infer that the user explicitly set the data_type (e.g., when loading from YAML).
+        This ensures is_altered will be set correctly in the after validator.
+        """
+        if isinstance(data, dict):
+            if data.get("data_type") is not None and "data_type_change" not in data:
+                data["data_type_change"] = True
+        return data
+
     @model_validator(mode="after")
     def set_default_new_name(self):
-        """If new_name is None, default it to old_name."""
+        """If new_name is None, default it to old_name. Also set is_altered if needed."""
         if self.new_name is None:
             self.new_name = self.old_name
         if self.old_name != self.new_name:
             self.is_altered = True
+        if self.data_type_change:
+            self.is_altered = True
         return self
 
     def __hash__(self):

flowfile_core/schemas/yaml_types.py

@@ -23,6 +23,19 @@ class SelectInputYaml(TypedDict, total=False):
     data_type: str
 
 
+class OutputFieldInfoYaml(TypedDict, total=False):
+    name: str
+    data_type: str
+    default_value: str
+
+
+class OutputFieldConfigYaml(TypedDict, total=False):
+    enabled: bool
+    validation_mode_behavior: str  # "add_missing", "raise_on_missing", or "select_only"
+    fields: list[OutputFieldInfoYaml]
+    validate_data_types: bool  # Enable data type validation
+
+
 class JoinInputsYaml(TypedDict):
     select: list[SelectInputYaml]
 
@@ -75,14 +88,15 @@ class OutputSettingsYaml(TypedDict, total=False):
     table_settings: dict
 
 
-class NodeSelectYaml(TypedDict):
+class NodeSelectYaml(TypedDict, total=False):
     cache_results: bool
     keep_missing: bool
     select_input: list[SelectInputYaml]
     sorted_by: str
+    output_field_config: OutputFieldConfigYaml
 
 
-class NodeJoinYaml(TypedDict):
+class NodeJoinYaml(TypedDict, total=False):
     cache_results: bool
     auto_generate_selection: bool
     verify_integrity: bool
@@ -90,9 +104,10 @@ class NodeJoinYaml(TypedDict):
     auto_keep_all: bool
     auto_keep_right: bool
     auto_keep_left: bool
+    output_field_config: OutputFieldConfigYaml
 
 
-class NodeCrossJoinYaml(TypedDict):
+class NodeCrossJoinYaml(TypedDict, total=False):
     cache_results: bool
     auto_generate_selection: bool
     verify_integrity: bool
@@ -100,9 +115,10 @@ class NodeCrossJoinYaml(TypedDict):
     auto_keep_all: bool
     auto_keep_right: bool
     auto_keep_left: bool
+    output_field_config: OutputFieldConfigYaml
 
 
-class NodeFuzzyMatchYaml(TypedDict):
+class NodeFuzzyMatchYaml(TypedDict, total=False):
     cache_results: bool
     auto_generate_selection: bool
     verify_integrity: bool
@@ -110,8 +126,10 @@ class NodeFuzzyMatchYaml(TypedDict):
     auto_keep_all: bool
     auto_keep_right: bool
     auto_keep_left: bool
+    output_field_config: OutputFieldConfigYaml
 
 
-class NodeOutputYaml(TypedDict):
+class NodeOutputYaml(TypedDict, total=False):
     cache_results: bool
     output_settings: OutputSettingsYaml
+    output_field_config: OutputFieldConfigYaml

flowfile_frame/adding_expr.py

@@ -4,10 +4,9 @@ from typing import TypeVar
 
 import polars as pl
 
+from flowfile_frame.callable_utils import process_callable_args
 from flowfile_frame.config import logger
-from flowfile_frame.utils import _get_function_source
 
-T = TypeVar("T")
 ExprT = TypeVar("ExprT", bound="Expr")
 PASSTHROUGH_METHODS = {"map_elements", "map_batches"}
 
@@ -36,46 +35,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
         if self.expr is None:
             raise ValueError(f"Cannot call '{method_name}' on Expr with no underlying polars expression.")
 
-        # Collect function sources and build representations
-        function_sources = []
-        args_representations = []
-        kwargs_representations = []
-
-        # Process positional arguments
-        for arg in args:
-            if callable(arg) and not isinstance(arg, type):
-                # Try to get function source
-                try:
-                    source, is_module_level = _get_function_source(arg)
-                    if source and hasattr(arg, "__name__") and arg.__name__ != "<lambda>":
-                        function_sources.append(source)
-                        # Use the function name in the representation
-                        args_representations.append(arg.__name__)
-                    else:
-                        # Fallback to repr if we can't get the source
-                        args_representations.append(repr(arg))
-                except:
-                    args_representations.append(repr(arg))
-            else:
-                args_representations.append(repr(arg))
-
-        # Process keyword arguments
-        for key, value in kwargs.items():
-            if callable(value) and not isinstance(value, type):
-                # Try to get function source
-                try:
-                    source, is_module_level = _get_function_source(value)
-                    if source and hasattr(value, "__name__") and value.__name__ != "<lambda>":
-                        function_sources.append(source)
-                        # Use the function name in the representation
-                        kwargs_representations.append(f"{key}={value.__name__}")
-                    else:
-                        # Fallback to repr if we can't get the source
-                        kwargs_representations.append(f"{key}={repr(value)}")
-                except:
-                    kwargs_representations.append(f"{key}={repr(value)}")
-            else:
-                kwargs_representations.append(f"{key}={repr(value)}")
+        processed = process_callable_args(args, kwargs)
 
         # Call the method on the underlying polars expression
         try:
@@ -84,22 +44,6 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
             logger.debug(f"Warning: Error in {method_name}() call: {e}")
             result_expr = None
 
-        # Format arguments for repr string
-        args_repr = ", ".join(args_representations)
-        kwargs_repr = ", ".join(kwargs_representations)
-
-        if args_repr and kwargs_repr:
-            params_repr = f"{args_repr}, {kwargs_repr}"
-        elif args_repr:
-            params_repr = args_repr
-        elif kwargs_repr:
-            params_repr = kwargs_repr
-        else:
-            params_repr = ""
-
-        # Create the repr string for this method call
-        new_repr = f"{self._repr_str}.{method_name}({params_repr})"
-
         # Methods that typically change the aggregation status or complexity
         agg_methods = {
             "sum",
@@ -167,7 +111,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
             result_expr=result_expr,
             is_complex=is_complex,
             method_name=method_name,
-            _function_sources=function_sources,  # Pass function sources
+            _function_sources=processed.function_sources,
         )
 
         # Set the agg_func if needed
@@ -220,76 +164,23 @@ def add_expr_methods(cls: type[ExprT]) -> type[ExprT]:
                                 f"Cannot call '{method_name}' on Expr with no underlying polars expression."
                             )
 
-                        # Collect function sources and build representations
-                        function_sources = []
-                        args_representations = []
-                        kwargs_representations = []
-                        convertable_to_code = True
-
-                        # Process positional arguments
-                        for i, arg in enumerate(args):
-                            if callable(arg) and not isinstance(arg, type):
-                                # Try to get function source
-                                try:
-                                    source, is_module_level = _get_function_source(arg)
-                                    if source and hasattr(arg, "__name__") and arg.__name__ != "<lambda>":
-                                        function_sources.append(source)
-                                        # Use the function name in the representation
-                                        args_representations.append(arg.__name__)
-                                        arg.__repr__ = lambda: arg.__name__
-
-                                    else:
-                                        # Lambda or unnamed function - not convertible
-                                        logger.warning(
-                                            f"Warning: Using anonymous functions in {method_name} is not convertable to UI code"
-                                        )
-                                        logger.warning(
-                                            "Consider using defined functions (def abc(a, b, c): return ...), "
-                                            "In a separate script"
-                                        )
-                                        convertable_to_code = False
-                                        args_representations.append(repr(arg))
-                                except:
-                                    args_representations.append(repr(arg))
-                            else:
-                                args_representations.append(repr(arg))
-
-                        # Process keyword arguments
-                        for key, value in kwargs.items():
-                            if callable(value) and not isinstance(value, type):
-                                # Try to get function source
-                                try:
-                                    source, is_module_level = _get_function_source(value)
-                                    if source and hasattr(value, "__name__") and value.__name__ != "<lambda>":
-                                        function_sources.append(source)
-                                        # Use the function name in the representation
-                                        kwargs_representations.append(f"{key}={value.__name__}")
-                                    else:
-                                        # Lambda or unnamed function - not convertible
-                                        convertable_to_code = False
-                                        kwargs_representations.append(f"{key}={repr(value)}")
-                                except:
-                                    kwargs_representations.append(f"{key}={repr(value)}")
-                            else:
-                                kwargs_representations.append(f"{key}={repr(value)}")
+                        processed = process_callable_args(args, kwargs)
+                        convertable_to_code = processed.all_resolved
+
+                        if not convertable_to_code:
+                            logger.warning(
+                                f"Warning: Using anonymous functions in {method_name} is not convertable to UI code"
+                            )
+                            logger.warning(
+                                "Consider using defined functions (def abc(a, b, c): return ...), "
+                                "In a separate script"
+                            )
 
                         # Call the underlying polars method
                         result_expr = getattr(self.expr, method_name)(*args, **kwargs)
-                        # Build parameter string
-                        args_repr = ", ".join(args_representations)
-                        kwargs_repr = ", ".join(kwargs_representations)
-
-                        if args_repr and kwargs_repr:
-                            params_repr = f"{args_repr}, {kwargs_repr}"
-                        elif args_repr:
-                            params_repr = args_repr
-                        elif kwargs_repr:
-                            params_repr = kwargs_repr
-                        else:
-                            params_repr = ""
+
                         # Create a representation string
-                        new_repr = f"{self._repr_str}.{method_name}({params_repr})"
-                        # self._repr_str = new_repr
+                        new_repr = f"{self._repr_str}.{method_name}({processed.params_repr})"
                         # Return a new expression with the convertable_to_code flag set appropriately
                         result = self._create_next_expr(
                             *args,
@@ -297,7 +188,8 @@ def add_expr_methods(cls: type[ExprT]) -> type[ExprT]:
                             result_expr=result_expr,
                             is_complex=True,
                             convertable_to_code=convertable_to_code,
-                            _function_sources=function_sources,  # Pass function sources
+                            _function_sources=processed.function_sources,
+                            _repr_override=new_repr,
                             **kwargs,
                         )
                         return result

flowfile_frame/callable_utils.py

@@ -0,0 +1,261 @@
+"""
+Utilities for resolving callable source code (lambdas and named functions)
+and processing callable arguments for code generation.
+
+This module centralizes the logic for extracting function definitions so
+that FlowFrame graph nodes store human-readable code instead of serialized
+LazyFrame blobs.
+"""
+
+import ast
+import inspect
+import textwrap
+from dataclasses import dataclass, field
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# Low-level extraction helpers
+# ---------------------------------------------------------------------------
+
+
+def _get_function_source(func) -> tuple[str | None, bool]:
+    """
+    Get the source code of a named function if possible.
+
+    Returns:
+        tuple: (source_code, is_module_level)
+    """
+    try:
+        source = inspect.getsource(func)
+
+        if func.__name__ == "<lambda>":
+            return None, False
+
+        is_module_level = func.__code__.co_flags & 0x10 == 0
+        source = textwrap.dedent(source)
+        return source, is_module_level
+    except (OSError, TypeError):
+        return None, False
+
+
+def _is_safely_representable(value: Any) -> bool:
+    """Check if a value can be safely round-tripped through repr()."""
+    if isinstance(value, (int, float, bool, str, bytes, type(None))):
+        return True
+    if isinstance(value, (list, tuple)):
+        return all(_is_safely_representable(item) for item in value)
+    if isinstance(value, dict):
+        return all(
+            _is_safely_representable(k) and _is_safely_representable(v)
+            for k, v in value.items()
+        )
+    if isinstance(value, set):
+        return all(_is_safely_representable(item) for item in value)
+    return False
+
+
+def _extract_lambda_source(func) -> tuple[str | None, str | None]:
+    """
+    Extract a lambda's source code and convert it to a named function definition.
+
+    Uses inspect.getsource() + AST parsing to find the lambda's argument list
+    and body, then generates a named function definition.  Closure variables
+    are captured as constant assignments so the generated code is self-contained.
+
+    Returns:
+        (function_definition_source, function_name) or (None, None) on failure.
+    """
+    try:
+        source = inspect.getsource(func)
+    except (OSError, TypeError):
+        return None, None
+
+    source = textwrap.dedent(source).strip()
+
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        return None, None
+
+    lambdas = [node for node in ast.walk(tree) if isinstance(node, ast.Lambda)]
+    if not lambdas:
+        return None, None
+
+    # Match the lambda to our function based on argument names
+    expected_args = list(func.__code__.co_varnames[: func.__code__.co_argcount])
+    matched_lambda = None
+    for lambda_node in lambdas:
+        node_args = [arg.arg for arg in lambda_node.args.args]
+        if node_args == expected_args:
+            matched_lambda = lambda_node
+            break
+
+    if matched_lambda is None:
+        matched_lambda = lambdas[0]
+
+    func_name = f"_lambda_fn_{abs(hash(func.__code__)) % 100000}"
+
+    args_str = ast.unparse(matched_lambda.args)
+    body_str = ast.unparse(matched_lambda.body)
+
+    # Capture closure variables
+    closure_defs: list[str] = []
+    if func.__code__.co_freevars and func.__closure__:
+        for var_name, cell in zip(func.__code__.co_freevars, func.__closure__):
+            try:
+                value = cell.cell_contents
+            except ValueError:
+                return None, None
+
+            if _is_safely_representable(value):
+                closure_defs.append(f"{var_name} = {repr(value)}")
+            elif callable(value) and hasattr(value, "__name__") and value.__name__ != "<lambda>":
+                # Closure variable is a named function — extract its source
+                source, _ = _get_function_source(value)
+                if source:
+                    closure_defs.append(source)
+                else:
+                    return None, None
+            elif callable(value) and hasattr(value, "__name__") and value.__name__ == "<lambda>":
+                # Closure variable is itself a lambda — recurse
+                inner_def, inner_name = _extract_lambda_source(value)
+                if inner_def and inner_name:
+                    # Assign the generated function to the variable name used in the outer lambda
+                    closure_defs.append(inner_def)
+                    closure_defs.append(f"{var_name} = {inner_name}")
+                else:
+                    return None, None
+            else:
+                # Cannot safely serialize this closure variable
+                return None, None
+
+    lines: list[str] = []
+    if closure_defs:
+        lines.extend(closure_defs)
+        lines.append("")
+    lines.append(f"def {func_name}({args_str}):")
+    lines.append(f"    return {body_str}")
+
+    return "\n".join(lines), func_name
+
+
+# ---------------------------------------------------------------------------
+# High-level resolution
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ResolvedCallable:
+    """Result of resolving a single callable for code generation."""
+
+    source: str | None
+    """Function definition source code, or ``None`` if extraction failed."""
+
+    name: str
+    """Name to use in the generated code (function name or ``repr(func)``)."""
+
+    resolved: bool
+    """Whether source code was successfully extracted."""
+
+
+def resolve_callable(func: Any) -> ResolvedCallable:
+    """
+    Resolve a callable (lambda or named function) to its source code.
+
+    * For lambdas: attempts AST extraction via ``_extract_lambda_source``.
+    * For named functions: uses ``_get_function_source``.
+    * Falls back to ``repr(func)`` if extraction fails.
+    """
+    if hasattr(func, "__name__") and func.__name__ == "<lambda>":
+        func_def, func_name = _extract_lambda_source(func)
+        if func_def and func_name:
+            return ResolvedCallable(source=func_def, name=func_name, resolved=True)
+        return ResolvedCallable(source=None, name=repr(func), resolved=False)
+
+    if hasattr(func, "__name__"):
+        try:
+            source, _ = _get_function_source(func)
+        except Exception:
+            source = None
+        if source:
+            return ResolvedCallable(source=source, name=func.__name__, resolved=True)
+        # Named function but source unavailable (e.g. built-in) — still use its name
+        return ResolvedCallable(source=None, name=func.__name__, resolved=True)
+
+    return ResolvedCallable(source=None, name=repr(func), resolved=False)
+
+
+# ---------------------------------------------------------------------------
+# Batch argument processing
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ProcessedArgs:
+    """Result of processing a function's args and kwargs for code generation."""
+
+    args_reprs: list[str] = field(default_factory=list)
+    """String representations for each positional argument."""
+
+    kwargs_reprs: list[str] = field(default_factory=list)
+    """String representations for each keyword argument (``key=value``)."""
+
+    function_sources: list[str] = field(default_factory=list)
+    """Collected function definition source strings."""
+
+    all_resolved: bool = True
+    """False if any callable could not be resolved to source code."""
+
+    @property
+    def params_repr(self) -> str:
+        """Join args and kwargs into a single comma-separated parameter string."""
+        args_str = ", ".join(self.args_reprs)
+        kwargs_str = ", ".join(self.kwargs_reprs)
+        if args_str and kwargs_str:
+            return f"{args_str}, {kwargs_str}"
+        return args_str or kwargs_str
+
+
+def process_callable_args(args: tuple, kwargs: dict) -> ProcessedArgs:
+    """
+    Process positional and keyword arguments, resolving any callables to source code.
+
+    Non-callable arguments are converted via ``repr()``.
+
+    Returns a :class:`ProcessedArgs` collecting the string representations,
+    extracted function sources, and an ``all_resolved`` flag.
+    """
+    result = ProcessedArgs()
+
+    for arg in args:
+        if callable(arg) and not isinstance(arg, type):
+            try:
+                resolved = resolve_callable(arg)
+                result.args_reprs.append(resolved.name)
+                if resolved.source:
+                    result.function_sources.append(resolved.source)
+                if not resolved.resolved:
+                    result.all_resolved = False
+            except Exception:
+                result.args_reprs.append(repr(arg))
+                result.all_resolved = False
+        else:
+            result.args_reprs.append(repr(arg))
+
+    for key, value in kwargs.items():
+        if callable(value) and not isinstance(value, type):
+            try:
+                resolved = resolve_callable(value)
+                result.kwargs_reprs.append(f"{key}={resolved.name}")
+                if resolved.source:
+                    result.function_sources.append(resolved.source)
+                if not resolved.resolved:
+                    result.all_resolved = False
+            except Exception:
+                result.kwargs_reprs.append(f"{key}={repr(value)}")
+                result.all_resolved = False
+        else:
+            result.kwargs_reprs.append(f"{key}={repr(value)}")
+
+    return result

flowfile_frame/database/connection_manager.py

@@ -186,7 +186,6 @@ def del_database_connection(connection_name: str) -> bool:
     Returns:
         True if the connection was deleted, False if it didn't exist.
     """
-    from flowfile_core.database.models import DatabaseConnection as DBConnectionModel
     from flowfile_core.database.models import Secret
 
     user_id = get_current_user_id()

flowfile_frame/expr.py

@@ -485,13 +485,17 @@ class Expr:
         convertable_to_code: bool = None,
         is_complex: bool,
         _function_sources: list[str] | None = None,
+        _repr_override: str | None = None,
         **kwargs,
     ) -> Expr:
         """Creates a new Expr instance, appending method call to repr string."""
-        # Filter out _function_sources from kwargs to avoid passing it to _repr_args
-        filtered_kwargs = {k: v for k, v in kwargs.items() if k != "_function_sources"}
-        args_repr = _repr_args(*args, **filtered_kwargs)
-        new_repr = f"{self._repr_str}.{method_name}({args_repr})"
+        if _repr_override is not None:
+            new_repr = _repr_override
+        else:
+            # Filter out _function_sources from kwargs to avoid passing it to _repr_args
+            filtered_kwargs = {k: v for k, v in kwargs.items() if k != "_function_sources"}
+            args_repr = _repr_args(*args, **filtered_kwargs)
+            new_repr = f"{self._repr_str}.{method_name}({args_repr})"
 
         if convertable_to_code is None:
             convertable_to_code = self.convertable_to_code