Flowfile 0.5.3__py3-none-any.whl → 0.5.6__py3-none-any.whl

flowfile_core/fileExplorer/funcs.py

@@ -406,15 +406,17 @@ def validate_file_path(user_path: str, allowed_base: Path) -> Optional[Path]:
 def validate_path_under_cwd(user_path: str) -> str:
     """Validate that a user-provided path resolves to within allowed directories.
 
+    In Electron mode (desktop app), users can access any file on their local system.
+    In Docker/package mode, paths are restricted to:
+    - Current working directory (for development/testing)
+    - Flowfile storage directory (~/.flowfile)
+    - User data directory (home directory in local mode, /data/user in Docker)
+
     Uses the exact pattern from CodeQL documentation for py/path-injection:
     - os.path.normpath for path normalization
     - os.path.join to combine base with user input
     - startswith check to ensure path stays within base
 
-    Allowed directories:
-    - Current working directory (for development/testing)
-    - Flowfile storage directory (~/.flowfile)
-
     Args:
         user_path: The user-provided path string
 
@@ -424,6 +426,19 @@ def validate_path_under_cwd(user_path: str) -> str:
     Raises:
         HTTPException: 403 if path escapes the allowed directories
     """
+    from flowfile_core.configs.settings import is_electron_mode
+
+    # In Electron mode, allow access to any local file path
+    # This is safe because Electron runs locally on the user's machine
+    if is_electron_mode():
+        # Normalize and resolve the path
+        normalized_path = os.path.normpath(os.path.expanduser(user_path))
+        # Block path traversal patterns even in Electron mode
+        if '..' in user_path:
+            raise HTTPException(403, 'Access denied: path traversal not allowed')
+        return normalized_path
+
+    # In Docker/package mode, enforce strict sandboxing
     # Try current working directory first
     base_path = os.path.normpath(os.getcwd())
     fullpath = os.path.normpath(os.path.join(base_path, user_path))
@@ -436,6 +451,13 @@ def validate_path_under_cwd(user_path: str) -> str:
     if fullpath.startswith(base_path):
         return fullpath
 
+    # Try user data directory (consistent with SecureFileExplorer sandbox)
+    # In local mode this is the home directory, in Docker it's /data/user
+    base_path = os.path.normpath(str(storage.user_data_directory))
+    fullpath = os.path.normpath(os.path.join(base_path, user_path))
+    if fullpath.startswith(base_path):
+        return fullpath
+
     raise HTTPException(403, 'Access denied')
 
 

flowfile_core/flowfile/code_generator/__init__.py

@@ -0,0 +1,11 @@
+from flowfile_core.flowfile.code_generator.code_generator import (
+    FlowGraphToPolarsConverter,
+    UnsupportedNodeError,
+    export_flow_to_polars,
+)
+
+__all__ = [
+    "FlowGraphToPolarsConverter",
+    "UnsupportedNodeError",
+    "export_flow_to_polars",
+]

flowfile_core/flowfile/code_generator/code_generator.py

@@ -1,7 +1,10 @@
+import inspect
+import typing
 import polars as pl
 from pl_fuzzy_frame_match.models import FuzzyMapping
 
 from flowfile_core.configs import logger
+from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, convert_pl_type_to_string
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_graph import FlowGraph
@@ -10,6 +13,18 @@ from flowfile_core.flowfile.util.execution_orderer import determine_execution_or
 from flowfile_core.schemas import input_schema, transform_schema
 
 
+class UnsupportedNodeError(Exception):
+    """Raised when code generation encounters a node type that cannot be converted to standalone code."""
+
+    def __init__(self, node_type: str, node_id: int, reason: str):
+        self.node_type = node_type
+        self.node_id = node_id
+        self.reason = reason
+        super().__init__(
+            f"Cannot generate code for node '{node_type}' (node_id={node_id}): {reason}"
+        )
+
+
 class FlowGraphToPolarsConverter:
     """
     Converts a FlowGraph into executable Polars code.
@@ -24,6 +39,8 @@ class FlowGraphToPolarsConverter:
     code_lines: list[str]
     output_nodes: list[tuple[int, str]] = []
     last_node_var: str | None = None
+    unsupported_nodes: list[tuple[int, str, str]]  # List of (node_id, node_type, reason)
+    custom_node_classes: dict[str, str]  # Maps custom node class name to source code
 
     def __init__(self, flow_graph: FlowGraph):
         self.flow_graph = flow_graph
@@ -32,6 +49,8 @@ class FlowGraphToPolarsConverter:
         self.code_lines: list[str] = []
         self.output_nodes = []
         self.last_node_var = None
+        self.unsupported_nodes = []
+        self.custom_node_classes = {}
 
     def convert(self) -> str:
         """
@@ -39,6 +58,10 @@ class FlowGraphToPolarsConverter:
 
         Returns:
             str: Complete Python code that can be executed standalone
+
+        Raises:
+            UnsupportedNodeError: If the graph contains nodes that cannot be converted
+                to standalone code (e.g., database nodes, explore_data, external_source).
         """
         # Get execution order
         execution_order = determine_execution_order(
@@ -50,6 +73,20 @@ class FlowGraphToPolarsConverter:
         for node in execution_order:
             self._generate_node_code(node)
 
+        # Check for unsupported nodes and raise an error with all of them listed
+        if self.unsupported_nodes:
+            error_messages = []
+            for node_id, node_type, reason in self.unsupported_nodes:
+                error_messages.append(f"  - Node {node_id} ({node_type}): {reason}")
+            raise UnsupportedNodeError(
+                node_type=self.unsupported_nodes[0][1],
+                node_id=self.unsupported_nodes[0][0],
+                reason=(
+                    f"The flow contains {len(self.unsupported_nodes)} node(s) that cannot be converted to code:\n"
+                    + "\n".join(error_messages)
+                ),
+            )
+
         # Combine everything
         return self._build_final_code()
 
@@ -73,13 +110,25 @@ class FlowGraphToPolarsConverter:
             self.last_node_var = var_name
         # Get input variable names
         input_vars = self._get_input_vars(node)
+
+        # Check if this is a user-defined node
+        if isinstance(settings, input_schema.UserDefinedNode) or getattr(settings, "is_user_defined", False):
+            self._handle_user_defined(node, var_name, input_vars)
+            return
+
         # Route to appropriate handler based on node type
         handler = getattr(self, f"_handle_{node_type}", None)
         if handler:
             handler(settings, var_name, input_vars)
         else:
-            self._add_comment(f"# TODO: Implement handler for node type: {node_type}")
-            raise Exception(f"No handler implemented for node type: {node_type}")
+            # Unknown node type - add to unsupported list
+            self.unsupported_nodes.append((
+                node.node_id,
+                node_type,
+                f"No code generator implemented for node type '{node_type}'"
+            ))
+            self._add_comment(f"# WARNING: Cannot generate code for node type '{node_type}' (node_id={node.node_id})")
+            self._add_comment(f"# This node type is not supported for code export")
 
     def _get_input_vars(self, node: FlowNode) -> dict[str, str]:
         """Get input variable names for a node."""
@@ -1067,6 +1116,291 @@ class FlowGraphToPolarsConverter:
         self._add_code(f"{var_name} = _polars_code_{var_name.replace('df_', '')}({args})")
         self._add_code("")
 
+    # Handlers for unsupported node types - these add nodes to the unsupported list
+
+    def _handle_explore_data(
+        self, settings: input_schema.NodeExploreData, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle explore_data nodes - these are skipped as they are interactive visualization only."""
+        # explore_data is just for visualization in the UI, it doesn't transform data
+        # So we skip it in code generation but don't fail - just add a comment
+        input_df = input_vars.get("main", "df")
+        self._add_comment(f"# Node {settings.node_id}: Explore Data (skipped - interactive visualization only)")
+        self._add_code(f"{var_name} = {input_df}  # Pass through unchanged")
+        self._add_code("")
+
+    def _handle_database_reader(
+        self, settings: input_schema.NodeDatabaseReader, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle database_reader nodes by generating code to read from database using a named connection."""
+        db_settings = settings.database_settings
+
+        # Only reference mode is supported for code generation
+        if db_settings.connection_mode != "reference":
+            self.unsupported_nodes.append((
+                settings.node_id,
+                "database_reader",
+                "Database Reader nodes with inline connections cannot be exported. "
+                "Please use a named connection (reference mode) instead."
+            ))
+            self._add_comment(f"# Node {settings.node_id}: Database Reader - Inline connections not supported")
+            return
+
+        if not db_settings.database_connection_name:
+            self.unsupported_nodes.append((
+                settings.node_id,
+                "database_reader",
+                "Database Reader node is missing a connection name"
+            ))
+            return
+
+        self.imports.add("import flowfile as ff")
+
+        connection_name = db_settings.database_connection_name
+        self._add_code(f"# Read from database using connection: {connection_name}")
+
+        if db_settings.query_mode == "query" and db_settings.query:
+            # Query mode - use triple quotes to preserve query formatting
+            self._add_code(f'{var_name} = ff.read_database(')
+            self._add_code(f'    "{connection_name}",')
+            self._add_code(f'    query="""')
+            # Add each line of the query with proper indentation
+            for line in db_settings.query.split("\n"):
+                self._add_code(f"        {line}")
+            self._add_code('    """,')
+            self._add_code(")")
+        else:
+            # Table mode
+            self._add_code(f'{var_name} = ff.read_database(')
+            self._add_code(f'    "{connection_name}",')
+            if db_settings.table_name:
+                self._add_code(f'    table_name="{db_settings.table_name}",')
+            if db_settings.schema_name:
+                self._add_code(f'    schema_name="{db_settings.schema_name}",')
+            self._add_code(")")
+
+        self._add_code("")
+
+    def _handle_database_writer(
+        self, settings: input_schema.NodeDatabaseWriter, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle database_writer nodes by generating code to write to database using a named connection."""
+        db_settings = settings.database_write_settings
+
+        # Only reference mode is supported for code generation
+        if db_settings.connection_mode != "reference":
+            self.unsupported_nodes.append((
+                settings.node_id,
+                "database_writer",
+                "Database Writer nodes with inline connections cannot be exported. "
+                "Please use a named connection (reference mode) instead."
+            ))
+            self._add_comment(f"# Node {settings.node_id}: Database Writer - Inline connections not supported")
+            return
+
+        if not db_settings.database_connection_name:
+            self.unsupported_nodes.append((
+                settings.node_id,
+                "database_writer",
+                "Database Writer node is missing a connection name"
+            ))
+            return
+
+        self.imports.add("import flowfile as ff")
+
+        connection_name = db_settings.database_connection_name
+        input_df = input_vars.get("main", "df")
+
+        self._add_code(f"# Write to database using connection: {connection_name}")
+        self._add_code(f"ff.write_database(")
+        self._add_code(f"    {input_df}.collect(),")
+        self._add_code(f'    "{connection_name}",')
+        self._add_code(f'    "{db_settings.table_name}",')
+        if db_settings.schema_name:
+            self._add_code(f'    schema_name="{db_settings.schema_name}",')
+        if db_settings.if_exists:
+            self._add_code(f'    if_exists="{db_settings.if_exists}",')
+        self._add_code(")")
+        self._add_code(f"{var_name} = {input_df}  # Pass through the input DataFrame")
+        self._add_code("")
+
+    def _handle_external_source(
+        self, settings: input_schema.NodeExternalSource, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle external_source nodes - these are not supported for code generation."""
+        self.unsupported_nodes.append((
+            settings.node_id,
+            "external_source",
+            "External Source nodes use dynamic data sources that cannot be included in generated code"
+        ))
+        self._add_comment(f"# Node {settings.node_id}: External Source - Not supported for code export")
+        self._add_comment("# (External data sources require runtime configuration)")
+
+    def _check_process_method_signature(self, custom_node_class: type) -> tuple[bool, bool]:
+        """
+        Check the process method signature to determine if collect/lazy is needed.
+
+        Returns:
+            Tuple of (needs_collect, needs_lazy):
+            - needs_collect: True if inputs need to be collected to DataFrame before passing to process()
+            - needs_lazy: True if output needs to be converted to LazyFrame after process()
+        """
+        needs_collect = True  # Default: assume needs DataFrame input
+        needs_lazy = True  # Default: assume returns DataFrame
+
+        process_method = getattr(custom_node_class, 'process', None)
+        if process_method is None:
+            return needs_collect, needs_lazy
+
+        try:
+            # Try to get type hints from the process method
+            type_hints = typing.get_type_hints(process_method)
+
+            # Check return type
+            return_type = type_hints.get('return')
+            if return_type is not None:
+                return_type_str = str(return_type)
+                if 'LazyFrame' in return_type_str:
+                    needs_lazy = False
+
+            # Check input parameter types (look for *inputs parameter or first param after self)
+            sig = inspect.signature(process_method)
+            params = list(sig.parameters.values())
+            for param in params[1:]:  # Skip 'self'
+                if param.annotation != inspect.Parameter.empty:
+                    param_type_str = str(param.annotation)
+                    if 'LazyFrame' in param_type_str:
+                        needs_collect = False
+                        break
+                # Also check the type_hints dict for this param
+                if param.name in type_hints:
+                    hint_str = str(type_hints[param.name])
+                    if 'LazyFrame' in hint_str:
+                        needs_collect = False
+                        break
+        except Exception as e:
+            # If we can't determine types, use defaults (collect + lazy)
+            logger.debug(f"Could not determine process method signature: {e}")
+
+        return needs_collect, needs_lazy
+
+    def _read_custom_node_source_file(self, custom_node_class: type) -> str | None:
+        """
+        Read the entire source file where a custom node class is defined.
+        This includes all class definitions in that file (settings schemas, etc.).
+
+        Returns:
+            The complete source code from the file, or None if not readable.
+        """
+        try:
+            source_file = inspect.getfile(custom_node_class)
+            with open(source_file, 'r') as f:
+                return f.read()
+        except (OSError, TypeError):
+            return None
+
+    def _handle_user_defined(
+        self, node: FlowNode, var_name: str, input_vars: dict[str, str]
+    ) -> None:
+        """Handle user-defined custom nodes by including their class definition and calling process()."""
+        node_type = node.node_type
+        settings = node.setting_input
+
+        # Get the custom node class from the registry
+        custom_node_class = CUSTOM_NODE_STORE.get(node_type)
+        if custom_node_class is None:
+            self.unsupported_nodes.append((
+                node.node_id,
+                node_type,
+                f"User-defined node type '{node_type}' not found in the custom node registry"
+            ))
+            self._add_comment(f"# Node {node.node_id}: User-defined node '{node_type}' - Not found in registry")
+            return
+
+        # Store the entire source file if we haven't already
+        class_name = custom_node_class.__name__
+        if class_name not in self.custom_node_classes:
+            # Read the entire source file - it contains everything we need
+            file_source = self._read_custom_node_source_file(custom_node_class)
+            if file_source:
+                # Remove import lines from the file since we handle imports separately
+                lines = file_source.split('\n')
+                non_import_lines = []
+                in_multiline_import = False
+                for line in lines:
+                    stripped = line.strip()
+                    # Track multi-line imports (using parentheses)
+                    if stripped.startswith('import ') or stripped.startswith('from '):
+                        if '(' in stripped and ')' not in stripped:
+                            in_multiline_import = True
+                        continue
+                    if in_multiline_import:
+                        if ')' in stripped:
+                            in_multiline_import = False
+                        continue
+                    # Skip comments at the very start (like "# Auto-generated custom node")
+                    if stripped.startswith('#') and not non_import_lines:
+                        continue
+                    non_import_lines.append(line)
+                # Remove leading empty lines
+                while non_import_lines and not non_import_lines[0].strip():
+                    non_import_lines.pop(0)
+                self.custom_node_classes[class_name] = '\n'.join(non_import_lines)
+            else:
+                # Fallback to just the class source
+                try:
+                    self.custom_node_classes[class_name] = inspect.getsource(custom_node_class)
+                except (OSError, TypeError) as e:
+                    self.unsupported_nodes.append((
+                        node.node_id,
+                        node_type,
+                        f"Could not retrieve source code for user-defined node: {e}"
+                    ))
+                    self._add_comment(f"# Node {node.node_id}: User-defined node '{node_type}' - Source code unavailable")
+                    return
+
+            # Add necessary imports
+            self.imports.add("from flowfile_core.flowfile.node_designer import CustomNodeBase, Section, NodeSettings, SingleSelect, MultiSelect, IncomingColumns, ColumnSelector, NumericInput, TextInput, DropdownSelector, TextArea, Toggle")
+
+        # Get settings values to initialize the node
+        settings_dict = getattr(settings, "settings", {}) or {}
+
+        # Check process method signature to determine if collect/lazy is needed
+        needs_collect, needs_lazy = self._check_process_method_signature(custom_node_class)
+
+        # Generate the code to instantiate and run the custom node
+        self._add_code(f"# User-defined node: {custom_node_class.model_fields.get('node_name', type('', (), {'default': node_type})).default}")
+        self._add_code(f"_custom_node_{node.node_id} = {class_name}()")
+
+        # If there are settings, apply them
+        if settings_dict:
+            self._add_code(f"_custom_node_{node.node_id}_settings = {repr(settings_dict)}")
+            self._add_code(f"if _custom_node_{node.node_id}.settings_schema:")
+            self._add_code(f"    _custom_node_{node.node_id}.settings_schema.populate_values(_custom_node_{node.node_id}_settings)")
+
+        # Prepare input arguments based on whether we need to collect
+        if len(input_vars) == 0:
+            input_args = ""
+        elif len(input_vars) == 1:
+            input_df = list(input_vars.values())[0]
+            input_args = f"{input_df}.collect()" if needs_collect else input_df
+        else:
+            arg_list = []
+            for key in sorted(input_vars.keys()):
+                if key.startswith("main"):
+                    if needs_collect:
+                        arg_list.append(f"{input_vars[key]}.collect()")
+                    else:
+                        arg_list.append(input_vars[key])
+            input_args = ", ".join(arg_list)
+
+        # Call the process method, adding .lazy() only if needed
+        if needs_lazy:
+            self._add_code(f"{var_name} = _custom_node_{node.node_id}.process({input_args}).lazy()")
+        else:
+            self._add_code(f"{var_name} = _custom_node_{node.node_id}.process({input_args})")
+        self._add_code("")
+
     # Helper methods
 
     def _add_code(self, line: str) -> None:
@@ -1286,6 +1620,17 @@ class FlowGraphToPolarsConverter:
         lines.append("")
         lines.append("")
 
+        # Add custom node class definitions if any
+        if self.custom_node_classes:
+            lines.append("# Custom Node Class Definitions")
+            lines.append("# These classes are user-defined nodes that were included in the flow")
+            lines.append("")
+            for class_name, source_code in self.custom_node_classes.items():
+                for source_line in source_code.split("\n"):
+                    lines.append(source_line)
+                lines.append("")
+            lines.append("")
+
         # Add main function
         lines.append("def run_etl_pipeline():")
         lines.append('    """')

flowfile_core/flowfile/flow_data_engine/flow_data_engine.py

@@ -991,8 +991,20 @@ class FlowDataEngine:
 
         df = self.data_frame.rename({c.old_name: c.new_name for c in group_columns})
         group_by_columns = [n_c.new_name for n_c in group_columns]
+
+        # Handle case where there are no aggregations - just get unique combinations of group columns
+        if len(aggregations) == 0:
+            return FlowDataEngine(
+                df.select(group_by_columns).unique(),
+                calculate_schema_stats=calculate_schema_stats,
+            )
+
+        grouped_df = df.group_by(*group_by_columns)
+        agg_exprs = [ac.agg_func(ac.old_name).alias(ac.new_name) for ac in aggregations]
+        result_df = grouped_df.agg(agg_exprs)
+
         return FlowDataEngine(
-            df.group_by(*group_by_columns).agg(ac.agg_func(ac.old_name).alias(ac.new_name) for ac in aggregations),
+            result_df,
             calculate_schema_stats=calculate_schema_stats,
         )
 

flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py

@@ -137,6 +137,12 @@ def get_results(file_ref: str) -> Status | None:
 
 
 def results_exists(file_ref: str):
+    from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
+
+    # Skip worker check if worker communication is disabled
+    if not OFFLOAD_TO_WORKER:
+        return False
+
     try:
         f = requests.get(f"{WORKER_URL}/status/{file_ref}")
         if f.status_code == 200:
@@ -159,6 +165,12 @@ def clear_task_from_worker(file_ref: str) -> bool:
     Returns:
         bool: True if the task was successfully cleared, False otherwise.
     """
+    from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
+
+    # Skip worker call if worker communication is disabled
+    if not OFFLOAD_TO_WORKER:
+        return False
+
     try:
         f = requests.delete(f"{WORKER_URL}/clear_task/{file_ref}")
         if f.status_code == 200:

flowfile_core/flowfile/flow_graph.py

@@ -1644,6 +1644,7 @@ class FlowGraph:
                     write_settings=node_cloud_storage_writer.cloud_storage_settings,
                     connection=full_cloud_storage_connection,
                     lf=df.data_frame,
+                    user_id=node_cloud_storage_writer.user_id,
                     flowfile_node_id=node_cloud_storage_writer.node_id,
                     flowfile_flow_id=self.flow_id,
                 )
@@ -1866,6 +1867,7 @@ class FlowGraph:
 
         if schema_callback is not None:
             node.schema_callback = schema_callback
+            node.user_provided_schema_callback = schema_callback
         return self
 
     def add_datasource(self, input_file: input_schema.NodeDatasource | input_schema.NodeManualInput) -> "FlowGraph":