Flowfile 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl

flowfile_core/flowfile/flow_node/models.py

@@ -108,14 +108,12 @@ class NodeStepSettings:
         streamable: If True, the node can process data in a streaming fashion.
         setup_errors: If True, indicates a non-blocking error occurred during setup.
         breaking_setup_errors: If True, indicates an error occurred that prevents execution.
-        execute_location: The preferred location for execution ('auto', 'local', 'remote').
     """
     cache_results: bool = False
     renew_schema: bool = True
     streamable: bool = True
     setup_errors: bool = False
     breaking_setup_errors: bool = False
-    execute_location: schemas.ExecutionLocationsLiteral = 'auto'
 
 
 class NodeStepInputs:

flowfile_core/flowfile/flow_node/schema_callback.py

@@ -1,71 +1,166 @@
-
 from typing import Callable, Any, Optional, Generic, TypeVar
 from concurrent.futures import ThreadPoolExecutor, Future
+import threading
 from flowfile_core.configs import logger
 
-
 T = TypeVar('T')
 
 
 class SingleExecutionFuture(Generic[T]):
-    """Single execution of a function in a separate thread with caching of the result."""
-    executor: ThreadPoolExecutor
-    future: Optional[Future[T]]
+    """Thread-safe single execution of a function with result caching.
+
+    Ensures a function is executed at most once even when called from multiple threads.
+    Subsequent calls return the cached result.
+    """
+
     func: Callable[[], T]
     on_error: Optional[Callable[[Exception], Any]]
-    result_value: Optional[T]
-    has_run_at_least_once: bool = False  # Indicates if the function has been run at least once
+    _lock: threading.RLock
+    _executor: Optional[ThreadPoolExecutor]
+    _future: Optional[Future[T]]
+    _result_value: Optional[T]
+    _exception: Optional[Exception]
+    _has_completed: bool
+    _has_started: bool
 
     def __init__(
-        self,
-        func: Callable[[], T],
-        on_error: Optional[Callable[[Exception], Any]] = None
+            self,
+            func: Callable[[], T],
+            on_error: Optional[Callable[[Exception], Any]] = None
     ) -> None:
         """Initialize with function and optional error handler."""
-        self.executor = ThreadPoolExecutor(max_workers=1)
-        self.future = None
         self.func = func
         self.on_error = on_error
-        self.result_value = None
-        self.has_run_at_least_once = False
+
+        # Thread safety
+        self._lock = threading.RLock()  # RLock allows re-entrant locking
+
+        # Execution state
+        self._executor = None
+        self._future = None
+        self._result_value = None
+        self._exception = None
+        self._has_completed = False
+        self._has_started = False
+
+    def _ensure_executor(self) -> ThreadPoolExecutor:
+        """Ensure executor exists, creating if necessary."""
+        if self._executor is None or self._executor._shutdown:
+            self._executor = ThreadPoolExecutor(max_workers=1)
+        return self._executor
 
     def start(self) -> None:
         """Start the function execution if not already started."""
-        if not self.future:
-            logger.info("single executor function started")
-            self.future = self.executor.submit(self.func)
+        with self._lock:
+            if self._has_started:
+                logger.info("Function already started or completed")
+                return
+
+            logger.info("Starting single executor function")
+            executor: ThreadPoolExecutor = self._ensure_executor()
+            self._future = executor.submit(self._func_wrapper)
+            self._has_started = True
+
+    def _func_wrapper(self) -> T:
+        """Wrapper to capture the result or exception."""
+        try:
+            result: T = self.func()
+            with self._lock:
+                self._result_value = result
+                self._has_completed = True
+            return result
+        except Exception as e:
+            with self._lock:
+                self._exception = e
+                self._has_completed = True
+            raise
 
     def cleanup(self) -> None:
-        """Clean up resources by clearing the future and shutting down the executor."""
-        self.has_run_at_least_once = True
-        self.executor.shutdown(wait=False)
+        """Clean up resources by shutting down the executor."""
+        with self._lock:
+            if self._executor and not self._executor._shutdown:
+                self._executor.shutdown(wait=False)
 
     def __call__(self) -> Optional[T]:
         """Execute function if not running and return its result."""
-        if self.result_value:
-            return self.result_value
-        if not self.future:
-            self.start()
-        else:
-            logger.info("Function already running or did complete")
-        try:
-            self.result_value = self.future.result()
-            logger.info("Done with the function")
-            return self.result_value
-        except Exception as e:
-            if self.on_error:
-                return self.on_error(e)
-            else:
-                raise e
-        finally:
-            self.cleanup()
+        with self._lock:
+            # If already completed, return cached result or raise cached exception
+            if self._has_completed:
+                if self._exception:
+                    if self.on_error:
+                        return self.on_error(self._exception)
+                    else:
+                        raise self._exception
+                return self._result_value
+
+            # Start if not already started
+            if not self._has_started:
+                self.start()
+
+        # Wait for completion outside the lock to avoid blocking other threads
+        if self._future:
+            try:
+                result: T = self._future.result()
+                logger.info("Function completed successfully")
+                return result
+            except Exception as e:
+                logger.error(f"Function raised exception: {e}")
+                if self.on_error:
+                    return self.on_error(e)
+                else:
+                    raise
+
+        return None
+
+    def reset(self) -> None:
+        """Reset the execution state, allowing the function to be run again."""
+        with self._lock:
+            logger.info("Resetting single execution future")
+
+            # Cancel any pending execution
+            if self._future and not self._future.done():
+                self._future.cancel()
 
-    def reset(self):
-        """Reset the future and result value."""
-        logger.info("Resetting the future and result value")
-        self.result_value = None
-        self.future = None
+            # Clean up old executor
+            if self._executor and not self._executor._shutdown:
+                self._executor.shutdown(wait=False)
+
+            # Reset state
+            self._executor = None
+            self._future = None
+            self._result_value = None
+            self._exception = None
+            self._has_completed = False
+            self._has_started = False
+
+    def is_running(self) -> bool:
+        """Check if the function is currently executing."""
+        with self._lock:
+            return bool(
+                self._has_started and
+                not self._has_completed and
+                self._future is not None and
+                not self._future.done()
+            )
+
+    def is_completed(self) -> bool:
+        """Check if the function has completed execution."""
+        with self._lock:
+            return self._has_completed
+
+    def get_result(self) -> Optional[T]:
+        """Get the cached result without triggering execution."""
+        with self._lock:
+            if self._exception:
+                if self.on_error:
+                    return self.on_error(self._exception)
+                else:
+                    raise self._exception
+            return self._result_value
 
     def __del__(self) -> None:
         """Ensure executor is shut down on deletion."""
-        self.cleanup()
+        try:
+            self.cleanup()
+        except Exception:
+            pass

flowfile_core/flowfile/graph_tree/graph_tree.py

@@ -0,0 +1,250 @@
+from pydantic import BaseModel
+
+from flowfile_core.flowfile.flow_node.flow_node import FlowNode
+
+from flowfile_core.flowfile.graph_tree.models import BranchInfo, InputInfo
+
+
+def calculate_depth(node_id: int, node_info: dict[int, BranchInfo], visited: set = None) -> int:
+    """Calculates the depth of each node."""
+
+    if visited is None:
+        visited = set()
+    if node_id in visited:
+        return node_info[node_id].depth
+    visited.add(node_id)
+    
+    max_input_depth = -1
+    inputs = node_info[node_id].inputs
+    
+    for main_id in inputs.main:
+        max_input_depth = max(max_input_depth, calculate_depth(main_id, node_info, visited))
+    if inputs.left:
+        max_input_depth = max(max_input_depth, calculate_depth(inputs.left, node_info, visited))
+    if inputs.right:
+        max_input_depth = max(max_input_depth, calculate_depth(inputs.right, node_info, visited))
+    
+    node_info[node_id].depth = max_input_depth + 1
+    return node_info[node_id].depth
+
+
+# Trace paths from each root
+def trace_path(node_id: int, node_info: dict[int, BranchInfo], merge_points: dict[int, list[int]],
+               current_path: list[int] | None = None):
+    """Define the trace of each node path"""
+    if current_path is None:
+        current_path = []
+    
+    current_path = current_path + [node_id]
+    outputs = node_info[node_id].outputs
+    
+    if not outputs:
+        # End of path
+        return [current_path]
+    
+    # If this node has multiple outputs or connects to a merge point, branch
+    all_paths = []
+    for output_id in outputs:
+        if output_id in merge_points and len(merge_points[output_id]) > 1:
+            # This is a merge point, end this path here
+            all_paths.append(current_path + [output_id])
+        else:
+            # Continue the path
+            all_paths.extend(trace_path(output_id, node_info, merge_points, current_path))
+    return all_paths
+
+
+def build_node_info(nodes: list[FlowNode]) -> dict[int, BranchInfo]:
+    """Builds node information used to construct the graph tree."""
+
+    node_info = {}
+    for node in nodes:
+        node_id = node.node_id
+
+        # Get node label
+        operation = node.node_type.replace("_", " ").title() if node.node_type else "Unknown"
+        label = f"{operation} (id={node_id})"
+        if hasattr(node, 'setting_input') and hasattr(node.setting_input, 'description'):
+            if node.setting_input.description:
+                desc = node.setting_input.description
+                if len(desc) > 20:  # Truncate long descriptions
+                    desc = desc[:17] + "..."
+                label = f"{operation} ({node_id}): {desc}"
+
+        # Get inputs and outputs
+        inputs = InputInfo(
+            main=[n.node_id for n in (node.node_inputs.main_inputs or [])],
+            left=node.node_inputs.left_input.node_id if node.node_inputs.left_input else None,
+            right=node.node_inputs.right_input.node_id if node.node_inputs.right_input else None
+        )
+        outputs = [n.node_id for n in node.leads_to_nodes]
+
+        node_info[node_id] = BranchInfo(
+            label=label,
+            short_label=f"{operation} ({node_id})",
+            inputs=inputs,
+            outputs=outputs,
+            depth=0
+        )
+    
+    return node_info
+
+
+def group_nodes_by_depth(node_info: dict[int, BranchInfo]) -> tuple[dict[int, list[int]], int]:
+    """Groups each node by depth"""
+    depth_groups = {}
+    max_depth = 0
+    for node_id, info in node_info.items():
+        depth = info.depth
+        max_depth = max(max_depth, depth)
+        if depth not in depth_groups:
+            depth_groups[depth] = []
+        depth_groups[depth].append(node_id)
+
+    return depth_groups, max_depth
+
+
+def define_node_connections(node_info: dict[int, BranchInfo]) -> dict[int, list[int]]:
+    """Defines node connections to merge"""
+    merge_points = {}  # target_id -> list of source_ids
+    for node_id, info in node_info.items():
+        for output_id in info.outputs:
+            if output_id not in merge_points:
+                merge_points[output_id] = []
+            merge_points[output_id].append(node_id)
+    
+    return merge_points
+
+
+def build_flow_paths(node_info: dict[int, BranchInfo], flow_starts: list[FlowNode],
+                     merge_points: dict[int, list[int]]):
+    """Build the flow paths to be drawn"""
+
+
+    # Find all root nodes (no inputs)
+    root_nodes = [nid for nid, info in node_info.items()
+                  if not info.inputs.main and not info.inputs.left and not info.inputs.right]
+    
+    if not root_nodes and flow_starts:
+        root_nodes = [n.node_id for n in flow_starts]
+    paths = []  # List of paths through the graph
+
+    # Get all paths
+    for root_id in root_nodes:
+        paths.extend(trace_path(root_id, node_info, merge_points))
+
+    return paths
+
+
+def group_paths(paths:list, merge_points:dict):
+    """Groups each node path."""
+    paths_by_merge = {}
+    standalone_paths = []
+
+    for path in paths:
+        if len(path) > 1 and path[-1] in merge_points and len(merge_points[path[-1]]) > 1:
+            merge_id = path[-1]
+            if merge_id not in paths_by_merge:
+                paths_by_merge[merge_id] = []
+            paths_by_merge[merge_id].append(path)
+        else:
+            standalone_paths.append(path)
+    return paths_by_merge, standalone_paths
+
+
+def draw_merged_paths(node_info: dict[int, BranchInfo],
+                      merge_points: dict[int, list[int]],
+                      paths_by_merge: dict[int, list[list[int]]],
+                      merge_drawn: set,
+                      drawn_nodes: set,
+                      lines: list[str]):
+    """Draws paths for each node that merges."""
+    for merge_id, merge_paths in paths_by_merge.items():
+        if merge_id in merge_drawn:
+            continue
+        merge_info = node_info[merge_id]
+        sources = merge_points[merge_id]
+
+        # Draw each source path leading to the merge
+        for i, source_id in enumerate(sources):
+            # Find the path containing this source
+            source_path = None
+            for path in merge_paths:
+                if source_id in path:
+
+                    source_path = path[:path.index(source_id) + 1]
+                    break
+
+            if source_path:
+                # Build the line for this path
+                line_parts = []
+                for j, nid in enumerate(source_path):
+                    if j == 0:
+                        line_parts.append(node_info[nid].label)
+                    else:
+                        line_parts.append(f" ──> {node_info[nid].short_label}")
+
+                # Add the merge arrow
+                if i == 0:
+                    # First source
+                    line = "".join(line_parts) + " ─────┐"
+                    lines.append(line)
+                elif i == len(sources) - 1:
+                    # Last source
+                    line = "".join(line_parts) + " ─────┴──> " + merge_info.label
+                    lines.append(line)
+
+                    # Continue with the rest of the path after merge
+                    remaining = node_info[merge_id].outputs
+                    while remaining:
+                        next_id = remaining[0]
+                        lines[-1] += f" ──> {node_info[next_id].label}"
+                        remaining = node_info[next_id].outputs
+                        drawn_nodes.add(next_id)
+                else:
+                    # Middle sources
+                    line = "".join(line_parts) + " ─────┤"
+                    lines.append(line)
+
+                for nid in source_path:
+                    drawn_nodes.add(nid)
+
+        drawn_nodes.add(merge_id)
+        merge_drawn.add(merge_id)
+        lines.append("")  # Add spacing between merge groups
+    return paths_by_merge
+
+
+def draw_standalone_paths(drawn_nodes: set[int], standalone_paths: list[list[int]], lines: list[str],
+                          node_info: dict[int, BranchInfo]):
+    """ Draws paths that do not merge."""
+    # Draw standalone paths
+    for path in standalone_paths:
+        if all(nid in drawn_nodes for nid in path):
+            continue
+        
+        line_parts = []
+        for i, node_id in enumerate(path):
+            if node_id not in drawn_nodes:
+                if i == 0:
+                    line_parts.append(node_info[node_id].label)
+                else:
+                    line_parts.append(f" ──> {node_info[node_id].short_label}")
+                drawn_nodes.add(node_id)
+        
+        if line_parts:
+            lines.append("".join(line_parts))
+
+
+def add_un_drawn_nodes(drawn_nodes: set[int], node_info: dict[int, BranchInfo],  lines: list[str]):
+    """Adds isolated nodes if exists."""
+    # Add any remaining undrawn nodes
+
+    for node_id in node_info:
+        if node_id not in drawn_nodes:
+            lines.append(node_info[node_id].label + " (isolated)")
+
+    lines.append("")
+    lines.append("=" * 80)
+    lines.append("Execution Order")
+    lines.append("=" * 80)

flowfile_core/flowfile/graph_tree/models.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel
+
+
+class InputInfo(BaseModel):
+    main: list[int]
+    right: int | None = None
+    left: int | None = None
+
+
+class BranchInfo(BaseModel):
+    label: str
+    short_label: str
+    inputs: InputInfo
+    outputs: list[int]
+    depth: int

flowfile_core/flowfile/handler.py

@@ -3,11 +3,25 @@ from dataclasses import dataclass
 from typing import Dict, List
 import os
 from pathlib import Path
+from datetime import datetime
 
 from flowfile_core.flowfile.manage.open_flowfile import open_flow
 from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.schemas.schemas import FlowSettings
 from flowfile_core.flowfile.utils import create_unique_id
+from shared.storage_config import storage
+
+
+def get_flow_save_location(flow_name: str) -> Path:
+    """Gets the initial save location for flow files"""
+    if ".flowfile" not in flow_name:
+        flow_name += ".flowfile"
+    return storage.temp_directory_for_flows / flow_name
+
+
+def create_flow_name() -> str:
+    """Creates a unique flow name"""
+    return datetime.now().strftime("%Y%m%d_%H_%M_%S")+"_flow.flowfile"
 
 
 @dataclass
@@ -57,7 +71,7 @@ class FlowfileHandler:
         else:
             raise Exception('Flow not found')
 
-    def add_flow(self, name: str, flow_path: str) -> int:
+    def add_flow(self, name: str = None, flow_path: str = None) -> int:
         """
         Creates a new flow with a reference to the flow path
         Args:
@@ -69,8 +83,13 @@ class FlowfileHandler:
 
         """
         next_id = create_unique_id()
-        flow_info = FlowSettings(name=name, flow_id=next_id, save_location='', path=flow_path)
-        _ = self.register_flow(flow_info)
+        if not name:
+            name = create_flow_name()
+        if not flow_path:
+            flow_path = get_flow_save_location(name)
+        flow_info = FlowSettings(name=name, flow_id=next_id, save_location=str(flow_path), path=str(flow_path))
+        flow = self.register_flow(flow_info)
+        flow.save_flow(flow.flow_settings.path)
         return next_id
 
     def get_flow_info(self, flow_id: int) -> FlowSettings:

flowfile_core/flowfile/manage/compatibility_enhancements.py

@@ -48,7 +48,7 @@ def ensure_compatibility(flow_storage_obj: schemas.FlowInformation, flow_path: s
         setattr(flow_storage_obj, 'flow_settings', flow_settings)
         flow_storage_obj = schemas.FlowInformation.model_validate(flow_storage_obj)
     elif not hasattr(getattr(flow_storage_obj, 'flow_settings'), 'execution_location'):
-        setattr(getattr(flow_storage_obj, 'flow_settings'), 'execution_location', 'auto')
+        setattr(getattr(flow_storage_obj, 'flow_settings'), 'execution_location', "remote")
     elif not hasattr(flow_storage_obj.flow_settings, 'is_running'):
         setattr(flow_storage_obj.flow_settings, 'is_running', False)
         setattr(flow_storage_obj.flow_settings, 'is_canceled', False)

flowfile_core/flowfile/{flow_data_engine/fuzzy_matching/settings_validator.py → schema_callbacks.py}

@@ -1,39 +1,90 @@
 
 from typing import List
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, PlType
-from flowfile_core.schemas import transform_schema
-from flowfile_core.schemas import input_schema
+
 from polars import datatypes
 import polars as pl
+
+from pl_fuzzy_frame_match.output_column_name_utils import set_name_in_fuzzy_mappings
+from pl_fuzzy_frame_match.pre_process import rename_fuzzy_right_mapping
+
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import fetch_unique_values
 from flowfile_core.configs.flow_logger import main_logger
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, PlType
+from flowfile_core.schemas import transform_schema
+from flowfile_core.schemas import input_schema
 
 
-def calculate_uniqueness(a: float, b: float) -> float:
-    return ((pow(a + 0.5, 2) + pow(b + 0.5, 2)) / 2 - pow(0.5, 2)) + 0.5 * abs(a - b)
+def _ensure_all_columns_have_select(left_cols: List[str],
+                                    right_cols: List[str],
+                                    fuzzy_match_input: transform_schema.FuzzyMatchInput):
+    """
+    Ensure that all columns in the left and right FlowDataEngines are included in the fuzzy match input's select
+     statements.
+    Args:
+        left_cols (List[str]): List of column names in the left FlowDataEngine.
+        right_cols (List[str]): List of column names in the right FlowDataEngine.
+        fuzzy_match_input (FuzzyMatchInput): Fuzzy match input configuration containing select statements.
+
+    Returns:
+        None
+    """
+    right_cols_in_select = {c.old_name for c in fuzzy_match_input.right_select.renames}
+    left_cols_in_select = {c.old_name for c in fuzzy_match_input.left_select.renames}
+
+    fuzzy_match_input.left_select.renames.extend(
+        [transform_schema.SelectInput(col) for col in left_cols if col not in left_cols_in_select])
+    fuzzy_match_input.right_select.renames.extend(
+        [transform_schema.SelectInput(col) for col in right_cols if col not in right_cols_in_select]
+    )
+
+
+def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: transform_schema.JoinInputs) -> None:
+    """
+    Ensure that the select columns in the fuzzy match input match the order of the incoming columns.
+    This function modifies the join_inputs object in-place.
+
+    Returns:
+        None
+    """
+    select_map = {select.new_name: select for select in join_inputs.renames}
+    ordered_renames = [select_map[col] for col in col_order if col in select_map]
+    join_inputs.renames = ordered_renames
 
 
 def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInput,
                                  left_schema: List[FlowfileColumn],
                                  right_schema: List[FlowfileColumn]):
-    print('calculating fuzzy match schema')
+    _ensure_all_columns_have_select(left_cols=[col.column_name for col in left_schema],
+                                    right_cols=[col.column_name for col in right_schema],
+                                    fuzzy_match_input=fm_input)
+    _order_join_inputs_based_on_col_order(col_order=[col.column_name for col in left_schema],
+                                          join_inputs=fm_input.left_select)
+    _order_join_inputs_based_on_col_order(col_order=[col.column_name for col in right_schema],
+                                          join_inputs=fm_input.right_select)
+    for column in fm_input.left_select.renames:
+        if column.join_key:
+            column.keep = True
+    for column in fm_input.right_select.renames:
+        if column.join_key:
+            column.keep = True
     left_schema_dict, right_schema_dict = ({ls.name: ls for ls in left_schema}, {rs.name: rs for rs in right_schema})
     fm_input.auto_rename()
-
+    right_renames = {column.old_name: column.new_name for column in fm_input.right_select.renames}
+    new_join_mapping = rename_fuzzy_right_mapping(fm_input.join_mapping, right_renames)
     output_schema = []
     for column in fm_input.left_select.renames:
         column_schema = left_schema_dict.get(column.old_name)
-        if column_schema and column.keep:
+        if column_schema and (column.keep or column.join_key):
             output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
                                                            example_values=column_schema.example_values))
     for column in fm_input.right_select.renames:
         column_schema = right_schema_dict.get(column.old_name)
-        if column_schema and column.keep:
+        if column_schema and (column.keep or column.join_key):
             output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
                                                            example_values=column_schema.example_values))
-
-    for i, fm in enumerate(fm_input.join_mapping):
-        output_schema.append(FlowfileColumn.from_input(f'fuzzy_score_{i}', 'Float64'))
+    set_name_in_fuzzy_mappings(new_join_mapping)
+    output_schema.extend([FlowfileColumn.from_input(fuzzy_mapping.output_column_name, 'Float64')
+                          for fuzzy_mapping in new_join_mapping])
     return output_schema
 
 
@@ -71,7 +122,8 @@ def pre_calculate_pivot_schema(node_input_schema: List[FlowfileColumn],
     val_column_schema = get_schema_of_column(node_input_schema, pivot_input.value_col)
     if output_fields is not None and len(output_fields) > 0:
         return index_columns_schema+[FlowfileColumn(PlType(Plcolumn_name=output_field.name,
-                                                           pl_datatype=output_field.data_type)) for output_field in output_fields]
+                                                           pl_datatype=output_field.data_type)) for output_field in
+                                     output_fields]
 
     else:
         max_unique_vals = 200
@@ -84,7 +136,11 @@ def pre_calculate_pivot_schema(node_input_schema: List[FlowfileColumn],
                                 f' Max unique values: {max_unique_vals}')
         pl_output_fields = []
         for val in unique_vals:
-            for agg in pivot_input.aggregations:
-                output_type = get_output_data_type_pivot(val_column_schema, agg)
-                pl_output_fields.append(PlType(column_name=f'{val}_{agg}', pl_datatype=output_type))
+            if len(pivot_input.aggregations) == 1:
+                output_type = get_output_data_type_pivot(val_column_schema, pivot_input.aggregations[0])
+                pl_output_fields.append(PlType(column_name=str(val), pl_datatype=output_type))
+            else:
+                for agg in pivot_input.aggregations:
+                    output_type = get_output_data_type_pivot(val_column_schema, agg)
+                    pl_output_fields.append(PlType(column_name=f'{val}_{agg}', pl_datatype=output_type))
         return index_columns_schema + [FlowfileColumn(pl_output_field) for pl_output_field in pl_output_fields]