Flowfile 0.3.9__py3-none-any.whl → 0.5.1__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/flow_data_engine.py

@@ -67,7 +67,7 @@ from flowfile_core.flowfile.sources.external_sources.base_class import ExternalD
 T = TypeVar('T', pl.DataFrame, pl.LazyFrame)
 
 
-def _handle_duplication_join_keys(left_df: T, right_df: T, join_input: transform_schemas.JoinInput) -> Tuple[T, T, Dict[str, str]]:
+def _handle_duplication_join_keys(left_df: T, right_df: T, join_manager: transform_schemas.JoinInputManager) -> Tuple[T, T, Dict[str, str]]:
     """Temporarily renames join keys to avoid conflicts during a join.
 
     This helper function checks the join type and renames the join key columns
@@ -86,20 +86,22 @@ def _handle_duplication_join_keys(left_df: T, right_df: T, join_input: transform
         - The (potentially modified) right DataFrame.
         - A dictionary mapping the temporary names back to their desired final names.
     """
+
     def _construct_temp_name(column_name: str) -> str:
         return "__FL_TEMP__"+column_name
-    if join_input.how == 'right':
+
+    if join_manager.how == 'right':
         left_df = left_df.with_columns(pl.col(jk.new_name).alias(_construct_temp_name(jk.new_name))
-                                       for jk in join_input.left_select.join_key_selects)
+                                       for jk in join_manager.left_manager.get_join_key_selects())
         reverse_actions = {
             _construct_temp_name(jk.new_name): transform_schemas.construct_join_key_name("left", jk.new_name)
-            for jk in join_input.left_select.join_key_selects}
-    elif join_input.how in ('left', 'inner'):
+            for jk in join_manager.left_manager.get_join_key_selects()}
+    elif join_manager.how in ('left', 'inner'):
         right_df = right_df.with_columns(pl.col(jk.new_name).alias(_construct_temp_name(jk.new_name))
-                                       for jk in join_input.right_select.join_key_selects)
+                                         for jk in join_manager.right_manager.get_join_key_selects())
         reverse_actions = {
             _construct_temp_name(jk.new_name): transform_schemas.construct_join_key_name("right", jk.new_name)
-            for jk in join_input.right_select.join_key_selects}
+            for jk in join_manager.right_manager.get_join_key_selects()}
     else:
         reverse_actions = {}
     return left_df, right_df, reverse_actions
@@ -193,7 +195,6 @@ class FlowDataEngine:
     _number_of_records_callback: Callable = None
     _data_callback: Callable = None
 
-
     def __init__(self,
                  raw_data: Union[List[Dict], List[Any], Dict[str, Any], 'ParquetFile', pl.DataFrame, pl.LazyFrame, input_schema.RawData] = None,
                  path_ref: str = None,
@@ -1147,7 +1148,7 @@ class FlowDataEngine:
         return cls(df, schema=schema, calculate_schema_stats=False, number_of_records=0)
 
     @classmethod
-    def create_from_path(cls, received_table: input_schema.ReceivedTableBase) -> "FlowDataEngine":
+    def create_from_path(cls, received_table: input_schema.ReceivedTable) -> "FlowDataEngine":
         """Creates a FlowDataEngine from a local file path.
 
         Supports various file types like CSV, Parquet, and Excel.
@@ -1579,7 +1580,6 @@ class FlowDataEngine:
             A new `FlowDataEngine` instance containing the sampled data.
         """
         logging.info(f'Getting sample of {n_rows} rows')
-
         if random:
             if self.lazy and self.external_source is not None:
                 self.collect_external()
@@ -1657,9 +1657,12 @@ class FlowDataEngine:
             An `ExternalFuzzyMatchFetcher` object that can be used to track the
             progress and retrieve the result of the fuzzy join.
         """
-        left_df, right_df = prepare_for_fuzzy_match(left=self, right=other, fuzzy_match_input=fuzzy_match_input)
+        fuzzy_match_input_manager = transform_schemas.FuzzyMatchInputManager(fuzzy_match_input)
+        left_df, right_df = prepare_for_fuzzy_match(left=self, right=other,
+                                                    fuzzy_match_input_manager=fuzzy_match_input_manager)
+
         return ExternalFuzzyMatchFetcher(left_df, right_df,
-                                         fuzzy_maps=fuzzy_match_input.fuzzy_maps,
+                                         fuzzy_maps=fuzzy_match_input_manager.fuzzy_maps,
                                          file_ref=file_ref + '_fm',
                                          wait_on_completion=False,
                                          flow_id=flow_id,
@@ -1674,10 +1677,12 @@ class FlowDataEngine:
                             ):
         if file_ref is None:
             file_ref = str(id(self)) + '_' + str(id(other))
+        fuzzy_match_input_manager = transform_schemas.FuzzyMatchInputManager(fuzzy_match_input)
 
-        left_df, right_df = prepare_for_fuzzy_match(left=self, right=other, fuzzy_match_input=fuzzy_match_input)
+        left_df, right_df = prepare_for_fuzzy_match(left=self, right=other,
+                                                    fuzzy_match_input_manager=fuzzy_match_input_manager)
         external_tracker = ExternalFuzzyMatchFetcher(left_df, right_df,
-                                                     fuzzy_maps=fuzzy_match_input.fuzzy_maps,
+                                                     fuzzy_maps=fuzzy_match_input_manager.fuzzy_maps,
                                                      file_ref=file_ref + '_fm',
                                                      wait_on_completion=False,
                                                      flow_id=flow_id,
@@ -1687,8 +1692,10 @@ class FlowDataEngine:
     def fuzzy_join(self, fuzzy_match_input: transform_schemas.FuzzyMatchInput,
                    other: "FlowDataEngine",
                    node_logger: NodeLogger = None) -> "FlowDataEngine":
-        left_df, right_df = prepare_for_fuzzy_match(left=self, right=other, fuzzy_match_input=fuzzy_match_input)
-        fuzzy_mappings = [FuzzyMapping(**fm.__dict__) for fm in fuzzy_match_input.fuzzy_maps]
+        fuzzy_match_input_manager = transform_schemas.FuzzyMatchInputManager(fuzzy_match_input)
+        left_df, right_df = prepare_for_fuzzy_match(left=self, right=other,
+                                                    fuzzy_match_input_manager=fuzzy_match_input_manager)
+        fuzzy_mappings = [FuzzyMapping(**fm.__dict__) for fm in fuzzy_match_input_manager.fuzzy_maps]
         return FlowDataEngine(fuzzy_match_dfs(left_df, right_df, fuzzy_maps=fuzzy_mappings,
                                               logger=node_logger.logger if node_logger else logger)
                               .lazy())
@@ -1713,24 +1720,22 @@ class FlowDataEngine:
             Exception: If `verify_integrity` is True and the join would result in
                 an excessively large number of records.
         """
-
         self.lazy = True
-
         other.lazy = True
-
-        verify_join_select_integrity(cross_join_input, left_columns=self.columns, right_columns=other.columns)
-        right_select = [v.old_name for v in cross_join_input.right_select.renames
+        cross_join_input_manager = transform_schemas.CrossJoinInputManager(cross_join_input)
+        verify_join_select_integrity(cross_join_input_manager.input, left_columns=self.columns, right_columns=other.columns)
+        right_select = [v.old_name for v in cross_join_input_manager.right_select.renames
                         if (v.keep or v.join_key) and v.is_available]
-        left_select = [v.old_name for v in cross_join_input.left_select.renames
+        left_select = [v.old_name for v in cross_join_input_manager.left_select.renames
                        if (v.keep or v.join_key) and v.is_available]
-
-        left = self.data_frame.select(left_select).rename(cross_join_input.left_select.rename_table)
-        right = other.data_frame.select(right_select).rename(cross_join_input.right_select.rename_table)
+        cross_join_input_manager.auto_rename(rename_mode="suffix")
+        left = self.data_frame.select(left_select).rename(cross_join_input_manager.left_select.rename_table)
+        right = other.data_frame.select(right_select).rename(cross_join_input_manager.right_select.rename_table)
 
         joined_df = left.join(right, how='cross')
 
         cols_to_delete_after = [col.new_name for col in
-                                cross_join_input.left_select.renames + cross_join_input.left_select.renames
+                                cross_join_input_manager.left_select.renames + cross_join_input_manager.left_select.renames
                                 if col.join_key and not col.keep and col.is_available]
 
         fl = FlowDataEngine(joined_df.drop(cols_to_delete_after), calculate_schema_stats=False, streamable=False)
@@ -1738,76 +1743,60 @@ class FlowDataEngine:
 
     def join(self, join_input: transform_schemas.JoinInput, auto_generate_selection: bool,
              verify_integrity: bool, other: "FlowDataEngine") -> "FlowDataEngine":
-        """Performs a standard SQL-style join with another DataFrame.
+        """Performs a standard SQL-style join with another DataFrame."""
+        # Create manager from input
+        join_manager = transform_schemas.JoinInputManager(join_input)
+        ensure_right_unselect_for_semi_and_anti_joins(join_manager.input)
+        for jk in join_manager.join_mapping:
+            if jk.left_col not in {c.old_name for c in join_manager.left_select.renames}:
+                join_manager.left_select.append(transform_schemas.SelectInput(jk.left_col, keep=False))
+            if jk.right_col not in {c.old_name for c in join_manager.right_select.renames}:
+                join_manager.right_select.append(transform_schemas.SelectInput(jk.right_col, keep=False))
+        verify_join_select_integrity(join_manager.input, left_columns=self.columns, right_columns=other.columns)
+        if not verify_join_map_integrity(join_manager.input, left_columns=self.schema, right_columns=other.schema):
+            raise Exception('Join is not valid by the data fields')
 
-        Supports various join types like 'inner', 'left', 'right', 'outer', 'semi', and 'anti'.
+        if auto_generate_selection:
+            join_manager.auto_rename()
 
-        Args:
-            join_input: A `JoinInput` object defining the join keys, join type,
-                and column selections.
-            auto_generate_selection: If True, automatically handles column renaming.
-            verify_integrity: If True, performs checks to prevent excessively large joins.
-            other: The right `FlowDataEngine` to join with.
+        # Use manager properties throughout
+        left = self.data_frame.select(join_manager.left_manager.get_select_cols()).rename(join_manager.left_manager.get_rename_table())
+        right = other.data_frame.select(join_manager.right_manager.get_select_cols()).rename(join_manager.right_manager.get_rename_table())
 
-        Returns:
-            A new `FlowDataEngine` with the joined data.
-
-        Raises:
-            Exception: If the join configuration is invalid or if `verify_integrity`
-                is True and the join is predicted to be too large.
-        """
-        ensure_right_unselect_for_semi_and_anti_joins(join_input)
-        verify_join_select_integrity(join_input, left_columns=self.columns, right_columns=other.columns)
-        if not verify_join_map_integrity(join_input, left_columns=self.schema, right_columns=other.schema):
-            raise Exception('Join is not valid by the data fields')
-        if auto_generate_selection:
-            join_input.auto_rename()
-        left = self.data_frame.select(get_select_columns(join_input.left_select.renames)).rename(join_input.left_select.rename_table)
-        right = other.data_frame.select(get_select_columns(join_input.right_select.renames)).rename(join_input.right_select.rename_table)
-        if verify_integrity and join_input.how != 'right':
-            n_records = get_join_count(left, right, left_on_keys=join_input.left_join_keys,
-                                       right_on_keys=join_input.right_join_keys, how=join_input.how)
-            if n_records > 1_000_000_000:
-                raise Exception("Join will result in too many records, ending process")
-        else:
-            n_records = -1
-        left, right, reverse_join_key_mapping = _handle_duplication_join_keys(left, right, join_input)
-        left, right = rename_df_table_for_join(left, right, join_input.get_join_key_renames())
-        if join_input.how == 'right':
+        left, right, reverse_join_key_mapping = _handle_duplication_join_keys(left, right, join_manager)
+        left, right = rename_df_table_for_join(left, right, join_manager.get_join_key_renames())
+        if join_manager.how == 'right':
             joined_df = right.join(
                 other=left,
-                left_on=join_input.right_join_keys,
-                right_on=join_input.left_join_keys,
+                left_on=join_manager.right_join_keys,
+                right_on=join_manager.left_join_keys,
                 how="left",
                 suffix="").rename(reverse_join_key_mapping)
         else:
             joined_df = left.join(
                 other=right,
-                left_on=join_input.left_join_keys,
-                right_on=join_input.right_join_keys,
-                how=join_input.how,
+                left_on=join_manager.left_join_keys,
+                right_on=join_manager.right_join_keys,
+                how=join_manager.how,
                 suffix="").rename(reverse_join_key_mapping)
-        left_cols_to_delete_after = [get_col_name_to_delete(col, 'left') for col in join_input.left_select.renames
-                                     if not col.keep
-                                     and col.is_available and col.join_key
-                                     ]
-        right_cols_to_delete_after = [get_col_name_to_delete(col, 'right') for col in join_input.right_select.renames
-                                      if not col.keep
-                                      and col.is_available and col.join_key
-                                      and join_input.how in ("left", "right", "inner", "cross", "outer")
-                                      ]
+
+        left_cols_to_delete_after = [get_col_name_to_delete(col, 'left')
+                                     for col in join_manager.input.left_select.renames
+                                     if not col.keep and col.is_available and col.join_key]
+
+        right_cols_to_delete_after = [get_col_name_to_delete(col, 'right')
+                                      for col in join_manager.input.right_select.renames
+                                      if not col.keep and col.is_available and col.join_key
+                                      and join_manager.how in ("left", "right", "inner", "cross", "outer")]
+
         if len(right_cols_to_delete_after + left_cols_to_delete_after) > 0:
             joined_df = joined_df.drop(left_cols_to_delete_after + right_cols_to_delete_after)
-        undo_join_key_remapping = get_undo_rename_mapping_join(join_input)
+
+        undo_join_key_remapping = get_undo_rename_mapping_join(join_manager)
         joined_df = joined_df.rename(undo_join_key_remapping)
 
-        if verify_integrity:
-            return FlowDataEngine(joined_df, calculate_schema_stats=True,
-                                  number_of_records=n_records, streamable=False)
-        else:
-            fl = FlowDataEngine(joined_df, calculate_schema_stats=False,
-                                number_of_records=0, streamable=False)
-            return fl
+        return FlowDataEngine(joined_df, calculate_schema_stats=False,
+                              number_of_records=0, streamable=False)
 
     def solve_graph(self, graph_solver_input: transform_schemas.GraphSolverInput) -> "FlowDataEngine":
         """Solves a graph problem represented by 'from' and 'to' columns.
@@ -2105,7 +2094,7 @@ class FlowDataEngine:
             A new `FlowDataEngine` instance with the applied formula.
         """
         expr = to_expr(func)
-        if output_data_type not in (None, "Auto"):
+        if output_data_type not in (None, transform_schemas.AUTO_DATA_TYPE):
             df = self.data_frame.with_columns(expr.cast(output_data_type).alias(col_name))
         else:
             df = self.data_frame.with_columns(expr.alias(col_name))
@@ -2134,8 +2123,8 @@ class FlowDataEngine:
                 data_type=output_fs.file_type,
                 path=output_fs.abs_file_path,
                 write_mode=output_fs.write_mode,
-                sheet_name=output_fs.output_excel_table.sheet_name,
-                delimiter=output_fs.output_csv_table.delimiter,
+                sheet_name=output_fs.sheet_name,
+                delimiter=output_fs.delimiter,
                 flow_id=flow_id,
                 node_id=node_id
             )
@@ -2149,8 +2138,8 @@ class FlowDataEngine:
                 data_type=output_fs.file_type,
                 path=output_fs.abs_file_path,
                 write_mode=output_fs.write_mode,
-                sheet_name=output_fs.output_excel_table.sheet_name,
-                delimiter=output_fs.output_csv_table.delimiter,
+                sheet_name=output_fs.sheet_name,
+                delimiter=output_fs.delimiter,
                 flow_id=flow_id,
                 node_id=node_id,
             )
@@ -2239,6 +2228,7 @@ class FlowDataEngine:
     def _calculate_schema(self) -> List[Dict]:
         """Calculates schema statistics."""
         if self.external_source is not None:
+
             self.collect_external()
         v = utils.calculate_schema(self.data_frame)
         return v
@@ -2256,6 +2246,7 @@ class FlowDataEngine:
     def create_from_path_worker(cls, received_table: input_schema.ReceivedTable, flow_id: int, node_id: int | str):
         """Creates a FlowDataEngine from a path in a worker process."""
         received_table.set_absolute_filepath()
+
         external_fetcher = ExternalCreateFetcher(received_table=received_table,
                                                  file_type=received_table.file_type, flow_id=flow_id, node_id=node_id)
         return cls(external_fetcher.get_result())

flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py

@@ -0,0 +1,4 @@
+from typing import Literal
+
+DataTypeGroup = Literal['numeric', 'str', 'date']
+ReadableDataTypeGroup = Literal['Numeric', 'String', 'Date', 'Other', 'Boolean', 'Binary', 'Complex']

flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py

@@ -1,44 +1,13 @@
 
 from dataclasses import dataclass
-from typing import Optional, Any, List, Dict, Literal, Iterable
+from typing import Optional, Any, List, Dict, Iterable
 
 from flowfile_core.schemas import input_schema
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.interface import ReadableDataTypeGroup, DataTypeGroup
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.type_registry import convert_pl_type_to_string
 import polars as pl
-# TODO: rename flow_file_column to flowfile_column
-DataTypeGroup = Literal['numeric', 'str', 'date']
-
-
-def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
-    if isinstance(pl_type, pl.List):
-        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
-        return f"pl.List({inner_str})"
-    elif isinstance(pl_type, pl.Array):
-        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
-        return f"pl.Array({inner_str})"
-    elif isinstance(pl_type, pl.Decimal):
-        precision = pl_type.precision if hasattr(pl_type, 'precision') else None
-        scale = pl_type.scale if hasattr(pl_type, 'scale') else None
-        if precision is not None and scale is not None:
-            return f"pl.Decimal({precision}, {scale})"
-        elif precision is not None:
-            return f"pl.Decimal({precision})"
-        else:
-            return "pl.Decimal()"
-    elif isinstance(pl_type, pl.Struct):
-        # Handle Struct with field definitions
-        fields = []
-        if hasattr(pl_type, 'fields'):
-            for field in pl_type.fields:
-                field_name = field.name
-                field_type = convert_pl_type_to_string(field.dtype, inner=True)
-                fields.append(f'pl.Field("{field_name}", {field_type})')
-        field_str = ", ".join(fields)
-        return f"pl.Struct([{field_str}])"
-    else:
-        # For base types, we want the full pl.TypeName format
-        return str(pl_type.base_type()) if not inner else f"pl.{pl_type}"
 
 
 @dataclass
@@ -52,6 +21,7 @@ class FlowfileColumn:
     number_of_empty_values: int
     number_of_unique_values: int
     example_values: str
+    data_type_group: ReadableDataTypeGroup
     __sql_type: Optional[Any]
     __is_unique: Optional[bool]
     __nullable: Optional[bool]
@@ -75,6 +45,7 @@ class FlowfileColumn:
         self.__is_unique = None
         self.__sql_type = None
         self.__perc_unique = None
+        self.data_type_group = self.get_readable_datatype_group()
 
     def __repr__(self):
         """
@@ -220,6 +191,20 @@ class FlowfileColumn:
             return 'numeric'
         elif self.data_type in ('datetime', 'date', 'Date', 'Datetime', 'Time'):
             return 'date'
+        else:
+            return 'str'
+
+    def get_readable_datatype_group(self) -> ReadableDataTypeGroup:
+        if self.data_type in ('Utf8', 'VARCHAR', 'CHAR', 'NVARCHAR', 'String'):
+            return 'String'
+        elif self.data_type in ('fixed_decimal', 'decimal', 'float', 'integer', 'boolean', 'double', 'Int16', 'Int32',
+                                'Int64', 'Float32', 'Float64', 'Decimal', 'Binary', 'Boolean', 'Uint8', 'Uint16',
+                                'Uint32', 'Uint64'):
+            return 'Numeric'
+        elif self.data_type in ('datetime', 'date', 'Date', 'Datetime', 'Time'):
+            return 'Date'
+        else:
+            return 'Other'
 
     def get_polars_type(self) -> PlType:
         pl_datatype = cast_str_to_polars_type(self.data_type)

flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py

@@ -0,0 +1,36 @@
+
+from typing import Type, Literal, List, Dict, Union, Tuple
+import polars as pl
+DataTypeGroup = Literal['numeric', 'string', 'datetime', 'boolean', 'binary', 'complex', 'unknown']
+
+
+def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
+    if isinstance(pl_type, pl.List):
+        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
+        return f"pl.List({inner_str})"
+    elif isinstance(pl_type, pl.Array):
+        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
+        return f"pl.Array({inner_str})"
+    elif isinstance(pl_type, pl.Decimal):
+        precision = pl_type.precision if hasattr(pl_type, 'precision') else None
+        scale = pl_type.scale if hasattr(pl_type, 'scale') else None
+        if precision is not None and scale is not None:
+            return f"pl.Decimal({precision}, {scale})"
+        elif precision is not None:
+            return f"pl.Decimal({precision})"
+        else:
+            return "pl.Decimal()"
+    elif isinstance(pl_type, pl.Struct):
+        # Handle Struct with field definitions
+        fields = []
+        if hasattr(pl_type, 'fields'):
+            for field in pl_type.fields:
+                field_name = field.name
+                field_type = convert_pl_type_to_string(field.dtype, inner=True)
+                fields.append(f'pl.Field("{field_name}", {field_type})')
+        field_str = ", ".join(fields)
+        return f"pl.Struct([{field_str}])"
+    else:
+        # For base types, we want the full pl.TypeName format
+        return str(pl_type.base_type()) if not inner else f"pl.{pl_type}"
+

flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py

@@ -1,4 +1,4 @@
-from flowfile_core.schemas.transform_schema import FuzzyMatchInput, SelectInput, JoinInputs
+from flowfile_core.schemas.transform_schema import FuzzyMatchInput, SelectInput, JoinInputs, FuzzyMatchInputManager
 from flowfile_core.flowfile.flow_data_engine.join import verify_join_select_integrity, verify_join_map_integrity
 import polars as pl
 from typing import TYPE_CHECKING, Tuple, List
@@ -15,37 +15,37 @@ def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: Joi
     Returns:
         None
     """
-    select_map = {select.new_name: select for select in join_inputs.renames}
+    select_map = {select.old_name: select for select in join_inputs.renames}
     ordered_renames = [select_map[col] for col in col_order if col in select_map]
     join_inputs.renames = ordered_renames
 
 
 def _ensure_all_columns_have_select(left: "FlowDataEngine",
                                     right: "FlowDataEngine",
-                                    fuzzy_match_input: FuzzyMatchInput):
+                                    fuzzy_match_input_manager: FuzzyMatchInputManager):
     """
     Ensure that all columns in the left and right FlowDataEngines are included in the fuzzy match input's select
      statements.
     Args:
-        left (FlowDataEngine):
-        right (FlowDataEngine):
-        fuzzy_match_input ():
+        left (FlowDataEngine): Left FlowDataEngine
+        right (FlowDataEngine): Right FlowDataEngine
+        fuzzy_match_input_manager (FuzzyMatchInputManager): Fuzzy match input manager
 
     Returns:
         None
     """
-    right_cols_in_select = {c.old_name for c in fuzzy_match_input.right_select.renames}
-    left_cols_in_select = {c.old_name for c in fuzzy_match_input.left_select.renames}
+    right_cols_in_select = {c.old_name for c in fuzzy_match_input_manager.right_select.renames}
+    left_cols_in_select = {c.old_name for c in fuzzy_match_input_manager.left_select.renames}
 
-    fuzzy_match_input.left_select.renames.extend(
+    fuzzy_match_input_manager.left_select.renames.extend(
         [SelectInput(col) for col in left.columns if col not in left_cols_in_select])
-    fuzzy_match_input.right_select.renames.extend(
+    fuzzy_match_input_manager.right_select.renames.extend(
         [SelectInput(col) for col in right.columns if col not in right_cols_in_select]
     )
 
 
 def prepare_for_fuzzy_match(left: "FlowDataEngine", right: "FlowDataEngine",
-                            fuzzy_match_input: FuzzyMatchInput) -> Tuple[pl.LazyFrame, pl.LazyFrame]:
+                            fuzzy_match_input_manager: FuzzyMatchInputManager) -> Tuple[pl.LazyFrame, pl.LazyFrame]:
     """
     Prepare two FlowDataEngines for fuzzy matching.
 
@@ -58,22 +58,22 @@ def prepare_for_fuzzy_match(left: "FlowDataEngine", right: "FlowDataEngine",
     """
     left.lazy = True
     right.lazy = True
-    _ensure_all_columns_have_select(left, right, fuzzy_match_input)
-    _order_join_inputs_based_on_col_order(left.columns, fuzzy_match_input.left_select)
-    _order_join_inputs_based_on_col_order(right.columns, fuzzy_match_input.right_select)
-
-    verify_join_select_integrity(fuzzy_match_input, left_columns=left.columns, right_columns=right.columns)
-    if not verify_join_map_integrity(fuzzy_match_input, left_columns=left.schema, right_columns=right.schema):
+    _ensure_all_columns_have_select(left, right, fuzzy_match_input_manager)
+    _order_join_inputs_based_on_col_order(left.columns, fuzzy_match_input_manager.left_select.join_inputs)
+    _order_join_inputs_based_on_col_order(right.columns, fuzzy_match_input_manager.right_select.join_inputs)
+    verify_join_select_integrity(fuzzy_match_input_manager.fuzzy_input, left_columns=left.columns, right_columns=right.columns)
+    if not verify_join_map_integrity(fuzzy_match_input_manager.fuzzy_input, left_columns=left.schema,
+                                     right_columns=right.schema):
         raise Exception('Join is not valid by the data fields')
-    fuzzy_match_input = fuzzy_match_input
-    fuzzy_match_input.auto_rename()
 
-    right_select = [v.old_name for v in fuzzy_match_input.right_select.renames if
+    fuzzy_match_input_manager.auto_rename()
+
+    right_select = [v.old_name for v in fuzzy_match_input_manager.right_select.renames if
                     (v.keep or v.join_key) and v.is_available]
-    left_select = [v.old_name for v in fuzzy_match_input.left_select.renames if
+    left_select = [v.old_name for v in fuzzy_match_input_manager.left_select.renames if
                    (v.keep or v.join_key) and v.is_available]
     left_df: pl.LazyFrame | pl.DataFrame = left.data_frame.select(left_select).rename(
-        fuzzy_match_input.left_select.rename_table)
+        fuzzy_match_input_manager.left_select.rename_table)
     right_df: pl.LazyFrame | pl.DataFrame = right.data_frame.select(right_select).rename(
-        fuzzy_match_input.right_select.rename_table)
+        fuzzy_match_input_manager.right_select.rename_table)
     return left_df, right_df

flowfile_core/flowfile/flow_data_engine/join/utils.py

@@ -16,7 +16,7 @@ def rename_df_table_for_join(left_df: T, right_df: T,  join_key_rename: transfor
             right_df.rename({r[0]: r[1] for r in join_key_rename.right.join_key_renames}))
 
 
-def get_undo_rename_mapping_join(join_input: transform_schemas.JoinInput) -> Dict[str, str]:
+def get_undo_rename_mapping_join(join_input: transform_schemas.JoinInputManager) -> Dict[str, str]:
     join_key_rename = join_input.get_join_key_renames(True)
     return {r[1]: r[0] for r in join_key_rename.right.join_key_renames + join_key_rename.left.join_key_renames}
 

flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py

@@ -4,9 +4,14 @@ from flowfile_core.schemas import transform_schema
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
 
 
-def verify_join_select_integrity(join_input: transform_schema.JoinInput | transform_schema.CrossJoinInput,
-                                 left_columns: List[str],
-                                 right_columns: List[str]):
+def verify_join_select_integrity(
+        join_input:
+        transform_schema.JoinInput |
+        transform_schema.CrossJoinInput |
+        transform_schema.FuzzyMatchInput |
+        transform_schema.JoinInputsManager,
+        left_columns: List[str],
+        right_columns: List[str]):
     """
     Verify column availability for join selection and update availability flags.
 
@@ -27,7 +32,7 @@ def verify_join_select_integrity(join_input: transform_schema.JoinInput | transf
             c.is_available = True
 
 
-def verify_join_map_integrity(join_input: transform_schema.JoinInput,
+def verify_join_map_integrity(join_input: transform_schema.JoinInput | transform_schema.FuzzyMatchInput | transform_schema.JoinInputManager,
                               left_columns: List[FlowfileColumn],
                               right_columns: List[FlowfileColumn]
                               ):