Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py

@@ -1,4 +1,4 @@
 from typing import Literal
 
-DataTypeGroup = Literal['numeric', 'str', 'date']
-ReadableDataTypeGroup = Literal['Numeric', 'String', 'Date', 'Other', 'Boolean', 'Binary', 'Complex']
+DataTypeGroup = Literal["numeric", "str", "date"]
+ReadableDataTypeGroup = Literal["Numeric", "String", "Date", "Other", "Boolean", "Binary", "Complex"]

flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py

@@ -1,13 +1,14 @@
-
+from collections.abc import Iterable
 from dataclasses import dataclass
-from typing import Optional, Any, List, Dict, Iterable
+from typing import Any
 
-from flowfile_core.schemas import input_schema
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
+import polars as pl
+
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.interface import DataTypeGroup, ReadableDataTypeGroup
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.interface import ReadableDataTypeGroup, DataTypeGroup
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.type_registry import convert_pl_type_to_string
-import polars as pl
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
+from flowfile_core.schemas import input_schema
 
 
 @dataclass
@@ -22,12 +23,12 @@ class FlowfileColumn:
     number_of_unique_values: int
     example_values: str
     data_type_group: ReadableDataTypeGroup
-    __sql_type: Optional[Any]
-    __is_unique: Optional[bool]
-    __nullable: Optional[bool]
-    __has_values: Optional[bool]
-    average_value: Optional[str]
-    __perc_unique: Optional[float]
+    __sql_type: Any | None
+    __is_unique: bool | None
+    __nullable: bool | None
+    __has_values: bool | None
+    average_value: str | None
+    __perc_unique: float | None
 
     def __init__(self, polars_type: PlType):
         self.data_type = convert_pl_type_to_string(polars_type.pl_datatype)
@@ -52,10 +53,12 @@ class FlowfileColumn:
         Provides a concise, developer-friendly representation of the object.
         Ideal for debugging and console inspection.
         """
-        return (f"FlowfileColumn(name='{self.column_name}', "
-                f"type={self.data_type}, "
-                f"size={self.size}, "
-                f"nulls={self.number_of_empty_values})")
+        return (
+            f"FlowfileColumn(name='{self.column_name}', "
+            f"type={self.data_type}, "
+            f"size={self.size}, "
+            f"nulls={self.number_of_empty_values})"
+        )
 
     def __str__(self):
         """
@@ -103,7 +106,7 @@ class FlowfileColumn:
             example_str = str(self.example_values)
             # Truncate long example strings for cleaner display
             if len(example_str) > 70:
-                example_str = example_str[:67] + '...'
+                example_str = example_str[:67] + "..."
             lines.append(f"  Examples: {example_str}")
 
         return f"{header}\n" + "\n".join(lines)
@@ -131,8 +134,7 @@ class FlowfileColumn:
 
     @classmethod
     def create_from_minimal_field_info(cls, minimal_field_info: input_schema.MinimalFieldInfo) -> "FlowfileColumn":
-        return cls.from_input(column_name=minimal_field_info.name,
-                              data_type=minimal_field_info.data_type)
+        return cls.from_input(column_name=minimal_field_info.name, data_type=minimal_field_info.data_type)
 
     @property
     def is_unique(self) -> bool:
@@ -170,41 +172,77 @@ class FlowfileColumn:
         return self.column_name
 
     def get_column_repr(self):
-        return dict(name=self.name,
-                    size=self.size,
-                    data_type=str(self.data_type),
-                    has_values=self.has_values,
-                    is_unique=self.is_unique,
-                    max_value=str(self.max_value),
-                    min_value=str(self.min_value),
-                    number_of_unique_values=self.number_of_unique_values,
-                    number_of_filled_values=self.number_of_filled_values,
-                    number_of_empty_values=self.number_of_empty_values,
-                    average_size=self.average_value)
+        return dict(
+            name=self.name,
+            size=self.size,
+            data_type=str(self.data_type),
+            has_values=self.has_values,
+            is_unique=self.is_unique,
+            max_value=str(self.max_value),
+            min_value=str(self.min_value),
+            number_of_unique_values=self.number_of_unique_values,
+            number_of_filled_values=self.number_of_filled_values,
+            number_of_empty_values=self.number_of_empty_values,
+            average_size=self.average_value,
+        )
 
     def generic_datatype(self) -> DataTypeGroup:
-        if self.data_type in ('Utf8', 'VARCHAR', 'CHAR', 'NVARCHAR', 'String'):
-            return 'str'
-        elif self.data_type in ('fixed_decimal', 'decimal', 'float', 'integer', 'boolean', 'double', 'Int16', 'Int32',
-                                'Int64', 'Float32', 'Float64', 'Decimal', 'Binary', 'Boolean', 'Uint8', 'Uint16',
-                                'Uint32', 'Uint64'):
-            return 'numeric'
-        elif self.data_type in ('datetime', 'date', 'Date', 'Datetime', 'Time'):
-            return 'date'
+        if self.data_type in ("Utf8", "VARCHAR", "CHAR", "NVARCHAR", "String"):
+            return "str"
+        elif self.data_type in (
+            "fixed_decimal",
+            "decimal",
+            "float",
+            "integer",
+            "boolean",
+            "double",
+            "Int16",
+            "Int32",
+            "Int64",
+            "Float32",
+            "Float64",
+            "Decimal",
+            "Binary",
+            "Boolean",
+            "Uint8",
+            "Uint16",
+            "Uint32",
+            "Uint64",
+        ):
+            return "numeric"
+        elif self.data_type in ("datetime", "date", "Date", "Datetime", "Time"):
+            return "date"
         else:
-            return 'str'
+            return "str"
 
     def get_readable_datatype_group(self) -> ReadableDataTypeGroup:
-        if self.data_type in ('Utf8', 'VARCHAR', 'CHAR', 'NVARCHAR', 'String'):
-            return 'String'
-        elif self.data_type in ('fixed_decimal', 'decimal', 'float', 'integer', 'boolean', 'double', 'Int16', 'Int32',
-                                'Int64', 'Float32', 'Float64', 'Decimal', 'Binary', 'Boolean', 'Uint8', 'Uint16',
-                                'Uint32', 'Uint64'):
-            return 'Numeric'
-        elif self.data_type in ('datetime', 'date', 'Date', 'Datetime', 'Time'):
-            return 'Date'
+        if self.data_type in ("Utf8", "VARCHAR", "CHAR", "NVARCHAR", "String"):
+            return "String"
+        elif self.data_type in (
+            "fixed_decimal",
+            "decimal",
+            "float",
+            "integer",
+            "boolean",
+            "double",
+            "Int16",
+            "Int32",
+            "Int64",
+            "Float32",
+            "Float64",
+            "Decimal",
+            "Binary",
+            "Boolean",
+            "Uint8",
+            "Uint16",
+            "Uint32",
+            "Uint64",
+        ):
+            return "Numeric"
+        elif self.data_type in ("datetime", "date", "Date", "Datetime", "Time"):
+            return "Date"
         else:
-            return 'Other'
+            return "Other"
 
     def get_polars_type(self) -> PlType:
         pl_datatype = cast_str_to_polars_type(self.data_type)
@@ -215,13 +253,15 @@ class FlowfileColumn:
         self.data_type = str(pl_type.pl_datatype.base_type())
 
 
-def convert_stats_to_column_info(stats: List[Dict]) -> List[FlowfileColumn]:
+def convert_stats_to_column_info(stats: list[dict]) -> list[FlowfileColumn]:
     return [FlowfileColumn.create_from_polars_type(PlType(**c)) for c in stats]
 
 
-def convert_pl_schema_to_raw_data_format(pl_schema: pl.Schema) -> List[input_schema.MinimalFieldInfo]:
-    return [FlowfileColumn.create_from_polars_type(PlType(column_name=k, pl_datatype=v)).get_minimal_field_info()
-            for k, v in pl_schema.items()]
+def convert_pl_schema_to_raw_data_format(pl_schema: pl.Schema) -> list[input_schema.MinimalFieldInfo]:
+    return [
+        FlowfileColumn.create_from_polars_type(PlType(column_name=k, pl_datatype=v)).get_minimal_field_info()
+        for k, v in pl_schema.items()
+    ]
 
 
 def assert_if_flowfile_schema(obj: Iterable) -> bool:

flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py

@@ -1,4 +1,5 @@
-from typing import Optional, Any
+from typing import Any
+
 from pydantic import BaseModel
 
 
@@ -9,16 +10,16 @@ class ColumnInfo:
 class PlType(BaseModel):
     column_name: str
     col_index: int = -1
-    count: Optional[int] = -1
-    null_count: Optional[int] = -1
-    mean: Optional[str] = ""
-    std: Optional[float] = -1
-    min: Optional[str] = ""
-    max: Optional[str] = ""
-    median: Optional[str] = 0
-    pl_datatype: Optional[Any]
-    n_unique: Optional[int] = -1
-    examples: Optional[str] = ""
+    count: int | None = -1
+    null_count: int | None = -1
+    mean: str | None = ""
+    std: float | None = -1
+    min: str | None = ""
+    max: str | None = ""
+    median: str | None = 0
+    pl_datatype: Any | None
+    n_unique: int | None = -1
+    examples: str | None = ""
 
     class Config:
         arbitrary_types_allowed = True

flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py

@@ -1,7 +1,8 @@
+from typing import Literal
 
-from typing import Type, Literal, List, Dict, Union, Tuple
 import polars as pl
-DataTypeGroup = Literal['numeric', 'string', 'datetime', 'boolean', 'binary', 'complex', 'unknown']
+
+DataTypeGroup = Literal["numeric", "string", "datetime", "boolean", "binary", "complex", "unknown"]
 
 
 def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
@@ -12,8 +13,8 @@ def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
         inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
         return f"pl.Array({inner_str})"
     elif isinstance(pl_type, pl.Decimal):
-        precision = pl_type.precision if hasattr(pl_type, 'precision') else None
-        scale = pl_type.scale if hasattr(pl_type, 'scale') else None
+        precision = pl_type.precision if hasattr(pl_type, "precision") else None
+        scale = pl_type.scale if hasattr(pl_type, "scale") else None
         if precision is not None and scale is not None:
             return f"pl.Decimal({precision}, {scale})"
         elif precision is not None:
@@ -23,7 +24,7 @@ def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
     elif isinstance(pl_type, pl.Struct):
         # Handle Struct with field definitions
         fields = []
-        if hasattr(pl_type, 'fields'):
+        if hasattr(pl_type, "fields"):
             for field in pl_type.fields:
                 field_name = field.name
                 field_type = convert_pl_type_to_string(field.dtype, inner=True)
@@ -33,4 +34,3 @@ def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
     else:
         # For base types, we want the full pl.TypeName format
         return str(pl_type.base_type()) if not inner else f"pl.{pl_type}"
-

flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py

@@ -1,21 +1,20 @@
 import polars as pl
 
-
 dtype_to_pl = {
-    'int': pl.Int64,
-    'integer': pl.Int64,
-    'char': pl.String,
-    'fixed decimal': pl.Float32,
-    'double': pl.Float64,
-    'float': pl.Float64,
-    'bool': pl.Boolean,
-    'byte': pl.UInt8,
-    'bit': pl.Binary,
-    'date': pl.Date,
-    'datetime': pl.Datetime,
-    'string': pl.String,
-    'str': pl.String,
-    'time': pl.Time,
+    "int": pl.Int64,
+    "integer": pl.Int64,
+    "char": pl.String,
+    "fixed decimal": pl.Float32,
+    "double": pl.Float64,
+    "float": pl.Float64,
+    "bool": pl.Boolean,
+    "byte": pl.UInt8,
+    "bit": pl.Binary,
+    "date": pl.Date,
+    "datetime": pl.Datetime,
+    "string": pl.String,
+    "str": pl.String,
+    "time": pl.Time,
 }
 
 
@@ -27,19 +26,17 @@ def safe_eval_pl_type(type_string: str):
     # Define allowed names in the evaluation namespace
     safe_dict = {
         # Polars module and types
-        'pl': pl,
-
+        "pl": pl,
         # Basic Python built-ins for literals
-        'int': int,
-        'str': str,
-        'float': float,
-        'bool': bool,
-        'list': list,
-        'dict': dict,
-        'tuple': tuple,
-
+        "int": int,
+        "str": str,
+        "float": float,
+        "bool": bool,
+        "list": list,
+        "dict": dict,
+        "tuple": tuple,
         # Disable dangerous built-ins
-        '__builtins__': {},
+        "__builtins__": {},
     }
 
     try:
@@ -52,10 +49,10 @@ dtype_to_pl_str = {k: v.__name__ for k, v in dtype_to_pl.items()}
 
 
 def get_polars_type(dtype: str):
-    if 'pl.' in dtype:
+    if "pl." in dtype:
         try:
             return safe_eval_pl_type(dtype)
-        except Exception as e:
+        except Exception:
             return pl.String
     pl_datetype = dtype_to_pl.get(dtype.lower())
     if pl_datetype is not None:
@@ -68,8 +65,7 @@ def get_polars_type(dtype: str):
 
 def cast_str_to_polars_type(dtype: str) -> pl.DataType:
     pl_type = get_polars_type(dtype)
-    if hasattr(pl_type, '__call__'):
+    if callable(pl_type):
         return pl_type()
     else:
         return pl_type
-

flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py

@@ -1,13 +1,15 @@
-from flowfile_core.schemas.transform_schema import FuzzyMatchInput, SelectInput, JoinInputs, FuzzyMatchInputManager
-from flowfile_core.flowfile.flow_data_engine.join import verify_join_select_integrity, verify_join_map_integrity
+from typing import TYPE_CHECKING
+
 import polars as pl
-from typing import TYPE_CHECKING, Tuple, List
+
+from flowfile_core.flowfile.flow_data_engine.join import verify_join_map_integrity, verify_join_select_integrity
+from flowfile_core.schemas.transform_schema import FuzzyMatchInputManager, JoinInputs, SelectInput
 
 if TYPE_CHECKING:
     from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
 
 
-def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: JoinInputs) -> None:
+def _order_join_inputs_based_on_col_order(col_order: list[str], join_inputs: JoinInputs) -> None:
     """
     Ensure that the select columns in the fuzzy match input match the order of the incoming columns.
     This function modifies the join_inputs object in-place.
@@ -20,9 +22,9 @@ def _order_join_inputs_based_on_col_order(col_order: List[str], join_inputs: Joi
     join_inputs.renames = ordered_renames
 
 
-def _ensure_all_columns_have_select(left: "FlowDataEngine",
-                                    right: "FlowDataEngine",
-                                    fuzzy_match_input_manager: FuzzyMatchInputManager):
+def _ensure_all_columns_have_select(
+    left: "FlowDataEngine", right: "FlowDataEngine", fuzzy_match_input_manager: FuzzyMatchInputManager
+):
     """
     Ensure that all columns in the left and right FlowDataEngines are included in the fuzzy match input's select
      statements.
@@ -38,14 +40,16 @@ def _ensure_all_columns_have_select(left: "FlowDataEngine",
     left_cols_in_select = {c.old_name for c in fuzzy_match_input_manager.left_select.renames}
 
     fuzzy_match_input_manager.left_select.renames.extend(
-        [SelectInput(col) for col in left.columns if col not in left_cols_in_select])
+        [SelectInput(col) for col in left.columns if col not in left_cols_in_select]
+    )
     fuzzy_match_input_manager.right_select.renames.extend(
         [SelectInput(col) for col in right.columns if col not in right_cols_in_select]
     )
 
 
-def prepare_for_fuzzy_match(left: "FlowDataEngine", right: "FlowDataEngine",
-                            fuzzy_match_input_manager: FuzzyMatchInputManager) -> Tuple[pl.LazyFrame, pl.LazyFrame]:
+def prepare_for_fuzzy_match(
+    left: "FlowDataEngine", right: "FlowDataEngine", fuzzy_match_input_manager: FuzzyMatchInputManager
+) -> tuple[pl.LazyFrame, pl.LazyFrame]:
     """
     Prepare two FlowDataEngines for fuzzy matching.
 
@@ -61,19 +65,26 @@ def prepare_for_fuzzy_match(left: "FlowDataEngine", right: "FlowDataEngine",
     _ensure_all_columns_have_select(left, right, fuzzy_match_input_manager)
     _order_join_inputs_based_on_col_order(left.columns, fuzzy_match_input_manager.left_select.join_inputs)
     _order_join_inputs_based_on_col_order(right.columns, fuzzy_match_input_manager.right_select.join_inputs)
-    verify_join_select_integrity(fuzzy_match_input_manager.fuzzy_input, left_columns=left.columns, right_columns=right.columns)
-    if not verify_join_map_integrity(fuzzy_match_input_manager.fuzzy_input, left_columns=left.schema,
-                                     right_columns=right.schema):
-        raise Exception('Join is not valid by the data fields')
+    verify_join_select_integrity(
+        fuzzy_match_input_manager.fuzzy_input, left_columns=left.columns, right_columns=right.columns
+    )
+    if not verify_join_map_integrity(
+        fuzzy_match_input_manager.fuzzy_input, left_columns=left.schema, right_columns=right.schema
+    ):
+        raise Exception("Join is not valid by the data fields")
 
     fuzzy_match_input_manager.auto_rename()
 
-    right_select = [v.old_name for v in fuzzy_match_input_manager.right_select.renames if
-                    (v.keep or v.join_key) and v.is_available]
-    left_select = [v.old_name for v in fuzzy_match_input_manager.left_select.renames if
-                   (v.keep or v.join_key) and v.is_available]
+    right_select = [
+        v.old_name for v in fuzzy_match_input_manager.right_select.renames if (v.keep or v.join_key) and v.is_available
+    ]
+    left_select = [
+        v.old_name for v in fuzzy_match_input_manager.left_select.renames if (v.keep or v.join_key) and v.is_available
+    ]
     left_df: pl.LazyFrame | pl.DataFrame = left.data_frame.select(left_select).rename(
-        fuzzy_match_input_manager.left_select.rename_table)
+        fuzzy_match_input_manager.left_select.rename_table
+    )
     right_df: pl.LazyFrame | pl.DataFrame = right.data_frame.select(right_select).rename(
-        fuzzy_match_input_manager.right_select.rename_table)
+        fuzzy_match_input_manager.right_select.rename_table
+    )
     return left_df, right_df

flowfile_core/flowfile/flow_data_engine/join/__init__.py

@@ -1,2 +1,2 @@
+from flowfile_core.flowfile.flow_data_engine.join.utils import *
 from flowfile_core.flowfile.flow_data_engine.join.verify_integrity import *
-from flowfile_core.flowfile.flow_data_engine.join.utils import *

flowfile_core/flowfile/flow_data_engine/join/utils.py

@@ -1,22 +1,24 @@
 # Standard library imports
-from typing import Dict, Tuple, TypeVar
+from typing import TypeVar
 
 # Third-party imports
 import polars as pl
 
-from flowfile_core.schemas import (
-    transform_schema as transform_schemas
-)
+from flowfile_core.schemas import transform_schema as transform_schemas
 
-T = TypeVar('T', pl.DataFrame, pl.LazyFrame)
+T = TypeVar("T", pl.DataFrame, pl.LazyFrame)
 
 
-def rename_df_table_for_join(left_df: T, right_df: T,  join_key_rename: transform_schemas.FullJoinKeyResponse) -> Tuple[T, T]:
-    return (left_df.rename({r[0]: r[1] for r in join_key_rename.left.join_key_renames}),
-            right_df.rename({r[0]: r[1] for r in join_key_rename.right.join_key_renames}))
+def rename_df_table_for_join(
+    left_df: T, right_df: T, join_key_rename: transform_schemas.FullJoinKeyResponse
+) -> tuple[T, T]:
+    return (
+        left_df.rename({r[0]: r[1] for r in join_key_rename.left.join_key_renames}),
+        right_df.rename({r[0]: r[1] for r in join_key_rename.right.join_key_renames}),
+    )
 
 
-def get_undo_rename_mapping_join(join_input: transform_schemas.JoinInputManager) -> Dict[str, str]:
+def get_undo_rename_mapping_join(join_input: transform_schemas.JoinInputManager) -> dict[str, str]:
     join_key_rename = join_input.get_join_key_renames(True)
     return {r[1]: r[0] for r in join_key_rename.right.join_key_renames + join_key_rename.left.join_key_renames}
 

flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py

@@ -1,17 +1,15 @@
-
-from typing import List
-from flowfile_core.schemas import transform_schema
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.schemas import transform_schema
 
 
 def verify_join_select_integrity(
-        join_input:
-        transform_schema.JoinInput |
-        transform_schema.CrossJoinInput |
-        transform_schema.FuzzyMatchInput |
-        transform_schema.JoinInputsManager,
-        left_columns: List[str],
-        right_columns: List[str]):
+    join_input: transform_schema.JoinInput
+    | transform_schema.CrossJoinInput
+    | transform_schema.FuzzyMatchInput
+    | transform_schema.JoinInputsManager,
+    left_columns: list[str],
+    right_columns: list[str],
+):
     """
     Verify column availability for join selection and update availability flags.
 
@@ -32,10 +30,11 @@ def verify_join_select_integrity(
             c.is_available = True
 
 
-def verify_join_map_integrity(join_input: transform_schema.JoinInput | transform_schema.FuzzyMatchInput | transform_schema.JoinInputManager,
-                              left_columns: List[FlowfileColumn],
-                              right_columns: List[FlowfileColumn]
-                              ):
+def verify_join_map_integrity(
+    join_input: transform_schema.JoinInput | transform_schema.FuzzyMatchInput | transform_schema.JoinInputManager,
+    left_columns: list[FlowfileColumn],
+    right_columns: list[FlowfileColumn],
+):
     """
     Verify data type compatibility for join mappings between tables.
 
@@ -56,4 +55,4 @@ def verify_join_map_integrity(join_input: transform_schema.JoinInput | transform
             return False
         if left_column_info.generic_datatype() != right_column_info.generic_datatype():
             return False
-    return True
+    return True

flowfile_core/flowfile/flow_data_engine/pivot_table.py

@@ -1,20 +1,18 @@
+from dataclasses import dataclass
+
 import polars as pl
 from polars.expr import Expr
-from dataclasses import dataclass
-from typing import List
 
 
 @dataclass
 class AggFunc:
-    __slots__ = ['func_name', 'func_expr']
+    __slots__ = ["func_name", "func_expr"]
     func_name: str
     func_expr: Expr
 
 
-AggFuncs = List[AggFunc]
+AggFuncs = list[AggFunc]
 
 pl.Expr.sum
 
-agg_funcs = ['sum', 'max', 'min', 'count', 'first', 'last', 'std', 'var', 'n_unique', 'list', 'list_agg']
-
-
+agg_funcs = ["sum", "max", "min", "count", "first", "last", "std", "var", "n_unique", "list", "list_agg"]