Flowfile 0.2.2__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py

@@ -0,0 +1,144 @@
+from dataclasses import dataclass
+from typing import Optional, Any, List, Dict, Literal
+from flowfile_core.schemas import input_schema
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import type_to_polars_str
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
+from polars import datatypes
+import polars as pl
+
+DataTypeGroup = Literal['numeric', 'str', 'date']
+
+
+@dataclass
+class FlowfileColumn:
+    column_name: str
+    data_type: str
+    size: int
+    max_value: str
+    min_value: str
+    col_index: int
+    number_of_empty_values: int
+    number_of_unique_values: int
+    example_values: str
+    __sql_type: Optional[Any]
+    __is_unique: Optional[bool]
+    __nullable: Optional[bool]
+    __has_values: Optional[bool]
+    average_value: Optional[str]
+    __perc_unique: Optional[float]
+
+    def __init__(self, polars_type: PlType):
+        self.data_type = str(polars_type.pl_datatype.base_type())
+        self.size = polars_type.count - polars_type.null_count
+        self.max_value = polars_type.max
+        self.min_value = polars_type.min
+        self.number_of_unique_values = polars_type.n_unique
+        self.number_of_empty_values = polars_type.null_count
+        self.example_values = polars_type.examples
+        self.column_name = polars_type.column_name
+        self.average_value = polars_type.mean
+        self.col_index = polars_type.col_index
+        self.__has_values = None
+        self.__nullable = None
+        self.__is_unique = None
+        self.__sql_type = None
+        self.__perc_unique = None
+
+    @classmethod
+    def create_from_polars_type(cls, polars_type: PlType, **kwargs) -> "FlowfileColumn":
+        for k, v in kwargs.items():
+            if hasattr(polars_type, k):
+                setattr(polars_type, k, v)
+        return cls(polars_type)
+
+    @classmethod
+    def from_input(cls, column_name: str, data_type: str, **kwargs) -> "FlowfileColumn":
+        pl_type = type_to_polars_str(data_type)
+        if pl_type is not None:
+            data_type = pl_type
+        return cls(PlType(column_name=column_name, pl_datatype=data_type, **kwargs))
+
+    @classmethod
+    def create_from_polars_dtype(cls, column_name: str, data_type: pl.DataType, **kwargs):
+        return cls(PlType(column_name=column_name, pl_datatype=data_type, **kwargs))
+
+    def get_minimal_field_info(self) -> input_schema.MinimalFieldInfo:
+        return input_schema.MinimalFieldInfo(name=self.column_name, data_type=self.data_type)
+
+    @classmethod
+    def create_from_minimal_field_info(cls, minimal_field_info: input_schema.MinimalFieldInfo) -> "FlowfileColumn":
+        return cls.from_input(column_name=minimal_field_info.name,
+                              data_type=minimal_field_info.data_type)
+
+    @property
+    def is_unique(self) -> bool:
+        if self.__is_unique is None:
+            if self.has_values:
+                self.__is_unique = self.number_of_unique_values == self.number_of_filled_values
+            else:
+                self.__is_unique = False
+        return self.__is_unique
+
+    @property
+    def perc_unique(self) -> float:
+        if self.__perc_unique is None:
+            self.__perc_unique = self.number_of_unique_values / self.number_of_filled_values
+        return self.__perc_unique
+
+    @property
+    def has_values(self) -> bool:
+        if not self.__has_values:
+            self.__has_values = self.number_of_unique_values > 0
+        return self.__has_values
+
+    @property
+    def number_of_filled_values(self):
+        return self.size
+
+    @property
+    def nullable(self):
+        if self.__nullable is None:
+            self.__nullable = self.number_of_empty_values > 0
+        return self.__nullable
+
+    @property
+    def name(self):
+        return self.column_name
+
+    def get_column_repr(self):
+        return dict(name=self.name,
+                    size=self.size,
+                    data_type=str(self.data_type),
+                    has_values=self.has_values,
+                    is_unique=self.is_unique,
+                    max_value=str(self.max_value),
+                    min_value=str(self.min_value),
+                    number_of_unique_values=self.number_of_unique_values,
+                    number_of_filled_values=self.number_of_filled_values,
+                    number_of_empty_values=self.number_of_empty_values,
+                    average_size=self.average_value)
+
+    def generic_datatype(self) -> DataTypeGroup:
+        if self.data_type in ('Utf8', 'VARCHAR', 'CHAR', 'NVARCHAR', 'String'):
+            return 'str'
+        elif self.data_type in ('fixed_decimal', 'decimal', 'float', 'integer', 'boolean', 'double', 'Int16', 'Int32',
+                                'Int64', 'Float32', 'Float64', 'Decimal', 'Binary', 'Boolean', 'Uint8', 'Uint16',
+                                'Uint32', 'Uint64'):
+            return 'numeric'
+        elif self.data_type in ('datetime', 'date', 'Date', 'Datetime', 'Time'):
+            return 'date'
+
+    def get_polars_type(self) -> PlType:
+        if hasattr(datatypes, self.data_type):
+            pl_datatype = getattr(datatypes, self.data_type)
+        else:
+            pl_datatype = None
+
+        return PlType(pl_datatype=pl_datatype, **self.__dict__)
+
+    def update_type_from_polars_type(self, pl_type: PlType):
+        self.data_type = str(pl_type.pl_datatype.base_type())
+
+
+def convert_stats_to_column_info(stats: List[Dict]) -> List[FlowfileColumn]:
+    return [FlowfileColumn.create_from_polars_type(PlType(**c)) for c in stats]

flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py

@@ -0,0 +1,24 @@
+from typing import Optional, Any
+from pydantic import BaseModel
+
+
+class ColumnInfo:
+    pass
+
+
+class PlType(BaseModel):
+    column_name: str
+    col_index: int = -1
+    count: Optional[int] = -1
+    null_count: Optional[int] = -1
+    mean: Optional[str] = ""
+    std: Optional[float] = -1
+    min: Optional[str] = ""
+    max: Optional[str] = ""
+    median: Optional[str] = 0
+    pl_datatype: Optional[Any]
+    n_unique: Optional[int] = -1
+    examples: Optional[str] = ""
+
+    class Config:
+        arbitrary_types_allowed = True

flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py

@@ -0,0 +1,36 @@
+import polars as pl
+
+
+dtype_to_pl = {
+    'int': pl.Int64,
+    'integer': pl.Int64,
+    'char': pl.String,
+    'fixed decimal': pl.Float32,
+    'double': pl.Float64,
+    'float': pl.Float64,
+    'bool': pl.Boolean,
+    'byte': pl.UInt8,
+    'bit': pl.Binary,
+    'date': pl.Date,
+    'datetime': pl.Datetime,
+    'string': pl.String,
+    'str': pl.String,
+    'time': pl.Time,
+}
+
+dtype_to_pl_str = {k: v.__name__ for k, v in dtype_to_pl.items()}
+
+
+def type_to_polars(dtype: str):
+    pl_datetype = dtype_to_pl.get(dtype.lower())
+    if pl_datetype is not None:
+        return pl_datetype
+    elif hasattr(pl, dtype):
+        return getattr(pl, dtype)
+    else:
+        return pl.String
+
+
+def type_to_polars_str(dtype: str) -> pl.DataType:
+    return type_to_polars(dtype)()
+

flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py

@@ -0,0 +1,38 @@
+from flowfile_core.schemas.transform_schema import FuzzyMatchInput
+from flowfile_core.flowfile.flow_data_engine.join import verify_join_select_integrity, verify_join_map_integrity
+import polars as pl
+from typing import TYPE_CHECKING, Tuple
+
+if TYPE_CHECKING:
+    from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+
+
+def prepare_for_fuzzy_match(left: "FlowDataEngine", right: "FlowDataEngine",
+                            fuzzy_match_input: FuzzyMatchInput) -> Tuple[pl.LazyFrame, pl.LazyFrame]:
+    """
+    Prepare two FlowDataEngines for fuzzy matching.
+
+    Args:
+        left: Left FlowDataEngine for fuzzy join
+        right: Right FlowDataEngine for fuzzy join
+        fuzzy_match_input: Parameters for fuzzy matching configuration
+    Returns:
+        Tuple[pl.LazyFrame, pl.LazyFrame]: Prepared left and right lazy frames
+    """
+
+    left.lazy = True
+    right.lazy = True
+    verify_join_select_integrity(fuzzy_match_input, left_columns=left.columns, right_columns=right.columns)
+    if not verify_join_map_integrity(fuzzy_match_input, left_columns=left.schema, right_columns=right.schema):
+        raise Exception('Join is not valid by the data fields')
+    fuzzy_match_input = fuzzy_match_input
+    fuzzy_match_input.auto_rename()
+    right_select = [v.old_name for v in fuzzy_match_input.right_select.renames if
+                    (v.keep or v.join_key) and v.is_available]
+    left_select = [v.old_name for v in fuzzy_match_input.left_select.renames if
+                   (v.keep or v.join_key) and v.is_available]
+    left_df: pl.LazyFrame | pl.DataFrame = left.data_frame.select(left_select).rename(
+        fuzzy_match_input.left_select.rename_table)
+    right_df: pl.LazyFrame | pl.DataFrame = right.data_frame.select(right_select).rename(
+        fuzzy_match_input.right_select.rename_table)
+    return left_df, right_df

flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py

@@ -0,0 +1,90 @@
+
+from typing import List
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, PlType
+from flowfile_core.schemas import transform_schema
+from flowfile_core.schemas import input_schema
+from polars import datatypes
+import polars as pl
+from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import fetch_unique_values
+from flowfile_core.configs.flow_logger import main_logger
+
+
+def calculate_uniqueness(a: float, b: float) -> float:
+    return ((pow(a + 0.5, 2) + pow(b + 0.5, 2)) / 2 - pow(0.5, 2)) + 0.5 * abs(a - b)
+
+
+def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInput,
+                                 left_schema: List[FlowfileColumn],
+                                 right_schema: List[FlowfileColumn]):
+    print('calculating fuzzy match schema')
+    left_schema_dict, right_schema_dict = ({ls.name: ls for ls in left_schema}, {rs.name: rs for rs in right_schema})
+    fm_input.auto_rename()
+
+    output_schema = []
+    for column in fm_input.left_select.renames:
+        column_schema = left_schema_dict.get(column.old_name)
+        if column_schema and column.keep:
+            output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
+                                                           example_values=column_schema.example_values))
+    for column in fm_input.right_select.renames:
+        column_schema = right_schema_dict.get(column.old_name)
+        if column_schema and column.keep:
+            output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
+                                                           example_values=column_schema.example_values))
+
+    for i, fm in enumerate(fm_input.join_mappings):
+        output_schema.append(FlowfileColumn.from_input(f'fuzzy_score_{i}', 'Float64'))
+    return output_schema
+
+
+def get_schema_of_column(node_input_schema: List[FlowfileColumn], col_name: str) -> FlowfileColumn|None:
+    for s in node_input_schema:
+        if s.name == col_name:
+            return s
+
+
+class InvalidSetup(ValueError):
+    """Error raised when pivot column has too many unique values."""
+    pass
+
+
+def get_output_data_type_pivot(schema: FlowfileColumn, agg_type: str) -> datatypes:
+    if agg_type in ('count', 'n_unique'):
+        output_type = datatypes.Float64  # count is always float
+    elif schema.generic_datatype() == 'numeric':
+        output_type = datatypes.Float64
+    elif schema.generic_datatype() == 'string':
+        output_type = datatypes.Utf8
+    elif schema.generic_datatype() == 'date':
+        output_type = datatypes.Datetime
+    else:
+        output_type = datatypes.Utf8
+    return output_type
+
+
+def pre_calculate_pivot_schema(node_input_schema: List[FlowfileColumn],
+                               pivot_input: transform_schema.PivotInput,
+                               output_fields: List[input_schema.MinimalFieldInfo] = None,
+                               input_lf: pl.LazyFrame = None) -> List[FlowfileColumn]:
+    index_columns_schema = [get_schema_of_column(node_input_schema, index_col) for index_col in
+                            pivot_input.index_columns]
+    val_column_schema = get_schema_of_column(node_input_schema, pivot_input.value_col)
+    if output_fields is not None and len(output_fields) > 0:
+        return index_columns_schema+[FlowfileColumn(PlType(Plcolumn_name=output_field.name,
+                                                           pl_datatype=output_field.data_type)) for output_field in output_fields]
+
+    else:
+        max_unique_vals = 200
+        unique_vals = fetch_unique_values(input_lf.select(pivot_input.pivot_column)
+                                          .unique()
+                                          .sort(pivot_input.pivot_column)
+                                          .limit(max_unique_vals).cast(pl.String))
+        if len(unique_vals) >= max_unique_vals:
+            main_logger.warning('Pivot column has too many unique values. Please consider using a different column.'
+                                f' Max unique values: {max_unique_vals}')
+        pl_output_fields = []
+        for val in unique_vals:
+            for agg in pivot_input.aggregations:
+                output_type = get_output_data_type_pivot(val_column_schema, agg)
+                pl_output_fields.append(PlType(column_name=f'{val}_{agg}', pl_datatype=output_type))
+        return index_columns_schema + [FlowfileColumn(pl_output_field) for pl_output_field in pl_output_fields]

flowfile_core/flowfile/flow_data_engine/join/__init__.py

@@ -0,0 +1 @@
+from flowfile_core.flowfile.flow_data_engine.join.verify_integrity import *

flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py

@@ -0,0 +1,54 @@
+
+from typing import List
+from flowfile_core.schemas import transform_schema
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+
+
+def verify_join_select_integrity(join_input: transform_schema.JoinInput | transform_schema.CrossJoinInput,
+                                 left_columns: List[str],
+                                 right_columns: List[str]):
+    """
+    Verify column availability for join selection and update availability flags.
+
+    Args:
+        join_input: Join configuration input containing column selections
+        left_columns: List of available column names in left table
+        right_columns: List of available column names in right table
+    """
+    for c in join_input.left_select.renames:
+        if c.old_name not in left_columns:
+            c.is_available = False
+        else:
+            c.is_available = True
+    for c in join_input.right_select.renames:
+        if c.old_name not in right_columns:
+            c.is_available = False
+        else:
+            c.is_available = True
+
+
+def verify_join_map_integrity(join_input: transform_schema.JoinInput,
+                              left_columns: List[FlowfileColumn],
+                              right_columns: List[FlowfileColumn]
+                              ):
+    """
+    Verify data type compatibility for join mappings between tables.
+
+    Args:
+        join_input: Join configuration with mappings between columns
+        left_columns: Schema columns from left table
+        right_columns: Schema columns from right table
+    Returns:
+        bool: True if join mapping is valid, False otherwise
+    """
+    join_mappings = join_input.join_mapping
+    left_column_dict = {lc.name: lc for lc in left_columns}
+    right_column_dict = {rc.name: rc for rc in right_columns}
+    for join_mapping in join_mappings:
+        left_column_info: FlowfileColumn | None = left_column_dict.get(join_mapping.left_col)
+        right_column_info: FlowfileColumn | None = right_column_dict.get(join_mapping.right_col)
+        if not left_column_info or not right_column_info:
+            return False
+        if left_column_info.generic_datatype() != right_column_info.generic_datatype():
+            return False
+    return True

flowfile_core/flowfile/flow_data_engine/pivot_table.py

@@ -0,0 +1,20 @@
+import polars as pl
+from polars.expr import Expr
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class AggFunc:
+    __slots__ = ['func_name', 'func_expr']
+    func_name: str
+    func_expr: Expr
+
+
+AggFuncs = List[AggFunc]
+
+pl.Expr.sum
+
+agg_funcs = ['sum', 'max', 'min', 'count', 'first', 'last', 'std', 'var', 'n_unique', 'list', 'list_agg']
+
+

flowfile_core/flowfile/flow_data_engine/polars_code_parser.py

@@ -0,0 +1,249 @@
+import polars as pl
+from typing import Dict, Any, Callable
+import textwrap
+import ast
+import time
+
+
+def remove_comments_and_docstrings(source: str) -> str:
+    """
+    Remove comments and docstrings from Python source code.
+
+    Args:
+        source: Python source code as string
+
+    Returns:
+        Cleaned Python source code
+    """
+    if not source.strip():
+        return ""
+
+    def remove_comments_from_line(line: str) -> str:
+        """Remove comments while preserving string literals."""
+        result = []
+        i = 0
+        in_string = False
+        string_char = None
+
+        while i < len(line):
+            char = line[i]
+
+            # Handle string boundaries
+            if char in ('"', "'"):
+                # Check for escaped quotes
+                if i > 0 and line[i - 1] == '\\':
+                    result.append(char)
+                    i += 1
+                    continue
+
+                if not in_string:
+                    # Check if it's the start of a string
+                    in_string = True
+                    string_char = char
+                elif string_char == char:
+                    # Check if it's the end of a string
+                    in_string = False
+                    string_char = None
+
+            # Only process comment characters outside strings
+            elif char == '#' and not in_string:
+                break
+
+            result.append(char)
+            i += 1
+
+        return ''.join(result).rstrip()
+
+    # First pass: handle comments
+    lines = [remove_comments_from_line(line) for line in source.splitlines()]
+    source = '\n'.join(line for line in lines if line.strip())
+
+    # Second pass: handle docstrings using AST
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        return source
+
+    class DocstringRemover(ast.NodeTransformer):
+        def visit_Module(self, node):
+            # Remove module-level docstrings
+            while (node.body and isinstance(node.body[0], ast.Expr)
+                   and isinstance(node.body[0].value, ast.Constant)
+                   and isinstance(node.body[0].value.value, str)):
+                node.body.pop(0)
+            return self.generic_visit(node)
+
+        def visit_FunctionDef(self, node):
+            # Remove function docstrings
+            if (node.body and isinstance(node.body[0], ast.Expr)
+                    and isinstance(node.body[0].value, ast.Constant)
+                    and isinstance(node.body[0].value.value, str)):
+                node.body.pop(0)
+            return self.generic_visit(node)
+
+        def visit_ClassDef(self, node):
+            # Remove class docstrings
+            if (node.body and isinstance(node.body[0], ast.Expr)
+                    and isinstance(node.body[0].value, ast.Constant)
+                    and isinstance(node.body[0].value.value, str)):
+                node.body.pop(0)
+            return self.generic_visit(node)
+
+        def visit_AsyncFunctionDef(self, node):
+            # Remove async function docstrings
+            if (node.body and isinstance(node.body[0], ast.Expr)
+                    and isinstance(node.body[0].value, ast.Constant)
+                    and isinstance(node.body[0].value.value, str)):
+                node.body.pop(0)
+            return self.generic_visit(node)
+
+        def visit_Expr(self, node):
+            # Remove standalone string literals
+            if isinstance(node.value, (ast.Str, ast.Constant)) and isinstance(getattr(node.value, 'value', None), str):
+                return None
+            return self.generic_visit(node)
+
+    try:
+        tree = DocstringRemover().visit(tree)
+        ast.fix_missing_locations(tree)
+        result = ast.unparse(tree)
+        # Remove empty lines
+        return '\n'.join(line for line in result.splitlines() if line.strip())
+    except Exception:
+        return source
+
+
+class PolarsCodeParser:
+    """
+    Securely executes Polars code with restricted access to Python functionality.
+    Supports multiple input DataFrames or no input DataFrames.
+    """
+
+    def __init__(self):
+        self.safe_globals = {
+            # Polars functionality
+            'pl': pl,
+            'col': pl.col,
+            'lit': pl.lit,
+            'expr': pl.expr,
+            # Basic Python built-ins
+            'print': print,
+            'len': len,
+            'range': range,
+            'enumerate': enumerate,
+            'zip': zip,
+            'list': list,
+            'dict': dict,
+            'set': set,
+            'str': str,
+            'int': int,
+            'float': float,
+            'bool': bool,
+            'True': True,
+            'False': False,
+            'None': None,
+            'time': time
+        }
+
+    @staticmethod
+    def _validate_code(code: str) -> None:
+        """
+        Validate code for security concerns before execution.
+        """
+        try:
+            tree = ast.parse(code)
+            for node in ast.walk(tree):
+                # Block imports
+                if isinstance(node, (ast.Import, ast.ImportFrom)):
+                    raise ValueError("Import statements are not allowed")
+
+                # Block exec/eval
+                if isinstance(node, ast.Call):
+                    if isinstance(node.func, ast.Name):
+                        if node.func.id in {'exec', 'eval', 'compile', '__import__'}:
+                            raise ValueError(f"Function '{node.func.id}' is not allowed")
+
+                # Block access to system attributes
+                if isinstance(node, ast.Attribute):
+                    if node.attr.startswith('__'):
+                        raise ValueError(f"Access to '{node.attr}' is not allowed")
+
+        except SyntaxError as e:
+            raise ValueError(f"Invalid Python syntax: {str(e)}")
+
+    def _wrap_in_function(self, code: str, num_inputs: int = 1) -> str:
+        """
+        Wraps code in a function definition that can accept multiple input DataFrames or none.
+
+        Args:
+            code: The code to wrap
+            num_inputs: Number of expected input DataFrames (0 for none)
+
+        Returns:
+            Wrapped code as a function
+        """
+        # Dedent the code first to handle various indentation styles
+        code = textwrap.dedent(code).strip()
+
+        # Create appropriate function signature based on number of inputs
+        if num_inputs == 0:
+            function_def = "def _transform():\n"
+        elif num_inputs == 1:
+            function_def = "def _transform(input_df):\n"
+        else:
+            params = ", ".join([f"input_df_{i+1}" for i in range(num_inputs)])
+            function_def = f"def _transform({params}):\n"
+
+        # Handle single line expressions
+        if '\n' not in code:
+            # For expressions that should return directly
+            if any(code.startswith(prefix) for prefix in ['pl.', 'col(', 'input_df', 'expr(']):
+                return function_def + f"    return {code}"
+            # For assignments
+            else:
+                return function_def + f"    {code}\n    return output_df"
+
+        # For multi-line code
+        indented_code = '\n'.join(f"    {line}" for line in code.split('\n'))
+        return function_def + indented_code + '\n    return output_df'
+
+    def get_executable(self, code: str, num_inputs: int = 1) -> Callable:
+        """
+        Securely get a function that can be executed with multiple DataFrames or none.
+
+        Args:
+            code: The code to execute
+            num_inputs: Number of expected input DataFrames (0 for none)
+
+        Returns:
+            Callable: A function that takes the specified number of DataFrames
+        """
+        # Validate and clean the code
+        code = remove_comments_and_docstrings(code)
+        code = textwrap.dedent(code).strip()
+        self._validate_code(code)
+
+        # Wrap the code in a function
+        wrapped_code = self._wrap_in_function(code, num_inputs)
+
+        try:
+            # Create namespace for execution
+            local_namespace: Dict[str, Any] = {}
+
+            exec(wrapped_code, self.safe_globals, local_namespace)
+
+            transform_func = local_namespace['_transform']
+            return transform_func
+        except Exception as e:
+            raise ValueError(f"Error executing code: {str(e)}")
+
+    def validate_code(self, code: str):
+        """
+        Validate code for security concerns before execution
+        """
+        code = remove_comments_and_docstrings(code)
+        code = textwrap.dedent(code).strip()
+        self._validate_code(code)
+
+
+polars_code_parser = PolarsCodeParser()