Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/polars_code_parser.py

@@ -1,9 +1,11 @@
-import polars as pl
-from typing import Dict, Any, Callable
-import textwrap
 import ast
+import textwrap
 import time
+from collections.abc import Callable
 from io import BytesIO
+from typing import Any
+
+import polars as pl
 
 
 def remove_comments_and_docstrings(source: str) -> str:
@@ -32,7 +34,7 @@ def remove_comments_and_docstrings(source: str) -> str:
             # Handle string boundaries
             if char in ('"', "'"):
                 # Check for escaped quotes
-                if i > 0 and line[i - 1] == '\\':
+                if i > 0 and line[i - 1] == "\\":
                     result.append(char)
                     i += 1
                     continue
@@ -47,17 +49,17 @@ def remove_comments_and_docstrings(source: str) -> str:
                     string_char = None
 
             # Only process comment characters outside strings
-            elif char == '#' and not in_string:
+            elif char == "#" and not in_string:
                 break
 
             result.append(char)
             i += 1
 
-        return ''.join(result).rstrip()
+        return "".join(result).rstrip()
 
     # First pass: handle comments
     lines = [remove_comments_from_line(line) for line in source.splitlines()]
-    source = '\n'.join(line for line in lines if line.strip())
+    source = "\n".join(line for line in lines if line.strip())
 
     # Second pass: handle docstrings using AST
     try:
@@ -68,39 +70,51 @@ def remove_comments_and_docstrings(source: str) -> str:
     class DocstringRemover(ast.NodeTransformer):
         def visit_Module(self, node):
             # Remove module-level docstrings
-            while (node.body and isinstance(node.body[0], ast.Expr)
-                   and isinstance(node.body[0].value, ast.Constant)
-                   and isinstance(node.body[0].value.value, str)):
+            while (
+                node.body
+                and isinstance(node.body[0], ast.Expr)
+                and isinstance(node.body[0].value, ast.Constant)
+                and isinstance(node.body[0].value.value, str)
+            ):
                 node.body.pop(0)
             return self.generic_visit(node)
 
         def visit_FunctionDef(self, node):
             # Remove function docstrings
-            if (node.body and isinstance(node.body[0], ast.Expr)
-                    and isinstance(node.body[0].value, ast.Constant)
-                    and isinstance(node.body[0].value.value, str)):
+            if (
+                node.body
+                and isinstance(node.body[0], ast.Expr)
+                and isinstance(node.body[0].value, ast.Constant)
+                and isinstance(node.body[0].value.value, str)
+            ):
                 node.body.pop(0)
             return self.generic_visit(node)
 
         def visit_ClassDef(self, node):
             # Remove class docstrings
-            if (node.body and isinstance(node.body[0], ast.Expr)
-                    and isinstance(node.body[0].value, ast.Constant)
-                    and isinstance(node.body[0].value.value, str)):
+            if (
+                node.body
+                and isinstance(node.body[0], ast.Expr)
+                and isinstance(node.body[0].value, ast.Constant)
+                and isinstance(node.body[0].value.value, str)
+            ):
                 node.body.pop(0)
             return self.generic_visit(node)
 
         def visit_AsyncFunctionDef(self, node):
             # Remove async function docstrings
-            if (node.body and isinstance(node.body[0], ast.Expr)
-                    and isinstance(node.body[0].value, ast.Constant)
-                    and isinstance(node.body[0].value.value, str)):
+            if (
+                node.body
+                and isinstance(node.body[0], ast.Expr)
+                and isinstance(node.body[0].value, ast.Constant)
+                and isinstance(node.body[0].value.value, str)
+            ):
                 node.body.pop(0)
             return self.generic_visit(node)
 
         def visit_Expr(self, node):
             # Remove standalone string literals
-            if isinstance(node.value, (ast.Str, ast.Constant)) and isinstance(getattr(node.value, 'value', None), str):
+            if isinstance(node.value, (ast.Str, ast.Constant)) and isinstance(getattr(node.value, "value", None), str):
                 return None
             return self.generic_visit(node)
 
@@ -109,7 +123,7 @@ def remove_comments_and_docstrings(source: str) -> str:
         ast.fix_missing_locations(tree)
         result = ast.unparse(tree)
         # Remove empty lines
-        return '\n'.join(line for line in result.splitlines() if line.strip())
+        return "\n".join(line for line in result.splitlines() if line.strip())
     except Exception:
         return source
 
@@ -122,62 +136,61 @@ class PolarsCodeParser:
 
     def __init__(self):
         import datetime
+
         self.safe_globals = {
             # Polars functionality
-            'pl': pl,
-            'col': pl.col,
-            'lit': pl.lit,
-            'expr': pl.expr,
-
+            "pl": pl,
+            "col": pl.col,
+            "lit": pl.lit,
+            "expr": pl.expr,
             # Polars datatypes - added directly
-            'Int8': pl.Int8,
-            'Int16': pl.Int16,
-            'Int32': pl.Int32,
-            'Int64': pl.Int64,
-            'Int128': pl.Int128,
-            'UInt8': pl.UInt8,
-            'UInt16': pl.UInt16,
-            'UInt32': pl.UInt32,
-            'UInt64': pl.UInt64,
-            'Float32': pl.Float32,
-            'Float64': pl.Float64,
-            'Boolean': pl.Boolean,
-            'String': pl.String,
-            'Utf8': pl.Utf8,
-            'Binary': pl.Binary,
-            'Null': pl.Null,
-            'List': pl.List,
-            'Array': pl.Array,
-            'Struct': pl.Struct,
-            'Object': pl.Object,
-            'Date': pl.Date,
-            'Time': pl.Time,
-            'Datetime': pl.Datetime,
-            'Duration': pl.Duration,
-            'Categorical': pl.Categorical,
-            'Decimal': pl.Decimal,
-            'Enum': pl.Enum,
-            'Unknown': pl.Unknown,
-
+            "Int8": pl.Int8,
+            "Int16": pl.Int16,
+            "Int32": pl.Int32,
+            "Int64": pl.Int64,
+            "Int128": pl.Int128,
+            "UInt8": pl.UInt8,
+            "UInt16": pl.UInt16,
+            "UInt32": pl.UInt32,
+            "UInt64": pl.UInt64,
+            "Float32": pl.Float32,
+            "Float64": pl.Float64,
+            "Boolean": pl.Boolean,
+            "String": pl.String,
+            "Utf8": pl.Utf8,
+            "Binary": pl.Binary,
+            "Null": pl.Null,
+            "List": pl.List,
+            "Array": pl.Array,
+            "Struct": pl.Struct,
+            "Object": pl.Object,
+            "Date": pl.Date,
+            "Time": pl.Time,
+            "Datetime": pl.Datetime,
+            "Duration": pl.Duration,
+            "Categorical": pl.Categorical,
+            "Decimal": pl.Decimal,
+            "Enum": pl.Enum,
+            "Unknown": pl.Unknown,
             # Basic Python built-ins
-            'print': print,
-            'len': len,
-            'range': range,
-            'enumerate': enumerate,
-            'zip': zip,
-            'list': list,
-            'dict': dict,
-            'set': set,
-            'str': str,
-            'int': int,
-            'float': float,
-            'bool': bool,
-            'True': True,
-            'False': False,
-            'None': None,
-            'time': time,
-            'BytesIO': BytesIO,
-            'datetime': datetime,
+            "print": print,
+            "len": len,
+            "range": range,
+            "enumerate": enumerate,
+            "zip": zip,
+            "list": list,
+            "dict": dict,
+            "set": set,
+            "str": str,
+            "int": int,
+            "float": float,
+            "bool": bool,
+            "True": True,
+            "False": False,
+            "None": None,
+            "time": time,
+            "BytesIO": BytesIO,
+            "datetime": datetime,
         }
 
     @staticmethod
@@ -195,12 +208,12 @@ class PolarsCodeParser:
                 # Block exec/eval
                 if isinstance(node, ast.Call):
                     if isinstance(node.func, ast.Name):
-                        if node.func.id in {'exec', 'eval', 'compile', '__import__'}:
+                        if node.func.id in {"exec", "eval", "compile", "__import__"}:
                             raise ValueError(f"Function '{node.func.id}' is not allowed")
 
                 # Block access to system attributes
                 if isinstance(node, ast.Attribute):
-                    if node.attr.startswith('__'):
+                    if node.attr.startswith("__"):
                         raise ValueError(f"Access to '{node.attr}' is not allowed")
 
         except SyntaxError as e:
@@ -230,17 +243,17 @@ class PolarsCodeParser:
             function_def = f"def _transform({params}):\n"
 
         # Handle single line expressions
-        if '\n' not in code:
+        if "\n" not in code:
             # For expressions that should return directly
-            if any(code.startswith(prefix) for prefix in ['pl.', 'col(', 'input_df', 'expr(']):
+            if any(code.startswith(prefix) for prefix in ["pl.", "col(", "input_df", "expr("]):
                 return function_def + f"    return {code}"
             # For assignments
             else:
                 return function_def + f"    {code}\n    return output_df"
 
         # For multi-line code
-        indented_code = '\n'.join(f"    {line}" for line in code.split('\n'))
-        return function_def + indented_code + '\n    return output_df'
+        indented_code = "\n".join(f"    {line}" for line in code.split("\n"))
+        return function_def + indented_code + "\n    return output_df"
 
     def get_executable(self, code: str, num_inputs: int = 1) -> Callable:
         """
@@ -262,11 +275,11 @@ class PolarsCodeParser:
         wrapped_code = self._wrap_in_function(code, num_inputs)
         try:
             # Create namespace for execution
-            local_namespace: Dict[str, Any] = {}
+            local_namespace: dict[str, Any] = {}
 
             exec(wrapped_code, self.safe_globals, local_namespace)
 
-            transform_func = local_namespace['_transform']
+            transform_func = local_namespace["_transform"]
             return transform_func
         except Exception as e:
             raise ValueError(f"Error executing code: {str(e)}")
@@ -280,4 +293,4 @@ class PolarsCodeParser:
         self._validate_code(code)
 
 
-polars_code_parser = PolarsCodeParser()
+polars_code_parser = PolarsCodeParser()

flowfile_core/flowfile/flow_data_engine/read_excel_tables.py

@@ -1,80 +1,77 @@
-from typing import Generator, List
-from openpyxl import Workbook, load_workbook
-from openpyxl.worksheet.worksheet import Worksheet
 import gc
+from collections.abc import Generator
+
 import polars as pl
-from flowfile_core.flowfile.flow_data_engine.utils import create_pl_df_type_save, get_data_type
+from openpyxl import Workbook, load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
+
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import dtype_to_pl_str
+from flowfile_core.flowfile.flow_data_engine.utils import create_pl_df_type_save, get_data_type
 
 
-def raw_data_openpyxl(file_path: str,
-                      sheet_name: str = None,
-                      min_row: int = None,
-                      max_row: int = None,
-                      min_col: int = None,
-                      max_col: int = None
-                      ) -> Generator[List, None, None]:
+def raw_data_openpyxl(
+    file_path: str,
+    sheet_name: str = None,
+    min_row: int = None,
+    max_row: int = None,
+    min_col: int = None,
+    max_col: int = None,
+) -> Generator[list, None, None]:
     workbook: Workbook = load_workbook(file_path, data_only=True, read_only=True)
     sheet_name = workbook.sheetnames[0] if sheet_name is None else sheet_name
     sheet: Worksheet = workbook[sheet_name]
-    for row in sheet.iter_rows(min_row=min_row,
-                               max_row=max_row,
-                               min_col=min_col,
-                               max_col=max_col,
-                               values_only=True):
+    for row in sheet.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True):
         yield row
     workbook.close()
     del workbook
     gc.collect()
 
 
-def get_calamine_xlsx_data_types(file_path: str, sheet_name: str, start_row: int=0, end_row: int=0):
+def get_calamine_xlsx_data_types(file_path: str, sheet_name: str, start_row: int = 0, end_row: int = 0):
     df = df_from_calamine_xlsx(file_path, sheet_name, start_row, end_row)
-    return [FlowfileColumn.from_input(n, str(dt), col_index=i) for i, (n, dt) in enumerate(zip(df.columns, df.dtypes))]
+    return [
+        FlowfileColumn.from_input(n, str(dt), col_index=i)
+        for i, (n, dt) in enumerate(zip(df.columns, df.dtypes, strict=False))
+    ]
 
 
 def df_from_calamine_xlsx(file_path: str, sheet_name: str, start_row: int = 0, end_row: int = 0) -> pl.DataFrame:
     read_options = {}
     if start_row > 0:
-        read_options['header_row'] = start_row
+        read_options["header_row"] = start_row
     if end_row > 0:
-        read_options['n_rows'] = end_row - start_row
-    return pl.read_excel(source=file_path,
-                         engine='calamine',
-                         sheet_name=sheet_name,
-                         read_options=read_options,
-                         raise_if_empty=False
-                         )
-
-
-def df_from_openpyxl(file_path: str,
-                     sheet_name: str = None,
-                     min_row: int = None,
-                     max_row: int = None,
-                     min_col: int = None,
-                     max_col: int = None,
-                     has_headers: bool = True
-                     ) -> pl.DataFrame:
-    data_iterator = raw_data_openpyxl(file_path=file_path,
-                                      sheet_name=sheet_name,
-                                      min_row=min_row,
-                                      max_row=max_row,
-                                      min_col=min_col,
-                                      max_col=max_col)
+        read_options["n_rows"] = end_row - start_row
+    return pl.read_excel(
+        source=file_path, engine="calamine", sheet_name=sheet_name, read_options=read_options, raise_if_empty=False
+    )
+
+
+def df_from_openpyxl(
+    file_path: str,
+    sheet_name: str = None,
+    min_row: int = None,
+    max_row: int = None,
+    min_col: int = None,
+    max_col: int = None,
+    has_headers: bool = True,
+) -> pl.DataFrame:
+    data_iterator = raw_data_openpyxl(
+        file_path=file_path, sheet_name=sheet_name, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col
+    )
     raw_data = list(data_iterator)
     if len(raw_data) > 0:
         if has_headers:
             columns = []
             for i, col in enumerate(raw_data[0]):
                 if col is None:
-                    col = f'_unnamed_column_{i}'
+                    col = f"_unnamed_column_{i}"
                 elif not isinstance(col, str):
                     col = str(col)
                 columns.append(col)
             columns = ensure_unique(columns)
             df = create_pl_df_type_save(raw_data[1:])
-            renames = {o: n for o, n in zip(df.columns, columns)}
+            renames = {o: n for o, n in zip(df.columns, columns, strict=False)}
             df = df.rename(renames)
 
         else:
@@ -84,32 +81,37 @@ def df_from_openpyxl(file_path: str,
         return pl.DataFrame()
 
 
-def get_open_xlsx_datatypes(file_path: str,
-                            sheet_name: str = None,
-                            min_row: int = None,
-                            max_row: int = None,
-                            min_col: int = None,
-                            max_col: int = None,
-                            has_headers: bool = True) -> List[FlowfileColumn]:
-    data_iterator = raw_data_openpyxl(file_path=file_path,
-                                      sheet_name=sheet_name,
-                                      min_row=min_row,
-                                      max_row=max_row,
-                                      min_col=min_col,
-                                      max_col=max_col)
+def get_open_xlsx_datatypes(
+    file_path: str,
+    sheet_name: str = None,
+    min_row: int = None,
+    max_row: int = None,
+    min_col: int = None,
+    max_col: int = None,
+    has_headers: bool = True,
+) -> list[FlowfileColumn]:
+    data_iterator = raw_data_openpyxl(
+        file_path=file_path, sheet_name=sheet_name, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col
+    )
     raw_data = data_iterator
     if has_headers:
-        columns = (f'_unnamed_column_{i}' if col is None else col for i, col in enumerate(next(raw_data)))
-        data_types = (dtype_to_pl_str.get(get_data_type(vals), 'String') for vals in zip(*raw_data))
-        schema = [FlowfileColumn.from_input(n, d, col_index = i) for i, (n, d) in enumerate(zip(columns, data_types))]
+        columns = (f"_unnamed_column_{i}" if col is None else col for i, col in enumerate(next(raw_data)))
+        data_types = (dtype_to_pl_str.get(get_data_type(vals), "String") for vals in zip(*raw_data, strict=False))
+        schema = [
+            FlowfileColumn.from_input(n, d, col_index=i)
+            for i, (n, d) in enumerate(zip(columns, data_types, strict=False))
+        ]
     else:
-        columns = (f'column_{i}' for i in range(len(next(raw_data))))
-        data_types = (dtype_to_pl_str.get(get_data_type(vals), 'String') for vals in zip(*raw_data))
-        schema = [FlowfileColumn.from_input(n, d, col_index=i) for i, (n, d) in enumerate(zip(columns, data_types))]
+        columns = (f"column_{i}" for i in range(len(next(raw_data))))
+        data_types = (dtype_to_pl_str.get(get_data_type(vals), "String") for vals in zip(*raw_data, strict=False))
+        schema = [
+            FlowfileColumn.from_input(n, d, col_index=i)
+            for i, (n, d) in enumerate(zip(columns, data_types, strict=False))
+        ]
     return schema
 
 
-def ensure_unique(lst: List[str]) -> List[str]:
+def ensure_unique(lst: list[str]) -> list[str]:
     """
     Ensures that all elements in the input list are unique by appending
     a version number (e.g., '_v1') to duplicates. It continues adding
@@ -140,4 +142,3 @@ def ensure_unique(lst: List[str]) -> List[str]:
             seen[item] = 1  # First occurrence of the item
 
     return result
-

flowfile_core/flowfile/flow_data_engine/sample_data.py

@@ -1,9 +1,11 @@
-from faker import Faker
+from collections.abc import Generator
 from functools import partial
 from math import ceil
 from random import randint
+from typing import Any
+
 import polars as pl
-from typing import List, Dict, Any, Generator
+from faker import Faker
 
 
 def create_fake_data(n_records: int = 1000, optimized: bool = True) -> pl.DataFrame:
@@ -45,22 +47,25 @@ def create_fake_data(n_records: int = 1000, optimized: bool = True) -> pl.DataFr
 
     def generate_phone_number():
         return fake.phone_number()
+
     data = []
     for i in range(max_n_records):
         name = generate_name()
-        data.append(dict(
-            ID=randint(1, 1000000),
-            Name=name,
-            Address=generate_address(),
-            City=cities[selector(min_range(7000))-1],
-            Email=generate_email(name),
-            Phone=generate_phone_number(),
-            DOB=dob[selector(min_range(100_000))-1],
-            Work=companies[selector(min_range(100_000))-1],
-            Zipcode=zipcodes[selector(min_range(200_000))-1],
-            Country=countries[selector(min_range(50))-1],
-            sales_data=sales_data[selector(max_n_records)-1]
-        ))
+        data.append(
+            dict(
+                ID=randint(1, 1000000),
+                Name=name,
+                Address=generate_address(),
+                City=cities[selector(min_range(7000)) - 1],
+                Email=generate_email(name),
+                Phone=generate_phone_number(),
+                DOB=dob[selector(min_range(100_000)) - 1],
+                Work=companies[selector(min_range(100_000)) - 1],
+                Zipcode=zipcodes[selector(min_range(200_000)) - 1],
+                Country=countries[selector(min_range(50)) - 1],
+                sales_data=sales_data[selector(max_n_records) - 1],
+            )
+        )
     if max_n_records < n_records:
         n_duplicates: int = ceil(n_records / max_n_records)
         output = []
@@ -71,8 +76,9 @@ def create_fake_data(n_records: int = 1000, optimized: bool = True) -> pl.DataFr
     return pl.DataFrame(data)
 
 
-
-def create_fake_data_raw(n_records: int = 1000, col_selection: List[str] = None) -> Generator[Dict[str, Any], None, None]:
+def create_fake_data_raw(
+    n_records: int = 1000, col_selection: list[str] = None
+) -> Generator[dict[str, Any], None, None]:
     fake = Faker()
     selector = partial(randint, 0)
 
@@ -112,7 +118,7 @@ def create_fake_data_raw(n_records: int = 1000, col_selection: List[str] = None)
         "Work": companies,
         "Zipcode": zipcodes,
         "Country": countries,
-        "sales_data": sales_data
+        "sales_data": sales_data,
     }
 
     # Filter the available columns based on col_selection
@@ -133,6 +139,6 @@ def create_fake_data_raw(n_records: int = 1000, col_selection: List[str] = None)
 
 def write_fake_data():
     df = create_fake_data()
-    df.write_parquet('backend/tests/data/fake_data.parquet')
-    df.write_csv('backend/tests/data/fake_data.csv')
-    df.write_excel('backend/tests/data/fake_data.xlsx')
+    df.write_parquet("backend/tests/data/fake_data.parquet")
+    df.write_csv("backend/tests/data/fake_data.csv")
+    df.write_excel("backend/tests/data/fake_data.xlsx")

flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py

@@ -1 +1 @@
-from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import *
+from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import *

flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py

@@ -1,8 +1,9 @@
-from typing import Any, Optional, Literal
-from pydantic import BaseModel
+from typing import Any, Literal
+
 from pl_fuzzy_frame_match.models import FuzzyMapping
+from pydantic import BaseModel
 
-OperationType = Literal['store', 'calculate_schema', 'calculate_number_of_records', 'write_output', 'store_sample']
+OperationType = Literal["store", "calculate_schema", "calculate_number_of_records", "write_output", "store_sample"]
 
 
 class PolarsOperation(BaseModel):
@@ -10,14 +11,14 @@ class PolarsOperation(BaseModel):
 
 
 class PolarsScript(PolarsOperation):
-    task_id: Optional[str] = None
-    cache_dir: Optional[str] = None
+    task_id: str | None = None
+    cache_dir: str | None = None
     operation_type: OperationType
 
 
 class FuzzyJoinInput(BaseModel):
-    task_id: Optional[str] = None
-    cache_dir: Optional[str] = None
+    task_id: str | None = None
+    cache_dir: str | None = None
     left_df_operation: PolarsOperation
     right_df_operation: PolarsOperation
     fuzzy_maps: list[FuzzyMapping]
@@ -27,10 +28,11 @@ class FuzzyJoinInput(BaseModel):
 
 class Status(BaseModel):
     background_task_id: str
-    status: Literal['Processing', 'Completed', 'Error', 'Unknown Error', 'Starting', 'Cancelled']  # Type alias for status
+    status: Literal[
+        "Processing", "Completed", "Error", "Unknown Error", "Starting", "Cancelled"
+    ]  # Type alias for status
     file_ref: str
     progress: int = 0
-    error_message: Optional[str] = None  # Add error_message field
+    error_message: str | None = None  # Add error_message field
     results: Any
-    result_type: Literal['polars', 'other'] = 'polars'
-
+    result_type: Literal["polars", "other"] = "polars"