Flowfile 0.2.2__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/read_excel_tables.py

@@ -0,0 +1,143 @@
+from typing import Generator, List
+from openpyxl import Workbook, load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
+import gc
+import polars as pl
+from flowfile_core.flowfile.flow_data_engine.utils import create_pl_df_type_save, get_data_type
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import dtype_to_pl_str
+
+
+def raw_data_openpyxl(file_path: str,
+                      sheet_name: str = None,
+                      min_row: int = None,
+                      max_row: int = None,
+                      min_col: int = None,
+                      max_col: int = None
+                      ) -> Generator[List, None, None]:
+    workbook: Workbook = load_workbook(file_path, data_only=True, read_only=True)
+    sheet_name = workbook.sheetnames[0] if sheet_name is None else sheet_name
+    sheet: Worksheet = workbook[sheet_name]
+    for row in sheet.iter_rows(min_row=min_row,
+                               max_row=max_row,
+                               min_col=min_col,
+                               max_col=max_col,
+                               values_only=True):
+        yield row
+    workbook.close()
+    del workbook
+    gc.collect()
+
+
+def get_calamine_xlsx_data_types(file_path: str, sheet_name: str, start_row: int=0, end_row: int=0):
+    df = df_from_calamine_xlsx(file_path, sheet_name, start_row, end_row)
+    return [FlowfileColumn.from_input(n, str(dt), col_index=i) for i, (n, dt) in enumerate(zip(df.columns, df.dtypes))]
+
+
+def df_from_calamine_xlsx(file_path: str, sheet_name: str, start_row: int = 0, end_row: int = 0) -> pl.DataFrame:
+    read_options = {}
+    if start_row > 0:
+        read_options['header_row'] = start_row
+    if end_row > 0:
+        read_options['n_rows'] = end_row - start_row
+    return pl.read_excel(source=file_path,
+                         engine='calamine',
+                         sheet_name=sheet_name,
+                         read_options=read_options,
+                         raise_if_empty=False
+                         )
+
+
+def df_from_openpyxl(file_path: str,
+                     sheet_name: str = None,
+                     min_row: int = None,
+                     max_row: int = None,
+                     min_col: int = None,
+                     max_col: int = None,
+                     has_headers: bool = True
+                     ) -> pl.DataFrame:
+    data_iterator = raw_data_openpyxl(file_path=file_path,
+                                      sheet_name=sheet_name,
+                                      min_row=min_row,
+                                      max_row=max_row,
+                                      min_col=min_col,
+                                      max_col=max_col)
+    raw_data = list(data_iterator)
+    if len(raw_data) > 0:
+        if has_headers:
+            columns = []
+            for i, col in enumerate(raw_data[0]):
+                if col is None:
+                    col = f'_unnamed_column_{i}'
+                elif not isinstance(col, str):
+                    col = str(col)
+                columns.append(col)
+            columns = ensure_unique(columns)
+            df = create_pl_df_type_save(raw_data[1:])
+            renames = {o: n for o, n in zip(df.columns, columns)}
+            df = df.rename(renames)
+
+        else:
+            df = create_pl_df_type_save(raw_data)
+        return df
+    else:
+        return pl.DataFrame()
+
+
+def get_open_xlsx_datatypes(file_path: str,
+                            sheet_name: str = None,
+                            min_row: int = None,
+                            max_row: int = None,
+                            min_col: int = None,
+                            max_col: int = None,
+                            has_headers: bool = True) -> List[FlowfileColumn]:
+    data_iterator = raw_data_openpyxl(file_path=file_path,
+                                      sheet_name=sheet_name,
+                                      min_row=min_row,
+                                      max_row=max_row,
+                                      min_col=min_col,
+                                      max_col=max_col)
+    raw_data = data_iterator
+    if has_headers:
+        columns = (f'_unnamed_column_{i}' if col is None else col for i, col in enumerate(next(raw_data)))
+        data_types = (dtype_to_pl_str.get(get_data_type(vals), 'String') for vals in zip(*raw_data))
+        schema = [FlowfileColumn.from_input(n, d, col_index = i) for i, (n, d) in enumerate(zip(columns, data_types))]
+    else:
+        columns = (f'column_{i}' for i in range(len(next(raw_data))))
+        data_types = (dtype_to_pl_str.get(get_data_type(vals), 'String') for vals in zip(*raw_data))
+        schema = [FlowfileColumn.from_input(n, d, col_index=i) for i, (n, d) in enumerate(zip(columns, data_types))]
+    return schema
+
+
+def ensure_unique(lst: List[str]) -> List[str]:
+    """
+    Ensures that all elements in the input list are unique by appending
+    a version number (e.g., '_v1') to duplicates. It continues adding
+    version numbers until all items in the list are unique.
+
+    Args:
+        lst (List[str]): A list of strings that may contain duplicates.
+
+    Returns:
+        List[str]: A new list where all elements are unique.
+    """
+    seen = {}
+    result = []
+
+    for item in lst:
+        if item in seen:
+            # Increment the version and append the version number
+            seen[item] += 1
+            new_item = f"{item}_v{seen[item]}"
+            # Ensure the new item is unique by checking for conflicts
+            while new_item in seen:
+                seen[new_item] += 1
+                new_item = f"{item}_v{seen[item]}"
+            result.append(new_item)
+            seen[new_item] = 1  # Mark the new unique item as seen
+        else:
+            result.append(item)
+            seen[item] = 1  # First occurrence of the item
+
+    return result
+

flowfile_core/flowfile/flow_data_engine/sample_data.py

@@ -0,0 +1,120 @@
+from faker import Faker
+from functools import partial
+from random import randint
+import polars as pl
+from typing import List, Dict, Any, Generator
+
+
+def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
+    fake = Faker()
+    selector = partial(randint,0)
+    min_range = partial(min, n_records)
+    # Pre-generation of static data
+    cities = [fake.city() for _ in range(min_range(7000))]
+    companies = [fake.company() for _ in range(min_range(100_000))]
+    zipcodes = [fake.zipcode() for _ in range(min_range(200_000))]
+    countries = [fake.country() for _ in range(min_range(50))]
+    street_names = [fake.street_name() for _ in range(min_range(100000))]
+    dob = [fake.date_of_birth() for _ in range(min_range(100_000))]
+    first_names = [fake.first_name() for _ in range(min_range(100_000))]
+    last_names = [fake.last_name() for _ in range(min_range(50_000))]
+    domain_names = [fake.domain_name() for _ in range(10)]
+    sales_data = [fake.random_int(0, 1000) for _ in range(n_records)]
+
+    def generate_name():
+        return f"{first_names[selector(min_range(100_000))-1]} {last_names[selector(min_range(50_000))-1]}"
+
+    def generate_address():
+        return f"{randint(100, 999)} {street_names[selector(min_range(100000))-1]}"
+
+    def generate_email(name):
+        return f"{name.lower().replace(' ', '_')}.{randint(1, 99)}@{domain_names[selector(10)-1]}"
+
+    def generate_phone_number():
+        return fake.phone_number()
+
+    data = []
+    for i in range(n_records):
+        name = generate_name()
+        data.append(dict(
+            ID=randint(1, 1000000),
+            Name=name,
+            Address=generate_address(),
+            City=cities[selector(min_range(7000))-1],
+            Email=generate_email(name),
+            Phone=generate_phone_number(),
+            DOB=dob[selector(min_range(100_000))-1],
+            Work=companies[selector(min_range(100_000))-1],
+            Zipcode=zipcodes[selector(min_range(200_000))-1],
+            Country=countries[selector(min_range(50))-1],
+            sales_data=sales_data[selector(n_records)-1]
+        ))
+
+    return pl.DataFrame(data)
+
+
+
+def create_fake_data_raw(n_records: int = 1000, col_selection: List[str] = None) -> Generator[Dict[str, Any], None, None]:
+    fake = Faker()
+    selector = partial(randint, 0)
+
+    # Caching static data to avoid regenerating it unnecessarily
+    cities = partial(fake.city)
+    companies = partial(fake.company)
+    zipcodes = partial(fake.zipcode)
+    countries = partial(fake.country)
+    street_names = partial(fake.street_name)
+    dob = partial(fake.date_of_birth)
+    first_names = partial(fake.first_name)
+    last_names = partial(fake.last_name)
+    domain_names = [fake.domain_name() for _ in range(10)]  # Pre-generate a small list
+    sales_data = lambda: fake.random_int(0, 1000)
+
+    def generate_name():
+        return f"{first_names()} {last_names()}"
+
+    def generate_address():
+        return f"{randint(100, 999)} {street_names()}"
+
+    def generate_email(_name):
+        return f"{_name.lower().replace(' ', '_')}.{randint(1, 99)}@{domain_names[selector(10)-1]}"
+
+    def generate_phone_number():
+        return fake.phone_number()
+
+    # Default columns if no selection is provided
+    all_columns = {
+        "ID": lambda: randint(1, 1000000),
+        "Name": generate_name,
+        "Address": generate_address,
+        "City": cities,
+        "Email": lambda name: generate_email(name),
+        "Phone": generate_phone_number,
+        "DOB": dob,
+        "Work": companies,
+        "Zipcode": zipcodes,
+        "Country": countries,
+        "sales_data": sales_data
+    }
+
+    # Filter the available columns based on col_selection
+    if col_selection is not None:
+        all_columns = {col: all_columns[col] for col in col_selection if col in all_columns}
+
+    # Use a generator to yield one record at a time
+    for _ in range(n_records):
+        record = {}
+        for col, generator in all_columns.items():
+            if col == "Email":  # Email depends on the Name column
+                name = record.get("Name", generate_name())
+                record["Email"] = generate_email(name)
+            else:
+                record[col] = generator()
+        yield record
+
+
+def write_fake_data():
+    df = create_fake_data()
+    df.write_parquet('backend/tests/data/fake_data.parquet')
+    df.write_csv('backend/tests/data/fake_data.csv')
+    df.write_excel('backend/tests/data/fake_data.xlsx')

flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py

@@ -0,0 +1 @@
+from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import *

flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py

@@ -0,0 +1,36 @@
+from typing import Any, Optional, Literal
+from pydantic import BaseModel
+from flowfile_core.schemas.transform_schema import FuzzyMap
+
+OperationType = Literal['store', 'calculate_schema', 'calculate_number_of_records', 'write_output', 'store_sample']
+
+
+class PolarsOperation(BaseModel):
+    operation: bytes
+
+
+class PolarsScript(PolarsOperation):
+    task_id: Optional[str] = None
+    cache_dir: Optional[str] = None
+    operation_type: OperationType
+
+
+class FuzzyJoinInput(BaseModel):
+    task_id: Optional[str] = None
+    cache_dir: Optional[str] = None
+    left_df_operation: PolarsOperation
+    right_df_operation: PolarsOperation
+    fuzzy_maps: list[FuzzyMap]
+    flowfile_node_id: int|str
+    flowfile_flow_id: int
+
+
+class Status(BaseModel):
+    background_task_id: str
+    status: Literal['Processing', 'Completed', 'Error', 'Unknown Error', 'Starting', 'Cancelled']  # Type alias for status
+    file_ref: str
+    progress: int = 0
+    error_message: Optional[str] = None  # Add error_message field
+    results: Any
+    result_type: Literal['polars', 'other'] = 'polars'
+