Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_worker/create/funcs.py

@@ -1,9 +1,16 @@
-import polars as pl
 import os
 
-from flowfile_worker.create.models import ReceivedTable, InputCsvTable, InputJsonTable, InputExcelTable, InputParquetTable
+import polars as pl
+
+from flowfile_worker.create.models import (
+    InputCsvTable,
+    InputExcelTable,
+    InputJsonTable,
+    InputParquetTable,
+    ReceivedTable,
+)
+from flowfile_worker.create.read_excel_tables import df_from_calamine_xlsx, df_from_openpyxl
 from flowfile_worker.create.utils import create_fake_data
-from flowfile_worker.create.read_excel_tables import df_from_openpyxl, df_from_calamine_xlsx
 
 
 def create_from_path_json(received_table: ReceivedTable):
@@ -13,42 +20,53 @@ def create_from_path_json(received_table: ReceivedTable):
     f = received_table.abs_file_path
     gbs_to_load = os.path.getsize(f) / 1024 / 1000 / 1000
     low_mem = gbs_to_load > 10
-    if input_table_settings.encoding.upper() == 'UTF8' or input_table_settings.encoding.upper() == 'UTF-8':
+    if input_table_settings.encoding.upper() == "UTF8" or input_table_settings.encoding.upper() == "UTF-8":
         try:
-            df = pl.scan_csv(f,
-                               low_memory=low_mem,
-                               try_parse_dates=True,
-                               separator=input_table_settings.delimiter,
-                               has_header=input_table_settings.has_headers,
-                               skip_rows=input_table_settings.starting_from_line,
-                               encoding='utf8',
-                               infer_schema_length=input_table_settings.infer_schema_length)
+            df = pl.scan_csv(
+                f,
+                low_memory=low_mem,
+                try_parse_dates=True,
+                separator=input_table_settings.delimiter,
+                has_header=input_table_settings.has_headers,
+                skip_rows=input_table_settings.starting_from_line,
+                encoding="utf8",
+                infer_schema_length=input_table_settings.infer_schema_length,
+            )
             df.head(1).collect()
             return df
         except:
             try:
-                df = pl.scan_csv(f, low_memory=low_mem,
-                                   separator=input_table_settings.delimiter,
-                                   has_header=input_table_settings.has_headers,
-                                   skip_rows=input_table_settings.starting_from_line,
-                                   encoding='utf8-lossy',
-                                   ignore_errors=True)
+                df = pl.scan_csv(
+                    f,
+                    low_memory=low_mem,
+                    separator=input_table_settings.delimiter,
+                    has_header=input_table_settings.has_headers,
+                    skip_rows=input_table_settings.starting_from_line,
+                    encoding="utf8-lossy",
+                    ignore_errors=True,
+                )
                 return df
             except:
-                df = pl.scan_csv(f, low_memory=low_mem,
-                                   separator=input_table_settings.delimiter,
-                                   has_header=input_table_settings.has_headers,
-                                   skip_rows=input_table_settings.starting_from_line,
-                                   encoding='utf8',
-                                   ignore_errors=True)
+                df = pl.scan_csv(
+                    f,
+                    low_memory=low_mem,
+                    separator=input_table_settings.delimiter,
+                    has_header=input_table_settings.has_headers,
+                    skip_rows=input_table_settings.starting_from_line,
+                    encoding="utf8",
+                    ignore_errors=True,
+                )
                 return df
     else:
-        df = pl.read_csv(f, low_memory=low_mem,
-                           separator=input_table_settings.delimiter,
-                           has_header=input_table_settings.has_headers,
-                           skip_rows=input_table_settings.starting_from_line,
-                           encoding=input_table_settings.encoding,
-                           ignore_errors=True)
+        df = pl.read_csv(
+            f,
+            low_memory=low_mem,
+            separator=input_table_settings.delimiter,
+            has_header=input_table_settings.has_headers,
+            skip_rows=input_table_settings.starting_from_line,
+            encoding=input_table_settings.encoding,
+            ignore_errors=True,
+        )
         return df
 
 
@@ -59,43 +77,53 @@ def create_from_path_csv(received_table: ReceivedTable) -> pl.DataFrame:
     input_table_settings: InputCsvTable = received_table.table_settings
     gbs_to_load = os.path.getsize(f) / 1024 / 1000 / 1000
     low_mem = gbs_to_load > 10
-    if input_table_settings.encoding.upper() == 'UTF8' or input_table_settings.encoding.upper() == 'UTF-8':
+    if input_table_settings.encoding.upper() == "UTF8" or input_table_settings.encoding.upper() == "UTF-8":
         try:
-            df = pl.scan_csv(f,
-                               low_memory=low_mem,
-                               try_parse_dates=True,
-                               separator=input_table_settings.delimiter,
-                               has_header=input_table_settings.has_headers,
-                               skip_rows=input_table_settings.starting_from_line,
-                               encoding='utf8',
-                               infer_schema_length=input_table_settings.infer_schema_length)
+            df = pl.scan_csv(
+                f,
+                low_memory=low_mem,
+                try_parse_dates=True,
+                separator=input_table_settings.delimiter,
+                has_header=input_table_settings.has_headers,
+                skip_rows=input_table_settings.starting_from_line,
+                encoding="utf8",
+                infer_schema_length=input_table_settings.infer_schema_length,
+            )
             df.head(1).collect()
             return df
         except:
             try:
-                df = pl.scan_csv(f, low_memory=low_mem,
-                                   separator=input_table_settings.delimiter,
-                                   has_header=input_table_settings.has_headers,
-                                   skip_rows=input_table_settings.starting_from_line,
-                                   encoding='utf8-lossy',
-                                   ignore_errors=True)
+                df = pl.scan_csv(
+                    f,
+                    low_memory=low_mem,
+                    separator=input_table_settings.delimiter,
+                    has_header=input_table_settings.has_headers,
+                    skip_rows=input_table_settings.starting_from_line,
+                    encoding="utf8-lossy",
+                    ignore_errors=True,
+                )
                 return df
             except:
-                df = pl.scan_csv(f, low_memory=low_mem,
-                                   separator=input_table_settings.delimiter,
-                                   has_header=input_table_settings.has_headers,
-                                   skip_rows=input_table_settings.starting_from_line,
-                                   encoding='utf8',
-                                   ignore_errors=True)
+                df = pl.scan_csv(
+                    f,
+                    low_memory=low_mem,
+                    separator=input_table_settings.delimiter,
+                    has_header=input_table_settings.has_headers,
+                    skip_rows=input_table_settings.starting_from_line,
+                    encoding="utf8",
+                    ignore_errors=True,
+                )
                 return df
     else:
-        df = pl.read_csv(f,
-                           low_memory=low_mem,
-                           separator=input_table_settings.delimiter,
-                           has_header=input_table_settings.has_headers,
-                           skip_rows=input_table_settings.starting_from_line,
-                           encoding=input_table_settings.encoding,
-                           ignore_errors=True)
+        df = pl.read_csv(
+            f,
+            low_memory=low_mem,
+            separator=input_table_settings.delimiter,
+            has_header=input_table_settings.has_headers,
+            skip_rows=input_table_settings.starting_from_line,
+            encoding=input_table_settings.encoding,
+            ignore_errors=True,
+        )
         return df
 
 
@@ -116,30 +144,36 @@ def create_from_path_excel(received_table: ReceivedTable):
     input_table_settings: InputExcelTable = received_table.table_settings
 
     if input_table_settings.type_inference:
-        engine = 'openpyxl'
+        engine = "openpyxl"
     elif input_table_settings.start_row > 0 and input_table_settings.start_column == 0:
-        engine = 'calamine' if input_table_settings.has_headers else 'xlsx2csv'
+        engine = "calamine" if input_table_settings.has_headers else "xlsx2csv"
     elif input_table_settings.start_column > 0 or input_table_settings.start_row > 0:
-        engine = 'openpyxl'
+        engine = "openpyxl"
     else:
-        engine = 'calamine'
+        engine = "calamine"
 
     sheet_name = input_table_settings.sheet_name
 
-    if engine == 'calamine':
-        df = df_from_calamine_xlsx(file_path=received_table.abs_file_path, sheet_name=sheet_name,
-                                   start_row=input_table_settings.start_row, end_row=input_table_settings.end_row)
+    if engine == "calamine":
+        df = df_from_calamine_xlsx(
+            file_path=received_table.abs_file_path,
+            sheet_name=sheet_name,
+            start_row=input_table_settings.start_row,
+            end_row=input_table_settings.end_row,
+        )
         if input_table_settings.end_column > 0:
             end_col_index = input_table_settings.end_column
             cols_to_select = [df.columns[i] for i in range(input_table_settings.start_column, end_col_index)]
             df = df.select(cols_to_select)
 
-    elif engine == 'xlsx2csv':
-        csv_options = {'has_header': input_table_settings.has_headers, 'skip_rows': input_table_settings.start_row}
-        df = pl.read_excel(source=received_table.abs_file_path,
-                           read_options=csv_options,
-                           engine='xlsx2csv',
-                           sheet_name=input_table_settings.sheet_name)
+    elif engine == "xlsx2csv":
+        csv_options = {"has_header": input_table_settings.has_headers, "skip_rows": input_table_settings.start_row}
+        df = pl.read_excel(
+            source=received_table.abs_file_path,
+            read_options=csv_options,
+            engine="xlsx2csv",
+            sheet_name=input_table_settings.sheet_name,
+        )
         end_col_index = input_table_settings.end_column if input_table_settings.end_column > 0 else len(df.columns)
         cols_to_select = [df.columns[i] for i in range(input_table_settings.start_column, end_col_index)]
         df = df.select(cols_to_select)
@@ -149,10 +183,13 @@ def create_from_path_excel(received_table: ReceivedTable):
     else:
         max_col = input_table_settings.end_column if input_table_settings.end_column > 0 else None
         max_row = input_table_settings.end_row + 1 if input_table_settings.end_row > 0 else None
-        df = df_from_openpyxl(file_path=received_table.abs_file_path,
-                              sheet_name=input_table_settings.sheet_name,
-                              min_row=input_table_settings.start_row + 1,
-                              min_col=input_table_settings.start_column + 1,
-                              max_row=max_row,
-                              max_col=max_col, has_headers=input_table_settings.has_headers)
+        df = df_from_openpyxl(
+            file_path=received_table.abs_file_path,
+            sheet_name=input_table_settings.sheet_name,
+            min_row=input_table_settings.start_row + 1,
+            min_col=input_table_settings.start_column + 1,
+            max_row=max_row,
+            max_col=max_col,
+            has_headers=input_table_settings.has_headers,
+        )
     return df

flowfile_worker/create/models.py

@@ -1,7 +1,8 @@
-from pydantic import BaseModel, Field, model_validator, field_validator
-from typing import List, Optional, Literal, Annotated
 import os
 from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, Field, field_validator, model_validator
 
 
 class MinimalFieldInfo(BaseModel):
@@ -11,19 +12,21 @@ class MinimalFieldInfo(BaseModel):
 
 class InputTableBase(BaseModel):
     """Base settings for input file operations."""
+
     file_type: str  # Will be overridden with Literal in subclasses
 
 
 class InputCsvTable(InputTableBase):
     """Defines settings for reading a CSV file."""
-    file_type: Literal['csv'] = 'csv'
-    reference: str = ''
+
+    file_type: Literal["csv"] = "csv"
+    reference: str = ""
     starting_from_line: int = 0
-    delimiter: str = ','
+    delimiter: str = ","
     has_headers: bool = True
-    encoding: str = 'utf-8'
-    parquet_ref: Optional[str] = None
-    row_delimiter: str = '\n'
+    encoding: str = "utf-8"
+    parquet_ref: str | None = None
+    row_delimiter: str = "\n"
     quote_char: str = '"'
     infer_schema_length: int = 10_000
     truncate_ragged_lines: bool = False
@@ -32,18 +35,21 @@ class InputCsvTable(InputTableBase):
 
 class InputJsonTable(InputCsvTable):
     """Defines settings for reading a JSON file."""
-    file_type: Literal['json'] = 'json'
+
+    file_type: Literal["json"] = "json"
 
 
 class InputParquetTable(InputTableBase):
     """Defines settings for reading a Parquet file."""
-    file_type: Literal['parquet'] = 'parquet'
+
+    file_type: Literal["parquet"] = "parquet"
 
 
 class InputExcelTable(InputTableBase):
     """Defines settings for reading an Excel file."""
-    file_type: Literal['excel'] = 'excel'
-    sheet_name: Optional[str] = None
+
+    file_type: Literal["excel"] = "excel"
+    sheet_name: str | None = None
     start_row: int = 0
     start_column: int = 0
     end_row: int = 0
@@ -51,60 +57,58 @@ class InputExcelTable(InputTableBase):
     has_headers: bool = True
     type_inference: bool = False
 
-    @model_validator(mode='after')
+    @model_validator(mode="after")
     def validate_range_values(self):
         """Validates that the Excel cell range is logical."""
         for attribute in [self.start_row, self.start_column, self.end_row, self.end_column]:
             if not isinstance(attribute, int) or attribute < 0:
                 raise ValueError("Row and column indices must be non-negative integers")
-        if (self.end_row > 0 and self.start_row > self.end_row) or \
-                (self.end_column > 0 and self.start_column > self.end_column):
+        if (self.end_row > 0 and self.start_row > self.end_row) or (
+            self.end_column > 0 and self.start_column > self.end_column
+        ):
             raise ValueError("Start row/column must not be greater than end row/column")
         return self
 
 
 # Create the discriminated union (similar to OutputTableSettings)
 InputTableSettings = Annotated[
-    InputCsvTable | InputJsonTable | InputParquetTable | InputExcelTable,
-    Field(discriminator='file_type')
+    InputCsvTable | InputJsonTable | InputParquetTable | InputExcelTable, Field(discriminator="file_type")
 ]
 
 
 # Now create the main ReceivedTable model
 class ReceivedTable(BaseModel):
     """Model for defining a table received from an external source."""
+
     # Metadata fields
-    id: Optional[int] = None
-    name: Optional[str] = None
+    id: int | None = None
+    name: str | None = None
     path: str  # This can be an absolute or relative path
-    directory: Optional[str] = None
+    directory: str | None = None
     analysis_file_available: bool = False
-    status: Optional[str] = None
-    fields: List[MinimalFieldInfo] = Field(default_factory=list)
-    abs_file_path: Optional[str] = None
+    status: str | None = None
+    fields: list[MinimalFieldInfo] = Field(default_factory=list)
+    abs_file_path: str | None = None
 
-    file_type: Literal['csv', 'json', 'parquet', 'excel']
+    file_type: Literal["csv", "json", "parquet", "excel"]
 
     table_settings: InputTableSettings
 
     @classmethod
-    def create_from_path(cls, path: str, file_type: Literal['csv', 'json', 'parquet', 'excel'] = 'csv'):
+    def create_from_path(cls, path: str, file_type: Literal["csv", "json", "parquet", "excel"] = "csv"):
         """Creates an instance from a file path string."""
         filename = Path(path).name
 
         # Create appropriate table_settings based on file_type
         settings_map = {
-            'csv': InputCsvTable(),
-            'json': InputJsonTable(),
-            'parquet': InputParquetTable(),
-            'excel': InputExcelTable(),
+            "csv": InputCsvTable(),
+            "json": InputJsonTable(),
+            "parquet": InputParquetTable(),
+            "excel": InputExcelTable(),
         }
 
         return cls(
-            name=filename,
-            path=path,
-            file_type=file_type,
-            table_settings=settings_map.get(file_type, InputCsvTable())
+            name=filename, path=path, file_type=file_type, table_settings=settings_map.get(file_type, InputCsvTable())
         )
 
     @property
@@ -124,31 +128,30 @@ class ReceivedTable(BaseModel):
             base_path = base_path / self.name
         self.abs_file_path = str(base_path.resolve())
 
-    @field_validator('table_settings', mode='before')
+    @field_validator("table_settings", mode="before")
     @classmethod
     def validate_table_settings(cls, v, info):
         """Ensures table_settings matches the file_type."""
         if v is None:
-            file_type = info.data.get('file_type', 'csv')
+            file_type = info.data.get("file_type", "csv")
             # Create default based on file_type
             settings_map = {
-                'csv': InputCsvTable(),
-                'json': InputJsonTable(),
-                'parquet': InputParquetTable(),
-                'excel': InputExcelTable(),
+                "csv": InputCsvTable(),
+                "json": InputJsonTable(),
+                "parquet": InputParquetTable(),
+                "excel": InputExcelTable(),
             }
             return settings_map.get(file_type, InputCsvTable())
 
         # If it's a dict, add file_type if missing
-        if isinstance(v, dict) and 'file_type' not in v:
-            v['file_type'] = info.data.get('file_type', 'csv')
+        if isinstance(v, dict) and "file_type" not in v:
+            v["file_type"] = info.data.get("file_type", "csv")
 
         return v
 
-    @model_validator(mode='after')
+    @model_validator(mode="after")
     def populate_abs_file_path(self):
         """Ensures the absolute file path is populated after validation."""
         if not self.abs_file_path:
             self.set_absolute_filepath()
         return self
-

flowfile_worker/create/pl_types.py

@@ -1,21 +1,20 @@
 import polars as pl
 
-
 dtype_to_pl = {
-    'int': pl.Int64,
-    'integer': pl.Int64,
-    'char': pl.String,
-    'fixed decimal': pl.Float32,
-    'double': pl.Float64,
-    'float': pl.Float64,
-    'bool': pl.Boolean,
-    'byte': pl.UInt8,
-    'bit': pl.Binary,
-    'date': pl.Date,
-    'datetime': pl.Datetime,
-    'string': pl.String,
-    'str': pl.String,
-    'time': pl.Time,
+    "int": pl.Int64,
+    "integer": pl.Int64,
+    "char": pl.String,
+    "fixed decimal": pl.Float32,
+    "double": pl.Float64,
+    "float": pl.Float64,
+    "bool": pl.Boolean,
+    "byte": pl.UInt8,
+    "bit": pl.Binary,
+    "date": pl.Date,
+    "datetime": pl.Datetime,
+    "string": pl.String,
+    "str": pl.String,
+    "time": pl.Time,
 }
 
 dtype_to_pl_str = {k: v.__name__ for k, v in dtype_to_pl.items()}

flowfile_worker/create/read_excel_tables.py

@@ -1,59 +1,55 @@
-from typing import Generator, List
-from openpyxl import Workbook, load_workbook
-from openpyxl.worksheet.worksheet import Worksheet
 import gc
+from collections.abc import Generator
+
 import polars as pl
+from openpyxl import Workbook, load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
+
 from flowfile_worker.create.utils import create_pl_df_type_save
 
 
-def raw_data_openpyxl(file_path: str,
-                      sheet_name: str = None,
-                      min_row: int = None,
-                      max_row: int = None,
-                      min_col: int = None,
-                      max_col: int = None
-                      ) -> Generator[List, None, None]:
+def raw_data_openpyxl(
+    file_path: str,
+    sheet_name: str = None,
+    min_row: int = None,
+    max_row: int = None,
+    min_col: int = None,
+    max_col: int = None,
+) -> Generator[list, None, None]:
     workbook: Workbook = load_workbook(file_path, data_only=True, read_only=True)
     sheet_name = workbook.sheetnames[0] if sheet_name is None else sheet_name
     sheet: Worksheet = workbook[sheet_name]
-    for row in sheet.iter_rows(min_row=min_row,
-                               max_row=max_row,
-                               min_col=min_col,
-                               max_col=max_col,
-                               values_only=True):
-        yield row
+    yield from sheet.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True)
     workbook.close()
     del workbook
     gc.collect()
 
 
-def df_from_openpyxl(file_path: str,
-                     sheet_name: str = None,
-                     min_row: int = None,
-                     max_row: int = None,
-                     min_col: int = None,
-                     max_col: int = None,
-                     has_headers: bool = True
-                     ) -> pl.DataFrame:
-    data_iterator = raw_data_openpyxl(file_path=file_path,
-                                      sheet_name=sheet_name,
-                                      min_row=min_row,
-                                      max_row=max_row,
-                                      min_col=min_col,
-                                      max_col=max_col)
+def df_from_openpyxl(
+    file_path: str,
+    sheet_name: str = None,
+    min_row: int = None,
+    max_row: int = None,
+    min_col: int = None,
+    max_col: int = None,
+    has_headers: bool = True,
+) -> pl.DataFrame:
+    data_iterator = raw_data_openpyxl(
+        file_path=file_path, sheet_name=sheet_name, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col
+    )
     raw_data = list(data_iterator)
     if len(raw_data) > 0:
         if has_headers:
             columns = []
             for i, col in enumerate(raw_data[0]):
                 if col is None:
-                    col = f'_unnamed_column_{i}'
+                    col = f"_unnamed_column_{i}"
                 elif not isinstance(col, str):
                     col = str(col)
                 columns.append(col)
             columns = ensure_unique(columns)
             df = create_pl_df_type_save(raw_data[1:])
-            renames = {o: n for o, n in zip(df.columns, columns)}
+            renames = {o: n for o, n in zip(df.columns, columns, strict=False)}
             df = df.rename(renames)
 
         else:
@@ -63,7 +59,7 @@ def df_from_openpyxl(file_path: str,
         return pl.DataFrame()
 
 
-def ensure_unique(lst: List[str]) -> List[str]:
+def ensure_unique(lst: list[str]) -> list[str]:
     """
     Ensures that all elements in the input list are unique by appending
     a version number (e.g., '_v1') to duplicates. It continues adding
@@ -99,12 +95,9 @@ def ensure_unique(lst: List[str]) -> List[str]:
 def df_from_calamine_xlsx(file_path: str, sheet_name: str, start_row: int = 0, end_row: int = 0) -> pl.DataFrame:
     read_options = {}
     if start_row > 0:
-        read_options['header_row'] = start_row
+        read_options["header_row"] = start_row
     if end_row > 0:
-        read_options['n_rows'] = end_row - start_row
-    return pl.read_excel(source=file_path,
-                         engine='calamine',
-                         sheet_name=sheet_name,
-                         read_options=read_options,
-                         raise_if_empty=False
-                         )
+        read_options["n_rows"] = end_row - start_row
+    return pl.read_excel(
+        source=file_path, engine="calamine", sheet_name=sheet_name, read_options=read_options, raise_if_empty=False
+    )