Flowfile 0.5.6__py3-none-any.whl → 0.6.1__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/flow_data_engine.py

@@ -7,7 +7,7 @@ from collections.abc import Callable, Generator, Iterable
 from copy import deepcopy
 from dataclasses import dataclass
 from math import ceil
-from typing import Any, Literal, TypeVar, Union
+from typing import Any, Literal, TypeVar
 
 import polars as pl
 
@@ -37,7 +37,11 @@ from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import (
     assert_if_flowfile_schema,
     convert_stats_to_column_info,
 )
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import (
+    cast_str_to_polars_type,
+    get_polars_type,
+    safe_eval_pl_type,
+)
 from flowfile_core.flowfile.flow_data_engine.fuzzy_matching.prepare_for_fuzzy_match import prepare_for_fuzzy_match
 from flowfile_core.flowfile.flow_data_engine.join import (
     get_col_name_to_delete,
@@ -171,7 +175,7 @@ class FlowDataEngine:
     name: str = None
     number_of_records: int = None
     errors: list = None
-    _schema: list["FlowfileColumn"] | None = None
+    _schema: list[FlowfileColumn] | None = None
 
     # Configuration attributes
     _optimize_memory: bool = False
@@ -204,13 +208,11 @@ class FlowDataEngine:
 
     def __init__(
         self,
-        raw_data: Union[
-            list[dict], list[Any], dict[str, Any], "ParquetFile", pl.DataFrame, pl.LazyFrame, input_schema.RawData
-        ] = None,
+        raw_data: list[dict] | list[Any] | dict[str, Any] | ParquetFile | pl.DataFrame | pl.LazyFrame | input_schema.RawData = None,
         path_ref: str = None,
         name: str = None,
         optimize_memory: bool = True,
-        schema: list["FlowfileColumn"] | list[str] | pl.Schema = None,
+        schema: list[FlowfileColumn] | list[str] | pl.Schema = None,
         number_of_records: int = None,
         calculate_schema_stats: bool = False,
         streamable: bool = True,
@@ -523,7 +525,7 @@ class FlowDataEngine:
     @classmethod
     def from_cloud_storage_obj(
         cls, settings: cloud_storage_schemas.CloudStorageReadSettingsInternal
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Creates a FlowDataEngine from an object in cloud storage.
 
         This method supports reading from various cloud storage providers like AWS S3,
@@ -607,7 +609,7 @@ class FlowDataEngine:
         storage_options: dict[str, Any],
         credential_provider: Callable | None,
         read_settings: cloud_storage_schemas.CloudStorageReadSettings,
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Reads Iceberg table(s) from cloud storage."""
         raise NotImplementedError("Failed to read Iceberg table from cloud storage: Not yet implemented")
 
@@ -618,7 +620,7 @@ class FlowDataEngine:
         storage_options: dict[str, Any],
         credential_provider: Callable | None,
         is_directory: bool,
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Reads Parquet file(s) from cloud storage."""
         try:
             # Use scan_parquet for lazy evaluation
@@ -656,7 +658,7 @@ class FlowDataEngine:
         storage_options: dict[str, Any],
         credential_provider: Callable | None,
         read_settings: cloud_storage_schemas.CloudStorageReadSettings,
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Reads a Delta Lake table from cloud storage."""
         try:
             logger.info("Reading Delta file from cloud storage...")
@@ -687,7 +689,7 @@ class FlowDataEngine:
         storage_options: dict[str, Any],
         credential_provider: Callable | None,
         read_settings: cloud_storage_schemas.CloudStorageReadSettings,
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Reads CSV file(s) from cloud storage."""
         try:
             scan_kwargs = {
@@ -730,7 +732,7 @@ class FlowDataEngine:
         storage_options: dict[str, Any],
         credential_provider: Callable | None,
         is_directory: bool,
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Reads JSON file(s) from cloud storage."""
         try:
             if is_directory:
@@ -821,6 +823,7 @@ class FlowDataEngine:
         if self.lazy and isinstance(df, pl.DataFrame):
             raise Exception("Cannot set a non-lazy dataframe to a lazy flowfile")
         self._data_frame = df
+        self._schema = None
 
     @staticmethod
     def _create_schema_stats_from_pl_schema(pl_schema: pl.Schema) -> list[dict]:
@@ -968,7 +971,7 @@ class FlowDataEngine:
 
     def do_group_by(
         self, group_by_input: transform_schemas.GroupByInput, calculate_schema_stats: bool = True
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Performs a group-by operation on the DataFrame.
 
         Args:
@@ -1008,7 +1011,7 @@ class FlowDataEngine:
             calculate_schema_stats=calculate_schema_stats,
         )
 
-    def do_sort(self, sorts: list[transform_schemas.SortByInput]) -> "FlowDataEngine":
+    def do_sort(self, sorts: list[transform_schemas.SortByInput]) -> FlowDataEngine:
         """Sorts the DataFrame by one or more columns.
 
         Args:
@@ -1027,7 +1030,7 @@ class FlowDataEngine:
 
     def change_column_types(
         self, transforms: list[transform_schemas.SelectInput], calculate_schema: bool = False
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Changes the data type of one or more columns.
 
         Args:
@@ -1040,7 +1043,7 @@ class FlowDataEngine:
         """
         dtypes = [dtype.base_type() for dtype in self.data_frame.collect_schema().dtypes()]
         idx_mapping = list(
-            (transform.old_name, self.cols_idx.get(transform.old_name), getattr(pl, transform.polars_type))
+            (transform.old_name, self.cols_idx.get(transform.old_name), get_polars_type(transform.polars_type))
             for transform in transforms
             if transform.data_type is not None
         )
@@ -1122,7 +1125,7 @@ class FlowDataEngine:
             return self.data_frame.to_dict(as_series=False)
 
     @classmethod
-    def create_from_external_source(cls, external_source: ExternalDataSource) -> "FlowDataEngine":
+    def create_from_external_source(cls, external_source: ExternalDataSource) -> FlowDataEngine:
         """Creates a FlowDataEngine from an external data source.
 
         Args:
@@ -1142,7 +1145,7 @@ class FlowDataEngine:
         return ff
 
     @classmethod
-    def create_from_sql(cls, sql: str, conn: Any) -> "FlowDataEngine":
+    def create_from_sql(cls, sql: str, conn: Any) -> FlowDataEngine:
         """Creates a FlowDataEngine by executing a SQL query.
 
         Args:
@@ -1155,7 +1158,7 @@ class FlowDataEngine:
         return cls(pl.read_sql(sql, conn))
 
     @classmethod
-    def create_from_schema(cls, schema: list[FlowfileColumn]) -> "FlowDataEngine":
+    def create_from_schema(cls, schema: list[FlowfileColumn]) -> FlowDataEngine:
         """Creates an empty FlowDataEngine from a schema definition.
 
         Args:
@@ -1172,7 +1175,7 @@ class FlowDataEngine:
         return cls(df, schema=schema, calculate_schema_stats=False, number_of_records=0)
 
     @classmethod
-    def create_from_path(cls, received_table: input_schema.ReceivedTable) -> "FlowDataEngine":
+    def create_from_path(cls, received_table: input_schema.ReceivedTable) -> FlowDataEngine:
         """Creates a FlowDataEngine from a local file path.
 
         Supports various file types like CSV, Parquet, and Excel.
@@ -1200,7 +1203,7 @@ class FlowDataEngine:
         return flow_file
 
     @classmethod
-    def create_random(cls, number_of_records: int = 1000) -> "FlowDataEngine":
+    def create_random(cls, number_of_records: int = 1000) -> FlowDataEngine:
         """Creates a FlowDataEngine with randomly generated data.
 
         Useful for testing and examples.
@@ -1214,7 +1217,7 @@ class FlowDataEngine:
         return cls(create_fake_data(number_of_records))
 
     @classmethod
-    def generate_enumerator(cls, length: int = 1000, output_name: str = "output_column") -> "FlowDataEngine":
+    def generate_enumerator(cls, length: int = 1000, output_name: str = "output_column") -> FlowDataEngine:
         """Generates a FlowDataEngine with a single column containing a sequence of integers.
 
         Args:
@@ -1277,7 +1280,7 @@ class FlowDataEngine:
 
         return flow_file_columns
 
-    def split(self, split_input: transform_schemas.TextToRowsInput) -> "FlowDataEngine":
+    def split(self, split_input: transform_schemas.TextToRowsInput) -> FlowDataEngine:
         """Splits a column's text values into multiple rows based on a delimiter.
 
         This operation is often referred to as "exploding" the DataFrame, as it
@@ -1304,7 +1307,7 @@ class FlowDataEngine:
 
         return FlowDataEngine(df)
 
-    def unpivot(self, unpivot_input: transform_schemas.UnpivotInput) -> "FlowDataEngine":
+    def unpivot(self, unpivot_input: transform_schemas.UnpivotInput) -> FlowDataEngine:
         """Converts the DataFrame from a wide to a long format.
 
         This is the inverse of a pivot operation, taking columns and transforming
@@ -1328,7 +1331,7 @@ class FlowDataEngine:
 
         return FlowDataEngine(result)
 
-    def do_pivot(self, pivot_input: transform_schemas.PivotInput, node_logger: NodeLogger = None) -> "FlowDataEngine":
+    def do_pivot(self, pivot_input: transform_schemas.PivotInput, node_logger: NodeLogger = None) -> FlowDataEngine:
         """Converts the DataFrame from a long to a wide format, aggregating values.
 
         Args:
@@ -1399,7 +1402,7 @@ class FlowDataEngine:
 
         return FlowDataEngine(df, calculate_schema_stats=False)
 
-    def do_filter(self, predicate: str) -> "FlowDataEngine":
+    def do_filter(self, predicate: str) -> FlowDataEngine:
         """Filters rows based on a predicate expression.
 
         Args:
@@ -1418,7 +1421,7 @@ class FlowDataEngine:
         df = self.data_frame.filter(f)
         return FlowDataEngine(df, schema=self.schema, streamable=self._streamable)
 
-    def add_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> "FlowDataEngine":
+    def add_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> FlowDataEngine:
         """Adds a record ID (row number) column to the DataFrame.
 
         Can generate a simple sequential ID or a grouped ID that resets for
@@ -1435,7 +1438,7 @@ class FlowDataEngine:
             return self._add_grouped_record_id(record_id_settings)
         return self._add_simple_record_id(record_id_settings)
 
-    def _add_grouped_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> "FlowDataEngine":
+    def _add_grouped_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> FlowDataEngine:
         """Adds a record ID column with grouping."""
         select_cols = [pl.col(record_id_settings.output_column_name)] + [pl.col(c) for c in self.columns]
 
@@ -1456,7 +1459,7 @@ class FlowDataEngine:
 
         return FlowDataEngine(df, schema=output_schema)
 
-    def _add_simple_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> "FlowDataEngine":
+    def _add_simple_record_id(self, record_id_settings: transform_schemas.RecordIdInput) -> FlowDataEngine:
         """Adds a simple sequential record ID column."""
         df = self.data_frame.with_row_index(record_id_settings.output_column_name, record_id_settings.offset)
 
@@ -1494,7 +1497,7 @@ class FlowDataEngine:
         """Returns a string representation of the FlowDataEngine."""
         return f"flow data engine\n{self.data_frame.__repr__()}"
 
-    def __call__(self) -> "FlowDataEngine":
+    def __call__(self) -> FlowDataEngine:
         """Makes the class instance callable, returning itself."""
         return self
 
@@ -1502,7 +1505,7 @@ class FlowDataEngine:
         """Returns the number of records in the table."""
         return self.number_of_records if self.number_of_records >= 0 else self.get_number_of_records()
 
-    def cache(self) -> "FlowDataEngine":
+    def cache(self) -> FlowDataEngine:
         """Caches the current DataFrame to disk and updates the internal reference.
 
         This triggers a background process to write the current LazyFrame's result
@@ -1557,7 +1560,7 @@ class FlowDataEngine:
             df = self.collect()
         return df.to_dicts()
 
-    def __get_sample__(self, n_rows: int = 100, streamable: bool = True) -> "FlowDataEngine":
+    def __get_sample__(self, n_rows: int = 100, streamable: bool = True) -> FlowDataEngine:
         """Internal method to get a sample of the data."""
         if not self.lazy:
             df = self.data_frame.lazy()
@@ -1581,7 +1584,7 @@ class FlowDataEngine:
         shuffle: bool = False,
         seed: int = None,
         execution_location: ExecutionLocationsLiteral | None = None,
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Gets a sample of rows from the DataFrame.
 
         Args:
@@ -1620,7 +1623,7 @@ class FlowDataEngine:
 
         return FlowDataEngine(sample_df, schema=self.schema)
 
-    def get_subset(self, n_rows: int = 100) -> "FlowDataEngine":
+    def get_subset(self, n_rows: int = 100) -> FlowDataEngine:
         """Gets the first `n_rows` from the DataFrame.
 
         Args:
@@ -1636,7 +1639,7 @@ class FlowDataEngine:
 
     def iter_batches(
         self, batch_size: int = 1000, columns: list | tuple | str = None
-    ) -> Generator["FlowDataEngine", None, None]:
+    ) -> Generator[FlowDataEngine, None, None]:
         """Iterates over the DataFrame in batches.
 
         Args:
@@ -1657,7 +1660,7 @@ class FlowDataEngine:
     def start_fuzzy_join(
         self,
         fuzzy_match_input: transform_schemas.FuzzyMatchInput,
-        other: "FlowDataEngine",
+        other: FlowDataEngine,
         file_ref: str,
         flow_id: int = -1,
         node_id: int | str = -1,
@@ -1696,7 +1699,7 @@ class FlowDataEngine:
     def fuzzy_join_external(
         self,
         fuzzy_match_input: transform_schemas.FuzzyMatchInput,
-        other: "FlowDataEngine",
+        other: FlowDataEngine,
         file_ref: str = None,
         flow_id: int = -1,
         node_id: int = -1,
@@ -1722,9 +1725,9 @@ class FlowDataEngine:
     def fuzzy_join(
         self,
         fuzzy_match_input: transform_schemas.FuzzyMatchInput,
-        other: "FlowDataEngine",
+        other: FlowDataEngine,
         node_logger: NodeLogger = None,
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         fuzzy_match_input_manager = transform_schemas.FuzzyMatchInputManager(fuzzy_match_input)
         left_df, right_df = prepare_for_fuzzy_match(
             left=self, right=other, fuzzy_match_input_manager=fuzzy_match_input_manager
@@ -1741,8 +1744,8 @@ class FlowDataEngine:
         cross_join_input: transform_schemas.CrossJoinInput,
         auto_generate_selection: bool,
         verify_integrity: bool,
-        other: "FlowDataEngine",
-    ) -> "FlowDataEngine":
+        other: FlowDataEngine,
+    ) -> FlowDataEngine:
         """Performs a cross join with another DataFrame.
 
         A cross join produces the Cartesian product of the two DataFrames.
@@ -1796,8 +1799,8 @@ class FlowDataEngine:
         join_input: transform_schemas.JoinInput,
         auto_generate_selection: bool,
         verify_integrity: bool,
-        other: "FlowDataEngine",
-    ) -> "FlowDataEngine":
+        other: FlowDataEngine,
+    ) -> FlowDataEngine:
         """Performs a standard SQL-style join with another DataFrame."""
         # Create manager from input
         join_manager = transform_schemas.JoinInputManager(join_input)
@@ -1864,7 +1867,7 @@ class FlowDataEngine:
 
         return FlowDataEngine(joined_df, calculate_schema_stats=False, number_of_records=0, streamable=False)
 
-    def solve_graph(self, graph_solver_input: transform_schemas.GraphSolverInput) -> "FlowDataEngine":
+    def solve_graph(self, graph_solver_input: transform_schemas.GraphSolverInput) -> FlowDataEngine:
         """Solves a graph problem represented by 'from' and 'to' columns.
 
         This is used for operations like finding connected components in a graph.
@@ -1883,7 +1886,7 @@ class FlowDataEngine:
         )
         return FlowDataEngine(lf)
 
-    def add_new_values(self, values: Iterable, col_name: str = None) -> "FlowDataEngine":
+    def add_new_values(self, values: Iterable, col_name: str = None) -> FlowDataEngine:
         """Adds a new column with the provided values.
 
         Args:
@@ -1897,7 +1900,7 @@ class FlowDataEngine:
             col_name = "new_values"
         return FlowDataEngine(self.data_frame.with_columns(pl.Series(values).alias(col_name)))
 
-    def get_record_count(self) -> "FlowDataEngine":
+    def get_record_count(self) -> FlowDataEngine:
         """Returns a new FlowDataEngine with a single column 'number_of_records'
         containing the total number of records.
 
@@ -1906,7 +1909,7 @@ class FlowDataEngine:
         """
         return FlowDataEngine(self.data_frame.select(pl.len().alias("number_of_records")))
 
-    def assert_equal(self, other: "FlowDataEngine", ordered: bool = True, strict_schema: bool = False):
+    def assert_equal(self, other: FlowDataEngine, ordered: bool = True, strict_schema: bool = False):
         """Asserts that this DataFrame is equal to another.
 
         Useful for testing.
@@ -2075,7 +2078,7 @@ class FlowDataEngine:
             [transform_schemas.SelectInput(old_name=c.name, data_type=c.data_type) for c in self.schema]
         )
 
-    def select_columns(self, list_select: list[str] | tuple[str] | str) -> "FlowDataEngine":
+    def select_columns(self, list_select: list[str] | tuple[str] | str) -> FlowDataEngine:
         """Selects a subset of columns from the DataFrame.
 
         Args:
@@ -2098,7 +2101,7 @@ class FlowDataEngine:
             streamable=self._streamable,
         )
 
-    def drop_columns(self, columns: list[str]) -> "FlowDataEngine":
+    def drop_columns(self, columns: list[str]) -> FlowDataEngine:
         """Drops specified columns from the DataFrame.
 
         Args:
@@ -2115,7 +2118,7 @@ class FlowDataEngine:
             self.data_frame.select(cols_for_select), number_of_records=self.number_of_records, schema=new_schema
         )
 
-    def reorganize_order(self, column_order: list[str]) -> "FlowDataEngine":
+    def reorganize_order(self, column_order: list[str]) -> FlowDataEngine:
         """Reorganizes columns into a specified order.
 
         Args:
@@ -2130,7 +2133,7 @@ class FlowDataEngine:
 
     def apply_flowfile_formula(
         self, func: str, col_name: str, output_data_type: pl.DataType = None
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Applies a formula to create a new column or transform an existing one.
 
         Args:
@@ -2149,7 +2152,7 @@ class FlowDataEngine:
 
         return FlowDataEngine(df2, number_of_records=self.number_of_records)
 
-    def apply_sql_formula(self, func: str, col_name: str, output_data_type: pl.DataType = None) -> "FlowDataEngine":
+    def apply_sql_formula(self, func: str, col_name: str, output_data_type: pl.DataType = None) -> FlowDataEngine:
         """Applies an SQL-style formula using `pl.sql_expr`.
 
         Args:
@@ -2170,7 +2173,7 @@ class FlowDataEngine:
 
     def output(
         self, output_fs: input_schema.OutputSettings, flow_id: int, node_id: int | str, execute_remote: bool = True
-    ) -> "FlowDataEngine":
+    ) -> FlowDataEngine:
         """Writes the DataFrame to an output file.
 
         Can execute the write operation locally or in a remote worker process.
@@ -2214,7 +2217,7 @@ class FlowDataEngine:
             logger.info("Finished writing output")
         return self
 
-    def make_unique(self, unique_input: transform_schemas.UniqueInput = None) -> "FlowDataEngine":
+    def make_unique(self, unique_input: transform_schemas.UniqueInput = None) -> FlowDataEngine:
         """Gets the unique rows from the DataFrame.
 
         Args:
@@ -2228,7 +2231,7 @@ class FlowDataEngine:
             return FlowDataEngine(self.data_frame.unique())
         return FlowDataEngine(self.data_frame.unique(unique_input.columns, keep=unique_input.strategy))
 
-    def concat(self, other: Iterable["FlowDataEngine"] | "FlowDataEngine") -> "FlowDataEngine":
+    def concat(self, other: Iterable[FlowDataEngine] | FlowDataEngine) -> FlowDataEngine:
         """Concatenates this DataFrame with one or more other DataFrames.
 
         Args:
@@ -2243,7 +2246,7 @@ class FlowDataEngine:
         dfs: list[pl.LazyFrame] | list[pl.DataFrame] = [self.data_frame] + [flt.data_frame for flt in other]
         return FlowDataEngine(pl.concat(dfs, how="diagonal_relaxed"))
 
-    def do_select(self, select_inputs: transform_schemas.SelectInputs, keep_missing: bool = True) -> "FlowDataEngine":
+    def do_select(self, select_inputs: transform_schemas.SelectInputs, keep_missing: bool = True) -> FlowDataEngine:
         """Performs a complex column selection, renaming, and reordering operation.
 
         Args:
@@ -2256,7 +2259,6 @@ class FlowDataEngine:
         """
         new_schema = deepcopy(self.schema)
         renames = [r for r in select_inputs.renames if r.is_available]
-
         if not keep_missing:
             drop_cols = set(self.data_frame.collect_schema().names()) - set(r.old_name for r in renames).union(
                 set(r.old_name for r in renames if not r.keep)
@@ -2322,7 +2324,7 @@ class FlowDataEngine:
         return cls(external_fetcher.get_result())
 
 
-def execute_polars_code(*flowfile_tables: "FlowDataEngine", code: str) -> "FlowDataEngine":
+def execute_polars_code(*flowfile_tables: FlowDataEngine, code: str) -> FlowDataEngine:
     """Executes arbitrary Polars code on one or more FlowDataEngine objects.
 
     This function takes a string of Python code that uses Polars and executes it.

flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py

@@ -5,32 +5,6 @@ import polars as pl
 DataTypeGroup = Literal["numeric", "string", "datetime", "boolean", "binary", "complex", "unknown"]
 
 
-def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
-    if isinstance(pl_type, pl.List):
-        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
-        return f"pl.List({inner_str})"
-    elif isinstance(pl_type, pl.Array):
-        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
-        return f"pl.Array({inner_str})"
-    elif isinstance(pl_type, pl.Decimal):
-        precision = pl_type.precision if hasattr(pl_type, "precision") else None
-        scale = pl_type.scale if hasattr(pl_type, "scale") else None
-        if precision is not None and scale is not None:
-            return f"pl.Decimal({precision}, {scale})"
-        elif precision is not None:
-            return f"pl.Decimal({precision})"
-        else:
-            return "pl.Decimal()"
-    elif isinstance(pl_type, pl.Struct):
-        # Handle Struct with field definitions
-        fields = []
-        if hasattr(pl_type, "fields"):
-            for field in pl_type.fields:
-                field_name = field.name
-                field_type = convert_pl_type_to_string(field.dtype, inner=True)
-                fields.append(f'pl.Field("{field_name}", {field_type})')
-        field_str = ", ".join(fields)
-        return f"pl.Struct([{field_str}])"
-    else:
-        # For base types, we want the full pl.TypeName format
-        return str(pl_type.base_type()) if not inner else f"pl.{pl_type}"
+def convert_pl_type_to_string(pl_type: pl.DataType) -> str:
+    """Convert a Polars DataType to its string representation."""
+    return str(pl_type)

flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py

@@ -21,20 +21,51 @@ dtype_to_pl = {
 def safe_eval_pl_type(type_string: str):
     """
     Safely evaluate a Polars type string with restricted namespace.
-    Only allows Polars types and basic Python literals.
+    Supports both formats:
+      - With pl. prefix: pl.List(pl.Int64)
+      - Without pl. prefix: List(Int64)
     """
     # Define allowed names in the evaluation namespace
     safe_dict = {
-        # Polars module and types
+        # Keep pl module for backwards compatibility with pl.X format
         "pl": pl,
-        # Basic Python built-ins for literals
-        "int": int,
-        "str": str,
-        "float": float,
-        "bool": bool,
-        "list": list,
-        "dict": dict,
-        "tuple": tuple,
+
+        # Polars types directly available (without pl. prefix)
+        "List": pl.List,
+        "Array": pl.Array,
+        "Struct": pl.Struct,
+        "Field": pl.Field,
+        "Decimal": pl.Decimal,
+
+        # Integer types
+        "Int8": pl.Int8,
+        "Int16": pl.Int16,
+        "Int32": pl.Int32,
+        "Int64": pl.Int64,
+        "Int128": pl.Int128,
+        "UInt8": pl.UInt8,
+        "UInt16": pl.UInt16,
+        "UInt32": pl.UInt32,
+        "UInt64": pl.UInt64,
+
+        # Float types
+        "Float32": pl.Float32,
+        "Float64": pl.Float64,
+
+        # Other types
+        "Boolean": pl.Boolean,
+        "String": pl.String,
+        "Utf8": pl.Utf8,
+        "Binary": pl.Binary,
+        "Date": pl.Date,
+        "Time": pl.Time,
+        "Datetime": pl.Datetime,
+        "Duration": pl.Duration,
+        "Categorical": pl.Categorical,
+        "Enum": pl.Enum,
+        "Null": pl.Null,
+        "Object": pl.Object,
+
         # Disable dangerous built-ins
         "__builtins__": {},
     }
@@ -57,10 +88,10 @@ def get_polars_type(dtype: str):
     pl_datetype = dtype_to_pl.get(dtype.lower())
     if pl_datetype is not None:
         return pl_datetype
-    elif hasattr(pl, dtype):
-        return getattr(pl, dtype)
-    else:
-        return pl.String
+    try:
+        return safe_eval_pl_type(dtype)
+    except Exception:
+        return pl.String  # Fallback to String if evaluation fails
 
 
 def cast_str_to_polars_type(dtype: str) -> pl.DataType:

flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py

@@ -1,13 +1,30 @@
-from typing import Any, Literal
+from base64 import b64decode, b64encode
+from typing import Annotated, Any, Literal
 
 from pl_fuzzy_frame_match.models import FuzzyMapping
-from pydantic import BaseModel
+from pydantic import BaseModel, BeforeValidator, PlainSerializer
 
 OperationType = Literal["store", "calculate_schema", "calculate_number_of_records", "write_output", "store_sample"]
 
 
+# Custom type for bytes that serializes to/from base64 string in JSON
+def _decode_bytes(v: Any) -> bytes:
+    if isinstance(v, bytes):
+        return v
+    if isinstance(v, str):
+        return b64decode(v)
+    raise ValueError(f"Expected bytes or base64 string, got {type(v)}")
+
+
+Base64Bytes = Annotated[
+    bytes,
+    BeforeValidator(_decode_bytes),
+    PlainSerializer(lambda x: b64encode(x).decode('ascii'), return_type=str),
+]
+
+
 class PolarsOperation(BaseModel):
-    operation: bytes
+    operation: Base64Bytes  # Automatically encodes/decodes base64 for JSON
 
 
 class PolarsScript(PolarsOperation):