Flowfile 0.3.2__tar.gz → 0.3.3__tar.gz

{flowfile-0.3.2 → flowfile-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: Flowfile
-Version: 0.3.2
+Version: 0.3.3
 Summary: Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)
 Author: Edward van Eechoud
 Author-email: evaneechoud@gmail.com

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile/__init__.py

@@ -18,8 +18,9 @@ os.environ['SINGLE_FILE_MODE'] = "1"
 from flowfile.web import start_server as start_web_ui
 from flowfile.api import open_graph_in_editor
 from flowfile_frame.flow_frame import (
-    FlowFrame, read_csv, read_parquet, from_dict, concat
+    FlowFrame
 )
+from flowfile_frame import read_csv, read_parquet, from_dict, concat
 from flowfile_frame.expr import (
     col, lit, column, cum_count, len,
     sum, min, max, mean, count, when

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile/web/__init__.py

@@ -135,6 +135,9 @@ def start_server(host="127.0.0.1", port=63578, open_browser=True):
 
     # Import core app
     from flowfile_core.main import run, app as core_app
+    from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
+
+    OFFLOAD_TO_WORKER.value = True
 
     # Extend the core app with web UI routes and worker functionality
     extend_app(core_app)

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/configs/__init__.py

@@ -11,16 +11,27 @@ logger = logging.getLogger('PipelineHandler')
 logger.setLevel(logging.INFO)
 logger.propagate = False
 
-# Create console handler with a specific format
-console_handler = logging.StreamHandler(sys.stdout)
+# Clear any existing handlers
+if logger.hasHandlers():
+    logger.handlers.clear()
+
+# Try to determine the best output stream
+output_stream = None
+if hasattr(sys.stdout, 'isatty') and sys.stdout.isatty():
+    output_stream = sys.stdout
+elif hasattr(sys.stderr, 'isatty') and sys.stderr.isatty():
+    output_stream = sys.stderr
+else:
+    # Use __stdout__ for debugger environments (PyDev, PyCharm, etc.)
+    output_stream = sys.__stdout__
+
+console_handler = logging.StreamHandler(output_stream)
 console_handler.setLevel(logging.INFO)
 
 # Create formatter
 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 console_handler.setFormatter(formatter)
 
-if logger.hasHandlers():
-    logger.handlers.clear()
 logger.addHandler(console_handler)
 
 # Create logs directory in temp at startup

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/configs/settings.py

@@ -5,10 +5,10 @@ import os
 import tempfile
 import argparse
 
-from databases import DatabaseURL
 from passlib.context import CryptContext
 from starlette.config import Config
-from starlette.datastructures import Secret
+
+from flowfile_core.configs.utils import MutableBool
 
 
 # Constants for server and worker configuration
@@ -18,6 +18,9 @@ DEFAULT_WORKER_PORT = 63579
 SINGLE_FILE_MODE: bool = os.environ.get("SINGLE_FILE_MODE", "0") == "1"
 
 
+OFFLOAD_TO_WORKER = MutableBool(True)
+
+
 def parse_args():
     """Parse command line arguments"""
     parser = argparse.ArgumentParser(description="Flowfile Backend Server")
@@ -79,7 +82,6 @@ args = parse_args()
 SERVER_HOST = args.host if args.host is not None else DEFAULT_SERVER_HOST
 SERVER_PORT = args.port if args.port is not None else DEFAULT_SERVER_PORT
 WORKER_PORT = args.worker_port if args.worker_port is not None else int(os.getenv("WORKER_PORT", DEFAULT_WORKER_PORT))
-# Worker configuration
 WORKER_HOST = os.getenv("WORKER_HOST", "0.0.0.0" if platform.system() != "Windows" else "127.0.0.1")
 
 config = Config(".env")

flowfile-0.3.3/flowfile_core/flowfile_core/configs/utils.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class MutableBool:
+    value: bool
+
+    def __bool__(self) -> bool:
+        """Allow direct boolean evaluation"""
+        return self.value
+
+    def __eq__(self, other) -> bool:
+        """Allow equality comparison with booleans"""
+        if isinstance(other, bool):
+            return self.value == other
+        elif isinstance(other, MutableBool):
+            return self.value == other.value
+        return NotImplemented

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/flowfile/FlowfileFlow.py

@@ -15,7 +15,7 @@ from flowfile_core.configs import logger
 from flowfile_core.configs.flow_logger import FlowLogger
 from flowfile_core.flowfile.sources.external_sources.factory import data_source_factory
 from flowfile_core.flowfile.sources.external_sources.airbyte_sources.settings import airbyte_settings_from_config
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import type_to_polars_str, FlowfileColumn
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import cast_str_to_polars_type, FlowfileColumn
 from flowfile_core.flowfile.flow_data_engine.fuzzy_matching.settings_validator import (calculate_fuzzy_match_schema,
                                                                                        pre_calculate_pivot_schema)
 from flowfile_core.utils.arrow_reader import get_read_top_n
@@ -25,7 +25,7 @@ from flowfile_core.flowfile.flow_data_engine.read_excel_tables import get_open_x
 from flowfile_core.flowfile.sources import external_sources
 from flowfile_core.schemas import input_schema, schemas, transform_schema
 from flowfile_core.schemas.output_model import TableExample, NodeData, NodeResult, RunInformation
-from flowfile_core.flowfile.utils import snake_case_to_camel_case
+from flowfile_core.flowfile.utils import snake_case_to_camel_case, _handle_raw_data
 from flowfile_core.flowfile.analytics.utils import create_graphic_walker_node_from_node_promise
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_core.flowfile.util.execution_orderer import determine_execution_order
@@ -34,7 +34,7 @@ from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_op
                                                                                                  ExternalDatabaseFetcher,
                                                                                                  ExternalDatabaseWriter,
                                                                                                  ExternalDfFetcher)
-from flowfile_core.secrets.secrets import get_encrypted_secret, decrypt_secret
+from flowfile_core.secret_manager.secret_manager import get_encrypted_secret, decrypt_secret
 from flowfile_core.flowfile.sources.external_sources.sql_source import utils as sql_utils, models as sql_models
 from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import SqlSource, BaseSqlSource
 from flowfile_core.flowfile.database_connection_manager.db_connections import get_local_database_connection
@@ -205,19 +205,12 @@ class FlowGraph:
         sample_size: int = 10000
 
         def analysis_preparation(flowfile_table: FlowDataEngine):
-            if flowfile_table.number_of_records < 0:
-
-                number_of_records = ExternalDfFetcher(
-                    lf=flowfile_table.data_frame,
-                    operation_type="calculate_number_of_records",
-                    flow_id=self.flow_id,
-                    node_id=node.node_id,
-                ).result
+            if flowfile_table.number_of_records <= 0:
+                number_of_records = flowfile_table.get_number_of_records(calculate_in_worker_process=True)
             else:
                 number_of_records = flowfile_table.number_of_records
             if number_of_records > sample_size:
                 flowfile_table = flowfile_table.get_sample(sample_size, random=True)
-
             external_sampler = ExternalDfFetcher(
                 lf=flowfile_table.data_frame,
                 file_ref="__gf_walker"+node.hash,
@@ -225,7 +218,7 @@ class FlowGraph:
                 node_id=node.node_id,
                 flow_id=self.flow_id,
             )
-            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref, 10000)
+            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref)
             return flowfile_table
 
         def schema_callback():
@@ -441,7 +434,7 @@ class FlowGraph:
     def add_formula(self, function_settings: input_schema.NodeFormula):
         error = ""
         if function_settings.function.field.data_type not in (None, "Auto"):
-            output_type = type_to_polars_str(function_settings.function.field.data_type)
+            output_type = cast_str_to_polars_type(function_settings.function.field.data_type)
         else:
             output_type = None
         if output_type not in (None, "Auto"):
@@ -486,7 +479,8 @@ class FlowGraph:
                            function=_func,
                            input_columns=[],
                            node_type='cross_join',
-                           setting_input=cross_join_settings)
+                           setting_input=cross_join_settings,
+                           input_node_ids=cross_join_settings.depending_on_ids)
         return self
 
     def add_join(self, join_settings: input_schema.NodeJoin) -> "FlowGraph":
@@ -1044,11 +1038,10 @@ class FlowGraph:
         return self
 
     def add_datasource(self, input_file: input_schema.NodeDatasource | input_schema.NodeManualInput):
-
         if isinstance(input_file, input_schema.NodeManualInput):
-            input_data = FlowDataEngine(input_file.raw_data)
+            _handle_raw_data(input_file)
+            input_data = FlowDataEngine(input_file.raw_data_format)
             ref = 'manual_input'
-
         else:
             input_data = FlowDataEngine(path_ref=input_file.file_ref)
             ref = 'datasource'
@@ -1061,7 +1054,9 @@ class FlowGraph:
 
             if not input_file.node_id in set(start_node.node_id for start_node in self._flow_starts):
                 self._flow_starts.append(node)
+
         else:
+            input_data.collect()
             node = FlowNode(input_file.node_id, function=input_data,
                             setting_input=input_file,
                             name=ref, node_type=ref, parent_uuid=self.uuid)

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/flowfile/database_connection_manager/db_connections.py

@@ -1,7 +1,7 @@
 from flowfile_core.schemas.input_schema import FullDatabaseConnection, FullDatabaseConnectionInterface
 from sqlalchemy.orm import Session
 from flowfile_core.database.models import DatabaseConnection as DBConnectionModel, Secret
-from flowfile_core.secrets.secrets import store_secret, SecretInput, decrypt_secret
+from flowfile_core.secret_manager.secret_manager import store_secret, SecretInput, decrypt_secret
 from flowfile_core.database.connection import get_db_context
 
 

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py

@@ -17,6 +17,7 @@ from pyarrow.parquet import ParquetFile
 # Local imports - Core
 from flowfile_core.configs import logger
 from flowfile_core.configs.flow_logger import NodeLogger
+from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
 from flowfile_core.schemas import (
     input_schema,
     transform_schema as transform_schemas
@@ -29,7 +30,7 @@ from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import (
     FlowfileColumn,
     convert_stats_to_column_info
 )
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import type_to_polars
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_data_engine.fuzzy_matching.prepare_for_fuzzy_match import prepare_for_fuzzy_match
 from flowfile_core.flowfile.flow_data_engine.join import (
     verify_join_select_integrity,
@@ -109,7 +110,7 @@ class FlowDataEngine:
     # flow_id: int = None  # TODO: Implement flow_id
 
     def __init__(self,
-                 raw_data: Union[List[Dict], List[Any], 'ParquetFile', pl.DataFrame, pl.LazyFrame] = None,
+                 raw_data: Union[List[Dict], List[Any], 'ParquetFile', pl.DataFrame, pl.LazyFrame, input_schema.RawData] = None,
                  path_ref: str = None,
                  name: str = None,
                  optimize_memory: bool = True,
@@ -147,7 +148,10 @@ class FlowDataEngine:
 
     def _handle_raw_data(self, raw_data, number_of_records, optimize_memory):
         """Process different types of input data."""
-        if isinstance(raw_data, pl.DataFrame):
+
+        if isinstance(raw_data, input_schema.RawData):
+            self._handle_raw_data_format(raw_data)
+        elif isinstance(raw_data, pl.DataFrame):
             self._handle_polars_dataframe(raw_data, number_of_records)
         elif isinstance(raw_data, pl.LazyFrame):
             self._handle_polars_lazy_frame(raw_data, number_of_records, optimize_memory)
@@ -190,6 +194,20 @@ class FlowDataEngine:
             self.number_of_records = 1
             self.data_frame = pl.DataFrame([data])
 
+    def _handle_raw_data_format(self, raw_data: input_schema.RawData):
+        """Create a FlowDataEngine from a RawData object."""
+        flowfile_schema = list(FlowfileColumn.create_from_minimal_field_info(c) for c in raw_data.columns)
+        polars_schema = pl.Schema([(flowfile_column.column_name, flowfile_column.get_polars_type().pl_datatype)
+                                   for flowfile_column in flowfile_schema])
+        try:
+            df = pl.DataFrame(raw_data.data, polars_schema)
+        except TypeError as e:
+            logger.warning(f"Could not parse the data with the schema:\n{e}")
+            df = pl.DataFrame(raw_data.data)
+        self.number_of_records = len(df)
+        self.data_frame = df.lazy()
+        self.lazy = True
+
     def _handle_list_input(self, data: List):
         """Handle list input."""
         number_of_records = len(data)
@@ -462,6 +480,9 @@ class FlowDataEngine:
             return self.data_frame.collect(engine="streaming" if self._streamable else "auto").to_dicts()
         return self.data_frame.to_dicts()
 
+    def to_dict(self) -> Dict[str, List]:
+        return self.data_frame.collect(engine="streaming" if self._streamable else "auto").to_dict(as_series=False)
+
     @classmethod
     def create_from_external_source(cls, external_source: ExternalDataSource) -> "FlowDataEngine":
         """Create a FlowDataEngine from an external data source."""
@@ -484,7 +505,7 @@ class FlowDataEngine:
         """Create a FlowDataEngine from a schema definition."""
         pl_schema = []
         for i, flow_file_column in enumerate(schema):
-            pl_schema.append((flow_file_column.name, type_to_polars(flow_file_column.data_type)))
+            pl_schema.append((flow_file_column.name, cast_str_to_polars_type(flow_file_column.data_type)))
             schema[i].col_index = i
         df = pl.LazyFrame(schema=pl_schema)
         return cls(df, schema=schema, calculate_schema_stats=False, number_of_records=0)
@@ -824,7 +845,7 @@ class FlowDataEngine:
         Returns:
             FlowDataEngine: New instance with sampled data
         """
-        n_records = min(n_rows, self.number_of_records)
+        n_records = min(n_rows, self.get_number_of_records(calculate_in_worker_process=True))
         logging.info(f'Getting sample of {n_rows} rows')
 
         if random:
@@ -1158,14 +1179,25 @@ class FlowDataEngine:
         self.number_of_records = 0
         self._lazy = True
 
-    def get_number_of_records(self, warn: bool = False, force_calculate: bool = False) -> int:
+    def _calculate_number_of_records_in_worker(self) -> int:
+        number_of_records = ExternalDfFetcher(
+            lf=self.data_frame,
+            operation_type="calculate_number_of_records",
+            flow_id=-1,
+            node_id=-1,
+            wait_on_completion=True
+        ).result
+        return number_of_records
+
+    def get_number_of_records(self, warn: bool = False, force_calculate: bool = False,
+                              calculate_in_worker_process: bool = False) -> int:
         """
         Get the total number of records in the DataFrame.
 
         Args:
             warn: Whether to warn about expensive operations
             force_calculate: Whether to force recalculation
-
+            calculate_in_worker_process: Whether to offload compute to the worker process
         Returns:
             int: Number of records
 
@@ -1174,22 +1206,24 @@ class FlowDataEngine:
         """
         if self.is_future and not self.is_collected:
             return -1
-
+        calculate_in_worker_process = False if not OFFLOAD_TO_WORKER.value else calculate_in_worker_process
         if self.number_of_records is None or self.number_of_records < 0 or force_calculate:
             if self._number_of_records_callback is not None:
                 self._number_of_records_callback(self)
 
             if self.lazy:
-                if warn:
-                    logger.warning('Calculating the number of records this can be expensive on a lazy frame')
-                try:
-                    self.number_of_records = self.data_frame.select(pl.len()).collect(
-                        engine="streaming" if self._streamable else "auto")[0, 0]
-                except Exception:
-                    raise Exception('Could not get number of records')
+                if calculate_in_worker_process:
+                    self.number_of_records = self._calculate_number_of_records_in_worker()
+                else:
+                    if warn:
+                        logger.warning('Calculating the number of records this can be expensive on a lazy frame')
+                    try:
+                        self.number_of_records = self.data_frame.select(pl.len()).collect(
+                            engine="streaming" if self._streamable else "auto")[0, 0]
+                    except Exception:
+                        raise ValueError('Could not get number of records')
             else:
                 self.number_of_records = self.data_frame.__len__()
-
         return self.number_of_records
 
     # Properties
@@ -1518,4 +1552,7 @@ def execute_polars_code(*flowfile_tables: "FlowDataEngine", code: str) -> "FlowD
         kwargs = {'input_df': flowfile_tables[0].data_frame}
     else:
         kwargs = {f'input_df_{i+1}': flowfile_table.data_frame for i, flowfile_table in enumerate(flowfile_tables)}
-    return FlowDataEngine(polars_executable(**kwargs))
+    df = polars_executable(**kwargs)
+    if isinstance(df, pl.DataFrame):
+        logger.warning("Got a non lazy DataFrame, possibly harming performance, if possible, try to use a lazy method")
+    return FlowDataEngine(df)

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import Optional, Any, List, Dict, Literal
 from flowfile_core.schemas import input_schema
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import type_to_polars_str
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
 from polars import datatypes
 import polars as pl
@@ -9,6 +9,37 @@ import polars as pl
 DataTypeGroup = Literal['numeric', 'str', 'date']
 
 
+def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
+    if isinstance(pl_type, pl.List):
+        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
+        return f"pl.List({inner_str})"
+    elif isinstance(pl_type, pl.Array):
+        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
+        return f"pl.Array({inner_str})"
+    elif isinstance(pl_type, pl.Decimal):
+        precision = pl_type.precision if hasattr(pl_type, 'precision') else None
+        scale = pl_type.scale if hasattr(pl_type, 'scale') else None
+        if precision is not None and scale is not None:
+            return f"pl.Decimal({precision}, {scale})"
+        elif precision is not None:
+            return f"pl.Decimal({precision})"
+        else:
+            return "pl.Decimal()"
+    elif isinstance(pl_type, pl.Struct):
+        # Handle Struct with field definitions
+        fields = []
+        if hasattr(pl_type, 'fields'):
+            for field in pl_type.fields:
+                field_name = field.name
+                field_type = convert_pl_type_to_string(field.dtype, inner=True)
+                fields.append(f'pl.Field("{field_name}", {field_type})')
+        field_str = ", ".join(fields)
+        return f"pl.Struct([{field_str}])"
+    else:
+        # For base types, we want the full pl.TypeName format
+        return str(pl_type.base_type()) if not inner else f"pl.{pl_type}"
+
+
 @dataclass
 class FlowfileColumn:
     column_name: str
@@ -28,7 +59,7 @@ class FlowfileColumn:
     __perc_unique: Optional[float]
 
     def __init__(self, polars_type: PlType):
-        self.data_type = str(polars_type.pl_datatype.base_type())
+        self.data_type = convert_pl_type_to_string(polars_type.pl_datatype)
         self.size = polars_type.count - polars_type.null_count
         self.max_value = polars_type.max
         self.min_value = polars_type.min
@@ -53,7 +84,7 @@ class FlowfileColumn:
 
     @classmethod
     def from_input(cls, column_name: str, data_type: str, **kwargs) -> "FlowfileColumn":
-        pl_type = type_to_polars_str(data_type)
+        pl_type = cast_str_to_polars_type(data_type)
         if pl_type is not None:
             data_type = pl_type
         return cls(PlType(column_name=column_name, pl_datatype=data_type, **kwargs))
@@ -129,12 +160,9 @@ class FlowfileColumn:
             return 'date'
 
     def get_polars_type(self) -> PlType:
-        if hasattr(datatypes, self.data_type):
-            pl_datatype = getattr(datatypes, self.data_type)
-        else:
-            pl_datatype = None
-
-        return PlType(pl_datatype=pl_datatype, **self.__dict__)
+        pl_datatype = cast_str_to_polars_type(self.data_type)
+        pl_type = PlType(pl_datatype=pl_datatype, **self.__dict__)
+        return pl_type
 
     def update_type_from_polars_type(self, pl_type: PlType):
         self.data_type = str(pl_type.pl_datatype.base_type())
@@ -142,3 +170,8 @@ class FlowfileColumn:
 
 def convert_stats_to_column_info(stats: List[Dict]) -> List[FlowfileColumn]:
     return [FlowfileColumn.create_from_polars_type(PlType(**c)) for c in stats]
+
+
+def convert_pl_schema_to_raw_data_format(pl_schema: pl.Schema) -> List[input_schema.MinimalFieldInfo]:
+    return [FlowfileColumn.create_from_polars_type(PlType(column_name=k, pl_datatype=v)).get_minimal_field_info()
+            for k, v in pl_schema.items()]

flowfile-0.3.3/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py

@@ -0,0 +1,75 @@
+import polars as pl
+
+
+dtype_to_pl = {
+    'int': pl.Int64,
+    'integer': pl.Int64,
+    'char': pl.String,
+    'fixed decimal': pl.Float32,
+    'double': pl.Float64,
+    'float': pl.Float64,
+    'bool': pl.Boolean,
+    'byte': pl.UInt8,
+    'bit': pl.Binary,
+    'date': pl.Date,
+    'datetime': pl.Datetime,
+    'string': pl.String,
+    'str': pl.String,
+    'time': pl.Time,
+}
+
+
+def safe_eval_pl_type(type_string: str):
+    """
+    Safely evaluate a Polars type string with restricted namespace.
+    Only allows Polars types and basic Python literals.
+    """
+    # Define allowed names in the evaluation namespace
+    safe_dict = {
+        # Polars module and types
+        'pl': pl,
+
+        # Basic Python built-ins for literals
+        'int': int,
+        'str': str,
+        'float': float,
+        'bool': bool,
+        'list': list,
+        'dict': dict,
+        'tuple': tuple,
+
+        # Disable dangerous built-ins
+        '__builtins__': {},
+    }
+
+    try:
+        return eval(type_string, safe_dict, {})
+    except Exception as e:
+        raise ValueError(f"Failed to safely evaluate type string '{type_string}': {e}")
+
+
+dtype_to_pl_str = {k: v.__name__ for k, v in dtype_to_pl.items()}
+
+
+def get_polars_type(dtype: str):
+    if 'pl.' in dtype:
+        try:
+            return safe_eval_pl_type(dtype)
+        except Exception as e:
+            return pl.String
+    pl_datetype = dtype_to_pl.get(dtype.lower())
+    if pl_datetype is not None:
+        return pl_datetype
+    elif hasattr(pl, dtype):
+        return getattr(pl, dtype)
+    else:
+        return pl.String
+
+
+def cast_str_to_polars_type(dtype: str) -> pl.DataType:
+    pl_type = get_polars_type(dtype)
+    if hasattr(pl_type, '__call__'):
+        return pl_type()
+    else:
+        return pl_type
+

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/flowfile/flow_data_engine/polars_code_parser.py

@@ -3,6 +3,7 @@ from typing import Dict, Any, Callable
 import textwrap
 import ast
 import time
+from io import BytesIO
 
 
 def remove_comments_and_docstrings(source: str) -> str:
@@ -174,6 +175,7 @@ class PolarsCodeParser:
             'False': False,
             'None': None,
             'time': time,
+            'BytesIO': BytesIO
         }
 
     @staticmethod
@@ -256,7 +258,6 @@ class PolarsCodeParser:
 
         # Wrap the code in a function
         wrapped_code = self._wrap_in_function(code, num_inputs)
-
         try:
             # Create namespace for execution
             local_namespace: Dict[str, Any] = {}

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/flowfile/flow_data_engine/sample_data.py

@@ -1,14 +1,27 @@
 from faker import Faker
 from functools import partial
+from math import ceil
 from random import randint
 import polars as pl
 from typing import List, Dict, Any, Generator
 
 
-def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
+def create_fake_data(n_records: int = 1000, optimized: bool = True) -> pl.DataFrame:
+    """
+
+    Args:
+        n_records (): Number of records to return
+        optimized (): Indicator if creation should be optimized, will result in more identical rows when True
+
+    Returns:
+        pl.DataFrame
+    """
     fake = Faker()
-    selector = partial(randint,0)
-    min_range = partial(min, n_records)
+    selector = partial(randint, 0)
+
+    max_n_records = min(10_000, n_records) if optimized else n_records
+
+    min_range = partial(min, max_n_records)
     # Pre-generation of static data
     cities = [fake.city() for _ in range(min_range(7000))]
     companies = [fake.company() for _ in range(min_range(100_000))]
@@ -19,7 +32,7 @@ def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
     first_names = [fake.first_name() for _ in range(min_range(100_000))]
     last_names = [fake.last_name() for _ in range(min_range(50_000))]
     domain_names = [fake.domain_name() for _ in range(10)]
-    sales_data = [fake.random_int(0, 1000) for _ in range(n_records)]
+    sales_data = [fake.random_int(0, 1000) for _ in range(max_n_records)]
 
     def generate_name():
         return f"{first_names[selector(min_range(100_000))-1]} {last_names[selector(min_range(50_000))-1]}"
@@ -32,9 +45,8 @@ def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
 
     def generate_phone_number():
         return fake.phone_number()
-
     data = []
-    for i in range(n_records):
+    for i in range(max_n_records):
         name = generate_name()
         data.append(dict(
             ID=randint(1, 1000000),
@@ -47,8 +59,14 @@ def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
             Work=companies[selector(min_range(100_000))-1],
             Zipcode=zipcodes[selector(min_range(200_000))-1],
             Country=countries[selector(min_range(50))-1],
-            sales_data=sales_data[selector(n_records)-1]
+            sales_data=sales_data[selector(max_n_records)-1]
         ))
+    if max_n_records < n_records:
+        n_duplicates: int = ceil(n_records / max_n_records)
+        output = []
+        for _ in range(n_duplicates):
+            output.extend(data)
+        data = output[:n_records]
 
     return pl.DataFrame(data)
 

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py

@@ -190,7 +190,7 @@ class BaseFetcher:
                 logger.info('Already running the fetching')
                 return
 
-            sleep_time = 1
+            sleep_time = .5
             self.running = True
             while not self.stop_event.is_set():
                 try:
@@ -205,7 +205,8 @@ class BaseFetcher:
                             break
                         elif status.status == 'Unknown Error':
                             self._handle_error(-1,
-                                               'There was an unknown error with the process, and the process got killed by the server')
+                                               'There was an unknown error with the process, '
+                                               'and the process got killed by the server')
                             break
                     else:
                         self._handle_error(2, r.text)
@@ -284,7 +285,7 @@ class ExternalDfFetcher(BaseFetcher):
 
     def __init__(self, flow_id: int, node_id: int | str, lf: pl.LazyFrame | pl.DataFrame, file_ref: str = None,
                  wait_on_completion: bool = True,
-                 operation_type: OperationType = 'store'):
+                 operation_type: OperationType = 'store', offload_to_worker: bool = True):
         super().__init__(file_ref=file_ref)
         lf = lf.lazy() if isinstance(lf, pl.DataFrame) else lf
         r = trigger_df_operation(lf=lf, file_ref=self.file_ref, operation_type=operation_type,

{flowfile-0.3.2 → flowfile-0.3.3}/flowfile_core/flowfile_core/flowfile/flow_data_engine/utils.py

@@ -210,3 +210,4 @@ def match_order(l: List[str], ref: List[str]) -> List[str]:
         org_order = ref_order.get(v, float('inf'))
         order.append(org_order)
     return [v for _, v in sorted(zip(order, l))]
+