Flowfile 0.3.2__py3-none-any.whl → 0.3.3.1__py3-none-any.whl

flowfile/__init__.py

@@ -7,7 +7,7 @@ This package ties together the FlowFile ecosystem components:
 - flowfile_worker: Computation engine
 """
 
-__version__ = "0.3.1"
+__version__ = "0.3.3.1"
 
 import os
 import logging
@@ -18,8 +18,9 @@ os.environ['SINGLE_FILE_MODE'] = "1"
 from flowfile.web import start_server as start_web_ui
 from flowfile.api import open_graph_in_editor
 from flowfile_frame.flow_frame import (
-    FlowFrame, read_csv, read_parquet, from_dict, concat
+    FlowFrame
 )
+from flowfile_frame import read_csv, read_parquet, from_dict, concat
 from flowfile_frame.expr import (
     col, lit, column, cum_count, len,
     sum, min, max, mean, count, when

flowfile/web/__init__.py

@@ -135,6 +135,9 @@ def start_server(host="127.0.0.1", port=63578, open_browser=True):
 
     # Import core app
     from flowfile_core.main import run, app as core_app
+    from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
+
+    OFFLOAD_TO_WORKER.value = True
 
     # Extend the core app with web UI routes and worker functionality
     extend_app(core_app)

{flowfile-0.3.2.dist-info → flowfile-0.3.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: Flowfile
-Version: 0.3.2
+Version: 0.3.3.1
 Summary: Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)
 Author: Edward van Eechoud
 Author-email: evaneechoud@gmail.com
@@ -23,10 +23,11 @@ Requires-Dist: methodtools (>=0.4.7,<0.5.0)
 Requires-Dist: openpyxl (>=3.1.2,<3.2.0)
 Requires-Dist: passlib (>=1.7.4,<1.8.0)
 Requires-Dist: pendulum (==2.1.2) ; python_version < "3.12"
-Requires-Dist: polars (>1.8.2,<=1.25.2)
+Requires-Dist: polars (>=1.8.2,<2.0.0) ; sys_platform != "win32"
+Requires-Dist: polars (>=1.8.2,<=1.25.2) ; sys_platform == "win32"
 Requires-Dist: polars-distance (>=0.4.3,<0.5.0)
 Requires-Dist: polars-ds (>=0.6.0)
-Requires-Dist: polars-expr-transformer (>0.4.7.0)
+Requires-Dist: polars-expr-transformer (>=0.4.9.0)
 Requires-Dist: polars-grouper (>=0.3.0,<0.4.0)
 Requires-Dist: polars_simed (>=0.3.4,<0.4.0)
 Requires-Dist: pyairbyte-flowfile (==0.20.2)

{flowfile-0.3.2.dist-info → flowfile-0.3.3.1.dist-info}/RECORD

@@ -1,11 +1,11 @@
 build_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 build_backends/main.py,sha256=hLmfqTeHLSTiwwZ5mUuoLQgtO40Igvl1_4NbnvzWSgI,9912
 build_backends/main_prd.py,sha256=JR2tYCMWM5ThooQjv5pw6nwVKMQjgsiHgKMhYn9NXWI,6927
-flowfile/__init__.py,sha256=B1vnUboOki3pP3BAmoQ0j62nEiB51X9kc9N8Qu7bgcg,2419
+flowfile/__init__.py,sha256=e-OVTnJOMsnqBbMt5WTGQmAYjHBt2YA6ODlkoIpCj20,2447
 flowfile/__main__.py,sha256=hAMeyORHhLnw1lIXe6-EYDwgUi2odW0Rb4eDkcNtBiM,2612
 flowfile/api.py,sha256=grbo8pUF8NNEGoqDQX-BTjWXC61mSIgJoD-w-GVeD4Y,15298
 flowfile/readme.md,sha256=n93Cpp9DPIBfe2jxKcUVokl5PTobxnE1mWHfRD018xE,4137
-flowfile/web/__init__.py,sha256=bViGSO1AmLSjCM4nnQVFaYUTFHYKj0XK4VjscGwZGTs,5462
+flowfile/web/__init__.py,sha256=w-nioI8PZzMIEU-wZ7fNDN8Q4sl8aNEMFRgDVU95p9o,5563
 flowfile/web/static/assets/AirbyteReader-1ac35765.css,sha256=GsNXZRBzBqcgSHWYHFfpQjYnQ1G90hCaWgThLCG80jI,6260
 flowfile/web/static/assets/AirbyteReader-2b1cf2d8.js,sha256=b9AhzC5wtvBqx0hRqHsZde55lh5sSsgT3vfZwDo84wY,38778
 flowfile/web/static/assets/CrossJoin-41efa4cb.css,sha256=Qe-ky2QI7rYfXMKV-bCB5HP0OJ6uBU74g9EEmcpXTlc,2838
@@ -140,11 +140,12 @@ flowfile_core/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 flowfile_core/auth/jwt.py,sha256=-j_eZd5z2ABP0KHo5PmmsuCyJUsvAFFlnANJiRCnNx4,4828
 flowfile_core/auth/models.py,sha256=ilQqy7ief1mwAP9xiNWbftCR9yyccECMa0Qsnnwax_g,648
 flowfile_core/auth/secrets.py,sha256=5TixLt9I64pR0OT4AoqgIzAmjCQNoGF4YPGuRhCWBH0,5840
-flowfile_core/configs/__init__.py,sha256=RSoO4aiWY-FzOUKLPY-mbwHS_YcbbNw5tNdVzrwi_ek,977
+flowfile_core/configs/__init__.py,sha256=W72OpTyVRF1kJyLPQFXT9tB6FoX-Bzvi5rTUusERfKI,1318
 flowfile_core/configs/flow_logger.py,sha256=Pk1yhaC58jjISMrgwhBFIue9Qj5XfYo8NfOefqsR-gA,15822
 flowfile_core/configs/node_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/configs/node_store/nodes.py,sha256=mVjHyJzkxrmg0PSAMdJC8CGyqD57NyWVEC3rhhzXosU,5960
-flowfile_core/configs/settings.py,sha256=6QI6wOhAwvRYLd3MMP7w3g6wgyT4DeE-zq6-I39Touo,3073
+flowfile_core/configs/settings.py,sha256=37SLuk2sL4E69SrYtN5ygJILIcc8CwAHLm5FiaTlC9Y,3065
+flowfile_core/configs/utils.py,sha256=BjyJCfO4gR_n9Sbs6dPlFfSKwxfxd3aKZ9zUH0Xs5GY,474
 flowfile_core/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/database/connection.py,sha256=Ul1gYnYvwvlTSgxlJr5JexYCdgyZFduXioFjY_Lw2U0,1410
 flowfile_core/database/init_db.py,sha256=vKQi340caayTqc1tR0CJcd621i4afVc0BLpJO-t4CFk,1236
@@ -152,7 +153,7 @@ flowfile_core/database/models.py,sha256=SUrOgefmGY4lrN9Cv67TEjEOQtyKoYEpkjIVCWs7
 flowfile_core/fileExplorer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/fileExplorer/funcs.py,sha256=9o1Wko-wPpKNapRSvwsC-FmofJ6VFmBU5v2aQLRU3sA,9048
 flowfile_core/fileExplorer/utils.py,sha256=RrXj1NdkB2K3v5khvXpEp-_OnN68k9Ex_9s7CDa3L5w,1703
-flowfile_core/flowfile/FlowfileFlow.py,sha256=3HvkfmN_aZCZ4JP9b1HL60Qf4QPmDLYj8dvi8d_JZ2M,72693
+flowfile_core/flowfile/FlowfileFlow.py,sha256=pJTbAxwFbZ9zbdEyAM6u_BpCe47r_IZpar5Gh0MqLJw,72714
 flowfile_core/flowfile/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/_extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/_extensions/real_time_interface.py,sha256=F9wGAFmu4gmC-svfwasDEaVYZBinBqYUZmCrEsWos44,1906
@@ -165,35 +166,35 @@ flowfile_core/flowfile/connection_manager/__init__.py,sha256=wLAGuQBA0lgN1tZleYZ
 flowfile_core/flowfile/connection_manager/_connection_manager.py,sha256=W9FWRAFUT1c2eHa2QhFpKNWA-Kps63o2vcGAH1zXSeo,2612
 flowfile_core/flowfile/connection_manager/models.py,sha256=o_2FK7aNdjMHfiGX7hIUz5uslfLar8KIcI_760WprzM,227
 flowfile_core/flowfile/database_connection_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/flowfile/database_connection_manager/db_connections.py,sha256=omQjmvwhL97d4WAgIEVBHhldOG58QySE1tq7f-timKc,5210
+flowfile_core/flowfile/database_connection_manager/db_connections.py,sha256=e0DpRZ-7wTKbqAs9icDkNwYdMxhAska0WNw1YRRhrPM,5224
 flowfile_core/flowfile/database_connection_manager/models.py,sha256=lVJSifqznQ8fKGWBEBCy_8JeXmdKF4pnAE5P5oXRrLM,379
 flowfile_core/flowfile/extensions.py,sha256=vVyM4sdUN5Eyez6IPt9poNWwvcJMJKwP4dLCq1ErQ5k,1795
 flowfile_core/flowfile/flow_data_engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/flow_data_engine/create/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/flow_data_engine/create/funcs.py,sha256=AC87vQHgna-stFJZPLWF8ErxFioxdCJAhVaXZGVOsY0,7317
-flowfile_core/flowfile/flow_data_engine/flow_data_engine.py,sha256=TY2QrFb_fFHksReXlLp4Tt0iqUjsKS6IVdz5z08bvhU,61682
+flowfile_core/flowfile/flow_data_engine/flow_data_engine.py,sha256=61js7mznpHROSxM-DFVXoC8JVXqHxhWFlAATwaNK3DM,63806
 flowfile_core/flowfile/flow_data_engine/flow_file_column/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py,sha256=e438RcbN9msjgsyoXyqsRXp7hiPO5xdw-F7cRfSOBCw,5533
+flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py,sha256=U7tLBGFngO0QeIJ5YyBt9UrGS8IhlclMfbIuaI_BfMo,7175
 flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py,sha256=xusyOLwSxevBk8-Uy9ZKISB_KOi0JeYfZ0wihcG-Qjk,530
-flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py,sha256=3AI4ZR4OhNZPaCHq646rRbu-2yLJzj_4gEjGu_5mTzA,775
+flowfile_core/flowfile/flow_data_engine/flow_file_column/utils.py,sha256=2KoODecEwNGSCLZLRLim3vAwVON0QZgv2m1gkGKHht0,1774
 flowfile_core/flowfile/flow_data_engine/fuzzy_matching/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/flow_data_engine/fuzzy_matching/prepare_for_fuzzy_match.py,sha256=67MxjjmOfNaDYYFELo-L-h7HNHWWiAd62VX-vDD7Sqs,1873
 flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py,sha256=SjIw-dV8SQUClxyCCO-6i-HC3avPd3yzQgLFO0L04nI,4550
 flowfile_core/flowfile/flow_data_engine/join/__init__.py,sha256=q4K5ZjGmFmGcuK545PKD1y_8WhyPjmEIyfeIkZ3WboM,75
 flowfile_core/flowfile/flow_data_engine/join/verify_integrity.py,sha256=TfjYEIu2G9I-N6NuXVNsQWbj57_n4WbZO2kphZcHOmc,2245
 flowfile_core/flowfile/flow_data_engine/pivot_table.py,sha256=seqJqbrhwK4gkO-EQ3MSQWseh5HR3D_slQtq6pjT_pw,366
-flowfile_core/flowfile/flow_data_engine/polars_code_parser.py,sha256=rA9X7WZYk1lRvT8YAtVfAIGdNqIsSj21RuD5rY1xqE8,9545
+flowfile_core/flowfile/flow_data_engine/polars_code_parser.py,sha256=R3ZNe7UlkYxWUkT8Sppo8NUCobn0N2Iq610wdmrS2dA,9598
 flowfile_core/flowfile/flow_data_engine/read_excel_tables.py,sha256=q7TccqyToowJEOU4j2tY40HshXYkDACkYE95bNqoiw8,5937
-flowfile_core/flowfile/flow_data_engine/sample_data.py,sha256=Y_9PuM5gBsb3bIrN-Cz70BNQlXuj88A_AmejYj01wys,4451
+flowfile_core/flowfile/flow_data_engine/sample_data.py,sha256=7r-SJBrgtatgy0r0vrd0HQAw-Su2GbIKP4TPzHRLDtk,5026
 flowfile_core/flowfile/flow_data_engine/subprocess_operations/__init__.py,sha256=04ZpOvZ6gF42vQ02a0Lim9w5EP7-xu2l_S2WrSoll-g,97
 flowfile_core/flowfile/flow_data_engine/subprocess_operations/models.py,sha256=0ZN8HfpTfXc5LxXjDc2CCl66xNpxf6n_6dwExIxWbjM,1079
-flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py,sha256=jPo9roN3_IikoRMSlDFK6KTshUwpRT6pqkQfFOXADvU,19962
+flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py,sha256=-JvAXS6OwySY-Zke0mCIzFj0hNhaUJFr9I5qdwny7yM,20045
 flowfile_core/flowfile/flow_data_engine/threaded_processes.py,sha256=15IPoqIoCfkPMb648o9hnOzNnZINhYQxJi5hNfQuRwE,1311
 flowfile_core/flowfile/flow_data_engine/types.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/flowfile/flow_data_engine/utils.py,sha256=JKOVKU-zatSk2oWXU8foR30MXooDJg4cODrw8c1uJqQ,7596
+flowfile_core/flowfile/flow_data_engine/utils.py,sha256=_LmTFZMwt03ZYwYgpPaRxe5pP-lg9k4UGBxLFYN2JPU,7597
 flowfile_core/flowfile/flow_graph_utils.py,sha256=7b8kHAdtIlohiaelXqicS4Bi4bp8GdIBH-1fSYWY5XI,10107
 flowfile_core/flowfile/flow_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/flowfile/flow_node/flow_node.py,sha256=sKFcSmxyBRUzCD_7ZJ9w1tEZ_odtS1WBI05frnpVl9o,35073
+flowfile_core/flowfile/flow_node/flow_node.py,sha256=sC0zPXsy25J5ll3qEJz-wJOe03dsRcuY-MOXYsww-fI,35081
 flowfile_core/flowfile/flow_node/models.py,sha256=jD2yUbrZYXy48nsUppuHUm8xqnrozhT37_oq9TBSYTw,4268
 flowfile_core/flowfile/flow_node/schema_callback.py,sha256=R9tQh20zC3Z492ne8OtsSpp9_jmDIAfyoX4YqB1s_2M,2288
 flowfile_core/flowfile/handler.py,sha256=wBeZmREY57ahuITLFTZ7FATIGXHK-3bQHyyM3eA-_s8,4105
@@ -208,7 +209,7 @@ flowfile_core/flowfile/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 flowfile_core/flowfile/sources/external_sources/__init__.py,sha256=bb9QikXEhkP0rdH68qRUJaPGA2lGFIauIWNlOBL-9uE,288
 flowfile_core/flowfile/sources/external_sources/airbyte_sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py,sha256=OBiy2G2AtPBlVhirZrRqTta013nAtQobpSKWEX0oCQU,5718
-flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py,sha256=iM156vf0rjtIHMLFgCjBJozbnXYOi5ohtQ66zxbumQ8,5566
+flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py,sha256=17lX1B8gyEt57S0Di75CSHZc7WEeyl-_b20LO_zeayU,5576
 flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py,sha256=3ZdWIGLCsbWmJb33wmmkQRIa7iEHpPrE7iuFag_TxBg,6288
 flowfile_core/flowfile/sources/external_sources/base_class.py,sha256=mQhplw0O4heef2yAFuvKAyQaZJxd7lDyohgGv-4yxOI,981
 flowfile_core/flowfile/sources/external_sources/custom_external_sources/__init__.py,sha256=cr2lrDCARyiSlGNgl-CWv0w1H0ORhoGE8HwW7sL-Yr0,97
@@ -218,19 +219,19 @@ flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_u
 flowfile_core/flowfile/sources/external_sources/factory.py,sha256=8um2kFuICqrOAeA1scLPIVDabAsOHabN2PartvfCpPU,919
 flowfile_core/flowfile/sources/external_sources/sql_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/sources/external_sources/sql_source/models.py,sha256=C6lhFKsF90ifoU9k421f2-K-FJwaLbKs9_yqYHWBzGE,4613
-flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py,sha256=jMHLUW2vU110z5_Pru3E2CSvov5PM4Yve2nflotawcs,12978
+flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py,sha256=202iHv9FthKUisYNXfbhGuX0nsAwdeoX2FliXjkBcgg,12992
 flowfile_core/flowfile/sources/external_sources/sql_source/utils.py,sha256=Cj-uKhHT6s_S8vmO7mONf0zrKb987ZzxVLISaUdtAVw,11765
 flowfile_core/flowfile/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/util/calculate_layout.py,sha256=iquA_-EGzX9mH1NasIZFNaYLfBcKqz5pHaTjHO1rYWE,5070
 flowfile_core/flowfile/util/execution_orderer.py,sha256=IJ-upXUruZfFj8z4J-2oDLz2hDSOnRIdgd_YLlRKU-c,5828
-flowfile_core/flowfile/utils.py,sha256=arSkifiXgbdWqrLcgQ6X3u-A9_6zDHocDuXEy6M3c18,3363
-flowfile_core/main.py,sha256=5rcWI1uFlYkUWd7T_ZHkolOmLlbIWjuDpzCyivw6og0,3879
+flowfile_core/flowfile/utils.py,sha256=hR008OuvSZNS9BBUlF3sJDePYhbS7wf-6Um6YLlU0OY,5224
+flowfile_core/main.py,sha256=b_0bbqMwTPpHkP5zooJVAvGVs4mLP5N1yb1oQSM1LDE,3914
 flowfile_core/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/routes/auth.py,sha256=-2SWMCDDKYSnriiYH3hOadnsHdmVckcIV_vA2mZfBu0,1251
 flowfile_core/routes/logs.py,sha256=LHGbCN10pjS8yAWHaG9eUHxMoGQlHjVWFSxtAnuQjrM,5868
 flowfile_core/routes/public.py,sha256=bRmRyphaCLTTdPjPn_M6NWPUEQKHB5eeKlKoOs7jWzA,217
 flowfile_core/routes/routes.py,sha256=-pJ8LQxX6_-EnkXGu3C-K3UcQI1ZUPNJ0X8REbpvcCI,23181
-flowfile_core/routes/secrets.py,sha256=QIrloBSclw9JGcpB-c4VHlNBVy2t-sAAB9RXK9ibNlk,3061
+flowfile_core/routes/secrets.py,sha256=SG5zNGk-XRAVod7GBLKXhjEpTuTqdlCHOCoqsMGi4T8,3075
 flowfile_core/run_lock.py,sha256=1bINUPwZe7v9rHRePTk0CaExscqf2pvk4hWoYFZWQqE,296
 flowfile_core/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/schemas/analysis_schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -238,27 +239,37 @@ flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py,sha256=1PQeiBFI
 flowfile_core/schemas/defaults.py,sha256=5FeYJIn1ZJ4Es8WQfSW3EkfoOnWsvifg9bvwZO7fklg,228
 flowfile_core/schemas/external_sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/schemas/external_sources/airbyte_schemas.py,sha256=xjMVevVronSvMJ1lQrBTv2ClkO2VSMSU8YCga_Mw6vk,594
-flowfile_core/schemas/input_schema.py,sha256=wUPFzq08Ig3SOsIZxDtlqOUXJSXk9Sj1YtRp0XA8_YU,13611
+flowfile_core/schemas/input_schema.py,sha256=W0b7WG--CwqkiXfDokn-ZGRcWpjy6qT7BCWLRNjuz1A,13885
 flowfile_core/schemas/models.py,sha256=W22swh1TZ1aAZY8R-OL0AiTXvIEUKtN9B1ZFc0dxZQI,5253
 flowfile_core/schemas/output_model.py,sha256=yenexlnHI9ecQhTDBPNChO2p25YNZ9btR5_cXd-ehLg,2665
 flowfile_core/schemas/schemas.py,sha256=_CXS-YUMOwJTlWAX4mPnhqHD7ZiQy3DM7rHB2eSbHUY,2647
-flowfile_core/schemas/transform_schema.py,sha256=7MooeHGWdSZnLsX7_WSXesML6Rpbi7rK6vUEwtVjBaY,21259
-flowfile_core/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/secrets/secrets.py,sha256=6G9hqo-GjTE7va6i6r3_ZmGo4Y26Aw_PY0W8L6pNo7E,2086
+flowfile_core/schemas/transform_schema.py,sha256=HRJhRwTQIxqOJ_zRW_OmGn63dDdcbP_CCdmgjFstGAQ,20072
+flowfile_core/secret_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+flowfile_core/secret_manager/secret_manager.py,sha256=6G9hqo-GjTE7va6i6r3_ZmGo4Y26Aw_PY0W8L6pNo7E,2086
 flowfile_core/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/utils/arrow_reader.py,sha256=SbDDzOmtHcZ8rH88v1jN0EZthaDqBmvCrbruX07Qdks,9366
 flowfile_core/utils/excel_file_manager.py,sha256=EIad2LenHu-3Yw1FcLmE0KgmLflnvNKt07FY6s6mPlE,452
 flowfile_core/utils/fileManager.py,sha256=LnJhK_pwjb9MIApG2e4Hp3L5Z7Wny8YYHaL9SkW8WlE,1371
 flowfile_core/utils/fl_executor.py,sha256=eNnNZHZ9451brzZD00_X8aoCHFl1hR1gVOIGxtE0Db4,1301
 flowfile_core/utils/utils.py,sha256=NkEu21OF1l5weu01g-dAVdJ6BRHdpT2jBrWBSi-wp3c,270
-flowfile_frame/__init__.py,sha256=ZEhs_YFgoA2CBGuvHIEY559Dk6q09clFscT3r7tITLM,1326
+flowfile_frame/__init__.py,sha256=L2-jTwdtPEtdIS4c1foVXwnh5T0jDzpkgn3CBJvOPHg,1544
 flowfile_frame/adapters.py,sha256=C6JZZKANoKbHHmwMaF9AqAcZvITvQeb2Dklcpg5JAdY,530
-flowfile_frame/expr.py,sha256=wzJLiHtAqBA4NUEyXW8UhNJJ2jiGDvWEX8RfWT5ICwQ,47370
-flowfile_frame/flow_frame.py,sha256=wXiXbQr4FUOvQl7iRQZkJ8Q5q1Vh7MLT7wHK84ngyAM,95588
-flowfile_frame/group_frame.py,sha256=MMGE2_DC8n1J2UxBBm_TyacGiRT_1V2CcWUvsIyhbIQ,9115
-flowfile_frame/join.py,sha256=pezHyNQMcaSPK9vhjaCaelMkgKdrvOQxwiROsa0fmN8,2480
-flowfile_frame/selectors.py,sha256=Ny5IpDP481ClNr5gI7_SjXzeqF16LsPcVQxiyUf5tUw,9130
-flowfile_frame/utils.py,sha256=LwsNm8tvqcyaAeGX1PNQbpt-NenDsM8jx9SbxhDslZI,1348
+flowfile_frame/adding_expr.py,sha256=K9KdF3WCbPy2YO5radyEBXQ0yi6SOI4nviaPwXmKW0k,12428
+flowfile_frame/config.py,sha256=Tl_4V8byUiO1y3ANesW2E9qeTtZvtKYJl8KU00hc1Cg,176
+flowfile_frame/expr.py,sha256=5Qz_IKV-5FvnlMGv-10D82cBKMlEyUCq2CXnsmJUrfk,56695
+flowfile_frame/expr.pyi,sha256=HR1qBfGIlei-6Lt3qztGUR_XfyiQJMUbUWCMY1pEBYo,87903
+flowfile_frame/expr_name.py,sha256=0VZZwC3xc6tiwOJPu7emQ86Xp2mVu7U-j-jrsMjbc2Y,9852
+flowfile_frame/flow_frame.py,sha256=My05BjxYIPiokRItV1asSSI3q5_QhKg0nrv0FaRIq68,88112
+flowfile_frame/flow_frame.pyi,sha256=8-P0drJFbtYWr9o9YwJzWtiHrDbcVOMZzC-3Mz0X37A,29725
+flowfile_frame/flow_frame_methods.py,sha256=yBDnDyRY7NAf1pYYD0wMIXZenNTi0u_WSHxfyVPp214,23298
+flowfile_frame/group_frame.py,sha256=VfYxEcvU9KfzTsgy4GiN8PZF83bRr1nCQ9c2DKTS9VI,11558
+flowfile_frame/join.py,sha256=YQCHmw8nYzvGzvrFc0jOmVMy-rahK3CPtuFdKB7loS8,2437
+flowfile_frame/lazy.py,sha256=F0wLzoE5c5EF0D5msjqNi0nTydepx_YtwHT0hDq6Q-I,28619
+flowfile_frame/lazy_methods.py,sha256=ysSmdDF6guDWN-ogBIpGoCHdjAUbnnl8CncjqDUdg_Y,7730
+flowfile_frame/list_name_space.py,sha256=7m7qxZ9GANrVM8U-gOq3hZ7aPGwSGD1hbI6kRJLc0Tg,15778
+flowfile_frame/selectors.py,sha256=Rnaa9AuMgzazt6ujB9yyvcbH9zA7JmYjs8rFqdZ2TYM,9186
+flowfile_frame/series.py,sha256=VkhLr03iA8LF2EifGoKdNvg1ajm7Ln9OgzkMurg_nR0,2199
+flowfile_frame/utils.py,sha256=t19WSnorytJHs8ut0a7le6SYkVjBv5NTFzU3Gv_kUQs,3780
 flowfile_worker/__init__.py,sha256=ZDdn3JCP7LWTiTsmntVIVduB4p2bUkJcZUKVEj7V9TU,1375
 flowfile_worker/configs.py,sha256=7fYtlj06vxDrMiRuMbwvSDOD1JRVMZqnPbcQFuikCJM,2714
 flowfile_worker/create/__init__.py,sha256=vkWy5uODffivUdxt3nNVALj6xgQK3HPBetqR-QqZ-uo,1643
@@ -295,8 +306,8 @@ test_utils/__init__.py,sha256=8WwOgIuKw6YtOc1GWR1DqIhQ8BhlLWqsMyQJSpxnzKk,66
 test_utils/postgres/__init__.py,sha256=y3V_6a9N1Pvm5NIBaA8CFf3i4mvPVY-H1teHA-rg0VU,33
 test_utils/postgres/commands.py,sha256=4oA8EHW3EqwGkG02HSqEGbXEBGM01sUW5FsyHm86W4k,4347
 test_utils/postgres/fixtures.py,sha256=kR8UBjQr3pgbe-xM-V8x8VseTHCPv0EmDEzPHl5Qc8Y,13507
-flowfile-0.3.2.dist-info/LICENSE,sha256=pCfLAA27jMHReYk_wGiirZxWRRXz_Bm7PVInRCa9P5g,1075
-flowfile-0.3.2.dist-info/METADATA,sha256=N49dUO71mSeEwObw0v4F6corTg9PHcfEn5mWjgQ8g10,7903
-flowfile-0.3.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-flowfile-0.3.2.dist-info/entry_points.txt,sha256=Q3CEYNk33UaWlA9D-8yXYH0FwjKBsrtNuzzzHxhwnNI,333
-flowfile-0.3.2.dist-info/RECORD,,
+flowfile-0.3.3.1.dist-info/LICENSE,sha256=pCfLAA27jMHReYk_wGiirZxWRRXz_Bm7PVInRCa9P5g,1075
+flowfile-0.3.3.1.dist-info/METADATA,sha256=0FNb6Bf0qD5R4iD1gVM6ACB59eFkdl2-5R5l_uP_RIE,7998
+flowfile-0.3.3.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+flowfile-0.3.3.1.dist-info/entry_points.txt,sha256=Q3CEYNk33UaWlA9D-8yXYH0FwjKBsrtNuzzzHxhwnNI,333
+flowfile-0.3.3.1.dist-info/RECORD,,

flowfile_core/configs/__init__.py

@@ -11,16 +11,27 @@ logger = logging.getLogger('PipelineHandler')
 logger.setLevel(logging.INFO)
 logger.propagate = False
 
-# Create console handler with a specific format
-console_handler = logging.StreamHandler(sys.stdout)
+# Clear any existing handlers
+if logger.hasHandlers():
+    logger.handlers.clear()
+
+# Try to determine the best output stream
+output_stream = None
+if hasattr(sys.stdout, 'isatty') and sys.stdout.isatty():
+    output_stream = sys.stdout
+elif hasattr(sys.stderr, 'isatty') and sys.stderr.isatty():
+    output_stream = sys.stderr
+else:
+    # Use __stdout__ for debugger environments (PyDev, PyCharm, etc.)
+    output_stream = sys.__stdout__
+
+console_handler = logging.StreamHandler(output_stream)
 console_handler.setLevel(logging.INFO)
 
 # Create formatter
 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 console_handler.setFormatter(formatter)
 
-if logger.hasHandlers():
-    logger.handlers.clear()
 logger.addHandler(console_handler)
 
 # Create logs directory in temp at startup

flowfile_core/configs/settings.py

@@ -5,10 +5,10 @@ import os
 import tempfile
 import argparse
 
-from databases import DatabaseURL
 from passlib.context import CryptContext
 from starlette.config import Config
-from starlette.datastructures import Secret
+
+from flowfile_core.configs.utils import MutableBool
 
 
 # Constants for server and worker configuration
@@ -18,6 +18,9 @@ DEFAULT_WORKER_PORT = 63579
 SINGLE_FILE_MODE: bool = os.environ.get("SINGLE_FILE_MODE", "0") == "1"
 
 
+OFFLOAD_TO_WORKER = MutableBool(True)
+
+
 def parse_args():
     """Parse command line arguments"""
     parser = argparse.ArgumentParser(description="Flowfile Backend Server")
@@ -79,7 +82,6 @@ args = parse_args()
 SERVER_HOST = args.host if args.host is not None else DEFAULT_SERVER_HOST
 SERVER_PORT = args.port if args.port is not None else DEFAULT_SERVER_PORT
 WORKER_PORT = args.worker_port if args.worker_port is not None else int(os.getenv("WORKER_PORT", DEFAULT_WORKER_PORT))
-# Worker configuration
 WORKER_HOST = os.getenv("WORKER_HOST", "0.0.0.0" if platform.system() != "Windows" else "127.0.0.1")
 
 config = Config(".env")

flowfile_core/configs/utils.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class MutableBool:
+    value: bool
+
+    def __bool__(self) -> bool:
+        """Allow direct boolean evaluation"""
+        return self.value
+
+    def __eq__(self, other) -> bool:
+        """Allow equality comparison with booleans"""
+        if isinstance(other, bool):
+            return self.value == other
+        elif isinstance(other, MutableBool):
+            return self.value == other.value
+        return NotImplemented

flowfile_core/flowfile/FlowfileFlow.py

@@ -15,7 +15,7 @@ from flowfile_core.configs import logger
 from flowfile_core.configs.flow_logger import FlowLogger
 from flowfile_core.flowfile.sources.external_sources.factory import data_source_factory
 from flowfile_core.flowfile.sources.external_sources.airbyte_sources.settings import airbyte_settings_from_config
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import type_to_polars_str, FlowfileColumn
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import cast_str_to_polars_type, FlowfileColumn
 from flowfile_core.flowfile.flow_data_engine.fuzzy_matching.settings_validator import (calculate_fuzzy_match_schema,
                                                                                        pre_calculate_pivot_schema)
 from flowfile_core.utils.arrow_reader import get_read_top_n
@@ -25,7 +25,7 @@ from flowfile_core.flowfile.flow_data_engine.read_excel_tables import get_open_x
 from flowfile_core.flowfile.sources import external_sources
 from flowfile_core.schemas import input_schema, schemas, transform_schema
 from flowfile_core.schemas.output_model import TableExample, NodeData, NodeResult, RunInformation
-from flowfile_core.flowfile.utils import snake_case_to_camel_case
+from flowfile_core.flowfile.utils import snake_case_to_camel_case, _handle_raw_data
 from flowfile_core.flowfile.analytics.utils import create_graphic_walker_node_from_node_promise
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_core.flowfile.util.execution_orderer import determine_execution_order
@@ -34,7 +34,7 @@ from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_op
                                                                                                  ExternalDatabaseFetcher,
                                                                                                  ExternalDatabaseWriter,
                                                                                                  ExternalDfFetcher)
-from flowfile_core.secrets.secrets import get_encrypted_secret, decrypt_secret
+from flowfile_core.secret_manager.secret_manager import get_encrypted_secret, decrypt_secret
 from flowfile_core.flowfile.sources.external_sources.sql_source import utils as sql_utils, models as sql_models
 from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import SqlSource, BaseSqlSource
 from flowfile_core.flowfile.database_connection_manager.db_connections import get_local_database_connection
@@ -205,19 +205,12 @@ class FlowGraph:
         sample_size: int = 10000
 
         def analysis_preparation(flowfile_table: FlowDataEngine):
-            if flowfile_table.number_of_records < 0:
-
-                number_of_records = ExternalDfFetcher(
-                    lf=flowfile_table.data_frame,
-                    operation_type="calculate_number_of_records",
-                    flow_id=self.flow_id,
-                    node_id=node.node_id,
-                ).result
+            if flowfile_table.number_of_records <= 0:
+                number_of_records = flowfile_table.get_number_of_records(calculate_in_worker_process=True)
             else:
                 number_of_records = flowfile_table.number_of_records
             if number_of_records > sample_size:
                 flowfile_table = flowfile_table.get_sample(sample_size, random=True)
-
             external_sampler = ExternalDfFetcher(
                 lf=flowfile_table.data_frame,
                 file_ref="__gf_walker"+node.hash,
@@ -225,7 +218,7 @@ class FlowGraph:
                 node_id=node.node_id,
                 flow_id=self.flow_id,
             )
-            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref, 10000)
+            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref)
             return flowfile_table
 
         def schema_callback():
@@ -441,7 +434,7 @@ class FlowGraph:
     def add_formula(self, function_settings: input_schema.NodeFormula):
         error = ""
         if function_settings.function.field.data_type not in (None, "Auto"):
-            output_type = type_to_polars_str(function_settings.function.field.data_type)
+            output_type = cast_str_to_polars_type(function_settings.function.field.data_type)
         else:
             output_type = None
         if output_type not in (None, "Auto"):
@@ -486,7 +479,8 @@ class FlowGraph:
                            function=_func,
                            input_columns=[],
                            node_type='cross_join',
-                           setting_input=cross_join_settings)
+                           setting_input=cross_join_settings,
+                           input_node_ids=cross_join_settings.depending_on_ids)
         return self
 
     def add_join(self, join_settings: input_schema.NodeJoin) -> "FlowGraph":
@@ -1044,11 +1038,10 @@ class FlowGraph:
         return self
 
     def add_datasource(self, input_file: input_schema.NodeDatasource | input_schema.NodeManualInput):
-
         if isinstance(input_file, input_schema.NodeManualInput):
-            input_data = FlowDataEngine(input_file.raw_data)
+            _handle_raw_data(input_file)
+            input_data = FlowDataEngine(input_file.raw_data_format)
             ref = 'manual_input'
-
         else:
             input_data = FlowDataEngine(path_ref=input_file.file_ref)
             ref = 'datasource'
@@ -1061,7 +1054,9 @@ class FlowGraph:
 
             if not input_file.node_id in set(start_node.node_id for start_node in self._flow_starts):
                 self._flow_starts.append(node)
+
         else:
+            input_data.collect()
             node = FlowNode(input_file.node_id, function=input_data,
                             setting_input=input_file,
                             name=ref, node_type=ref, parent_uuid=self.uuid)

flowfile_core/flowfile/database_connection_manager/db_connections.py

@@ -1,7 +1,7 @@
 from flowfile_core.schemas.input_schema import FullDatabaseConnection, FullDatabaseConnectionInterface
 from sqlalchemy.orm import Session
 from flowfile_core.database.models import DatabaseConnection as DBConnectionModel, Secret
-from flowfile_core.secrets.secrets import store_secret, SecretInput, decrypt_secret
+from flowfile_core.secret_manager.secret_manager import store_secret, SecretInput, decrypt_secret
 from flowfile_core.database.connection import get_db_context
 
 

flowfile_core/flowfile/flow_data_engine/flow_data_engine.py

@@ -17,6 +17,7 @@ from pyarrow.parquet import ParquetFile
 # Local imports - Core
 from flowfile_core.configs import logger
 from flowfile_core.configs.flow_logger import NodeLogger
+from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
 from flowfile_core.schemas import (
     input_schema,
     transform_schema as transform_schemas
@@ -29,7 +30,7 @@ from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import (
     FlowfileColumn,
     convert_stats_to_column_info
 )
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import type_to_polars
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_data_engine.fuzzy_matching.prepare_for_fuzzy_match import prepare_for_fuzzy_match
 from flowfile_core.flowfile.flow_data_engine.join import (
     verify_join_select_integrity,
@@ -109,7 +110,7 @@ class FlowDataEngine:
     # flow_id: int = None  # TODO: Implement flow_id
 
     def __init__(self,
-                 raw_data: Union[List[Dict], List[Any], 'ParquetFile', pl.DataFrame, pl.LazyFrame] = None,
+                 raw_data: Union[List[Dict], List[Any], 'ParquetFile', pl.DataFrame, pl.LazyFrame, input_schema.RawData] = None,
                  path_ref: str = None,
                  name: str = None,
                  optimize_memory: bool = True,
@@ -147,7 +148,10 @@ class FlowDataEngine:
 
     def _handle_raw_data(self, raw_data, number_of_records, optimize_memory):
         """Process different types of input data."""
-        if isinstance(raw_data, pl.DataFrame):
+
+        if isinstance(raw_data, input_schema.RawData):
+            self._handle_raw_data_format(raw_data)
+        elif isinstance(raw_data, pl.DataFrame):
             self._handle_polars_dataframe(raw_data, number_of_records)
         elif isinstance(raw_data, pl.LazyFrame):
             self._handle_polars_lazy_frame(raw_data, number_of_records, optimize_memory)
@@ -190,6 +194,20 @@ class FlowDataEngine:
             self.number_of_records = 1
             self.data_frame = pl.DataFrame([data])
 
+    def _handle_raw_data_format(self, raw_data: input_schema.RawData):
+        """Create a FlowDataEngine from a RawData object."""
+        flowfile_schema = list(FlowfileColumn.create_from_minimal_field_info(c) for c in raw_data.columns)
+        polars_schema = pl.Schema([(flowfile_column.column_name, flowfile_column.get_polars_type().pl_datatype)
+                                   for flowfile_column in flowfile_schema])
+        try:
+            df = pl.DataFrame(raw_data.data, polars_schema)
+        except TypeError as e:
+            logger.warning(f"Could not parse the data with the schema:\n{e}")
+            df = pl.DataFrame(raw_data.data)
+        self.number_of_records = len(df)
+        self.data_frame = df.lazy()
+        self.lazy = True
+
     def _handle_list_input(self, data: List):
         """Handle list input."""
         number_of_records = len(data)
@@ -462,6 +480,9 @@ class FlowDataEngine:
             return self.data_frame.collect(engine="streaming" if self._streamable else "auto").to_dicts()
         return self.data_frame.to_dicts()
 
+    def to_dict(self) -> Dict[str, List]:
+        return self.data_frame.collect(engine="streaming" if self._streamable else "auto").to_dict(as_series=False)
+
     @classmethod
     def create_from_external_source(cls, external_source: ExternalDataSource) -> "FlowDataEngine":
         """Create a FlowDataEngine from an external data source."""
@@ -484,7 +505,7 @@ class FlowDataEngine:
         """Create a FlowDataEngine from a schema definition."""
         pl_schema = []
         for i, flow_file_column in enumerate(schema):
-            pl_schema.append((flow_file_column.name, type_to_polars(flow_file_column.data_type)))
+            pl_schema.append((flow_file_column.name, cast_str_to_polars_type(flow_file_column.data_type)))
             schema[i].col_index = i
         df = pl.LazyFrame(schema=pl_schema)
         return cls(df, schema=schema, calculate_schema_stats=False, number_of_records=0)
@@ -824,7 +845,7 @@ class FlowDataEngine:
         Returns:
             FlowDataEngine: New instance with sampled data
         """
-        n_records = min(n_rows, self.number_of_records)
+        n_records = min(n_rows, self.get_number_of_records(calculate_in_worker_process=True))
         logging.info(f'Getting sample of {n_rows} rows')
 
         if random:
@@ -1158,14 +1179,25 @@ class FlowDataEngine:
         self.number_of_records = 0
         self._lazy = True
 
-    def get_number_of_records(self, warn: bool = False, force_calculate: bool = False) -> int:
+    def _calculate_number_of_records_in_worker(self) -> int:
+        number_of_records = ExternalDfFetcher(
+            lf=self.data_frame,
+            operation_type="calculate_number_of_records",
+            flow_id=-1,
+            node_id=-1,
+            wait_on_completion=True
+        ).result
+        return number_of_records
+
+    def get_number_of_records(self, warn: bool = False, force_calculate: bool = False,
+                              calculate_in_worker_process: bool = False) -> int:
         """
         Get the total number of records in the DataFrame.
 
         Args:
             warn: Whether to warn about expensive operations
             force_calculate: Whether to force recalculation
-
+            calculate_in_worker_process: Whether to offload compute to the worker process
         Returns:
             int: Number of records
 
@@ -1174,22 +1206,24 @@ class FlowDataEngine:
         """
         if self.is_future and not self.is_collected:
             return -1
-
+        calculate_in_worker_process = False if not OFFLOAD_TO_WORKER.value else calculate_in_worker_process
         if self.number_of_records is None or self.number_of_records < 0 or force_calculate:
             if self._number_of_records_callback is not None:
                 self._number_of_records_callback(self)
 
             if self.lazy:
-                if warn:
-                    logger.warning('Calculating the number of records this can be expensive on a lazy frame')
-                try:
-                    self.number_of_records = self.data_frame.select(pl.len()).collect(
-                        engine="streaming" if self._streamable else "auto")[0, 0]
-                except Exception:
-                    raise Exception('Could not get number of records')
+                if calculate_in_worker_process:
+                    self.number_of_records = self._calculate_number_of_records_in_worker()
+                else:
+                    if warn:
+                        logger.warning('Calculating the number of records this can be expensive on a lazy frame')
+                    try:
+                        self.number_of_records = self.data_frame.select(pl.len()).collect(
+                            engine="streaming" if self._streamable else "auto")[0, 0]
+                    except Exception:
+                        raise ValueError('Could not get number of records')
             else:
                 self.number_of_records = self.data_frame.__len__()
-
         return self.number_of_records
 
     # Properties
@@ -1518,4 +1552,7 @@ def execute_polars_code(*flowfile_tables: "FlowDataEngine", code: str) -> "FlowD
         kwargs = {'input_df': flowfile_tables[0].data_frame}
     else:
         kwargs = {f'input_df_{i+1}': flowfile_table.data_frame for i, flowfile_table in enumerate(flowfile_tables)}
-    return FlowDataEngine(polars_executable(**kwargs))
+    df = polars_executable(**kwargs)
+    if isinstance(df, pl.DataFrame):
+        logger.warning("Got a non lazy DataFrame, possibly harming performance, if possible, try to use a lazy method")
+    return FlowDataEngine(df)