Flowfile 0.3.7__py3-none-any.whl → 0.3.8__py3-none-any.whl

flowfile/__init__.py

@@ -7,7 +7,7 @@ This package ties together the FlowFile ecosystem components:
 - flowfile_worker: Computation engine
 """
 
-__version__ = "0.3.7"
+__version__ = "0.3.8"
 
 import os
 import logging

flowfile/api.py

@@ -206,7 +206,7 @@ def check_if_in_single_mode() -> bool:
     try:
         response: requests.Response = requests.get(f"{FLOWFILE_BASE_URL}/single_mode", timeout=1)
         if response.ok:
-            return response.json() == "1"
+            return response.json()
     except Exception:
         pass
     return False
@@ -400,6 +400,8 @@ def _open_flow_in_browser(flow_id: int) -> None:
         logger.info(f"Unified mode detected. Opening imported flow in browser: {flow_url}")
         try:
             time.sleep(0.5)
+            logger.info("Attempting to open browser tab for flow...")
+            logger.info("Opening URL in browser: %s", flow_url)
             webbrowser.open_new_tab(flow_url)
         except Exception as wb_err:
             logger.warning(f"Could not automatically open browser tab: {wb_err}")
@@ -452,7 +454,7 @@ def open_graph_in_editor(flow_graph: FlowGraph, storage_location: Optional[str]
             return False
 
         flow_id = import_flow_to_editor(flow_file_path, auth_token)
-
+        print(flow_id, "flow_id", flow_in_single_mode, automatically_open_browser)
         if flow_id is not None:
             if flow_in_single_mode and automatically_open_browser:
                 _open_flow_in_browser(flow_id)

flowfile/web/__init__.py

@@ -51,6 +51,8 @@ def extend_app(app: FastAPI):
 
     @app.get("/single_mode")
     async def in_single_mode() -> bool:
+        print("Checking if single file mode is enabled")
+        print(os.environ.get('FLOWFILE_SINGLE_FILE_MODE'))
         return os.environ.get('FLOWFILE_SINGLE_FILE_MODE', "0") == "1"
 
     @app.get("/ui", include_in_schema=False)

{flowfile-0.3.7.dist-info → flowfile-0.3.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: Flowfile
-Version: 0.3.7
+Version: 0.3.8
 Summary: Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)
 Author: Edward van Eechoud
 Author-email: evaneechoud@gmail.com

{flowfile-0.3.7.dist-info → flowfile-0.3.8.dist-info}/RECORD

@@ -1,10 +1,10 @@
 build_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 build_backends/main.py,sha256=hLmfqTeHLSTiwwZ5mUuoLQgtO40Igvl1_4NbnvzWSgI,9912
 build_backends/main_prd.py,sha256=JR2tYCMWM5ThooQjv5pw6nwVKMQjgsiHgKMhYn9NXWI,6927
-flowfile/__init__.py,sha256=kQ10o5ueV_t_wRXShd1FTlLmCpGxeS4InxA6VurC7UE,4025
+flowfile/__init__.py,sha256=ba9c-bTEliEzkOy8awqjFHYOesK6JpuJHAwdYIISALo,4025
 flowfile/__main__.py,sha256=cpWeAL9Xw2qHfm52ZCAQzQhjoOAmVFSXPL-8MrnWAxA,2540
-flowfile/api.py,sha256=TDxw2g6cF_A94jTDAwTEnVyve99i3UJMjC9D0z8KgSg,18720
-flowfile/web/__init__.py,sha256=DMbHVQ2yVopshBQr5dBIn7hqrxO9RqCdwc_CA9Kd6F0,5780
+flowfile/api.py,sha256=kN03OrC_X0448DyPPVWGk6b9Y4pi3LAMqwsOKhWbDlI,18929
+flowfile/web/__init__.py,sha256=InpX02yACH4ZJLVKFbpnJw7rg3k9KwtOKhxLsuHsJEc,5896
 flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css,sha256=Lf3OL7Vgg7ns0m6C0FiCNY0h4vgKALPrvTTkETjQmdc,1621
 flowfile/web/static/assets/CloudConnectionManager-c20a740f.js,sha256=HCr-EGu2ZPo3ZEcEU5ILcDd8zHCQaFjuZh4FpgGUSqQ,35768
 flowfile/web/static/assets/CloudStorageReader-29d14fcc.css,sha256=KdFPzPpEdf69gKbDLTPG_bk5QvJKaFXLloUjfzg67-s,3154
@@ -147,9 +147,9 @@ flowfile_core/configs/flow_logger.py,sha256=Pk1yhaC58jjISMrgwhBFIue9Qj5XfYo8NfOe
 flowfile_core/configs/node_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/configs/node_store/nodes.py,sha256=cZmevDcRYthaHxM04BE2WjsswQdMpaOQHNus-sRBQlc,6388
 flowfile_core/configs/settings.py,sha256=5EAZKlq7oehakL3F0uT_VI_xXGkcIK9NL2-KAZq_Na8,3199
-flowfile_core/configs/utils.py,sha256=BjyJCfO4gR_n9Sbs6dPlFfSKwxfxd3aKZ9zUH0Xs5GY,474
+flowfile_core/configs/utils.py,sha256=DBLpZfvgzTQmF3c8ABediMBcORIdEtcyy2L8eIZa4vM,601
 flowfile_core/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/database/connection.py,sha256=g6oqL8d47eED58lJleL6mVdTTzSZTb2BY-E-6P0zWRY,3057
+flowfile_core/database/connection.py,sha256=CVqbQEQ5i-I9Dnybh_ZBRBc52I5ZfrWtR4Zqod73Rc0,2999
 flowfile_core/database/init_db.py,sha256=9j4osTUrjsX3Y4z8hRVFDTCNzQxXqxZqXG2rwb6Mz5I,1235
 flowfile_core/database/models.py,sha256=7PlhE9Xe5dMvI5mqAlPWGS-LAZbFWq111EfCCXRT-ec,3440
 flowfile_core/fileExplorer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -173,10 +173,10 @@ flowfile_core/flowfile/database_connection_manager/db_connections.py,sha256=dVYr
 flowfile_core/flowfile/database_connection_manager/models.py,sha256=lVJSifqznQ8fKGWBEBCy_8JeXmdKF4pnAE5P5oXRrLM,379
 flowfile_core/flowfile/extensions.py,sha256=gUKTzNs6YBayHF51-ZacMknvrE1S8OXc89qGIdqPl2w,1814
 flowfile_core/flowfile/flow_data_engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py,sha256=9QlduMjmFhhwEvzwqkbS3eANvmMasn1C_lj_knSfDJw,10493
+flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py,sha256=QE2h9ULd1zyqQ_Ph4yqpDMekTOtqH2ypu-NwhVfmgVc,10414
 flowfile_core/flowfile/flow_data_engine/create/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/flow_data_engine/create/funcs.py,sha256=Hvk4L3aOppt3RyKBEqFutx1t2RF8qA6QdrS1lPnJJkY,7822
-flowfile_core/flowfile/flow_data_engine/flow_data_engine.py,sha256=Wg9UkvtKN9t8dVfPyxOC0cotwHfh1EVJ5Rki3JzP0lI,100068
+flowfile_core/flowfile/flow_data_engine/flow_data_engine.py,sha256=ycGWuohlkD3x0IPQQvGPcl9BHkt-fc8DOOxURZwPPQA,100062
 flowfile_core/flowfile/flow_data_engine/flow_file_column/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py,sha256=auDG4EeLcnXlsTe3IlU6t8jLsVUEre0-QcoQRACt0xU,7484
 flowfile_core/flowfile/flow_data_engine/flow_file_column/polars_type.py,sha256=xusyOLwSxevBk8-Uy9ZKISB_KOi0JeYfZ0wihcG-Qjk,530
@@ -197,10 +197,10 @@ flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operati
 flowfile_core/flowfile/flow_data_engine/threaded_processes.py,sha256=15IPoqIoCfkPMb648o9hnOzNnZINhYQxJi5hNfQuRwE,1311
 flowfile_core/flowfile/flow_data_engine/types.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/flowfile/flow_data_engine/utils.py,sha256=Nkw4MM6SX9veOAC8bthWTLUqA_aIFLtDnJG3bTNcwoc,6628
-flowfile_core/flowfile/flow_graph.py,sha256=UqEk3sLDL1K4EucMgySw75B45Mj5Qnjbzl3k1LJ4w7o,91002
+flowfile_core/flowfile/flow_graph.py,sha256=OUKfAio_HNB8eQz5ofGDTSXPXUb9_otTdG34DTizcdA,91219
 flowfile_core/flowfile/flow_graph_utils.py,sha256=nqfQdkFo4DM1WpbMuL1tZps51dKFWZi2yu_UgNUxtNU,10057
 flowfile_core/flowfile/flow_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/flowfile/flow_node/flow_node.py,sha256=L9p0grp59WBFEtw6NxNK5nHOcXwIqs7XVG9eG90-oOQ,51948
+flowfile_core/flowfile/flow_node/flow_node.py,sha256=frRM97Ac4BcXAtYzorLDBCUU7g7aT2nHfigC-gq6Al0,52159
 flowfile_core/flowfile/flow_node/models.py,sha256=2inXz2Ov5gl2NzGh6HGtrKM3ow93iyEixsqX2w3pgdM,9929
 flowfile_core/flowfile/flow_node/schema_callback.py,sha256=ed8OLzRgQluC6aa2CcvSWq7WC3ZNZnLvDk9rXWwIux4,2410
 flowfile_core/flowfile/handler.py,sha256=Ov25vblx6oiMM4atPScqk6OnuEZ1PEPgCCO6mqT0y8I,3210
@@ -247,12 +247,12 @@ flowfile_core/schemas/transform_schema.py,sha256=zwvomSAEgoEyQTdUbp_Mnawz2zVIuTv
 flowfile_core/secret_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flowfile_core/secret_manager/secret_manager.py,sha256=6G9hqo-GjTE7va6i6r3_ZmGo4Y26Aw_PY0W8L6pNo7E,2086
 flowfile_core/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flowfile_core/utils/arrow_reader.py,sha256=SbDDzOmtHcZ8rH88v1jN0EZthaDqBmvCrbruX07Qdks,9366
+flowfile_core/utils/arrow_reader.py,sha256=IeRaNEJvCVb-bVOmZnrZhzxFGWOVnH3p4MnXGxvUo_8,9541
 flowfile_core/utils/excel_file_manager.py,sha256=EIad2LenHu-3Yw1FcLmE0KgmLflnvNKt07FY6s6mPlE,452
 flowfile_core/utils/fileManager.py,sha256=LnJhK_pwjb9MIApG2e4Hp3L5Z7Wny8YYHaL9SkW8WlE,1371
 flowfile_core/utils/fl_executor.py,sha256=eNnNZHZ9451brzZD00_X8aoCHFl1hR1gVOIGxtE0Db4,1301
 flowfile_core/utils/utils.py,sha256=XPanIHWQjNLkd-lC7mKoWdjP3TAR_GZwYIbZ5_KF5o0,1326
-flowfile_core/utils/validate_setup.py,sha256=1kaintlS0MA-PDO6IKgW0NGGai7WVUeAM_Vnvbm1Mfk,1737
+flowfile_core/utils/validate_setup.py,sha256=3g6JtMgda-N7Ts_xlOSS8HtW4ea3eDsFq2UbguRwzdI,1641
 flowfile_frame/__init__.py,sha256=JDEFbzBQ-YtB129IppN06v-8KOYb3SADtOTDgCpPh0k,2044
 flowfile_frame/adapters.py,sha256=8H1z6UpYQ3aeuSYNs8GbWfL5BUzVbGJbGzi6b9uVBt4,528
 flowfile_frame/adding_expr.py,sha256=K9KdF3WCbPy2YO5radyEBXQ0yi6SOI4nviaPwXmKW0k,12428
@@ -260,11 +260,11 @@ flowfile_frame/cloud_storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 flowfile_frame/cloud_storage/frame_helpers.py,sha256=4x5I5WVKpT1PXoPvEJd28MmsHDCJNhZxAdG6MKOSA_4,2271
 flowfile_frame/cloud_storage/secret_manager.py,sha256=lIjOf2u6ZWrXFFYCrHK2XH51p2DEl9m7Pr1uLKe1DUI,2540
 flowfile_frame/config.py,sha256=Tl_4V8byUiO1y3ANesW2E9qeTtZvtKYJl8KU00hc1Cg,176
-flowfile_frame/expr.py,sha256=OmpQZ7xleBoBmSLwlZ1pp1_WVd38bpqCs8OGQL0vx7s,59264
+flowfile_frame/expr.py,sha256=r340IF6t2cZkUeXmhdU-wOxpiNpjiyQ4dEABO_VZxk8,59743
 flowfile_frame/expr.pyi,sha256=kUMj6IRlczjInzUb6WXkBrxboCRhWdkxGlpiR30zCKw,89316
 flowfile_frame/expr_name.py,sha256=0VZZwC3xc6tiwOJPu7emQ86Xp2mVu7U-j-jrsMjbc2Y,9852
-flowfile_frame/flow_frame.py,sha256=NCny_WgHzM2tnqdJGZTDGrFOj1jiMAJWvFgSK0N7tv4,101530
-flowfile_frame/flow_frame.pyi,sha256=P0DPw5xx6nXoL7hbUOS5t06OmGZjY7P5ZgHO1iCvYIw,33956
+flowfile_frame/flow_frame.py,sha256=Rvz8SCRtLEtqWCjKeTiqoSGfGX2IBCnjqeeFx7wllUI,101545
+flowfile_frame/flow_frame.pyi,sha256=HChIEnLil9_8sFAI61jqA9ldhfDZheIaHY9HFp-Vnr0,33664
 flowfile_frame/flow_frame_methods.py,sha256=mHFltRFeKQ4m2aDsU0HcpAhACR0gdLCeONvBFeG-wJk,29054
 flowfile_frame/group_frame.py,sha256=aLpiT9ZEidVDJ04ORr57SPxR7WfgmziBPkBGXB9oR54,11632
 flowfile_frame/join.py,sha256=YQCHmw8nYzvGzvrFc0jOmVMy-rahK3CPtuFdKB7loS8,2437
@@ -313,8 +313,8 @@ test_utils/s3/commands.py,sha256=m-r6rWZZP7Q7bzXwuOId1QbQkw8aq8-3qcjH8wpSqxI,142
 test_utils/s3/data_generator.py,sha256=qEVN6paBvPf1W4Pxjb341WOLvTrx9D20MyJ944TAHc8,10939
 test_utils/s3/demo_data_generator.py,sha256=YuYnbEmuKBUsuuBMjvq9SCmm11pYJc75u-OqpOGMr5A,7682
 test_utils/s3/fixtures.py,sha256=n5qTGbW1LASFusYbeNh5_53aKui2Xp0yhT70_VXJ4NU,7778
-flowfile-0.3.7.dist-info/LICENSE,sha256=pCfLAA27jMHReYk_wGiirZxWRRXz_Bm7PVInRCa9P5g,1075
-flowfile-0.3.7.dist-info/METADATA,sha256=nmpxLVZlB17JVxvG3ZbxHmBAnAdxDOXenZLOKrU_B4g,8814
-flowfile-0.3.7.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-flowfile-0.3.7.dist-info/entry_points.txt,sha256=3VMU1A7kmYy8DSPw7axzpy00DEaVRoVagE4jyZFAz2U,425
-flowfile-0.3.7.dist-info/RECORD,,
+flowfile-0.3.8.dist-info/LICENSE,sha256=pCfLAA27jMHReYk_wGiirZxWRRXz_Bm7PVInRCa9P5g,1075
+flowfile-0.3.8.dist-info/METADATA,sha256=KxNspVyvQnG6Tp0njD_vr6sK34tXp1a18rUxdF6FHGg,8814
+flowfile-0.3.8.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+flowfile-0.3.8.dist-info/entry_points.txt,sha256=3VMU1A7kmYy8DSPw7axzpy00DEaVRoVagE4jyZFAz2U,425
+flowfile-0.3.8.dist-info/RECORD,,

flowfile_core/configs/utils.py

@@ -16,3 +16,8 @@ class MutableBool:
         elif isinstance(other, MutableBool):
             return self.value == other.value
         return NotImplemented
+
+    def set(self, value):
+        """Set the value of the MutableBool"""
+        self.value = bool(value)
+        return self

flowfile_core/database/connection.py

@@ -26,8 +26,6 @@ def get_app_data_dir() -> Path:
             base_dir = os.path.join(os.path.expanduser("~"), ".local", "share")
 
     app_dir = Path(base_dir) / app_name
-
-    print(f"Using application data directory: {app_dir}")
     app_dir.mkdir(parents=True, exist_ok=True)
 
     return app_dir
@@ -48,7 +46,7 @@ def get_database_url():
     app_dir = get_app_data_dir()
 
     db_path = app_dir / "flowfile.db"
-    logger.info(f"Using database URL: sqlite:///{db_path}")
+    logger.debug(f"Using database URL: sqlite:///{db_path}")
     return f"sqlite:///{db_path}"
 
 

flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py

@@ -68,7 +68,6 @@ class CloudStorageReader:
     def _get_s3_storage_options(connection: 'FullCloudStorageConnection') -> Dict[str, Any]:
         """Build S3-specific storage options."""
         auth_method = connection.auth_method
-        print(f"Building S3 storage options for auth_method: '{auth_method}'")
         if auth_method == "aws-cli":
             return create_storage_options_from_boto_credentials(
                 profile_name=connection.connection_name,

flowfile_core/flowfile/flow_data_engine/flow_data_engine.py

@@ -1956,7 +1956,7 @@ class FlowDataEngine:
         """
         if self.is_future and not self.is_collected:
             return -1
-        calculate_in_worker_process = False if not OFFLOAD_TO_WORKER.value else calculate_in_worker_process
+        calculate_in_worker_process = False if not OFFLOAD_TO_WORKER else calculate_in_worker_process
         if self.number_of_records is None or self.number_of_records < 0 or force_calculate:
             if self._number_of_records_callback is not None:
                 self._number_of_records_callback(self)

flowfile_core/flowfile/flow_graph.py

@@ -300,8 +300,6 @@ class FlowGraph:
 
         return print(tree)
 
-
-
     def apply_layout(self, y_spacing: int = 150, x_spacing: int = 200, initial_y: int = 100):
         """Calculates and applies a layered layout to all nodes in the graph.
 
@@ -490,7 +488,8 @@ class FlowGraph:
                 node_id=node.node_id,
                 flow_id=self.flow_id,
             )
-            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref)
+            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref,
+                                                                  n=min(sample_size, number_of_records))
             return flowfile_table
 
         def schema_callback():
@@ -1581,12 +1580,13 @@ class FlowGraph:
             execution_order = determine_execution_order(all_nodes=[node for node in self.nodes if
                                                                    node not in skip_nodes],
                                                         flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
-
             skip_node_message(self.flow_logger, skip_nodes)
             execution_order_message(self.flow_logger, execution_order)
             performance_mode = self.flow_settings.execution_mode == 'Performance'
             if self.flow_settings.execution_location == 'local':
                 OFFLOAD_TO_WORKER.value = False
+            elif self.flow_settings.execution_location == 'remote':
+                OFFLOAD_TO_WORKER.value = True
             for node in execution_order:
                 node_logger = self.flow_logger.get_node_logger(node.node_id)
                 if self.flow_settings.is_canceled:

flowfile_core/flowfile/flow_node/flow_node.py

@@ -5,7 +5,7 @@ from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEng
 from flowfile_core.utils.arrow_reader import get_read_top_n
 from flowfile_core.schemas import input_schema, schemas
 from flowfile_core.configs.flow_logger import NodeLogger
-from flowfile_core.configs.settings import SINGLE_FILE_MODE
+from flowfile_core.configs.settings import SINGLE_FILE_MODE, OFFLOAD_TO_WORKER
 
 from flowfile_core.schemas.output_model import TableExample, FileColumn, NodeData
 from flowfile_core.flowfile.utils import get_hash
@@ -724,9 +724,19 @@ class FlowNode:
         Raises:
             Exception: Propagates exceptions from the execution.
         """
-        if self.results.resulting_data is None and not performance_mode:
-            self.results.resulting_data = self.get_resulting_data()
-            self.results.example_data_generator = lambda: self.get_resulting_data().get_sample(100).to_arrow()
+        def example_data_generator():
+            example_data = None
+
+            def get_example_data():
+                nonlocal example_data
+                if example_data is None:
+                    example_data = resulting_data.get_sample(100).to_arrow()
+                return example_data
+            return get_example_data
+        resulting_data = self.get_resulting_data()
+
+        if not performance_mode:
+            self.results.example_data_generator = example_data_generator()
             self.node_schema.result_schema = self.results.resulting_data.schema
             self.node_stats.has_completed_last_run = True
 
@@ -899,8 +909,8 @@ class FlowNode:
                     else:
                         self.results.errors = str(e)
                         node_logger.error(f'Error with running the node: {e}')
-            elif ((run_location == 'local' or SINGLE_FILE_MODE) and (not self.node_stats.has_run_with_current_setup
-                                                                     or self.node_template.node_group == "output")):
+            elif ((run_location == 'local' or SINGLE_FILE_MODE) and
+                  (not self.node_stats.has_run_with_current_setup or self.node_template.node_group == "output")):
                 try:
                     node_logger.info('Executing fully locally')
                     self.execute_full_local(performance_mode)

flowfile_core/utils/arrow_reader.py

@@ -138,11 +138,16 @@ def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> Tuple[List[
     rows_collected = 0
 
     for batch in iter_batches(reader, n, rows_collected):
-        batches.append(batch)
+
         rows_collected += batch.num_rows
         logger.debug(f"Collected batch: total rows now {rows_collected}")
         if rows_collected >= n:
+            if rows_collected > n:
+                batches.append(batch.slice(0, n - (rows_collected - batch.num_rows)))
+            else:
+                batches.append(batch)
             break
+        batches.append(batch)
 
     logger.info(f"Finished collecting {len(batches)} batches with {rows_collected} total rows")
     return batches, rows_collected
@@ -217,7 +222,7 @@ def read_top_n(file_path: str, n: int = 1000, strict: bool = False) -> pa.Table:
 
         table = pa.Table.from_batches(batches)  # type: ignore
         logger.info(f"Successfully read {rows_collected} rows from {file_path}")
-        return table
+    return table
 
 
 def get_read_top_n(file_path: str, n: int = 1000, strict: bool = False) -> Callable[[], pa.Table]:
@@ -244,4 +249,4 @@ def get_read_top_n(file_path: str, n: int = 1000, strict: bool = False) -> Calla
         >>> table = reader_func()
     """
     logger.info(f"Creating reader function for {file_path} with n={n}, strict={strict}")
-    return lambda: read_top_n(file_path, n, strict)
+    return lambda: read_top_n(file_path, n, strict)

flowfile_core/utils/validate_setup.py

@@ -34,8 +34,6 @@ def validate_setup():
         check_if_node_has_add_function_in_flow_graph(node)
         check_if_node_has_input_schema_definition(node)
 
-    print("All nodes have corresponding functions in FlowGraph and input schema definitions.")
-
 
 if __name__ == "__main__":
     validate_setup()

flowfile_frame/expr.py

@@ -490,6 +490,20 @@ class Expr:
         result.agg_func = "sum"
         return result
 
+    def unique_counts(self):
+        """
+        Return the number of unique values in the column.
+
+        Returns
+        -------
+        Expr
+            A new expression with the unique counts
+        """
+        result_expr = self.expr.unique_counts() if self.expr is not None else None
+        result = self._create_next_expr(method_name="unique_counts", result_expr=result_expr, is_complex=self.is_complex)
+        result.agg_func = "unique_counts"
+        return result
+
     def implode(self):
         result_expr = self.expr.implode() if self.expr is not None else None
         result = self._create_next_expr(method_name="implode", result_expr=result_expr, is_complex=self.is_complex)

flowfile_frame/flow_frame.py

@@ -565,7 +565,7 @@ class FlowFrame:
             coalesce: bool = None,
             maintain_order: Literal[None, "left", "right", "left_right", "right_left"] = None,
             description: str = None,
-    ):
+    ) -> "FlowFrame":
         """
         Add a join operation to the Logical Plan.
 

flowfile_frame/flow_frame.pyi

@@ -80,8 +80,8 @@ class FlowFrame:
 
     def __ne__(self, other: object) -> typing.NoReturn: ...
 
-    # Create and configure a new FlowFrame instance, mimicking Polars' flexible constructor.
-    def __new__(cls, data: typing.Union[LazyFrame, collections.abc.Mapping[str, typing.Union[collections.abc.Sequence[object], collections.abc.Mapping[str, collections.abc.Sequence[object]], ForwardRef('Series')]], collections.abc.Sequence[typing.Any], ForwardRef('np.ndarray[Any, Any]'), ForwardRef('pa.Table'), ForwardRef('pd.DataFrame'), ForwardRef('ArrowArrayExportable'), ForwardRef('ArrowStreamExportable'), ForwardRef('torch.Tensor')] = None, schema: typing.Union[collections.abc.Mapping[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType'), type[int], type[float], type[bool], type[str], type['date'], type['time'], type['datetime'], type['timedelta'], type[list[typing.Any]], type[tuple[typing.Any, ...]], type[bytes], type[object], type['Decimal'], type[None], NoneType]], collections.abc.Sequence[typing.Union[str, tuple[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType'), type[int], type[float], type[bool], type[str], type['date'], type['time'], type['datetime'], type['timedelta'], type[list[typing.Any]], type[tuple[typing.Any, ...]], type[bytes], type[object], type['Decimal'], type[None], NoneType]]]], NoneType] = None, schema_overrides: collections.abc.Mapping[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType')]] | None = None, strict: bool = True, orient: typing.Optional[typing.Literal['col', 'row']] = None, infer_schema_length: int | None = 100, nan_to_null: bool = False, flow_graph: typing.Optional[flowfile_core.flowfile.flow_graph.FlowGraph] = None, node_id: typing.Optional[int] = None, parent_node_id: typing.Optional[int] = None, override_initial: bool = False) -> Self: ...
+    # Unified constructor for FlowFrame.
+    def __new__(cls, data: typing.Union[LazyFrame, collections.abc.Mapping[str, typing.Union[collections.abc.Sequence[object], collections.abc.Mapping[str, collections.abc.Sequence[object]], ForwardRef('Series')]], collections.abc.Sequence[typing.Any], ForwardRef('np.ndarray[Any, Any]'), ForwardRef('pa.Table'), ForwardRef('pd.DataFrame'), ForwardRef('ArrowArrayExportable'), ForwardRef('ArrowStreamExportable'), ForwardRef('torch.Tensor')] = None, schema: typing.Union[collections.abc.Mapping[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType'), type[int], type[float], type[bool], type[str], type['date'], type['time'], type['datetime'], type['timedelta'], type[list[typing.Any]], type[tuple[typing.Any, ...]], type[bytes], type[object], type['Decimal'], type[None], NoneType]], collections.abc.Sequence[typing.Union[str, tuple[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType'), type[int], type[float], type[bool], type[str], type['date'], type['time'], type['datetime'], type['timedelta'], type[list[typing.Any]], type[tuple[typing.Any, ...]], type[bytes], type[object], type['Decimal'], type[None], NoneType]]]], NoneType] = None, schema_overrides: collections.abc.Mapping[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType')]] | None = None, strict: bool = True, orient: typing.Optional[typing.Literal['col', 'row']] = None, infer_schema_length: int | None = 100, nan_to_null: bool = False, flow_graph: typing.Optional[flowfile_core.flowfile.flow_graph.FlowGraph] = None, node_id: typing.Optional[int] = None, parent_node_id: typing.Optional[int] = None, **kwargs) -> Self: ...
 
     def __repr__(self) -> Any: ...
 
@@ -118,9 +118,6 @@ class FlowFrame:
     # Execute join using Polars code approach.
     def _execute_polars_code_join(self, other: FlowFrame, new_node_id: int, on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column], left_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column], right_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column], left_columns: typing.Optional[typing.List[str]], right_columns: typing.Optional[typing.List[str]], how: str, suffix: str, validate: str, nulls_equal: bool, coalesce: bool, maintain_order: typing.Literal[None, 'left', 'right', 'left_right', 'right_left'], description: str) -> 'FlowFrame': ...
 
-    # Internal constructor to create a FlowFrame instance that wraps an
-    def _from_existing_node(self, data: LazyFrame, flow_graph: FlowGraph, node_id: int, parent_node_id: typing.Optional[int] = None) -> 'FlowFrame': ...
-
     # Generates the `input_df.sort(...)` Polars code string using pure expression strings.
     def _generate_sort_polars_code(self, pure_sort_expr_strs: typing.List[str], descending_values: typing.List[bool], nulls_last_values: typing.List[bool], multithreaded: bool, maintain_order: bool) -> str: ...
 
@@ -231,7 +228,7 @@ class FlowFrame:
     def interpolate(self, description: Optional[str] = None) -> 'FlowFrame': ...
 
     # Add a join operation to the Logical Plan.
-    def join(self, other, on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, how: str = 'inner', left_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, right_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, suffix: str = '_right', validate: str = None, nulls_equal: bool = False, coalesce: bool = None, maintain_order: typing.Literal[None, 'left', 'right', 'left_right', 'right_left'] = None, description: str = None) -> Any: ...
+    def join(self, other, on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, how: str = 'inner', left_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, right_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, suffix: str = '_right', validate: str = None, nulls_equal: bool = False, coalesce: bool = None, maintain_order: typing.Literal[None, 'left', 'right', 'left_right', 'right_left'] = None, description: str = None) -> 'FlowFrame': ...
 
     # Perform an asof join.
     def join_asof(self, other: FlowFrame, left_on: str | None | Expr = None, right_on: str | None | Expr = None, on: str | None | Expr = None, by_left: str | Sequence[str] | None = None, by_right: str | Sequence[str] | None = None, by: str | Sequence[str] | None = None, strategy: AsofJoinStrategy = 'backward', suffix: str = '_right', tolerance: str | int | float | timedelta | None = None, allow_parallel: bool = True, force_parallel: bool = False, coalesce: bool = True, allow_exact_matches: bool = True, check_sortedness: bool = True, description: Optional[str] = None) -> 'FlowFrame': ...