Flowfile 0.3.7__tar.gz → 0.3.8__tar.gz

{flowfile-0.3.7 → flowfile-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: Flowfile
-Version: 0.3.7
+Version: 0.3.8
 Summary: Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)
 Author: Edward van Eechoud
 Author-email: evaneechoud@gmail.com

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile/flowfile/__init__.py

@@ -7,7 +7,7 @@ This package ties together the FlowFile ecosystem components:
 - flowfile_worker: Computation engine
 """
 
-__version__ = "0.3.7"
+__version__ = "0.3.8"
 
 import os
 import logging

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile/flowfile/api.py

@@ -206,7 +206,7 @@ def check_if_in_single_mode() -> bool:
     try:
         response: requests.Response = requests.get(f"{FLOWFILE_BASE_URL}/single_mode", timeout=1)
         if response.ok:
-            return response.json() == "1"
+            return response.json()
     except Exception:
         pass
     return False
@@ -400,6 +400,8 @@ def _open_flow_in_browser(flow_id: int) -> None:
         logger.info(f"Unified mode detected. Opening imported flow in browser: {flow_url}")
         try:
             time.sleep(0.5)
+            logger.info("Attempting to open browser tab for flow...")
+            logger.info("Opening URL in browser: %s", flow_url)
             webbrowser.open_new_tab(flow_url)
         except Exception as wb_err:
             logger.warning(f"Could not automatically open browser tab: {wb_err}")
@@ -452,7 +454,7 @@ def open_graph_in_editor(flow_graph: FlowGraph, storage_location: Optional[str]
             return False
 
         flow_id = import_flow_to_editor(flow_file_path, auth_token)
-
+        print(flow_id, "flow_id", flow_in_single_mode, automatically_open_browser)
         if flow_id is not None:
             if flow_in_single_mode and automatically_open_browser:
                 _open_flow_in_browser(flow_id)

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile/flowfile/web/__init__.py

@@ -51,6 +51,8 @@ def extend_app(app: FastAPI):
 
     @app.get("/single_mode")
     async def in_single_mode() -> bool:
+        print("Checking if single file mode is enabled")
+        print(os.environ.get('FLOWFILE_SINGLE_FILE_MODE'))
         return os.environ.get('FLOWFILE_SINGLE_FILE_MODE', "0") == "1"
 
     @app.get("/ui", include_in_schema=False)

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_core/flowfile_core/configs/utils.py

@@ -16,3 +16,8 @@ class MutableBool:
         elif isinstance(other, MutableBool):
             return self.value == other.value
         return NotImplemented
+
+    def set(self, value):
+        """Set the value of the MutableBool"""
+        self.value = bool(value)
+        return self

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_core/flowfile_core/database/connection.py

@@ -26,8 +26,6 @@ def get_app_data_dir() -> Path:
             base_dir = os.path.join(os.path.expanduser("~"), ".local", "share")
 
     app_dir = Path(base_dir) / app_name
-
-    print(f"Using application data directory: {app_dir}")
     app_dir.mkdir(parents=True, exist_ok=True)
 
     return app_dir
@@ -48,7 +46,7 @@ def get_database_url():
     app_dir = get_app_data_dir()
 
     db_path = app_dir / "flowfile.db"
-    logger.info(f"Using database URL: sqlite:///{db_path}")
+    logger.debug(f"Using database URL: sqlite:///{db_path}")
     return f"sqlite:///{db_path}"
 
 

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_core/flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py

@@ -68,7 +68,6 @@ class CloudStorageReader:
     def _get_s3_storage_options(connection: 'FullCloudStorageConnection') -> Dict[str, Any]:
         """Build S3-specific storage options."""
         auth_method = connection.auth_method
-        print(f"Building S3 storage options for auth_method: '{auth_method}'")
         if auth_method == "aws-cli":
             return create_storage_options_from_boto_credentials(
                 profile_name=connection.connection_name,

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py

@@ -1956,7 +1956,7 @@ class FlowDataEngine:
         """
         if self.is_future and not self.is_collected:
             return -1
-        calculate_in_worker_process = False if not OFFLOAD_TO_WORKER.value else calculate_in_worker_process
+        calculate_in_worker_process = False if not OFFLOAD_TO_WORKER else calculate_in_worker_process
         if self.number_of_records is None or self.number_of_records < 0 or force_calculate:
             if self._number_of_records_callback is not None:
                 self._number_of_records_callback(self)

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_core/flowfile_core/flowfile/flow_graph.py

@@ -300,8 +300,6 @@ class FlowGraph:
 
         return print(tree)
 
-
-
     def apply_layout(self, y_spacing: int = 150, x_spacing: int = 200, initial_y: int = 100):
         """Calculates and applies a layered layout to all nodes in the graph.
 
@@ -490,7 +488,8 @@ class FlowGraph:
                 node_id=node.node_id,
                 flow_id=self.flow_id,
             )
-            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref)
+            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref,
+                                                                  n=min(sample_size, number_of_records))
             return flowfile_table
 
         def schema_callback():
@@ -1581,12 +1580,13 @@ class FlowGraph:
             execution_order = determine_execution_order(all_nodes=[node for node in self.nodes if
                                                                    node not in skip_nodes],
                                                         flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
-
             skip_node_message(self.flow_logger, skip_nodes)
             execution_order_message(self.flow_logger, execution_order)
             performance_mode = self.flow_settings.execution_mode == 'Performance'
             if self.flow_settings.execution_location == 'local':
                 OFFLOAD_TO_WORKER.value = False
+            elif self.flow_settings.execution_location == 'remote':
+                OFFLOAD_TO_WORKER.value = True
             for node in execution_order:
                 node_logger = self.flow_logger.get_node_logger(node.node_id)
                 if self.flow_settings.is_canceled:

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_core/flowfile_core/flowfile/flow_node/flow_node.py

@@ -5,7 +5,7 @@ from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEng
 from flowfile_core.utils.arrow_reader import get_read_top_n
 from flowfile_core.schemas import input_schema, schemas
 from flowfile_core.configs.flow_logger import NodeLogger
-from flowfile_core.configs.settings import SINGLE_FILE_MODE
+from flowfile_core.configs.settings import SINGLE_FILE_MODE, OFFLOAD_TO_WORKER
 
 from flowfile_core.schemas.output_model import TableExample, FileColumn, NodeData
 from flowfile_core.flowfile.utils import get_hash
@@ -724,9 +724,19 @@ class FlowNode:
         Raises:
             Exception: Propagates exceptions from the execution.
         """
-        if self.results.resulting_data is None and not performance_mode:
-            self.results.resulting_data = self.get_resulting_data()
-            self.results.example_data_generator = lambda: self.get_resulting_data().get_sample(100).to_arrow()
+        def example_data_generator():
+            example_data = None
+
+            def get_example_data():
+                nonlocal example_data
+                if example_data is None:
+                    example_data = resulting_data.get_sample(100).to_arrow()
+                return example_data
+            return get_example_data
+        resulting_data = self.get_resulting_data()
+
+        if not performance_mode:
+            self.results.example_data_generator = example_data_generator()
             self.node_schema.result_schema = self.results.resulting_data.schema
             self.node_stats.has_completed_last_run = True
 
@@ -899,8 +909,8 @@ class FlowNode:
                     else:
                         self.results.errors = str(e)
                         node_logger.error(f'Error with running the node: {e}')
-            elif ((run_location == 'local' or SINGLE_FILE_MODE) and (not self.node_stats.has_run_with_current_setup
-                                                                     or self.node_template.node_group == "output")):
+            elif ((run_location == 'local' or SINGLE_FILE_MODE) and
+                  (not self.node_stats.has_run_with_current_setup or self.node_template.node_group == "output")):
                 try:
                     node_logger.info('Executing fully locally')
                     self.execute_full_local(performance_mode)

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_core/flowfile_core/utils/arrow_reader.py

@@ -138,11 +138,16 @@ def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> Tuple[List[
     rows_collected = 0
 
     for batch in iter_batches(reader, n, rows_collected):
-        batches.append(batch)
+
         rows_collected += batch.num_rows
         logger.debug(f"Collected batch: total rows now {rows_collected}")
         if rows_collected >= n:
+            if rows_collected > n:
+                batches.append(batch.slice(0, n - (rows_collected - batch.num_rows)))
+            else:
+                batches.append(batch)
             break
+        batches.append(batch)
 
     logger.info(f"Finished collecting {len(batches)} batches with {rows_collected} total rows")
     return batches, rows_collected
@@ -217,7 +222,7 @@ def read_top_n(file_path: str, n: int = 1000, strict: bool = False) -> pa.Table:
 
         table = pa.Table.from_batches(batches)  # type: ignore
         logger.info(f"Successfully read {rows_collected} rows from {file_path}")
-        return table
+    return table
 
 
 def get_read_top_n(file_path: str, n: int = 1000, strict: bool = False) -> Callable[[], pa.Table]:
@@ -244,4 +249,4 @@ def get_read_top_n(file_path: str, n: int = 1000, strict: bool = False) -> Calla
         >>> table = reader_func()
     """
     logger.info(f"Creating reader function for {file_path} with n={n}, strict={strict}")
-    return lambda: read_top_n(file_path, n, strict)
+    return lambda: read_top_n(file_path, n, strict)

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_core/flowfile_core/utils/validate_setup.py

@@ -34,8 +34,6 @@ def validate_setup():
         check_if_node_has_add_function_in_flow_graph(node)
         check_if_node_has_input_schema_definition(node)
 
-    print("All nodes have corresponding functions in FlowGraph and input schema definitions.")
-
 
 if __name__ == "__main__":
     validate_setup()

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_frame/flowfile_frame/expr.py

@@ -490,6 +490,20 @@ class Expr:
         result.agg_func = "sum"
         return result
 
+    def unique_counts(self):
+        """
+        Return the number of unique values in the column.
+
+        Returns
+        -------
+        Expr
+            A new expression with the unique counts
+        """
+        result_expr = self.expr.unique_counts() if self.expr is not None else None
+        result = self._create_next_expr(method_name="unique_counts", result_expr=result_expr, is_complex=self.is_complex)
+        result.agg_func = "unique_counts"
+        return result
+
     def implode(self):
         result_expr = self.expr.implode() if self.expr is not None else None
         result = self._create_next_expr(method_name="implode", result_expr=result_expr, is_complex=self.is_complex)

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_frame/flowfile_frame/flow_frame.py

@@ -565,7 +565,7 @@ class FlowFrame:
             coalesce: bool = None,
             maintain_order: Literal[None, "left", "right", "left_right", "right_left"] = None,
             description: str = None,
-    ):
+    ) -> "FlowFrame":
         """
         Add a join operation to the Logical Plan.
 

{flowfile-0.3.7 → flowfile-0.3.8}/flowfile_frame/flowfile_frame/flow_frame.pyi

@@ -80,8 +80,8 @@ class FlowFrame:
 
     def __ne__(self, other: object) -> typing.NoReturn: ...
 
-    # Create and configure a new FlowFrame instance, mimicking Polars' flexible constructor.
-    def __new__(cls, data: typing.Union[LazyFrame, collections.abc.Mapping[str, typing.Union[collections.abc.Sequence[object], collections.abc.Mapping[str, collections.abc.Sequence[object]], ForwardRef('Series')]], collections.abc.Sequence[typing.Any], ForwardRef('np.ndarray[Any, Any]'), ForwardRef('pa.Table'), ForwardRef('pd.DataFrame'), ForwardRef('ArrowArrayExportable'), ForwardRef('ArrowStreamExportable'), ForwardRef('torch.Tensor')] = None, schema: typing.Union[collections.abc.Mapping[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType'), type[int], type[float], type[bool], type[str], type['date'], type['time'], type['datetime'], type['timedelta'], type[list[typing.Any]], type[tuple[typing.Any, ...]], type[bytes], type[object], type['Decimal'], type[None], NoneType]], collections.abc.Sequence[typing.Union[str, tuple[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType'), type[int], type[float], type[bool], type[str], type['date'], type['time'], type['datetime'], type['timedelta'], type[list[typing.Any]], type[tuple[typing.Any, ...]], type[bytes], type[object], type['Decimal'], type[None], NoneType]]]], NoneType] = None, schema_overrides: collections.abc.Mapping[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType')]] | None = None, strict: bool = True, orient: typing.Optional[typing.Literal['col', 'row']] = None, infer_schema_length: int | None = 100, nan_to_null: bool = False, flow_graph: typing.Optional[flowfile_core.flowfile.flow_graph.FlowGraph] = None, node_id: typing.Optional[int] = None, parent_node_id: typing.Optional[int] = None, override_initial: bool = False) -> Self: ...
+    # Unified constructor for FlowFrame.
+    def __new__(cls, data: typing.Union[LazyFrame, collections.abc.Mapping[str, typing.Union[collections.abc.Sequence[object], collections.abc.Mapping[str, collections.abc.Sequence[object]], ForwardRef('Series')]], collections.abc.Sequence[typing.Any], ForwardRef('np.ndarray[Any, Any]'), ForwardRef('pa.Table'), ForwardRef('pd.DataFrame'), ForwardRef('ArrowArrayExportable'), ForwardRef('ArrowStreamExportable'), ForwardRef('torch.Tensor')] = None, schema: typing.Union[collections.abc.Mapping[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType'), type[int], type[float], type[bool], type[str], type['date'], type['time'], type['datetime'], type['timedelta'], type[list[typing.Any]], type[tuple[typing.Any, ...]], type[bytes], type[object], type['Decimal'], type[None], NoneType]], collections.abc.Sequence[typing.Union[str, tuple[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType'), type[int], type[float], type[bool], type[str], type['date'], type['time'], type['datetime'], type['timedelta'], type[list[typing.Any]], type[tuple[typing.Any, ...]], type[bytes], type[object], type['Decimal'], type[None], NoneType]]]], NoneType] = None, schema_overrides: collections.abc.Mapping[str, typing.Union[ForwardRef('DataTypeClass'), ForwardRef('DataType')]] | None = None, strict: bool = True, orient: typing.Optional[typing.Literal['col', 'row']] = None, infer_schema_length: int | None = 100, nan_to_null: bool = False, flow_graph: typing.Optional[flowfile_core.flowfile.flow_graph.FlowGraph] = None, node_id: typing.Optional[int] = None, parent_node_id: typing.Optional[int] = None, **kwargs) -> Self: ...
 
     def __repr__(self) -> Any: ...
 
@@ -118,9 +118,6 @@ class FlowFrame:
     # Execute join using Polars code approach.
     def _execute_polars_code_join(self, other: FlowFrame, new_node_id: int, on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column], left_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column], right_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column], left_columns: typing.Optional[typing.List[str]], right_columns: typing.Optional[typing.List[str]], how: str, suffix: str, validate: str, nulls_equal: bool, coalesce: bool, maintain_order: typing.Literal[None, 'left', 'right', 'left_right', 'right_left'], description: str) -> 'FlowFrame': ...
 
-    # Internal constructor to create a FlowFrame instance that wraps an
-    def _from_existing_node(self, data: LazyFrame, flow_graph: FlowGraph, node_id: int, parent_node_id: typing.Optional[int] = None) -> 'FlowFrame': ...
-
     # Generates the `input_df.sort(...)` Polars code string using pure expression strings.
     def _generate_sort_polars_code(self, pure_sort_expr_strs: typing.List[str], descending_values: typing.List[bool], nulls_last_values: typing.List[bool], multithreaded: bool, maintain_order: bool) -> str: ...
 
@@ -231,7 +228,7 @@ class FlowFrame:
     def interpolate(self, description: Optional[str] = None) -> 'FlowFrame': ...
 
     # Add a join operation to the Logical Plan.
-    def join(self, other, on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, how: str = 'inner', left_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, right_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, suffix: str = '_right', validate: str = None, nulls_equal: bool = False, coalesce: bool = None, maintain_order: typing.Literal[None, 'left', 'right', 'left_right', 'right_left'] = None, description: str = None) -> Any: ...
+    def join(self, other, on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, how: str = 'inner', left_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, right_on: typing.Union[typing.List[str | flowfile_frame.expr.Column], str, flowfile_frame.expr.Column] = None, suffix: str = '_right', validate: str = None, nulls_equal: bool = False, coalesce: bool = None, maintain_order: typing.Literal[None, 'left', 'right', 'left_right', 'right_left'] = None, description: str = None) -> 'FlowFrame': ...
 
     # Perform an asof join.
     def join_asof(self, other: FlowFrame, left_on: str | None | Expr = None, right_on: str | None | Expr = None, on: str | None | Expr = None, by_left: str | Sequence[str] | None = None, by_right: str | Sequence[str] | None = None, by: str | Sequence[str] | None = None, strategy: AsofJoinStrategy = 'backward', suffix: str = '_right', tolerance: str | int | float | timedelta | None = None, allow_parallel: bool = True, force_parallel: bool = False, coalesce: bool = True, allow_exact_matches: bool = True, check_sortedness: bool = True, description: Optional[str] = None) -> 'FlowFrame': ...

{flowfile-0.3.7 → flowfile-0.3.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "Flowfile"
-version = "0.3.7"
+version = "0.3.8"
 description = "Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)"
 readme = "readme-pypi.md"
 authors = ["Edward van Eechoud <evaneechoud@gmail.com>"]