Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_frame/adding_expr.py

@@ -1,11 +1,14 @@
-import polars as pl
+from collections.abc import Callable
 from functools import wraps
-from typing import Callable, TypeVar, Type
-from flowfile_frame.utils import _get_function_source
+from typing import TypeVar
+
+import polars as pl
+
 from flowfile_frame.config import logger
+from flowfile_frame.utils import _get_function_source
 
-T = TypeVar('T')
-ExprT = TypeVar('ExprT', bound='Expr')
+T = TypeVar("T")
+ExprT = TypeVar("ExprT", bound="Expr")
 PASSTHROUGH_METHODS = {"map_elements", "map_batches"}
 
 
@@ -29,12 +32,9 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
 
     @wraps(original_method)
     def wrapper(self: Expr, *args, **kwargs):
-        from flowfile_frame.expr import Expr
         # Check if we have a valid underlying expression
         if self.expr is None:
-            raise ValueError(
-                f"Cannot call '{method_name}' on Expr with no underlying polars expression."
-            )
+            raise ValueError(f"Cannot call '{method_name}' on Expr with no underlying polars expression.")
 
         # Collect function sources and build representations
         function_sources = []
@@ -47,7 +47,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
                 # Try to get function source
                 try:
                     source, is_module_level = _get_function_source(arg)
-                    if source and hasattr(arg, '__name__') and arg.__name__ != '<lambda>':
+                    if source and hasattr(arg, "__name__") and arg.__name__ != "<lambda>":
                         function_sources.append(source)
                         # Use the function name in the representation
                         args_representations.append(arg.__name__)
@@ -65,7 +65,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
                 # Try to get function source
                 try:
                     source, is_module_level = _get_function_source(value)
-                    if source and hasattr(value, '__name__') and value.__name__ != '<lambda>':
+                    if source and hasattr(value, "__name__") and value.__name__ != "<lambda>":
                         function_sources.append(source)
                         # Use the function name in the representation
                         kwargs_representations.append(f"{key}={value.__name__}")
@@ -102,16 +102,56 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
 
         # Methods that typically change the aggregation status or complexity
         agg_methods = {
-            "sum", "mean", "min", "max", "median", "first", "last", "std", "var",
-            "count", "n_unique", "quantile", "implode", "explode"
+            "sum",
+            "mean",
+            "min",
+            "max",
+            "median",
+            "first",
+            "last",
+            "std",
+            "var",
+            "count",
+            "n_unique",
+            "quantile",
+            "implode",
+            "explode",
         }
         # Methods that typically make expressions complex
         complex_methods = {
-            "filter", "map", "shift", "fill_null", "fill_nan", "round", "abs", "alias",
-            "cast", "is_between", "over", "sort", "arg_sort", "arg_unique", "arg_min",
-            "arg_max", "rolling", "interpolate", "ewm_mean", "ewm_std", "ewm_var",
-            "backward_fill", "forward_fill", "rank", "diff", "clip", "dot", "mode",
-            "drop_nulls", "drop_nans", "take", "gather", "filter", "shift_and_fill"
+            "filter",
+            "map",
+            "shift",
+            "fill_null",
+            "fill_nan",
+            "round",
+            "abs",
+            "alias",
+            "cast",
+            "is_between",
+            "over",
+            "sort",
+            "arg_sort",
+            "arg_unique",
+            "arg_min",
+            "arg_max",
+            "rolling",
+            "interpolate",
+            "ewm_mean",
+            "ewm_std",
+            "ewm_var",
+            "backward_fill",
+            "forward_fill",
+            "rank",
+            "diff",
+            "clip",
+            "dot",
+            "mode",
+            "drop_nulls",
+            "drop_nans",
+            "take",
+            "gather",
+            "shift_and_fill",
         }
 
         # Determine new agg_func status
@@ -127,7 +167,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
             result_expr=result_expr,
             is_complex=is_complex,
             method_name=method_name,
-            _function_sources=function_sources  # Pass function sources
+            _function_sources=function_sources,  # Pass function sources
         )
 
         # Set the agg_func if needed
@@ -139,7 +179,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
     return wrapper
 
 
-def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
+def add_expr_methods(cls: type[ExprT]) -> type[ExprT]:
     """
     Class decorator that adds all polars Expr methods to a custom Expr class.
 
@@ -160,8 +200,7 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
     existing_methods = set(dir(cls))
 
     skip_methods = {
-        name for name in dir(pl.Expr)
-        if name.startswith('_') or isinstance(getattr(pl.Expr, name, None), property)
+        name for name in dir(pl.Expr) if name.startswith("_") or isinstance(getattr(pl.Expr, name, None), property)
     }
 
     # Add all public Expr methods that don't already exist
@@ -193,20 +232,21 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
                                 # Try to get function source
                                 try:
                                     source, is_module_level = _get_function_source(arg)
-                                    if source and hasattr(arg, '__name__') and arg.__name__ != '<lambda>':
-
+                                    if source and hasattr(arg, "__name__") and arg.__name__ != "<lambda>":
                                         function_sources.append(source)
                                         # Use the function name in the representation
                                         args_representations.append(arg.__name__)
                                         arg.__repr__ = lambda: arg.__name__
 
                                     else:
-
                                         # Lambda or unnamed function - not convertible
                                         logger.warning(
-                                            f"Warning: Using anonymous functions in {method_name} is not convertable to UI code")
-                                        logger.warning(f"Consider using defined functions (def abc(a, b, c): return ...), "
-                                                       f"In a separate script")
+                                            f"Warning: Using anonymous functions in {method_name} is not convertable to UI code"
+                                        )
+                                        logger.warning(
+                                            "Consider using defined functions (def abc(a, b, c): return ...), "
+                                            "In a separate script"
+                                        )
                                         convertable_to_code = False
                                         args_representations.append(repr(arg))
                                 except:
@@ -220,7 +260,7 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
                                 # Try to get function source
                                 try:
                                     source, is_module_level = _get_function_source(value)
-                                    if source and hasattr(value, '__name__') and value.__name__ != '<lambda>':
+                                    if source and hasattr(value, "__name__") and value.__name__ != "<lambda>":
                                         function_sources.append(source)
                                         # Use the function name in the representation
                                         kwargs_representations.append(f"{key}={value.__name__}")
@@ -258,7 +298,7 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
                             is_complex=True,
                             convertable_to_code=convertable_to_code,
                             _function_sources=function_sources,  # Pass function sources
-                            **kwargs
+                            **kwargs,
                         )
                         return result
 
@@ -271,8 +311,9 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
                 setattr(cls, name, wrapped_method)
 
     overlap = {
-        name for name in existing_methods
-        if name in dir(pl.Expr) and not name.startswith('_') and callable(getattr(pl.Expr, name))
+        name
+        for name in existing_methods
+        if name in dir(pl.Expr) and not name.startswith("_") and callable(getattr(pl.Expr, name))
     }
     if overlap:
         logger.debug(f"Preserved existing methods in {cls.__name__}: {', '.join(sorted(overlap))}")

flowfile_frame/cloud_storage/frame_helpers.py

@@ -1,39 +1,43 @@
-from typing import Optional, Literal
+from typing import Literal
 
-from polars._typing import (CsvEncoding)
+from polars._typing import CsvEncoding
 
 from flowfile_core.flowfile.flow_graph import FlowGraph
-from flowfile_core.schemas import input_schema, cloud_storage_schemas
+from flowfile_core.schemas import cloud_storage_schemas, input_schema
 from flowfile_frame.cloud_storage.secret_manager import get_current_user_id
 from flowfile_frame.utils import generate_node_id
 
 
-def add_write_ff_to_cloud_storage(path: str,
-                                  flow_graph: Optional[FlowGraph],
-                                  depends_on_node_id: int,
-                                  *,
-                                  connection_name: Optional[str] = None,
-                                  write_mode: Literal["overwrite", "append"] = "overwrite",
-                                  file_format: Literal["csv", "parquet", "json", "delta"] = "parquet",
-                                  csv_delimiter: str = ";",
-                                  csv_encoding: CsvEncoding = "utf8",
-                                  parquet_compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy",
-                                  description: Optional[str] = None
-                              ) -> int:
+def add_write_ff_to_cloud_storage(
+    path: str,
+    flow_graph: FlowGraph | None,
+    depends_on_node_id: int,
+    *,
+    connection_name: str | None = None,
+    write_mode: Literal["overwrite", "append"] = "overwrite",
+    file_format: Literal["csv", "parquet", "json", "delta"] = "parquet",
+    csv_delimiter: str = ";",
+    csv_encoding: CsvEncoding = "utf8",
+    parquet_compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy",
+    description: str | None = None,
+) -> int:
     node_id = generate_node_id()
     flow_id = flow_graph.flow_id
     settings = input_schema.NodeCloudStorageWriter(
         flow_id=flow_id,
         node_id=node_id,
-        cloud_storage_settings=cloud_storage_schemas.CloudStorageWriteSettings(resource_path=path,
-                                                                               connection_name=connection_name,
-                                                                               file_format=file_format,
-                                                                               write_mode=write_mode,
-                                                                               csv_delimiter=csv_delimiter,
-                                                                               csv_encoding=csv_encoding,
-                                                                               parquet_compression=parquet_compression),
+        cloud_storage_settings=cloud_storage_schemas.CloudStorageWriteSettings(
+            resource_path=path,
+            connection_name=connection_name,
+            file_format=file_format,
+            write_mode=write_mode,
+            csv_delimiter=csv_delimiter,
+            csv_encoding=csv_encoding,
+            parquet_compression=parquet_compression,
+        ),
         user_id=get_current_user_id(),
         depending_on_id=depends_on_node_id,
-        description=description)
+        description=description,
+    )
     flow_graph.add_cloud_storage_writer(settings)
     return node_id

flowfile_frame/cloud_storage/secret_manager.py

@@ -1,21 +1,17 @@
-from typing import List
-
-from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection, FullCloudStorageConnectionInterface
-from flowfile_core.flowfile.database_connection_manager.db_connections import (store_cloud_connection,
-                                                                               get_all_cloud_connections_interface,
-                                                                               delete_cloud_connection)
+from flowfile_core.auth.jwt import create_access_token, get_current_user_sync
 from flowfile_core.database.connection import get_db_context
-from flowfile_core.auth.jwt import  get_current_user_sync, create_access_token
-from asyncio import run
+from flowfile_core.flowfile.database_connection_manager.db_connections import (
+    delete_cloud_connection,
+    get_all_cloud_connections_interface,
+    store_cloud_connection,
+)
+from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection, FullCloudStorageConnectionInterface
 
 
 def get_current_user_id() -> int | None:
     access_token = create_access_token(data={"sub": "local_user"})
     with get_db_context() as db:
-        current_user_id = get_current_user_sync(
-            access_token,
-            db
-        ).id
+        current_user_id = get_current_user_sync(access_token, db).id
     return current_user_id
 
 
@@ -32,15 +28,8 @@ def create_cloud_storage_connection(connection: FullCloudStorageConnection) -> N
     access_token = create_access_token(data={"sub": "local_user"})
 
     with get_db_context() as db:
-        current_user_id = get_current_user_sync(
-            access_token,
-            db
-        ).id
-        store_cloud_connection(
-            db,
-            connection,
-            current_user_id
-        )
+        current_user_id = get_current_user_sync(access_token, db).id
+        store_cloud_connection(db, connection, current_user_id)
 
 
 def create_cloud_storage_connection_if_not_exists(connection: FullCloudStorageConnection) -> None:
@@ -58,12 +47,9 @@ def create_cloud_storage_connection_if_not_exists(connection: FullCloudStorageCo
         create_cloud_storage_connection(connection)
 
 
-def get_all_available_cloud_storage_connections() -> List[FullCloudStorageConnectionInterface]:
+def get_all_available_cloud_storage_connections() -> list[FullCloudStorageConnectionInterface]:
     with get_db_context() as db:
-        all_connections = get_all_cloud_connections_interface(
-            db,
-            get_current_user_id()
-        )
+        all_connections = get_all_cloud_connections_interface(db, get_current_user_id())
     return all_connections
 
 

flowfile_frame/config.py

@@ -1,9 +1,6 @@
 import logging
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='[%(levelname)s] %(message)s'
-)
+logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
 
 # Create and export the logger
-logger = logging.getLogger('flow_frame')
+logger = logging.getLogger("flow_frame")