Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

flowfile_frame/__init__.py

@@ -1,69 +1,135 @@
 # flowframe/__init__.py
 """A Polars-like API for building ETL graphs."""
 
-from importlib.metadata import version
+from importlib.metadata import PackageNotFoundError, version
 
-# Core classes
-from flowfile_frame.flow_frame import FlowFrame   # noqa: F401
 from pl_fuzzy_frame_match.models import FuzzyMapping  # noqa: F401
+from polars.datatypes import (  # noqa: F401
+    Array,
+    Binary,
+    # Other primitive types
+    Boolean,
+    # Special types
+    Categorical,
+    # Type classes
+    DataType,
+    DataTypeClass,
+    # Date/time types
+    Date,
+    Datetime,
+    Decimal,
+    Duration,
+    Enum,
+    Field,
+    # Float types
+    Float32,
+    Float64,
+    # Integer types
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    Int128,
+    IntegerType,
+    # Complex types
+    List,
+    Null,
+    Object,
+    String,
+    Struct,
+    TemporalType,
+    Time,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Unknown,
+    Utf8,
+)
 
-from flowfile_frame.utils import create_flow_graph  # noqa: F401
-
-# Commonly used functions
-from flowfile_frame.expr import (  # noqa: F401
-    col, lit, column,
-    cum_count, len,
-    sum, min, max, mean, count, when, implode, last, corr, cov, first
+from flowfile_frame.cloud_storage.secret_manager import (
+    create_cloud_storage_connection,
+    create_cloud_storage_connection_if_not_exists,
+    del_cloud_storage_connection,
+    get_all_available_cloud_storage_connections,
 )
 
-from flowfile_frame.lazy import (fold)
+# Database I/O
+from flowfile_frame.database import (
+    create_database_connection,
+    create_database_connection_if_not_exists,
+    del_database_connection,
+    get_all_available_database_connections,
+    get_database_connection_by_name,
+    read_database,
+    write_database,
+)
 
-# Selector utilities
-from flowfile_frame.selectors import (  # noqa: F401
-    numeric, float_, integer, string, temporal,
-    datetime, date, time, duration, boolean,
-    categorical, object_, list_, struct, all_,
-    by_dtype, contains, starts_with, ends_with, matches
+# Commonly used functions
+from flowfile_frame.expr import (  # noqa: F401
+    col,
+    column,
+    corr,
+    count,
+    cov,
+    cum_count,
+    first,
+    implode,
+    last,
+    len,
+    lit,
+    max,
+    mean,
+    min,
+    sum,
+    when,
 )
 
-from flowfile_frame.series import Series
+# Core classes
+from flowfile_frame.flow_frame import FlowFrame  # noqa: F401
 
 # File I/O
 from flowfile_frame.flow_frame_methods import (  # noqa: F401
-    read_csv, read_parquet, from_dict, concat,  scan_csv, scan_parquet, scan_json_from_cloud_storage,
-    scan_parquet_from_cloud_storage,
+    concat,
+    from_dict,
+    read_csv,
+    read_parquet,
+    scan_csv,
     scan_csv_from_cloud_storage,
-    scan_delta)
-
-from flowfile_frame.cloud_storage.secret_manager import (del_cloud_storage_connection,
-                                                         create_cloud_storage_connection,
-                                                         get_all_available_cloud_storage_connections,
-                                                         create_cloud_storage_connection_if_not_exists)
-
-from polars.datatypes import (  # noqa: F401
-    # Integer types
-    Int8, Int16, Int32, Int64, Int128,
-    UInt8, UInt16, UInt32, UInt64,
-    IntegerType,
-
-    # Float types
-    Float32, Float64,
-
-    # Other primitive types
-    Boolean, String, Utf8, Binary, Null,
-
-    # Complex types
-    List, Array, Struct, Object,
-
-    # Date/time types
-    Date, Time, Datetime, Duration,
-    TemporalType,
-
-    # Special types
-    Categorical, Decimal, Enum, Unknown,
+    scan_delta,
+    scan_json_from_cloud_storage,
+    scan_parquet,
+    scan_parquet_from_cloud_storage,
+)
+from flowfile_frame.lazy import fold
 
-    # Type classes
-    DataType, DataTypeClass, Field
+# Selector utilities
+from flowfile_frame.selectors import (  # noqa: F401
+    all_,
+    boolean,
+    by_dtype,
+    categorical,
+    contains,
+    date,
+    datetime,
+    duration,
+    ends_with,
+    float_,
+    integer,
+    list_,
+    matches,
+    numeric,
+    object_,
+    starts_with,
+    string,
+    struct,
+    temporal,
+    time,
 )
+from flowfile_frame.series import Series
+from flowfile_frame.utils import create_flow_graph  # noqa: F401
 
-__version__ = version("Flowfile")
+try:
+    __version__ = version("Flowfile")
+except PackageNotFoundError:
+    __version__ = "0.5.0"

flowfile_frame/adapters.py

@@ -2,16 +2,9 @@
 """Adapters to connect FlowFrame with the flowfile-core library."""
 
 # Import from your existing project
-from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 from flowfile_core.schemas import input_schema, schemas, transform_schema
 
 # Export these for use in FlowFrame
-__all__ = [
-    'FlowGraph',
-    'add_connection',
-    'FlowDataEngine',
-    'input_schema',
-    'schemas',
-    'transform_schema'
-]
+__all__ = ["FlowGraph", "add_connection", "FlowDataEngine", "input_schema", "schemas", "transform_schema"]

flowfile_frame/adding_expr.py

@@ -1,11 +1,14 @@
-import polars as pl
+from collections.abc import Callable
 from functools import wraps
-from typing import Callable, TypeVar, Type
-from flowfile_frame.utils import _get_function_source
+from typing import TypeVar
+
+import polars as pl
+
 from flowfile_frame.config import logger
+from flowfile_frame.utils import _get_function_source
 
-T = TypeVar('T')
-ExprT = TypeVar('ExprT', bound='Expr')
+T = TypeVar("T")
+ExprT = TypeVar("ExprT", bound="Expr")
 PASSTHROUGH_METHODS = {"map_elements", "map_batches"}
 
 
@@ -29,12 +32,9 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
 
     @wraps(original_method)
     def wrapper(self: Expr, *args, **kwargs):
-        from flowfile_frame.expr import Expr
         # Check if we have a valid underlying expression
         if self.expr is None:
-            raise ValueError(
-                f"Cannot call '{method_name}' on Expr with no underlying polars expression."
-            )
+            raise ValueError(f"Cannot call '{method_name}' on Expr with no underlying polars expression.")
 
         # Collect function sources and build representations
         function_sources = []
@@ -47,7 +47,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
                 # Try to get function source
                 try:
                     source, is_module_level = _get_function_source(arg)
-                    if source and hasattr(arg, '__name__') and arg.__name__ != '<lambda>':
+                    if source and hasattr(arg, "__name__") and arg.__name__ != "<lambda>":
                         function_sources.append(source)
                         # Use the function name in the representation
                         args_representations.append(arg.__name__)
@@ -65,7 +65,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
                 # Try to get function source
                 try:
                     source, is_module_level = _get_function_source(value)
-                    if source and hasattr(value, '__name__') and value.__name__ != '<lambda>':
+                    if source and hasattr(value, "__name__") and value.__name__ != "<lambda>":
                         function_sources.append(source)
                         # Use the function name in the representation
                         kwargs_representations.append(f"{key}={value.__name__}")
@@ -102,16 +102,56 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
 
         # Methods that typically change the aggregation status or complexity
         agg_methods = {
-            "sum", "mean", "min", "max", "median", "first", "last", "std", "var",
-            "count", "n_unique", "quantile", "implode", "explode"
+            "sum",
+            "mean",
+            "min",
+            "max",
+            "median",
+            "first",
+            "last",
+            "std",
+            "var",
+            "count",
+            "n_unique",
+            "quantile",
+            "implode",
+            "explode",
         }
         # Methods that typically make expressions complex
         complex_methods = {
-            "filter", "map", "shift", "fill_null", "fill_nan", "round", "abs", "alias",
-            "cast", "is_between", "over", "sort", "arg_sort", "arg_unique", "arg_min",
-            "arg_max", "rolling", "interpolate", "ewm_mean", "ewm_std", "ewm_var",
-            "backward_fill", "forward_fill", "rank", "diff", "clip", "dot", "mode",
-            "drop_nulls", "drop_nans", "take", "gather", "filter", "shift_and_fill"
+            "filter",
+            "map",
+            "shift",
+            "fill_null",
+            "fill_nan",
+            "round",
+            "abs",
+            "alias",
+            "cast",
+            "is_between",
+            "over",
+            "sort",
+            "arg_sort",
+            "arg_unique",
+            "arg_min",
+            "arg_max",
+            "rolling",
+            "interpolate",
+            "ewm_mean",
+            "ewm_std",
+            "ewm_var",
+            "backward_fill",
+            "forward_fill",
+            "rank",
+            "diff",
+            "clip",
+            "dot",
+            "mode",
+            "drop_nulls",
+            "drop_nans",
+            "take",
+            "gather",
+            "shift_and_fill",
         }
 
         # Determine new agg_func status
@@ -127,7 +167,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
             result_expr=result_expr,
             is_complex=is_complex,
             method_name=method_name,
-            _function_sources=function_sources  # Pass function sources
+            _function_sources=function_sources,  # Pass function sources
         )
 
         # Set the agg_func if needed
@@ -139,7 +179,7 @@ def create_expr_method_wrapper(method_name: str, original_method: Callable) -> C
     return wrapper
 
 
-def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
+def add_expr_methods(cls: type[ExprT]) -> type[ExprT]:
     """
     Class decorator that adds all polars Expr methods to a custom Expr class.
 
@@ -160,8 +200,7 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
     existing_methods = set(dir(cls))
 
     skip_methods = {
-        name for name in dir(pl.Expr)
-        if name.startswith('_') or isinstance(getattr(pl.Expr, name, None), property)
+        name for name in dir(pl.Expr) if name.startswith("_") or isinstance(getattr(pl.Expr, name, None), property)
     }
 
     # Add all public Expr methods that don't already exist
@@ -193,20 +232,21 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
                                 # Try to get function source
                                 try:
                                     source, is_module_level = _get_function_source(arg)
-                                    if source and hasattr(arg, '__name__') and arg.__name__ != '<lambda>':
-
+                                    if source and hasattr(arg, "__name__") and arg.__name__ != "<lambda>":
                                         function_sources.append(source)
                                         # Use the function name in the representation
                                         args_representations.append(arg.__name__)
                                         arg.__repr__ = lambda: arg.__name__
 
                                     else:
-
                                         # Lambda or unnamed function - not convertible
                                         logger.warning(
-                                            f"Warning: Using anonymous functions in {method_name} is not convertable to UI code")
-                                        logger.warning(f"Consider using defined functions (def abc(a, b, c): return ...), "
-                                                       f"In a separate script")
+                                            f"Warning: Using anonymous functions in {method_name} is not convertable to UI code"
+                                        )
+                                        logger.warning(
+                                            "Consider using defined functions (def abc(a, b, c): return ...), "
+                                            "In a separate script"
+                                        )
                                         convertable_to_code = False
                                         args_representations.append(repr(arg))
                                 except:
@@ -220,7 +260,7 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
                                 # Try to get function source
                                 try:
                                     source, is_module_level = _get_function_source(value)
-                                    if source and hasattr(value, '__name__') and value.__name__ != '<lambda>':
+                                    if source and hasattr(value, "__name__") and value.__name__ != "<lambda>":
                                         function_sources.append(source)
                                         # Use the function name in the representation
                                         kwargs_representations.append(f"{key}={value.__name__}")
@@ -258,7 +298,7 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
                             is_complex=True,
                             convertable_to_code=convertable_to_code,
                             _function_sources=function_sources,  # Pass function sources
-                            **kwargs
+                            **kwargs,
                         )
                         return result
 
@@ -271,8 +311,9 @@ def add_expr_methods(cls: Type[ExprT]) -> Type[ExprT]:
                 setattr(cls, name, wrapped_method)
 
     overlap = {
-        name for name in existing_methods
-        if name in dir(pl.Expr) and not name.startswith('_') and callable(getattr(pl.Expr, name))
+        name
+        for name in existing_methods
+        if name in dir(pl.Expr) and not name.startswith("_") and callable(getattr(pl.Expr, name))
     }
     if overlap:
         logger.debug(f"Preserved existing methods in {cls.__name__}: {', '.join(sorted(overlap))}")

flowfile_frame/cloud_storage/frame_helpers.py

@@ -1,39 +1,43 @@
-from typing import Optional, Literal
+from typing import Literal
 
-from polars._typing import (CsvEncoding)
+from polars._typing import CsvEncoding
 
 from flowfile_core.flowfile.flow_graph import FlowGraph
-from flowfile_core.schemas import input_schema, cloud_storage_schemas
+from flowfile_core.schemas import cloud_storage_schemas, input_schema
 from flowfile_frame.cloud_storage.secret_manager import get_current_user_id
 from flowfile_frame.utils import generate_node_id
 
 
-def add_write_ff_to_cloud_storage(path: str,
-                                  flow_graph: Optional[FlowGraph],
-                                  depends_on_node_id: int,
-                                  *,
-                                  connection_name: Optional[str] = None,
-                                  write_mode: Literal["overwrite", "append"] = "overwrite",
-                                  file_format: Literal["csv", "parquet", "json", "delta"] = "parquet",
-                                  csv_delimiter: str = ";",
-                                  csv_encoding: CsvEncoding = "utf8",
-                                  parquet_compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy",
-                                  description: Optional[str] = None
-                              ) -> int:
+def add_write_ff_to_cloud_storage(
+    path: str,
+    flow_graph: FlowGraph | None,
+    depends_on_node_id: int,
+    *,
+    connection_name: str | None = None,
+    write_mode: Literal["overwrite", "append"] = "overwrite",
+    file_format: Literal["csv", "parquet", "json", "delta"] = "parquet",
+    csv_delimiter: str = ";",
+    csv_encoding: CsvEncoding = "utf8",
+    parquet_compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy",
+    description: str | None = None,
+) -> int:
     node_id = generate_node_id()
     flow_id = flow_graph.flow_id
     settings = input_schema.NodeCloudStorageWriter(
         flow_id=flow_id,
         node_id=node_id,
-        cloud_storage_settings=cloud_storage_schemas.CloudStorageWriteSettings(resource_path=path,
-                                                                               connection_name=connection_name,
-                                                                               file_format=file_format,
-                                                                               write_mode=write_mode,
-                                                                               csv_delimiter=csv_delimiter,
-                                                                               csv_encoding=csv_encoding,
-                                                                               parquet_compression=parquet_compression),
+        cloud_storage_settings=cloud_storage_schemas.CloudStorageWriteSettings(
+            resource_path=path,
+            connection_name=connection_name,
+            file_format=file_format,
+            write_mode=write_mode,
+            csv_delimiter=csv_delimiter,
+            csv_encoding=csv_encoding,
+            parquet_compression=parquet_compression,
+        ),
         user_id=get_current_user_id(),
         depending_on_id=depends_on_node_id,
-        description=description)
+        description=description,
+    )
     flow_graph.add_cloud_storage_writer(settings)
     return node_id

flowfile_frame/cloud_storage/secret_manager.py

@@ -1,21 +1,17 @@
-from typing import List
-
-from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection, FullCloudStorageConnectionInterface
-from flowfile_core.flowfile.database_connection_manager.db_connections import (store_cloud_connection,
-                                                                               get_all_cloud_connections_interface,
-                                                                               delete_cloud_connection)
+from flowfile_core.auth.jwt import create_access_token, get_current_user_sync
 from flowfile_core.database.connection import get_db_context
-from flowfile_core.auth.jwt import  get_current_user_sync, create_access_token
-from asyncio import run
+from flowfile_core.flowfile.database_connection_manager.db_connections import (
+    delete_cloud_connection,
+    get_all_cloud_connections_interface,
+    store_cloud_connection,
+)
+from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection, FullCloudStorageConnectionInterface
 
 
 def get_current_user_id() -> int | None:
     access_token = create_access_token(data={"sub": "local_user"})
     with get_db_context() as db:
-        current_user_id = get_current_user_sync(
-            access_token,
-            db
-        ).id
+        current_user_id = get_current_user_sync(access_token, db).id
     return current_user_id
 
 
@@ -32,15 +28,8 @@ def create_cloud_storage_connection(connection: FullCloudStorageConnection) -> N
     access_token = create_access_token(data={"sub": "local_user"})
 
     with get_db_context() as db:
-        current_user_id = get_current_user_sync(
-            access_token,
-            db
-        ).id
-        store_cloud_connection(
-            db,
-            connection,
-            current_user_id
-        )
+        current_user_id = get_current_user_sync(access_token, db).id
+        store_cloud_connection(db, connection, current_user_id)
 
 
 def create_cloud_storage_connection_if_not_exists(connection: FullCloudStorageConnection) -> None:
@@ -58,12 +47,9 @@ def create_cloud_storage_connection_if_not_exists(connection: FullCloudStorageCo
         create_cloud_storage_connection(connection)
 
 
-def get_all_available_cloud_storage_connections() -> List[FullCloudStorageConnectionInterface]:
+def get_all_available_cloud_storage_connections() -> list[FullCloudStorageConnectionInterface]:
     with get_db_context() as db:
-        all_connections = get_all_cloud_connections_interface(
-            db,
-            get_current_user_id()
-        )
+        all_connections = get_all_cloud_connections_interface(db, get_current_user_id())
     return all_connections
 
 

flowfile_frame/config.py

@@ -1,9 +1,6 @@
 import logging
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='[%(levelname)s] %(message)s'
-)
+logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
 
 # Create and export the logger
-logger = logging.getLogger('flow_frame')
+logger = logging.getLogger("flow_frame")

flowfile_frame/database/__init__.py

@@ -0,0 +1,36 @@
+"""Database module for flowfile_frame.
+
+This module provides functions for:
+- Managing database connections (create, list, delete)
+- Reading from databases
+- Writing to databases
+"""
+
+from flowfile_frame.database.connection_manager import (
+    create_database_connection,
+    create_database_connection_if_not_exists,
+    del_database_connection,
+    get_all_available_database_connections,
+    get_database_connection_by_name,
+)
+from flowfile_frame.database.frame_helpers import (
+    add_read_from_database,
+    add_write_to_database,
+    read_database,
+    write_database,
+)
+
+__all__ = [
+    # Connection management
+    "create_database_connection",
+    "create_database_connection_if_not_exists",
+    "del_database_connection",
+    "get_all_available_database_connections",
+    "get_database_connection_by_name",
+    # FlowGraph helpers
+    "add_read_from_database",
+    "add_write_to_database",
+    # Direct read/write
+    "read_database",
+    "write_database",
+]