Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_core/schemas/yaml_types.py

@@ -0,0 +1,117 @@
+from typing import TypedDict
+
+# === Transform Schema YAML Types ===
+
+
+class BasicFilterYaml(TypedDict, total=False):
+    field: str
+    operator: str
+    value: str
+    value2: str  # For BETWEEN operator
+
+
+class FilterInputYaml(TypedDict, total=False):
+    mode: str  # "basic" or "advanced"
+    basic_filter: BasicFilterYaml
+    advanced_filter: str
+
+
+class SelectInputYaml(TypedDict, total=False):
+    old_name: str
+    new_name: str
+    keep: bool
+    data_type: str
+
+
+class JoinInputsYaml(TypedDict):
+    select: list[SelectInputYaml]
+
+
+class JoinMapYaml(TypedDict):
+    left_col: str
+    right_col: str
+
+
+class JoinInputYaml(TypedDict):
+    join_mapping: list[JoinMapYaml]
+    left_select: JoinInputsYaml
+    right_select: JoinInputsYaml
+    how: str
+
+
+class CrossJoinInputYaml(TypedDict):
+    left_select: JoinInputsYaml
+    right_select: JoinInputsYaml
+
+
+class FuzzyMappingYaml(TypedDict, total=False):
+    left_col: str
+    right_col: str
+    threshold_score: float
+    fuzzy_type: str
+    perc_unique: float
+    output_column_name: str
+    valid: bool
+
+
+class FuzzyMatchInputYaml(TypedDict):
+    join_mapping: list[FuzzyMappingYaml]
+    left_select: JoinInputsYaml
+    right_select: JoinInputsYaml
+    how: str
+    aggregate_output: bool
+
+
+# === Input Schema YAML Types ===
+
+
+class OutputSettingsYaml(TypedDict, total=False):
+    name: str
+    directory: str
+    file_type: str
+    write_mode: str
+    abs_file_path: str
+    fields: list[str]
+    table_settings: dict
+
+
+class NodeSelectYaml(TypedDict):
+    cache_results: bool
+    keep_missing: bool
+    select_input: list[SelectInputYaml]
+    sorted_by: str
+
+
+class NodeJoinYaml(TypedDict):
+    cache_results: bool
+    auto_generate_selection: bool
+    verify_integrity: bool
+    join_input: JoinInputYaml
+    auto_keep_all: bool
+    auto_keep_right: bool
+    auto_keep_left: bool
+
+
+class NodeCrossJoinYaml(TypedDict):
+    cache_results: bool
+    auto_generate_selection: bool
+    verify_integrity: bool
+    cross_join_input: CrossJoinInputYaml
+    auto_keep_all: bool
+    auto_keep_right: bool
+    auto_keep_left: bool
+
+
+class NodeFuzzyMatchYaml(TypedDict):
+    cache_results: bool
+    auto_generate_selection: bool
+    verify_integrity: bool
+    join_input: FuzzyMatchInputYaml
+    auto_keep_all: bool
+    auto_keep_right: bool
+    auto_keep_left: bool
+
+
+class NodeOutputYaml(TypedDict):
+    cache_results: bool
+    output_settings: OutputSettingsYaml

flowfile_core/secret_manager/secret_manager.py

@@ -1,13 +1,13 @@
-
 from cryptography.fernet import Fernet
+from fastapi.exceptions import HTTPException
+from pydantic import SecretStr
 from sqlalchemy import and_
 from sqlalchemy.orm import Session
+
+from flowfile_core.auth.models import SecretInput
+from flowfile_core.auth.secrets import get_master_key
 from flowfile_core.database import models as db_models
 from flowfile_core.database.connection import get_db_context
-from flowfile_core.auth.secrets import get_master_key
-from pydantic import SecretStr
-from flowfile_core.auth.models import SecretInput
-from fastapi.exceptions import HTTPException
 
 
 def encrypt_secret(secret_value):
@@ -24,11 +24,14 @@ def decrypt_secret(encrypted_value) -> SecretStr:
     return SecretStr(f.decrypt(encrypted_value.encode()).decode())
 
 
-def get_encrypted_secret(current_user_id: int, secret_name: str) -> str|None:
+def get_encrypted_secret(current_user_id: int, secret_name: str) -> str | None:
     with get_db_context() as db:
         user_id = current_user_id
-        db_secret = db.query(db_models.Secret).filter(and_(db_models.Secret.user_id == user_id,
-                                                      db_models.Secret.name == secret_name)).first()
+        db_secret = (
+            db.query(db_models.Secret)
+            .filter(and_(db_models.Secret.user_id == user_id, db_models.Secret.name == secret_name))
+            .first()
+        )
         if db_secret:
             return db_secret.encrypted_value
         else:
@@ -43,7 +46,7 @@ def store_secret(db: Session, secret: SecretInput, user_id: int) -> db_models.Se
         name=secret.name,
         encrypted_value=encrypted_value,
         iv="",  # Not used with Fernet
-        user_id=user_id
+        user_id=user_id,
     )
     db.add(db_secret)
     db.commit()
@@ -52,10 +55,11 @@ def store_secret(db: Session, secret: SecretInput, user_id: int) -> db_models.Se
 
 
 def delete_secret(db: Session, secret_name: str, user_id: int) -> None:
-    db_secret = db.query(db_models.Secret).filter(
-        db_models.Secret.user_id == user_id,
-        db_models.Secret.name == secret_name
-    ).first()
+    db_secret = (
+        db.query(db_models.Secret)
+        .filter(db_models.Secret.user_id == user_id, db_models.Secret.name == secret_name)
+        .first()
+    )
 
     if not db_secret:
         raise HTTPException(status_code=404, detail="Secret not found")

flowfile_core/{flowfile/node_designer/data_types.py → types.py}

@@ -18,12 +18,41 @@ Usage:
 """
 
 from enum import Enum
-from typing import List, Union
+from typing import Literal, Union
+
 import polars as pl
 
+DataTypeStr = Literal[
+    "Int8",
+    "Int16",
+    "Int32",
+    "Int64",
+    "UInt8",
+    "UInt16",
+    "UInt32",
+    "UInt64",
+    "Float32",
+    "Float64",
+    "Decimal",
+    "String",
+    "Date",
+    "Datetime",
+    "Time",
+    "Duration",
+    "Boolean",
+    "Binary",
+    "List",
+    "Struct",
+    "Array",
+    "Integer",
+    "Double",
+    "Utf8",
+]
+
 
 class TypeGroup(str, Enum):
     """High-level type groups for column selection."""
+
     Numeric = "Numeric"
     String = "String"
     Date = "Date"
@@ -41,6 +70,7 @@ class TypeGroup(str, Enum):
 
 class DataType(str, Enum):
     """Specific data types for fine-grained control."""
+
     # Numeric types
     Int8 = "Int8"
     Int16 = "Int16"
@@ -140,7 +170,7 @@ TypeSpec = Union[
     TypeGroup,
     DataType,
     str,
-    List[Union[TypeGroup, DataType, str, type[pl.DataType], pl.DataType]],
+    list[TypeGroup | DataType | str | type[pl.DataType] | pl.DataType],
     type[pl.DataType],
-    pl.DataType
+    pl.DataType,
 ]

flowfile_core/utils/arrow_reader.py

@@ -1,5 +1,7 @@
+from collections.abc import Callable, Iterator
+
 import pyarrow as pa
-from typing import List, Iterator, Tuple, Callable
+
 from flowfile_core.configs import logger
 
 
@@ -36,7 +38,7 @@ def open_validated_file(file_path: str, n: int) -> pa.OSFile:
         logger.error(f"Invalid file_path type: {type(file_path)}")
         raise TypeError("file_path must be a string")
     try:
-        file = pa.OSFile(file_path, 'rb')
+        file = pa.OSFile(file_path, "rb")
         logger.info(f"Successfully opened file: {file_path}")
         return file
     except FileNotFoundError:
@@ -112,7 +114,7 @@ def iter_batches(reader: pa.ipc.RecordBatchFileReader, n: int, rows_collected: i
             break
 
 
-def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> Tuple[List[pa.RecordBatch], int]:
+def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> tuple[list[pa.RecordBatch], int]:
     """
     Collect record batches from a reader up to a specified number of rows.
 
@@ -134,11 +136,10 @@ def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> Tuple[List[
         >>> print(f"Collected {row_count} rows in {len(batches)} batches")
     """
     logger.debug(f"Collecting batches up to {n} rows")
-    batches: List[pa.RecordBatch] = []
+    batches: list[pa.RecordBatch] = []
     rows_collected = 0
 
     for batch in iter_batches(reader, n, rows_collected):
-
         rows_collected += batch.num_rows
         logger.debug(f"Collected batch: total rows now {rows_collected}")
         if rows_collected >= n:
@@ -178,7 +179,7 @@ def read(file_path: str) -> pa.Table:
     logger.info(f"Reading entire file: {file_path}")
     with open_validated_file(file_path, 0) as source:
         reader = create_reader(source)
-        batches, total_rows = collect_batches(reader, float('inf'))
+        batches, total_rows = collect_batches(reader, float("inf"))
         table = pa.Table.from_batches(batches)  # type: ignore
         logger.info(f"Successfully read {total_rows} rows from {file_path}")
         return table

flowfile_core/utils/excel_file_manager.py

@@ -1,13 +1,13 @@
 import os
-import fastexcel
-from typing import List
 from functools import lru_cache
 
+import fastexcel
+
 from flowfile_core.configs import logger
 
 
 @lru_cache(maxsize=32)
-def get_sheet_names(file_path: str) -> List[str] | None:
+def get_sheet_names(file_path: str) -> list[str] | None:
     if not os.path.exists(file_path):
         logger.error(f"File does not exist: {file_path}")
         return

flowfile_core/utils/fileManager.py

@@ -1,13 +1,13 @@
 import os
-from flowfile_core.schemas.input_schema import NewDirectory, RemoveItem, RemoveItemsInput
-from typing import Tuple, Optional
+
 from flowfile_core.configs import logger
+from flowfile_core.schemas.input_schema import NewDirectory, RemoveItem, RemoveItemsInput
 
 local_database_connection = None
 
 
-def create_dir(new_directory: NewDirectory) -> Tuple[bool, Optional[Exception]]:
-    full_path: str = os.path.join(new_directory.source_path,new_directory.dir_name)
+def create_dir(new_directory: NewDirectory) -> tuple[bool, Exception | None]:
+    full_path: str = os.path.join(new_directory.source_path, new_directory.dir_name)
     try:
         os.mkdir(full_path)
         logger.info("Successfully created a new folder")
@@ -16,7 +16,7 @@ def create_dir(new_directory: NewDirectory) -> Tuple[bool, Optional[Exception]]:
         return False, e
 
 
-def remove_path(path: str) -> Tuple[bool, Optional[Exception]]:
+def remove_path(path: str) -> tuple[bool, Exception | None]:
     try:
         os.remove(path)
         logger.info(f"Succesfully removed {path}")
@@ -35,11 +35,11 @@ def remove_item(item_to_remove: RemoveItem):
         os.rmdir(item_to_remove.path)
 
 
-def remove_paths(remove_items: RemoveItemsInput) -> Tuple[bool, Optional[Exception]]:
+def remove_paths(remove_items: RemoveItemsInput) -> tuple[bool, Exception | None]:
     try:
         for path in remove_items.paths:
             remove_item(path)
-        logger.info(f'Successfully removed {remove_items.paths}')
+        logger.info(f"Successfully removed {remove_items.paths}")
         return True, None
     except Exception as e:
         return False, e

flowfile_core/utils/fl_executor.py

@@ -1,9 +1,9 @@
-from flowfile_core.configs import logger
+from functools import wraps
 from inspect import isfunction
+
 from loky import get_reusable_executor
-from functools import wraps
-from concurrent.futures import ThreadPoolExecutor
-import atexit
+
+from flowfile_core.configs import logger
 
 
 # process_executor: Uses loky for process-based parallelism
@@ -13,18 +13,18 @@ def process_executor(wait_on_completion: bool = False, max_workers: int = 12):
     def executor(f):
         @wraps(f)
         def inner(*args, **kwargs):
-            logger.debug(f'Added task {f.__name__} to a process executor')
-            logger.debug(f'max_workers: {max_workers}')
+            logger.debug(f"Added task {f.__name__} to a process executor")
+            logger.debug(f"max_workers: {max_workers}")
 
             # Create a new executor with the required number of workers
             func_executor = get_reusable_executor(max_workers=max_workers, timeout=2, kill_workers=False, reuse=True)
             r = func_executor.submit(f, *args, **kwargs)
             if wait_on_completion:
                 result = r.result()
-                logger.info(f'done executing {f.__name__}')
+                logger.info(f"done executing {f.__name__}")
                 return result
 
-            logger.info(f'done submitting {f.__name__} to a process executor')
+            logger.info(f"done submitting {f.__name__} to a process executor")
             return r
 
         return inner
@@ -34,5 +34,3 @@ def process_executor(wait_on_completion: bool = False, max_workers: int = 12):
         wait_on_completion = False
         return executor(f)
     return executor
-
-

flowfile_core/utils/utils.py

@@ -1,14 +1,14 @@
 import re
 from itertools import chain
-from typing import List, Dict
+
 
 def camel_case_to_snake_case(text: str) -> str:
     # Use a regular expression to find capital letters and replace them with _ followed by the lowercase letter
-    transformed_text = re.sub(r'(?<!^)(?=[A-Z])', '_', text).lower()
+    transformed_text = re.sub(r"(?<!^)(?=[A-Z])", "_", text).lower()
     return transformed_text
 
 
-def ensure_similarity_dicts(datas: List[Dict], respect_order: bool = True):
+def ensure_similarity_dicts(datas: list[dict], respect_order: bool = True):
     all_cols = (data.keys() for data in datas)
     if not respect_order:
         unique_cols = set(chain(*all_cols))
@@ -44,4 +44,4 @@ def standardize_col_dtype(vals):
     elif int in types and float in types:
         return vals
     else:
-        return [convert_to_string(v) for v in vals]
+        return [convert_to_string(v) for v in vals]

flowfile_core/utils/validate_setup.py

@@ -1,11 +1,12 @@
 """This script runs on run time and checks if all the nodes that are created have a function in the flow_graph as well
 as have a component in flowfile_frontend"""
 
-from flowfile_core.schemas import input_schema
-from flowfile_core.flowfile.flow_graph import FlowGraph
-from flowfile_core.configs.node_store import nodes_list, NodeTemplate
 import inspect
 
+from flowfile_core.configs.node_store import NodeTemplate, nodes_list
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.schemas import input_schema
+
 
 def check_if_node_has_add_function_in_flow_graph(node: NodeTemplate):
     func_name = "add_" + node.item
@@ -17,7 +18,7 @@ def check_if_node_has_add_function_in_flow_graph(node: NodeTemplate):
 
 
 def check_if_node_has_input_schema_definition(node: NodeTemplate):
-    if "node"+node.item.replace("_","") not in {k.lower() for k in inspect.getmodule(input_schema).__dict__.keys()}:
+    if "node" + node.item.replace("_", "") not in {k.lower() for k in inspect.getmodule(input_schema).__dict__.keys()}:
         raise ValueError(
             f"Node {node.name} ({node.item}) does not have a corresponding input schema definition in input_schema.py."
             "Check if the schema is implemented or if the node item is correct."

flowfile_frame/__init__.py

@@ -1,67 +1,124 @@
 # flowframe/__init__.py
 """A Polars-like API for building ETL graphs."""
 
-# Core classes
-from flowfile_frame.flow_frame import FlowFrame   # noqa: F401
+from importlib.metadata import PackageNotFoundError, version
+
 from pl_fuzzy_frame_match.models import FuzzyMapping  # noqa: F401
+from polars.datatypes import (  # noqa: F401
+    Array,
+    Binary,
+    # Other primitive types
+    Boolean,
+    # Special types
+    Categorical,
+    # Type classes
+    DataType,
+    DataTypeClass,
+    # Date/time types
+    Date,
+    Datetime,
+    Decimal,
+    Duration,
+    Enum,
+    Field,
+    # Float types
+    Float32,
+    Float64,
+    # Integer types
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    Int128,
+    IntegerType,
+    # Complex types
+    List,
+    Null,
+    Object,
+    String,
+    Struct,
+    TemporalType,
+    Time,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Unknown,
+    Utf8,
+)
 
-from flowfile_frame.utils import create_flow_graph  # noqa: F401
+from flowfile_frame.cloud_storage.secret_manager import (
+    create_cloud_storage_connection,
+    create_cloud_storage_connection_if_not_exists,
+    del_cloud_storage_connection,
+    get_all_available_cloud_storage_connections,
+)
 
 # Commonly used functions
 from flowfile_frame.expr import (  # noqa: F401
-    col, lit, column,
-    cum_count, len,
-    sum, min, max, mean, count, when, implode, last, corr, cov, first
+    col,
+    column,
+    corr,
+    count,
+    cov,
+    cum_count,
+    first,
+    implode,
+    last,
+    len,
+    lit,
+    max,
+    mean,
+    min,
+    sum,
+    when,
 )
 
-from flowfile_frame.lazy import (fold)
-
-# Selector utilities
-from flowfile_frame.selectors import (  # noqa: F401
-    numeric, float_, integer, string, temporal,
-    datetime, date, time, duration, boolean,
-    categorical, object_, list_, struct, all_,
-    by_dtype, contains, starts_with, ends_with, matches
-)
-
-from flowfile_frame.series import Series
+# Core classes
+from flowfile_frame.flow_frame import FlowFrame  # noqa: F401
 
 # File I/O
 from flowfile_frame.flow_frame_methods import (  # noqa: F401
-    read_csv, read_parquet, from_dict, concat,  scan_csv, scan_parquet, scan_json_from_cloud_storage,
-    scan_parquet_from_cloud_storage,
+    concat,
+    from_dict,
+    read_csv,
+    read_parquet,
+    scan_csv,
     scan_csv_from_cloud_storage,
-    scan_delta)
-
-from flowfile_frame.cloud_storage.secret_manager import (del_cloud_storage_connection,
-                                                         create_cloud_storage_connection,
-                                                         get_all_available_cloud_storage_connections,
-                                                         create_cloud_storage_connection_if_not_exists)
-
-from polars.datatypes import (  # noqa: F401
-    # Integer types
-    Int8, Int16, Int32, Int64, Int128,
-    UInt8, UInt16, UInt32, UInt64,
-    IntegerType,
-
-    # Float types
-    Float32, Float64,
-
-    # Other primitive types
-    Boolean, String, Utf8, Binary, Null,
-
-    # Complex types
-    List, Array, Struct, Object,
-
-    # Date/time types
-    Date, Time, Datetime, Duration,
-    TemporalType,
-
-    # Special types
-    Categorical, Decimal, Enum, Unknown,
+    scan_delta,
+    scan_json_from_cloud_storage,
+    scan_parquet,
+    scan_parquet_from_cloud_storage,
+)
+from flowfile_frame.lazy import fold
 
-    # Type classes
-    DataType, DataTypeClass, Field
+# Selector utilities
+from flowfile_frame.selectors import (  # noqa: F401
+    all_,
+    boolean,
+    by_dtype,
+    categorical,
+    contains,
+    date,
+    datetime,
+    duration,
+    ends_with,
+    float_,
+    integer,
+    list_,
+    matches,
+    numeric,
+    object_,
+    starts_with,
+    string,
+    struct,
+    temporal,
+    time,
 )
+from flowfile_frame.series import Series
+from flowfile_frame.utils import create_flow_graph  # noqa: F401
 
-__version__ = "0.1.0"
+try:
+    __version__ = version("Flowfile")
+except PackageNotFoundError:
+    __version__ = "0.5.0"

flowfile_frame/adapters.py

@@ -2,16 +2,9 @@
 """Adapters to connect FlowFrame with the flowfile-core library."""
 
 # Import from your existing project
-from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 from flowfile_core.schemas import input_schema, schemas, transform_schema
 
 # Export these for use in FlowFrame
-__all__ = [
-    'FlowGraph',
-    'add_connection',
-    'FlowDataEngine',
-    'input_schema',
-    'schemas',
-    'transform_schema'
-]
+__all__ = ["FlowGraph", "add_connection", "FlowDataEngine", "input_schema", "schemas", "transform_schema"]