Flowfile 0.3.9__py3-none-any.whl → 0.5.1__py3-none-any.whl

flowfile_core/schemas/yaml_types.py

@@ -0,0 +1,103 @@
+from typing import TypedDict, List
+
+
+# === Transform Schema YAML Types ===
+
+class SelectInputYaml(TypedDict, total=False):
+    old_name: str
+    new_name: str
+    keep: bool
+    data_type: str
+
+
+class JoinInputsYaml(TypedDict):
+    select: List[SelectInputYaml]
+
+
+class JoinMapYaml(TypedDict):
+    left_col: str
+    right_col: str
+
+
+class JoinInputYaml(TypedDict):
+    join_mapping: List[JoinMapYaml]
+    left_select: JoinInputsYaml
+    right_select: JoinInputsYaml
+    how: str
+
+
+class CrossJoinInputYaml(TypedDict):
+    left_select: JoinInputsYaml
+    right_select: JoinInputsYaml
+
+
+class FuzzyMappingYaml(TypedDict, total=False):
+    left_col: str
+    right_col: str
+    threshold_score: float
+    fuzzy_type: str
+    perc_unique: float
+    output_column_name: str
+    valid: bool
+
+
+class FuzzyMatchInputYaml(TypedDict):
+    join_mapping: List[FuzzyMappingYaml]
+    left_select: JoinInputsYaml
+    right_select: JoinInputsYaml
+    how: str
+    aggregate_output: bool
+
+
+# === Input Schema YAML Types ===
+
+class OutputSettingsYaml(TypedDict, total=False):
+    name: str
+    directory: str
+    file_type: str
+    write_mode: str
+    abs_file_path: str
+    fields: List[str]
+    table_settings: dict
+
+
+class NodeSelectYaml(TypedDict):
+    cache_results: bool
+    keep_missing: bool
+    select_input: List[SelectInputYaml]
+    sorted_by: str
+
+
+class NodeJoinYaml(TypedDict):
+    cache_results: bool
+    auto_generate_selection: bool
+    verify_integrity: bool
+    join_input: JoinInputYaml
+    auto_keep_all: bool
+    auto_keep_right: bool
+    auto_keep_left: bool
+
+
+class NodeCrossJoinYaml(TypedDict):
+    cache_results: bool
+    auto_generate_selection: bool
+    verify_integrity: bool
+    cross_join_input: CrossJoinInputYaml
+    auto_keep_all: bool
+    auto_keep_right: bool
+    auto_keep_left: bool
+
+
+class NodeFuzzyMatchYaml(TypedDict):
+    cache_results: bool
+    auto_generate_selection: bool
+    verify_integrity: bool
+    join_input: FuzzyMatchInputYaml
+    auto_keep_all: bool
+    auto_keep_right: bool
+    auto_keep_left: bool
+
+
+class NodeOutputYaml(TypedDict):
+    cache_results: bool
+    output_settings: OutputSettingsYaml

flowfile_core/types.py

@@ -0,0 +1,156 @@
+# types.py - Public API for type specifications
+"""
+Public type system for column selection and data type specification.
+
+Usage:
+    from flowfile_core.types import Types
+
+    # Use type groups
+    ColumnSelector(data_types=Types.Numeric)
+    ColumnSelector(data_types=Types.String)
+
+    # Use specific types
+    ColumnSelector(data_types=Types.Int64)
+    ColumnSelector(data_types=Types.Float)
+
+    # Mix and match
+    ColumnSelector(data_types=[Types.Numeric, Types.String])
+"""
+
+from enum import Enum
+from typing import List, Literal, Union
+import polars as pl
+
+
+DataTypeStr = Literal[
+    "Int8", "Int16", "Int32", "Int64",
+    "UInt8", "UInt16", "UInt32", "UInt64",
+    "Float32", "Float64", "Decimal",
+    "String",
+    "Date", "Datetime", "Time", "Duration",
+    "Boolean", "Binary", "List", "Struct", "Array", "Integer", "Double", "Utf8"
+]
+
+
+class TypeGroup(str, Enum):
+    """High-level type groups for column selection."""
+    Numeric = "Numeric"
+    String = "String"
+    Date = "Date"
+    Boolean = "Boolean"
+    Binary = "Binary"
+    Complex = "Complex"
+    All = "ALL"
+
+    def __str__(self) -> str:
+        return self.value
+
+    def __repr__(self) -> str:
+        return f"Types.{self.name}"
+
+
+class DataType(str, Enum):
+    """Specific data types for fine-grained control."""
+    # Numeric types
+    Int8 = "Int8"
+    Int16 = "Int16"
+    Int32 = "Int32"
+    Int64 = "Int64"
+    UInt8 = "UInt8"
+    UInt16 = "UInt16"
+    UInt32 = "UInt32"
+    UInt64 = "UInt64"
+    Float32 = "Float32"
+    Float64 = "Float64"
+    Decimal = "Decimal"
+
+    # String types
+    String = "String"
+    Categorical = "Categorical"
+
+    # Date types
+    Date = "Date"
+    Datetime = "Datetime"
+    Time = "Time"
+    Duration = "Duration"
+
+    # Other types
+    Boolean = "Boolean"
+    Binary = "Binary"
+    List = "List"
+    Struct = "Struct"
+    Array = "Array"
+
+    def __str__(self) -> str:
+        return self.value
+
+    def __repr__(self) -> str:
+        return f"Types.{self.name}"
+
+
+class Types:
+    """
+    Main entry point for type specifications.
+
+    Examples:
+        Types.Numeric     # All numeric columns
+        Types.String      # All string columns
+        Types.Int64       # 64-bit integers only
+        Types.Float       # Alias for Float64
+        Types.All         # All column types
+    """
+
+    # Type groups (most common use case)
+    Numeric = TypeGroup.Numeric
+    String = TypeGroup.String
+    AnyDate = TypeGroup.Date
+    Boolean = TypeGroup.Boolean
+    Binary = TypeGroup.Binary
+    Complex = TypeGroup.Complex
+    All = TypeGroup.All
+
+    # Specific numeric types
+    Int = DataType.Int64  # Default integer
+    Int8 = DataType.Int8
+    Int16 = DataType.Int16
+    Int32 = DataType.Int32
+    Int64 = DataType.Int64
+    UInt8 = DataType.UInt8
+    UInt16 = DataType.UInt16
+    UInt32 = DataType.UInt32
+    UInt64 = DataType.UInt64
+
+    Float = DataType.Float64  # Default float
+    Float32 = DataType.Float32
+    Float64 = DataType.Float64
+    Decimal = DataType.Decimal
+
+    # String types
+    Str = DataType.String
+    Text = DataType.String  # Alias
+    Categorical = DataType.Categorical
+    Cat = DataType.Categorical  # Short alias
+
+    # Date/time types
+    Date = DataType.Date
+    Datetime = DataType.Datetime
+    Time = DataType.Time
+    Duration = DataType.Duration
+
+    # Other types
+    Bool = DataType.Boolean
+    Bytes = DataType.Binary
+    List = DataType.List
+    Struct = DataType.Struct
+    Array = DataType.Array
+
+
+# Type alias for better type hints
+TypeSpec = Union[
+    TypeGroup,
+    DataType,
+    str,
+    List[Union[TypeGroup, DataType, str, type[pl.DataType], pl.DataType]],
+    type[pl.DataType],
+    pl.DataType
+]

flowfile_core/utils/validate_setup.py

@@ -3,7 +3,7 @@ as have a component in flowfile_frontend"""
 
 from flowfile_core.schemas import input_schema
 from flowfile_core.flowfile.flow_graph import FlowGraph
-from flowfile_core.configs.node_store.nodes import nodes_list, NodeTemplate
+from flowfile_core.configs.node_store import nodes_list, NodeTemplate
 import inspect
 
 
@@ -31,6 +31,8 @@ def validate_setup():
     Raises ValueError if any node is missing either.
     """
     for node in nodes_list:
+        if node.custom_node:
+            continue
         check_if_node_has_add_function_in_flow_graph(node)
         check_if_node_has_input_schema_definition(node)
 

flowfile_frame/__init__.py

@@ -1,6 +1,8 @@
 # flowframe/__init__.py
 """A Polars-like API for building ETL graphs."""
 
+from importlib.metadata import version
+
 # Core classes
 from flowfile_frame.flow_frame import FlowFrame   # noqa: F401
 from pl_fuzzy_frame_match.models import FuzzyMapping  # noqa: F401
@@ -64,4 +66,4 @@ from polars.datatypes import (  # noqa: F401
     DataType, DataTypeClass, Field
 )
 
-__version__ = "0.1.0"
+__version__ = version("Flowfile")

flowfile_frame/flow_frame.py

@@ -10,7 +10,7 @@ from flowfile_frame.lazy_methods import add_lazyframe_methods
 from polars._typing import (CsvEncoding, FrameInitTypes, SchemaDefinition, SchemaDict, Orientation)
 from collections.abc import Iterator
 
-from pl_fuzzy_frame_match import FuzzyMapping, fuzzy_match_dfs
+from pl_fuzzy_frame_match import FuzzyMapping
 
 from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 from flowfile_core.flowfile.flow_graph_utils import combine_flow_graphs_with_mapping
@@ -27,6 +27,8 @@ from flowfile_frame.join import _normalize_columns_to_list, _create_join_mapping
 from flowfile_frame.utils import _check_if_convertible_to_code
 from flowfile_frame.config import logger
 from flowfile_frame.cloud_storage.frame_helpers import add_write_ff_to_cloud_storage
+from collections.abc import Mapping
+
 
 
 def can_be_expr(param: inspect.Parameter) -> bool:
@@ -624,7 +626,6 @@ class FlowFrame:
         left_columns, right_columns = self._parse_join_columns(
             on, left_on, right_on, how
         )
-
         # Step 5: Validate column lists have same length (except for cross join)
         if how != 'cross' and left_columns is not None and right_columns is not None:
             if len(left_columns) != len(right_columns):
@@ -796,33 +797,36 @@ class FlowFrame:
     ) -> "FlowFrame":
         """Execute join using native FlowFile join nodes."""
         # Create select inputs for both frames
+
         left_select = transform_schema.SelectInputs.create_from_pl_df(self.data)
         right_select = transform_schema.SelectInputs.create_from_pl_df(other.data)
         # Create appropriate join input based on join type
         if how == 'cross':
             join_input = transform_schema.CrossJoinInput(
-                left_select=left_select.renames,
+                left_select=transform_schema.JoinInputs(renames=left_select.renames),
                 right_select=right_select.renames,
             )
+            join_input_manager = transform_schema.CrossJoinInputManager(join_input)
+
         else:
             join_input = transform_schema.JoinInput(
                 join_mapping=join_mappings,
-                left_select=left_select.renames,
+                left_select=transform_schema.JoinInputs(renames=left_select.renames),
                 right_select=right_select.renames,
                 how=how,
             )
+            join_input_manager = transform_schema.JoinInputManager(join_input)
 
         # Configure join input
-        join_input.auto_rename()
-        for right_column in right_select.renames:
+        for right_column in join_input_manager.right_select.renames:
             if right_column.join_key:
                 right_column.keep = False
 
         # Create and add appropriate node
         if how == 'cross':
-            self._add_cross_join_node(new_node_id, join_input, description, other)
+            self._add_cross_join_node(new_node_id, join_input_manager.to_cross_join_input(), description, other)
         else:
-            self._add_regular_join_node(new_node_id, join_input, description, other)
+            self._add_regular_join_node(new_node_id, join_input_manager.to_join_input(), description, other)
 
         # Add connections
         self._add_connection(self.node_id, new_node_id, "main")
@@ -890,13 +894,18 @@ class FlowFrame:
         self.flow_graph.add_record_count(node_number_of_records)
         return self._create_child_frame(new_node_id)
 
-    def select(self, *columns: Union[str, Expr, Selector], description: Optional[str] = None) -> "FlowFrame":
+    def rename(self, mapping: Mapping[str, str], *, strict: bool = True,
+               description: str = None) -> "FlowFrame":
+        """Rename columns based on a mapping or function."""
+        return self.select([col(old_name).alias(new_name) for old_name, new_name in mapping.items()],
+                           description=description, _keep_missing=True)
+
+    def select(self, *columns: Union[str, Expr, Selector], description: Optional[str] = None, _keep_missing: bool = False) -> "FlowFrame":
         """
         Select columns from the frame.
         """
         columns_iterable = list(_parse_inputs_as_iterable(columns))
         new_node_id = generate_node_id()
-
         if (len(columns_iterable) == 1 and isinstance(columns_iterable[0], Expr)
                 and str(columns_iterable[0]) == "pl.Expr(len()).alias('number_of_records')"):
             return self._add_number_of_records(new_node_id, description)
@@ -914,7 +923,6 @@ class FlowFrame:
         for expr_input in effective_columns_iterable:
             current_expr_obj = expr_input
             is_simple_col_for_native = False
-
             if isinstance(expr_input, str):
                 current_expr_obj = col(expr_input)
                 selected_col_names_for_native.append(transform_schema.SelectInput(old_name=expr_input))
@@ -942,14 +950,18 @@ class FlowFrame:
         if can_use_native_node:
             existing_cols = self.columns
             selected_col_names = {select_col.old_name for select_col in selected_col_names_for_native}
-            dropped_columns = [transform_schema.SelectInput(c, keep=False) for c in existing_cols if
+            not_selected_columns = [transform_schema.SelectInput(c, keep=_keep_missing) for c in existing_cols if
                                c not in selected_col_names]
-            selected_col_names_for_native.extend(dropped_columns)
+            selected_col_names_for_native.extend(not_selected_columns)
+            if _keep_missing:
+                lookup_selection = {_col.old_name: _col for _col in selected_col_names_for_native}
+                selected_col_names_for_native = [lookup_selection.get(_col) for
+                                                 _col in existing_cols if _col in lookup_selection]
             select_settings = input_schema.NodeSelect(
                 flow_id=self.flow_graph.flow_id,
                 node_id=new_node_id,
                 select_input=selected_col_names_for_native,
-                keep_missing=False,
+                keep_missing=_keep_missing,
                 pos_x=200,
                 pos_y=100,
                 is_setup=True,
@@ -1130,16 +1142,11 @@ class FlowFrame:
         file_name = file_str.split(os.sep)[-1]
         use_polars_code = bool(kwargs.items()) or not is_path_input
 
-        output_parquet_table = input_schema.OutputParquetTable(
-            file_type="parquet"
-        )
         output_settings = input_schema.OutputSettings(
             file_type='parquet',
             name=file_name,
             directory=file_str if is_path_input else str(file_str),
-            output_parquet_table=output_parquet_table,
-            output_csv_table=input_schema.OutputCsvTable(),
-            output_excel_table=input_schema.OutputExcelTable()
+            table_settings=input_schema.OutputParquetTable()
         )
 
         if is_path_input:
@@ -1210,10 +1217,10 @@ class FlowFrame:
             file_type='csv',
             name=file_name,
             directory=file_str if is_path_input else str(file_str),
-            output_csv_table=input_schema.OutputCsvTable(
-                file_type="csv", delimiter=separator, encoding=encoding),
-            output_excel_table=input_schema.OutputExcelTable(),
-            output_parquet_table=input_schema.OutputParquetTable()
+            table_settings=input_schema.OutputCsvTable(
+                delimiter=separator,
+                encoding=encoding
+            )
         )
         if is_path_input:
             try:

flowfile_frame/flow_frame_methods.py

@@ -186,15 +186,17 @@ def read_csv(
             file_type='csv',
             path=current_source_path_for_native,
             name=Path(current_source_path_for_native).name,
-            delimiter=separator,
-            has_headers=has_header,
-            encoding=encoding,
-            starting_from_line=skip_rows,
-            quote_char=quote_char if quote_char is not None else '"',
-            infer_schema_length=actual_infer_schema_length if actual_infer_schema_length is not None else 10000,
-            truncate_ragged_lines=truncate_ragged_lines,
-            ignore_errors=ignore_errors,
-            row_delimiter=eol_char
+            table_settings=input_schema.InputCsvTable(
+                delimiter=separator,
+                has_headers=has_header,
+                encoding=encoding,
+                starting_from_line=skip_rows,
+                quote_char=quote_char if quote_char is not None else '"',
+                infer_schema_length=actual_infer_schema_length if actual_infer_schema_length is not None else 10000,
+                truncate_ragged_lines=truncate_ragged_lines,
+                ignore_errors=ignore_errors,
+                row_delimiter=eol_char
+            )
         )
         if convert_to_absolute_path:
             try:
@@ -407,6 +409,7 @@ def read_parquet(source, *, flow_graph: FlowGraph = None, description: str = Non
         file_type='parquet',
         path=source,
         name=Path(source).name,
+        table_settings=input_schema.InputParquetTable()
     )
     if convert_to_absolute_path:
         received_table.path = received_table.abs_file_path

flowfile_worker/__init__.py

@@ -1,15 +1,18 @@
 from typing import Dict
-import tempfile
 import threading
 import multiprocessing
-import os
-import shutil
-multiprocessing.set_start_method('spawn', force=True)
+from shared.storage_config import storage
+from importlib.metadata import version
+
+__version__ = version("Flowfile")
 
+multiprocessing.set_start_method('spawn', force=True)
 
 from multiprocessing import get_context
 from flowfile_worker.models import Status
+
 mp_context = get_context("spawn")
+
 status_dict: Dict[str, Status] = dict()
 process_dict = dict()
 
@@ -17,39 +20,10 @@ status_dict_lock = threading.Lock()
 process_dict_lock = threading.Lock()
 
 
-class SharedTempDirectory:
-    """A class that mimics tempfile.TemporaryDirectory but uses a fixed directory"""
-    def __init__(self, dir_path):
-        self._path = dir_path
-        os.makedirs(self._path, exist_ok=True)
-
-    @property
-    def name(self):
-        return self._path
-
-    def cleanup(self):
-        """Remove all contents of the temp directory"""
-        try:
-            shutil.rmtree(self._path)
-            os.makedirs(self._path, exist_ok=True)
-            print(f"Cleaned up temporary directory: {self._path}")
-        except Exception as e:
-            print(f"Error during cleanup: {e}")
-
-    def __enter__(self):
-        return self.name
-
-    def __exit__(self, exc, value, tb):
-        self.cleanup()
-
-
 CACHE_EXPIRATION_TIME = 24 * 60 * 60
 
 
-TEMP_DIR = os.getenv('TEMP_DIR')
-if TEMP_DIR:
-    CACHE_DIR = SharedTempDirectory(TEMP_DIR)
-else:
-    CACHE_DIR = tempfile.TemporaryDirectory()
+CACHE_DIR = storage.cache_directory
+
 
 PROCESS_MEMORY_USAGE: Dict[str, float] = dict()

flowfile_worker/create/__init__.py

@@ -1,29 +1,11 @@
-from flowfile_worker.create.models import (ReceivedCsvTable, ReceivedParquetTable, ReceivedExcelTable,
-                                           ReceivedJsonTable)
+
 from flowfile_worker.create.funcs import (create_from_path_csv, create_from_path_parquet, create_from_path_excel,
                                           create_from_path_json)
-from typing import Dict, Literal
+from typing import Literal
 
-ReceivedTableCollection = ReceivedCsvTable | ReceivedParquetTable | ReceivedJsonTable | ReceivedExcelTable
 FileType = Literal['csv', 'parquet', 'json', 'excel']
 
-
-def received_table_parser(received_table_raw: Dict, file_type: FileType) -> ReceivedTableCollection:
-    match file_type:
-        case 'csv':
-            received_table = ReceivedCsvTable.model_validate(received_table_raw)
-        case 'parquet':
-            received_table = ReceivedParquetTable.model_validate(received_table_raw)
-        case 'excel':
-            received_table = ReceivedExcelTable.model_validate(received_table_raw)
-        case 'json':
-            return ReceivedJsonTable.model_validate(received_table_raw)
-        case _:
-            raise ValueError(f'Unsupported file type: {file_type}')
-    return received_table
-
-
-def table_creator_factory_method(file_type: Literal['csv', 'parquet', 'json', 'excel']) -> callable:
+def table_creator_factory_method(file_type: FileType) -> callable:
     match file_type:
         case 'csv':
             return create_from_path_csv