Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

flowfile_core/flowfile/setting_generator/settings.py

@@ -1,10 +1,11 @@
+from collections.abc import Callable, Iterable
+from functools import wraps
+
+from pl_fuzzy_frame_match.models import FuzzyMapping
 
+from flowfile_core.flowfile.setting_generator.setting_generator import SettingGenerator, SettingUpdator
 from flowfile_core.schemas import input_schema, transform_schema
-from typing import Callable, Iterable
-from functools import wraps
 from flowfile_core.schemas.output_model import NodeData
-from flowfile_core.flowfile.setting_generator.setting_generator import SettingGenerator, SettingUpdator
-from pl_fuzzy_frame_match.models import FuzzyMapping
 
 setting_generator = SettingGenerator()
 setting_updator = SettingUpdator()
@@ -16,6 +17,7 @@ def setting_generator_method(f: callable) -> Callable:
         if node_data.setting_input is None or isinstance(node_data.setting_input, input_schema.NodePromise):
             f(node_data)
         return node_data
+
     setting_generator.add_setting_generator_func(inner)
     return inner
 
@@ -38,18 +40,19 @@ def join(node_data: "NodeData") -> NodeData:
         if len(overlapping_cols) > 0:
             join_key = overlapping_cols[0]
         else:
-            join_key = ''
+            join_key = ""
         join_input_manager = transform_schema.JoinInputManager(
-            transform_schema.JoinInput(join_mapping=join_key,
-                                       left_select=node_data.main_input.columns,
-                                       right_select=node_data.right_input.columns
-                                       )
+            transform_schema.JoinInput(
+                join_mapping=join_key,
+                left_select=node_data.main_input.columns,
+                right_select=node_data.right_input.columns,
+            )
         )
         join_input_manager.auto_rename()
         ji = join_input_manager.to_join_input()
-        node_data.setting_input = input_schema.NodeJoin(flow_id=node_data.flow_id,
-                                                        node_id=node_data.node_id,
-                                                        join_input=ji)
+        node_data.setting_input = input_schema.NodeJoin(
+            flow_id=node_data.flow_id, node_id=node_data.node_id, join_input=ji
+        )
     return node_data
 
 
@@ -57,24 +60,31 @@ def join(node_data: "NodeData") -> NodeData:
 def cross_join(node_data: "NodeData") -> NodeData:
     if node_data.right_input and node_data.main_input:
         cj_input_manager = transform_schema.CrossJoinInputManager(
-            transform_schema.CrossJoinInput(left_select=node_data.main_input.columns,
-                                            right_select=node_data.right_input.columns)
+            transform_schema.CrossJoinInput(
+                left_select=node_data.main_input.columns, right_select=node_data.right_input.columns
+            )
         )
         cj_input_manager.auto_rename()
         cj = cj_input_manager.to_cross_join_input()
-        node_data.setting_input = input_schema.NodeCrossJoin(flow_id=node_data.flow_id,
-                                                             node_id=node_data.node_id,
-                                                             cross_join_input=cj)
+        node_data.setting_input = input_schema.NodeCrossJoin(
+            flow_id=node_data.flow_id, node_id=node_data.node_id, cross_join_input=cj
+        )
     return node_data
 
 
 @setting_generator_method
 def filter(node_data: "NodeData") -> NodeData:
     if node_data.main_input:
-        fi = transform_schema.FilterInput(basic_filter=transform_schema.BasicFilter(), filter_type='advanced')
-        node_data.setting_input = input_schema.NodeFilter(flow_id=node_data.flow_id,
-                                                          node_id=node_data.node_id,
-                                                          filter_input=fi)
+        # Default to basic mode with an empty basic filter
+        basic_filter = transform_schema.BasicFilter(
+            field="",
+            operator=transform_schema.FilterOperator.EQUALS,
+            value="",
+        )
+        fi = transform_schema.FilterInput(basic_filter=basic_filter, mode="basic")
+        node_data.setting_input = input_schema.NodeFilter(
+            flow_id=node_data.flow_id, node_id=node_data.node_id, filter_input=fi
+        )
     return node_data
 
 
@@ -86,26 +96,26 @@ def join(node_data: NodeData):
         right_columns = set(node_data.right_input.columns)
         left_select = setting_input.join_input.left_select
         right_select = setting_input.join_input.right_select
+        # Update is_available based on whether column exists in input
         for ls in left_select.renames:
-            if ls.old_name not in right_columns:
-                left_select.remove_select_input(ls.old_name)
+            ls.is_available = ls.old_name in left_columns
         for rs in right_select.renames:
-            if rs.old_name not in right_columns:
-                right_select.remove_select_input(rs.old_name)
-        existing_columns_right = set(r.old_name for r in right_select.renames if r.is_available)
-        existing_columns_left = set(r.old_name for r in left_select.renames if r.is_available)
+            rs.is_available = rs.old_name in right_columns
+        # Check ALL columns in renames to prevent duplicates
+        existing_columns_left = set(r.old_name for r in left_select.renames)
+        existing_columns_right = set(r.old_name for r in right_select.renames)
         missing_incoming_left_columns = [ilc for ilc in left_columns if ilc not in existing_columns_left]
         missing_incoming_right_columns = [irc for irc in right_columns if irc not in existing_columns_right]
-        if not hasattr(setting_input, 'auto_keep_left'):
+        if not hasattr(setting_input, "auto_keep_left"):
             setting_input.auto_keep_left = False
-        if not hasattr(setting_input, 'auto_keep_right'):
+        if not hasattr(setting_input, "auto_keep_right"):
             setting_input.auto_keep_right = False
         for milc in missing_incoming_left_columns:
             select_input = transform_schema.SelectInput(old_name=milc, keep=setting_input.auto_keep_left)
-            setting_input.join_input.add_new_select_column(select_input, 'left')
+            setting_input.join_input.add_new_select_column(select_input, "left")
         for mirc in missing_incoming_right_columns:
             select_input = transform_schema.SelectInput(old_name=mirc, keep=setting_input.auto_keep_right)
-            setting_input.join_input.add_new_select_column(select_input, 'right')
+            setting_input.join_input.add_new_select_column(select_input, "right")
     return node_data
 
 
@@ -117,31 +127,32 @@ def cross_join(node_data: NodeData):
         right_columns = set(node_data.right_input.columns)
         left_select = setting_input.cross_join_input.left_select
         right_select = setting_input.cross_join_input.right_select
+        # Update is_available based on whether column exists in input
         for ls in left_select.renames:
-            if ls.old_name not in right_columns:
-                left_select.remove_select_input(ls.old_name)
+            ls.is_available = ls.old_name in left_columns
         for rs in right_select.renames:
-            if rs.old_name not in right_columns:
-                right_select.remove_select_input(rs.old_name)
-        existing_columns_right = set(r.old_name for r in right_select.renames if r.is_available)
-        existing_columns_left = set(r.old_name for r in left_select.renames if r.is_available)
+            rs.is_available = rs.old_name in right_columns
+        # Check ALL columns in renames to prevent duplicates
+        existing_columns_left = set(r.old_name for r in left_select.renames)
+        existing_columns_right = set(r.old_name for r in right_select.renames)
         missing_incoming_left_columns = [ilc for ilc in left_columns if ilc not in existing_columns_left]
         missing_incoming_right_columns = [irc for irc in right_columns if irc not in existing_columns_right]
-        if not hasattr(setting_input, 'auto_keep_left'):
+        if not hasattr(setting_input, "auto_keep_left"):
             setting_input.auto_keep_left = False
-        if not hasattr(setting_input, 'auto_keep_right'):
+        if not hasattr(setting_input, "auto_keep_right"):
             setting_input.auto_keep_right = False
         for milc in missing_incoming_left_columns:
             select_input = transform_schema.SelectInput(old_name=milc, keep=setting_input.auto_keep_left)
-            setting_input.cross_join_input.add_new_select_column(select_input, 'left')
+            setting_input.cross_join_input.add_new_select_column(select_input, "left")
         for mirc in missing_incoming_right_columns:
             select_input = transform_schema.SelectInput(old_name=mirc, keep=setting_input.auto_keep_right)
-            setting_input.cross_join_input.add_new_select_column(select_input, 'right')
+            setting_input.cross_join_input.add_new_select_column(select_input, "right")
     return node_data
 
 
-def check_if_fuzzy_match_is_valid(left_columns: Iterable[str], right_columns: Iterable[str],
-                                  fuzzy_map: FuzzyMapping) -> bool:
+def check_if_fuzzy_match_is_valid(
+    left_columns: Iterable[str], right_columns: Iterable[str], fuzzy_map: FuzzyMapping
+) -> bool:
     if fuzzy_map.left_col not in left_columns:
         return False
     if fuzzy_map.right_col not in right_columns:
@@ -159,24 +170,24 @@ def fuzzy_match(node_data: NodeData):
         right_select = setting_input.join_input.right_select
         for fuzzy_map in setting_input.join_input.join_mapping:
             fuzzy_map.valid = check_if_fuzzy_match_is_valid(left_columns, right_columns, fuzzy_map)
+        # Update is_available based on whether column exists in input
         for ls in left_select.renames:
-            if ls.old_name not in right_columns:
-                left_select.remove_select_input(ls.old_name)
+            ls.is_available = ls.old_name in left_columns
         for rs in right_select.renames:
-            if rs.old_name not in right_columns:
-                right_select.remove_select_input(rs.old_name)
-        existing_columns_right = set(r.old_name for r in right_select.renames if r.is_available)
-        existing_columns_left = set(r.old_name for r in left_select.renames if r.is_available)
+            rs.is_available = rs.old_name in right_columns
+        # Check ALL columns in renames to prevent duplicates
+        existing_columns_left = set(r.old_name for r in left_select.renames)
+        existing_columns_right = set(r.old_name for r in right_select.renames)
         missing_incoming_left_columns = [ilc for ilc in left_columns if ilc not in existing_columns_left]
         missing_incoming_right_columns = [irc for irc in right_columns if irc not in existing_columns_right]
-        if not hasattr(setting_input, 'auto_keep_left'):
+        if not hasattr(setting_input, "auto_keep_left"):
             setting_input.auto_keep_left = False
-        if not hasattr(setting_input, 'auto_keep_right'):
+        if not hasattr(setting_input, "auto_keep_right"):
             setting_input.auto_keep_right = False
         for milc in missing_incoming_left_columns:
             select_input = transform_schema.SelectInput(old_name=milc, keep=setting_input.auto_keep_left)
-            setting_input.join_input.add_new_select_column(select_input, 'left')
+            setting_input.join_input.add_new_select_column(select_input, "left")
         for mirc in missing_incoming_right_columns:
             select_input = transform_schema.SelectInput(old_name=mirc, keep=setting_input.auto_keep_right)
-            setting_input.join_input.add_new_select_column(select_input, 'right')
+            setting_input.join_input.add_new_select_column(select_input, "right")
     return node_data

flowfile_core/flowfile/sources/external_sources/base_class.py

@@ -1,31 +1,34 @@
 from abc import ABC, abstractmethod
-from typing import Generator, Callable, List, Any, Optional, Dict
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from collections.abc import Callable, Generator
+from typing import Any
+
 import polars as pl
 
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+
 
 class ExternalDataSource(ABC):
-    schema: Optional[List[FlowfileColumn]]
-    data_getter: Optional[Callable]
+    schema: list[FlowfileColumn] | None
+    data_getter: Callable | None
     is_collected: bool
     cache_store: Any
     _type: str
-    initial_data_getter: Optional[Callable]
+    initial_data_getter: Callable | None
 
     @abstractmethod
     def __init__(self):
         pass
 
     @abstractmethod
-    def get_initial_data(self) -> List[Dict[str, Any]]:
+    def get_initial_data(self) -> list[dict[str, Any]]:
         pass
 
     @abstractmethod
-    def get_iter(self) -> Generator[Dict[str, Any], None, None]:
+    def get_iter(self) -> Generator[dict[str, Any], None, None]:
         pass
 
     @abstractmethod
-    def get_sample(self, n: int = 10000) -> Generator[Dict[str, Any], None, None]:
+    def get_sample(self, n: int = 10000) -> Generator[dict[str, Any], None, None]:
         pass
 
     @abstractmethod
@@ -34,6 +37,5 @@ class ExternalDataSource(ABC):
 
     @staticmethod
     @abstractmethod
-    def parse_schema(*args, **kwargs) -> List[FlowfileColumn]:
+    def parse_schema(*args, **kwargs) -> list[FlowfileColumn]:
         pass
-

flowfile_core/flowfile/sources/external_sources/custom_external_sources/external_source.py

@@ -1,16 +1,19 @@
-from typing import Any, Dict, Generator, List, Optional, Callable
+from collections.abc import Callable, Generator
+from typing import Any
+
+import polars as pl
+
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
-from flowfile_core.schemas import input_schema
 from flowfile_core.flowfile.sources.external_sources.base_class import ExternalDataSource
-import polars as pl
+from flowfile_core.schemas import input_schema
 
 
 class CustomExternalSourceSettings:
     data_getter: Generator
-    initial_data_getter: Optional[Callable] = None
-    orientation: str = 'row'
+    initial_data_getter: Callable | None = None
+    orientation: str = "row"
 
-    def __init__(self, data_getter: Generator, initial_data_getter: Optional[Callable] = None, orientation: str = 'row'):
+    def __init__(self, data_getter: Generator, initial_data_getter: Callable | None = None, orientation: str = "row"):
         self.data_getter = data_getter
         self.initial_data_getter = initial_data_getter
         self.orientation = orientation
@@ -18,15 +21,18 @@ class CustomExternalSourceSettings:
 
 class CustomExternalSource(ExternalDataSource):
     data_getter: Generator = None
-    schema: Optional[List[FlowfileColumn]] = None
-    cache_store: List = None
+    schema: list[FlowfileColumn] | None = None
+    cache_store: list = None
     is_collected: bool = False
 
-    def __init__(self, data_getter: Generator[Any, None, None],
-                 initial_data_getter: Callable = None,
-                 orientation: str = 'row',
-                 schema: List = None,
-                 **kwargs):
+    def __init__(
+        self,
+        data_getter: Generator[Any, None, None],
+        initial_data_getter: Callable = None,
+        orientation: str = "row",
+        schema: list = None,
+        **kwargs,
+    ):
         self.cache_store = list()
         self.data_getter = data_getter
         self.collected = False
@@ -38,23 +44,27 @@ class CustomExternalSource(ExternalDataSource):
         else:
             self.schema = None
 
-        if not initial_data_getter and orientation == 'row':
+        if not initial_data_getter and orientation == "row":
+
             def initial_data_getter():
                 if len(self.cache_store) == 0:
                     self.cache_store.append(next(data_getter, None))
                 return self.cache_store
+
             self.initial_data_getter = initial_data_getter
         elif initial_data_getter:
             self.initial_data_getter = initial_data_getter
         elif self.schema:
+
             def initial_data_getter():
                 return [{d.column_name: None for d in self.schema}]
+
             self.initial_data_getter = initial_data_getter
         else:
             self.initial_data_getter = None
 
     @staticmethod
-    def parse_schema(schema: List[Any]) -> List[FlowfileColumn]:
+    def parse_schema(schema: list[Any]) -> list[FlowfileColumn]:
         if len(schema) == 0:
             return []
         first_col = schema[0]
@@ -63,7 +73,7 @@ class CustomExternalSource(ExternalDataSource):
         elif isinstance(first_col, (list, tuple)):
             return [FlowfileColumn.from_input(column_name=col[0], data_type=col[1]) for col in schema]
         elif isinstance(first_col, str):
-            return [FlowfileColumn.from_input(column_name=col, data_type='varchar') for col in schema]
+            return [FlowfileColumn.from_input(column_name=col, data_type="varchar") for col in schema]
         elif isinstance(first_col, input_schema.MinimalFieldInfo):
             return [FlowfileColumn.from_input(column_name=col.name, data_type=col.data_type) for col in schema]
         elif isinstance(first_col, FlowfileColumn):
@@ -76,7 +86,7 @@ class CustomExternalSource(ExternalDataSource):
             return self.initial_data_getter()
         return []
 
-    def get_iter(self) -> Generator[Dict[str, Any], None, None]:
+    def get_iter(self) -> Generator[dict[str, Any], None, None]:
         if self.collected:
             return
         for data in self.cache_store:

flowfile_core/flowfile/sources/external_sources/custom_external_sources/sample_users.py

@@ -1,10 +1,13 @@
-from typing import Dict, Any, Generator
+from collections.abc import Generator
 from time import sleep
-from flowfile_core.schemas.input_schema import SampleUsers
+from typing import Any
+
 import requests
 
+from flowfile_core.schemas.input_schema import SampleUsers
 
-def getter(data: SampleUsers) -> Generator[Dict[str, Any], None, None]:
+
+def getter(data: SampleUsers) -> Generator[dict[str, Any], None, None]:
     """
     Sample users generator function. This is a minimal example of a generator function that yields user data and can
     be used in a flowfile. The function simulates a delay to mimic the behavior of an external data source.
@@ -17,13 +20,10 @@ def getter(data: SampleUsers) -> Generator[Dict[str, Any], None, None]:
     index_pos = 0
     for i in range(data.size):
         sleep(0.01)
-        headers = {
-            'x-api-key': 'reqres-free-v1'
-        }
+        headers = {"x-api-key": "reqres-free-v1"}
 
         response = requests.get("https://reqres.in/api/users", headers=headers).json()
-        for v in response['data']:
-            v['index'] = index_pos
+        for v in response["data"]:
+            v["index"] = index_pos
             index_pos += 1
             yield v
-

flowfile_core/flowfile/sources/external_sources/factory.py

@@ -16,4 +16,3 @@ def data_source_factory(source_type: str, **kwargs) -> CustomExternalSource:
         return CustomExternalSource(**kwargs)
     else:
         raise ValueError(f"Unknown source type: {source_type}")
-

flowfile_core/flowfile/sources/external_sources/sql_source/models.py

@@ -1,34 +1,42 @@
-from typing import Literal, Optional, TYPE_CHECKING
-from pydantic import BaseModel, SecretStr
-from flowfile_core.schemas.input_schema import (DatabaseConnection,
-                                                NodeDatabaseReader,
-                                                FullDatabaseConnection,
-                                                NodeDatabaseWriter)
 import base64
+from typing import Literal
+
 import polars as pl
+from pydantic import BaseModel
+
+from flowfile_core.schemas.input_schema import (
+    DatabaseConnection,
+    FullDatabaseConnection,
+    NodeDatabaseReader,
+    NodeDatabaseWriter,
+)
 
 
 class ExtDatabaseConnection(DatabaseConnection):
     """Database connection configuration with password handling."""
+
     password: str = None
 
 
 class DatabaseExternalWriteSettings(BaseModel):
     """Settings for SQL sink."""
+
     connection: ExtDatabaseConnection
     table_name: str
-    if_exists: Optional[Literal['append', 'replace', 'fail']] = 'append'
+    if_exists: Literal["append", "replace", "fail"] | None = "append"
     flowfile_flow_id: int = 1
     flowfile_node_id: int | str = -1
     operation: str
 
     @classmethod
-    def create_from_from_node_database_writer(cls, node_database_writer: NodeDatabaseWriter,
-                                              password: str,
-                                              table_name: str,
-                                              lf: pl.LazyFrame,
-                                              database_reference_settings: FullDatabaseConnection = None,
-                                              ) -> 'DatabaseExternalWriteSettings':
+    def create_from_from_node_database_writer(
+        cls,
+        node_database_writer: NodeDatabaseWriter,
+        password: str,
+        table_name: str,
+        lf: pl.LazyFrame,
+        database_reference_settings: FullDatabaseConnection = None,
+    ) -> "DatabaseExternalWriteSettings":
         """
         Create DatabaseExternalWriteSettings from NodeDatabaseWriter.
         Args:
@@ -45,28 +53,33 @@ class DatabaseExternalWriteSettings(BaseModel):
         else:
             database_connection = {k: v for k, v in database_reference_settings.model_dump().items() if k != "password"}
 
-        ext_database_connection = ExtDatabaseConnection(**database_connection,
-                                                        password=password)
-        return cls(connection=ext_database_connection,
-                   table_name=table_name,
-                   if_exists=node_database_writer.database_write_settings.if_exists,
-                   flowfile_flow_id=node_database_writer.flow_id,
-                   flowfile_node_id=node_database_writer.node_id,
-                   operation=base64.b64encode(lf.serialize()).decode())
+        ext_database_connection = ExtDatabaseConnection(**database_connection, password=password)
+        return cls(
+            connection=ext_database_connection,
+            table_name=table_name,
+            if_exists=node_database_writer.database_write_settings.if_exists,
+            flowfile_flow_id=node_database_writer.flow_id,
+            flowfile_node_id=node_database_writer.node_id,
+            operation=base64.b64encode(lf.serialize()).decode(),
+        )
 
 
 class DatabaseExternalReadSettings(BaseModel):
     """Settings for SQL source."""
+
     connection: ExtDatabaseConnection
     query: str
     flowfile_flow_id: int = 1
     flowfile_node_id: int | str = -1
 
     @classmethod
-    def create_from_from_node_database_reader(cls, node_database_reader: NodeDatabaseReader,
-                                              password: str,
-                                              query: str,
-                                              database_reference_settings: FullDatabaseConnection = None) -> 'DatabaseExternalReadSettings':
+    def create_from_from_node_database_reader(
+        cls,
+        node_database_reader: NodeDatabaseReader,
+        password: str,
+        query: str,
+        database_reference_settings: FullDatabaseConnection = None,
+    ) -> "DatabaseExternalReadSettings":
         """
         Create DatabaseExternalReadSettings from NodeDatabaseReader.
         Args:
@@ -82,9 +95,10 @@ class DatabaseExternalReadSettings(BaseModel):
         else:
             database_connection = {k: v for k, v in database_reference_settings.model_dump().items() if k != "password"}
 
-        ext_database_connection = ExtDatabaseConnection(**database_connection,
-                                                        password=password)
-        return cls(connection=ext_database_connection,
-                   query=query,
-                   flowfile_flow_id=node_database_reader.flow_id,
-                   flowfile_node_id=node_database_reader.node_id)
+        ext_database_connection = ExtDatabaseConnection(**database_connection, password=password)
+        return cls(
+            connection=ext_database_connection,
+            query=query,
+            flowfile_flow_id=node_database_reader.flow_id,
+            flowfile_node_id=node_database_reader.node_id,
+        )