Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py

@@ -1,9 +1,10 @@
+
 from dataclasses import dataclass
-from typing import Optional, Any, List, Dict, Literal
+from typing import Optional, Any, List, Dict, Literal, Iterable
+
 from flowfile_core.schemas import input_schema
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
-from polars import datatypes
 import polars as pl
 # TODO: rename flow_file_column to flowfile_column
 DataTypeGroup = Literal['numeric', 'str', 'date']
@@ -175,3 +176,12 @@ def convert_stats_to_column_info(stats: List[Dict]) -> List[FlowfileColumn]:
 def convert_pl_schema_to_raw_data_format(pl_schema: pl.Schema) -> List[input_schema.MinimalFieldInfo]:
     return [FlowfileColumn.create_from_polars_type(PlType(column_name=k, pl_datatype=v)).get_minimal_field_info()
             for k, v in pl_schema.items()]
+
+
+def assert_if_flowfile_schema(obj: Iterable) -> bool:
+    """
+    Assert that the object is a valid iterable of FlowfileColumn objects.
+    """
+    if isinstance(obj, (list, set, tuple)):
+        return all(isinstance(item, FlowfileColumn) for item in obj)
+    return False

flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py

@@ -32,7 +32,7 @@ def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInput,
             output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
                                                            example_values=column_schema.example_values))
 
-    for i, fm in enumerate(fm_input.join_mappings):
+    for i, fm in enumerate(fm_input.join_mapping):
         output_schema.append(FlowfileColumn.from_input(f'fuzzy_score_{i}', 'Float64'))
     return output_schema
 

flowfile_core/flowfile/flow_data_engine/join/__init__.py

@@ -1 +1,2 @@
-from flowfile_core.flowfile.flow_data_engine.join.verify_integrity import *
+from flowfile_core.flowfile.flow_data_engine.join.verify_integrity import *
+from flowfile_core.flowfile.flow_data_engine.join.utils import *

flowfile_core/flowfile/flow_data_engine/join/utils.py

@@ -0,0 +1,25 @@
+# Standard library imports
+from typing import Dict, Tuple, TypeVar
+
+# Third-party imports
+import polars as pl
+
+from flowfile_core.schemas import (
+    transform_schema as transform_schemas
+)
+
+T = TypeVar('T', pl.DataFrame, pl.LazyFrame)
+
+
+def rename_df_table_for_join(left_df: T, right_df: T,  join_key_rename: transform_schemas.FullJoinKeyResponse) -> Tuple[T, T]:
+    return (left_df.rename({r[0]: r[1] for r in join_key_rename.left.join_key_renames}),
+            right_df.rename({r[0]: r[1] for r in join_key_rename.right.join_key_renames}))
+
+
+def get_undo_rename_mapping_join(join_input: transform_schemas.JoinInput) -> Dict[str, str]:
+    join_key_rename = join_input.get_join_key_renames(True)
+    return {r[1]: r[0] for r in join_key_rename.right.join_key_renames + join_key_rename.left.join_key_renames}
+
+
+def get_col_name_to_delete(col: transform_schemas.SelectInput, side: transform_schemas.SideLit):
+    return col.new_name if not col.join_key else transform_schemas.construct_join_key_name(side, col.new_name)

flowfile_core/flowfile/flow_data_engine/polars_code_parser.py

@@ -121,6 +121,7 @@ class PolarsCodeParser:
     """
 
     def __init__(self):
+        import datetime
         self.safe_globals = {
             # Polars functionality
             'pl': pl,
@@ -175,7 +176,8 @@ class PolarsCodeParser:
             'False': False,
             'None': None,
             'time': time,
-            'BytesIO': BytesIO
+            'BytesIO': BytesIO,
+            'datetime': datetime,
         }
 
     @staticmethod

flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py

@@ -18,9 +18,9 @@ from flowfile_core.flowfile.flow_data_engine.subprocess_operations.models import
     PolarsOperation,
     Status
 )
-from flowfile_core.flowfile.sources.external_sources.airbyte_sources.models import AirbyteSettings
 from flowfile_core.flowfile.sources.external_sources.sql_source.models import (DatabaseExternalReadSettings,
                                                                                DatabaseExternalWriteSettings)
+from flowfile_core.schemas.cloud_storage_schemas import CloudStorageWriteSettingsWorkerInterface
 from flowfile_core.schemas.input_schema import (
     ReceivedCsvTable,
     ReceivedExcelTable,
@@ -81,13 +81,6 @@ def trigger_create_operation(flow_id: int, node_id: int | str, received_table: R
     return Status(**f.json())
 
 
-def trigger_airbyte_collector(airbyte_settings: AirbyteSettings):
-    f = requests.post(url=f'{WORKER_URL}/store_airbyte_result', data=airbyte_settings.model_dump_json())
-    if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
-    return Status(**f.json())
-
-
 def trigger_database_read_collector(database_external_read_settings: DatabaseExternalReadSettings):
     f = requests.post(url=f'{WORKER_URL}/store_database_read_result',
                       data=database_external_read_settings.model_dump_json())
@@ -104,6 +97,14 @@ def trigger_database_write(database_external_write_settings: DatabaseExternalWri
     return Status(**f.json())
 
 
+def trigger_cloud_storage_write(database_external_write_settings: CloudStorageWriteSettingsWorkerInterface):
+    f = requests.post(url=f'{WORKER_URL}/write_data_to_cloud',
+                      data=database_external_write_settings.model_dump_json())
+    if not f.ok:
+        raise Exception(f'Could not cache the data, {f.text}')
+    return Status(**f.json())
+
+
 def get_results(file_ref: str) -> Status | None:
     f = requests.get(f'{WORKER_URL}/status/{file_ref}')
     if f.status_code == 200:
@@ -113,11 +114,15 @@ def get_results(file_ref: str) -> Status | None:
 
 
 def results_exists(file_ref: str):
-    f = requests.get(f'{WORKER_URL}/status/{file_ref}')
-    if f.status_code == 200:
-        if f.json()['status'] == 'Completed':
-            return True
-    return False
+    try:
+        f = requests.get(f'{WORKER_URL}/status/{file_ref}')
+        if f.status_code == 200:
+            if f.json()['status'] == 'Completed':
+                return True
+        return False
+    except requests.RequestException as e:
+        logger.error(f"Failed to check results existence: {str(e)}")
+        return False
 
 
 def get_df_result(encoded_df: str) -> pl.LazyFrame:
@@ -336,15 +341,6 @@ class ExternalCreateFetcher(BaseFetcher):
             _ = self.get_result()
 
 
-class ExternalAirbyteFetcher(BaseFetcher):
-    def __init__(self, airbyte_settings: AirbyteSettings, wait_on_completion: bool = True):
-        r = trigger_airbyte_collector(airbyte_settings)
-        super().__init__(file_ref=r.background_task_id)
-        self.running = r.status == 'Processing'
-        if wait_on_completion:
-            _ = self.get_result()
-
-
 class ExternalDatabaseFetcher(BaseFetcher):
     def __init__(self, database_external_read_settings: DatabaseExternalReadSettings,
                  wait_on_completion: bool = True):
@@ -365,6 +361,17 @@ class ExternalDatabaseWriter(BaseFetcher):
             _ = self.get_result()
 
 
+class ExternalCloudWriter(BaseFetcher):
+
+    def __init__(self, cloud_storage_write_settings: CloudStorageWriteSettingsWorkerInterface,
+                 wait_on_completion: bool = True):
+        r = trigger_cloud_storage_write(database_external_write_settings=cloud_storage_write_settings)
+        super().__init__(file_ref=r.background_task_id)
+        self.running = r.status == 'Processing'
+        if wait_on_completion:
+            _ = self.get_result()
+
+
 class ExternalExecutorTracker:
     result: Optional[pl.LazyFrame]
     started: bool = False

flowfile_core/flowfile/flow_data_engine/utils.py

@@ -3,30 +3,13 @@ from flowfile_core.configs.settings import AVAILABLE_RAM, WORKER_URL
 from flowfile_core.configs import logger
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations import ExternalDfFetcher
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations import Status
+from flowfile_core.utils.utils import standardize_col_dtype
 import os
 from typing import List, Dict, Iterable, Callable, Any
-from itertools import chain
 import requests
 from base64 import encodebytes
 
 
-def convert_to_string(v):
-    try:
-        return str(v)
-    except:
-        return None
-
-
-def standardize_col_dtype(vals):
-    types = set(type(val) for val in vals)
-    if len(types) == 1:
-        return vals
-    elif int in types and float in types:
-        return vals
-    else:
-        return [convert_to_string(v) for v in vals]
-
-
 def get_data_type(vals: Iterable[Any]):
     types = set(type(val) for val in vals)
     if len(types) == 1:
@@ -37,28 +20,6 @@ def get_data_type(vals: Iterable[Any]):
         return 'str'
 
 
-def ensure_similarity_dicts(datas: List[Dict], respect_order: bool = True):
-    all_cols = (data.keys() for data in datas)
-    if not respect_order:
-        unique_cols = set(chain(*all_cols))
-    else:
-        col_store = set()
-        unique_cols = list()
-        for row in all_cols:
-            for col in row:
-                if col not in col_store:
-                    unique_cols.append(col)
-                    col_store.update((col,))
-    output = []
-    for data in datas:
-        new_record = dict()
-        for col in unique_cols:
-            val = data.get(col)
-            new_record[col] = val
-        output.append(new_record)
-    return output
-
-
 def calculate_schema(lf: pl.LazyFrame) -> List[Dict]:
     r = ExternalDfFetcher(lf=lf, operation_type='calculate_schema', wait_on_completion=False, flow_id=-1, node_id=-1)
     schema_stats: List[Dict] = r.get_result()