Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

flowfile_core/schemas/transform_schema.py

@@ -4,8 +4,11 @@ import polars as pl
 from polars import selectors
 from copy import deepcopy
 
+from typing import NamedTuple
+
 
 def get_func_type_mapping(func: str):
+    """Infers the output data type of common aggregation functions."""
     if func in ["mean", "avg", "median", "std", "var"]:
         return "Float64"
     elif func in ['min', 'max', 'first', 'last', "cumsum", "sum"]:
@@ -17,16 +20,45 @@ def get_func_type_mapping(func: str):
 
 
 def string_concat(*column: str):
+    """A simple wrapper to concatenate string columns in Polars."""
     return pl.col(column).cast(pl.Utf8).str.concat(delimiter=',')
 
 
-JoinStrategy = Literal['inner', 'left', 'right', 'full', 'semi', 'anti', 'cross']
+SideLit = Literal["left", "right"]
+JoinStrategy = Literal['inner', 'left', 'right', 'full', 'semi', 'anti', 'cross', 'outer']
 FuzzyTypeLiteral = Literal['levenshtein', 'jaro', 'jaro_winkler', 'hamming', 'damerau_levenshtein', 'indel']
 
 
+def construct_join_key_name(side: SideLit, column_name: str) -> str:
+    """Creates a temporary, unique name for a join key column."""
+    return "_FLOWFILE_JOIN_KEY_" + side.upper() + "_" + column_name
+
+
+class JoinKeyRename(NamedTuple):
+    """Represents the renaming of a join key from its original to a temporary name."""
+    original_name: str
+    temp_name: str
+
+
+class JoinKeyRenameResponse(NamedTuple):
+    """Contains a list of join key renames for one side of a join."""
+    side: SideLit
+    join_key_renames: List[JoinKeyRename]
+
+
+class FullJoinKeyResponse(NamedTuple):
+    """Holds the join key rename responses for both sides of a join."""
+    left: JoinKeyRenameResponse
+    right: JoinKeyRenameResponse
+
+
 @dataclass
 class SelectInput:
-    # __slots__ = ['old_name', 'new_name', 'keep', 'data_type', 'data_type_change', 'join_key']
+    """Defines how a single column should be selected, renamed, or type-cast.
+
+    This is a core building block for any operation that involves column manipulation.
+    It holds all the configuration for a single field in a selection operation.
+    """
     old_name: str
     original_position: Optional[int] = None
     new_name: Optional[str] = None
@@ -58,6 +90,7 @@ class SelectInput:
 
     @property
     def polars_type(self) -> str:
+        """Translates a user-friendly type name to a Polars data type string."""
         if self.data_type.lower() == 'string':
             return 'Utf8'
         elif self.data_type.lower() == 'integer':
@@ -69,7 +102,7 @@ class SelectInput:
 
 @dataclass
 class FieldInput:
-    # __slots__ = ['old_name', 'new_name', 'keep', 'data_type', 'data_type_change', 'join_key']
+    """Represents a single field with its name and data type, typically for defining an output column."""
     name: str
     data_type: Optional[str] = None
 
@@ -80,19 +113,22 @@ class FieldInput:
 
 @dataclass
 class FunctionInput:
+    """Defines a formula to be applied, including the output field information."""
     field: FieldInput
     function: str
 
 
 @dataclass
 class BasicFilter:
+    """Defines a simple, single-condition filter (e.g., 'column' 'equals' 'value')."""
     field: str = ''
-    filter_type: str = ''  # equals, in, not in, smaller, larger
+    filter_type: str = ''
     filter_value: str = ''
 
 
 @dataclass
 class FilterInput:
+    """Defines the settings for a filter operation, supporting basic or advanced (expression-based) modes."""
     advanced_filter: str = ''
     basic_filter: BasicFilter = None
     filter_type: str = 'basic'
@@ -100,49 +136,98 @@ class FilterInput:
 
 @dataclass
 class SelectInputs:
+    """A container for a list of `SelectInput` objects, providing helper methods for managing selections."""
     renames: List[SelectInput]
 
     @property
     def old_cols(self) -> Set:
+        """Returns a set of original column names to be kept in the selection."""
         return set(v.old_name for v in self.renames if v.keep)
 
     @property
     def new_cols(self) -> Set:
-        return set(v.new_name for v in self.renames if v.keep or v.join_key)
+        """Returns a set of new (renamed) column names to be kept in the selection."""
+        return set(v.new_name for v in self.renames if v.keep)
 
     @property
     def rename_table(self):
-        return {v.old_name: v.new_name for v in self.renames if (v.keep or v.join_key) and v.is_available}
+        """Generates a dictionary for use in Polars' `.rename()` method."""
+        return {v.old_name: v.new_name for v in self.renames if v.is_available and (v.keep or v.join_key)}
 
     def get_select_cols(self, include_join_key: bool = True):
+        """Gets a list of original column names to select from the source DataFrame."""
         return [v.old_name for v in self.renames if v.keep or (v.join_key and include_join_key)]
 
-    def __add__(self, other: SelectInput):
+    def __add__(self, other: "SelectInput"):
+        """Allows adding a SelectInput using the '+' operator."""
         self.renames.append(other)
 
-    def append(self, other: SelectInput):
+    def append(self, other: "SelectInput"):
+        """Appends a new SelectInput to the list of renames."""
         self.renames.append(other)
 
     def remove_select_input(self, old_key: str):
+        """Removes a SelectInput from the list based on its original name."""
         self.renames = [rename for rename in self.renames if rename.old_name != old_key]
 
+    def unselect_field(self, old_key: str):
+        """Marks a field to be dropped from the final selection by setting `keep` to False."""
+        for rename in self.renames:
+            if old_key == rename.old_name:
+                rename.keep = False
+
     @classmethod
-    def create_from_list(cls, col_list: str):
+    def create_from_list(cls, col_list: List[str]):
+        """Creates a SelectInputs object from a simple list of column names."""
         return cls([SelectInput(c) for c in col_list])
 
     @classmethod
     def create_from_pl_df(cls, df: pl.DataFrame | pl.LazyFrame):
+        """Creates a SelectInputs object from a Polars DataFrame's columns."""
         return cls([SelectInput(c) for c in df.columns])
 
+    def get_select_input_on_old_name(self, old_name: str) -> SelectInput | None:
+        return next((v for v in self.renames if v.old_name == old_name), None)
+
+    def get_select_input_on_new_name(self, old_name: str) -> SelectInput | None:
+        return next((v for v in self.renames if v.new_name == old_name), None)
+
+
+class JoinInputs(SelectInputs):
+    """Extends `SelectInputs` with functionality specific to join operations, like handling join keys."""
+
+    def __init__(self, renames: List[SelectInput]):
+        self.renames = renames
+
+    @property
+    def join_key_selects(self) -> List[SelectInput]:
+        """Returns only the `SelectInput` objects that are marked as join keys."""
+        return [v for v in self.renames if v.join_key]
+
+    def get_join_key_renames(self, side: SideLit, filter_drop: bool = False) -> JoinKeyRenameResponse:
+        """Gets the temporary rename mapping for all join keys on one side of a join."""
+        return JoinKeyRenameResponse(
+            side,
+            [JoinKeyRename(jk.new_name,
+                           construct_join_key_name(side, jk.new_name))
+             for jk in self.join_key_selects if jk.keep or not filter_drop]
+        )
+
+    def get_join_key_rename_mapping(self, side: SideLit) -> Dict[str, str]:
+        """Returns a dictionary mapping original join key names to their temporary names."""
+        return {jkr[0]: jkr[1] for jkr in self.get_join_key_renames(side)[1]}
+
 
 @dataclass
 class JoinMap:
+    """Defines a single mapping between a left and right column for a join key."""
     left_col: str
     right_col: str
 
 
 @dataclass
 class FuzzyMap(JoinMap):
+    """Extends `JoinMap` with settings for fuzzy string matching, such as the algorithm and similarity threshold."""
     threshold_score: Optional[float] = 80.0
     fuzzy_type: Optional[FuzzyTypeLiteral] = 'levenshtein'
     perc_unique: Optional[float] = 0.0
@@ -167,22 +252,26 @@ class FuzzyMap(JoinMap):
 
 
 class JoinSelectMixin:
-    """Mixin for common join selection functionality"""
+    """A mixin providing common methods for join-like operations that involve left and right inputs."""
+    left_select: JoinInputs = None
+    right_select: JoinInputs = None
 
     @staticmethod
-    def parse_select(select: List[SelectInput] | List[str] | List[Dict]) -> SelectInputs:
+    def parse_select(select: List[SelectInput] | List[str] | List[Dict]) -> JoinInputs | None:
+        """Parses various input formats into a standardized `JoinInputs` object."""
         if all(isinstance(c, SelectInput) for c in select):
-            return SelectInputs(select)
+            return JoinInputs(select)
         elif all(isinstance(c, dict) for c in select):
-            return SelectInputs([SelectInput(**c) for c in select])
+            return JoinInputs([SelectInput(**c.__dict__) for c in select])
         elif isinstance(select, dict):
             renames = select.get('renames')
             if renames:
-                return SelectInputs([SelectInput(**c) for c in renames])
+                return JoinInputs([SelectInput(**c) for c in renames])
         elif all(isinstance(c, str) for c in select):
-            return SelectInputs([SelectInput(s, s) for s in select])
+            return JoinInputs([SelectInput(s, s) for s in select])
 
     def auto_generate_new_col_name(self, old_col_name: str, side: str) -> str:
+        """Generates a new, non-conflicting column name by adding a suffix if necessary."""
         current_names = self.left_select.new_cols & self.right_select.new_cols
         if old_col_name not in current_names:
             return old_col_name
@@ -192,6 +281,7 @@ class JoinSelectMixin:
             old_col_name = f'{side}_{old_col_name}'
 
     def add_new_select_column(self, select_input: SelectInput, side: str):
+        """Adds a new column to the selection for either the left or right side."""
         selects = self.right_select if side == 'right' else self.left_select
         select_input.new_name = self.auto_generate_new_col_name(select_input.old_name, side=side)
         selects.__add__(select_input)
@@ -199,19 +289,23 @@ class JoinSelectMixin:
 
 @dataclass
 class CrossJoinInput(JoinSelectMixin):
+    """Defines the settings for a cross join operation, including column selections for both inputs."""
     left_select: SelectInputs = None
     right_select: SelectInputs = None
 
     def __init__(self, left_select: List[SelectInput] | List[str],
                  right_select: List[SelectInput] | List[str]):
+        """Initializes the CrossJoinInput with selections for left and right tables."""
         self.left_select = self.parse_select(left_select)
         self.right_select = self.parse_select(right_select)
 
     @property
     def overlapping_records(self):
+        """Finds column names that would conflict after the join."""
         return self.left_select.new_cols & self.right_select.new_cols
 
     def auto_rename(self):
+        """Automatically renames columns on the right side to prevent naming conflicts."""
         overlapping_records = self.overlapping_records
         while len(overlapping_records) > 0:
             for right_col in self.right_select.renames:
@@ -222,13 +316,15 @@ class CrossJoinInput(JoinSelectMixin):
 
 @dataclass
 class JoinInput(JoinSelectMixin):
+    """Defines the settings for a standard SQL-style join, including keys, strategy, and selections."""
     join_mapping: List[JoinMap]
-    left_select: SelectInputs = None
-    right_select: SelectInputs = None
+    left_select: JoinInputs = None
+    right_select: JoinInputs = None
     how: JoinStrategy = 'inner'
 
     @staticmethod
-    def parse_join_mapping(join_mapping: List[JoinMap] | Tuple[str, str] | str) -> List[JoinMap]:
+    def parse_join_mapping(join_mapping: any) -> List[JoinMap]:
+        """Parses various input formats for join keys into a standardized list of `JoinMap` objects."""
         if isinstance(join_mapping, (tuple, list)):
             assert len(join_mapping) > 0
             if all(isinstance(jm, dict) for jm in join_mapping):
@@ -251,39 +347,63 @@ class JoinInput(JoinSelectMixin):
                  left_select: List[SelectInput] | List[str],
                  right_select: List[SelectInput] | List[str],
                  how: JoinStrategy = 'inner'):
+        """Initializes the JoinInput with keys, selections, and join strategy."""
         self.join_mapping = self.parse_join_mapping(join_mapping)
         self.left_select = self.parse_select(left_select)
         self.right_select = self.parse_select(right_select)
+        self.set_join_keys()
+        self.how = how
+
+    def set_join_keys(self):
+        """Marks the `SelectInput` objects corresponding to join keys."""
         [setattr(v, "join_key", v.old_name in self._left_join_keys) for v in self.left_select.renames]
         [setattr(v, "join_key", v.old_name in self._right_join_keys) for v in self.right_select.renames]
-        self.how = how
+
+    def get_join_key_renames(self, filter_drop: bool = False) -> FullJoinKeyResponse:
+        """Gets the temporary rename mappings for the join keys on both sides."""
+        return FullJoinKeyResponse(self.left_select.get_join_key_renames(side="left", filter_drop=filter_drop),
+                                   self.right_select.get_join_key_renames(side="right", filter_drop=filter_drop))
+
+    def get_names_for_table_rename(self) -> List[JoinMap]:
+        new_mappings: List[JoinMap] = []
+        left_rename_table, right_rename_table = self.left_select.rename_table, self.right_select.rename_table
+        for join_map in self.join_mapping:
+            new_mappings.append(JoinMap(left_rename_table.get(join_map.left_col, join_map.left_col),
+                                        right_rename_table.get(join_map.right_col, join_map.right_col)
+                                        )
+                                )
+        return new_mappings
 
     @property
     def _left_join_keys(self) -> Set:
+        """Returns a set of the left-side join key column names."""
         return set(jm.left_col for jm in self.join_mapping)
 
     @property
     def _right_join_keys(self) -> Set:
+        """Returns a set of the right-side join key column names."""
         return set(jm.right_col for jm in self.join_mapping)
 
     @property
-    def left_join_keys(self) -> List:
-        return [self.left_select.rename_table.get(jm.left_col) for jm in self.join_mapping]
+    def left_join_keys(self) -> List[str]:
+        """Returns an ordered list of the left-side join key column names to be used in the join."""
+        return [jm.left_col for jm in self.used_join_mapping]
 
     @property
-    def right_join_keys(self) -> List:
-        return [self.right_select.rename_table.get(jm.right_col, jm.right_col) for jm in self.join_mapping]
+    def right_join_keys(self) -> List[str]:
+        """Returns an ordered list of the right-side join key column names to be used in the join."""
+        return [jm.right_col for jm in self.used_join_mapping]
 
     @property
     def overlapping_records(self):
         if self.how in ('left', 'right', 'inner'):
-            #  Never consider join keys as overlapping records since they will be dropped after the join
-            return ((self.left_select.new_cols & self.right_select.new_cols) -
-                    (set(self.left_join_keys) & set(self.right_join_keys)))
+            return self.left_select.new_cols & self.right_select.new_cols
         else:
             return self.left_select.new_cols & self.right_select.new_cols
 
     def auto_rename(self):
+        """Automatically renames columns on the right side to prevent naming conflicts."""
+        self.set_join_keys()
         overlapping_records = self.overlapping_records
         while len(overlapping_records) > 0:
             for right_col in self.right_select.renames:
@@ -292,13 +412,16 @@ class JoinInput(JoinSelectMixin):
             overlapping_records = self.overlapping_records
 
     @property
-    def join_mappings(self):
-        new_mappings = []
+    def used_join_mapping(self) -> List[JoinMap]:
+        """Returns the final join mapping after applying all renames and transformations."""
+        new_mappings: List[JoinMap] = []
         left_rename_table, right_rename_table = self.left_select.rename_table, self.right_select.rename_table
+        left_join_rename_mapping: Dict[str, str] = self.left_select.get_join_key_rename_mapping("left")
+        right_join_rename_mapping: Dict[str, str] = self.right_select.get_join_key_rename_mapping("right")
         for join_map in self.join_mapping:
             # del self.right_select.rename_table, self.left_select.rename_table
-            new_mappings.append(JoinMap(left_rename_table.get(join_map.left_col),
-                                        right_rename_table.get(join_map.right_col)
+            new_mappings.append(JoinMap(left_join_rename_mapping.get(left_rename_table.get(join_map.left_col, join_map.left_col)),
+                                        right_join_rename_mapping.get(right_rename_table.get(join_map.right_col, join_map.right_col))
                                         )
                                 )
         return new_mappings
@@ -306,6 +429,7 @@ class JoinInput(JoinSelectMixin):
 
 @dataclass
 class FuzzyMatchInput(JoinInput):
+    """Extends `JoinInput` with settings specific to fuzzy matching, such as the matching algorithm and threshold."""
     join_mapping: List[FuzzyMap]
     aggregate_output: bool = False
 
@@ -332,7 +456,7 @@ class FuzzyMatchInput(JoinInput):
         return fuzz_mapping
 
     def __init__(self, join_mapping: List[FuzzyMap] | Tuple[str, str] | str, left_select: List[SelectInput] | List[str],
-                 right_select: List[SelectInput] | List[str], aggregate_output: bool = False, how: str = 'inner'):
+                 right_select: List[SelectInput] | List[str], aggregate_output: bool = False, how: JoinStrategy = 'inner'):
         self.join_mapping = self.parse_fuzz_mapping(join_mapping)
         self.left_select = self.parse_select(left_select)
         self.right_select = self.parse_select(right_select)
@@ -353,6 +477,7 @@ class FuzzyMatchInput(JoinInput):
 
     @property
     def fuzzy_maps(self) -> List[FuzzyMap]:
+        """Returns the final fuzzy mappings after applying all column renames."""
         new_mappings = []
         left_rename_table, right_rename_table = self.left_select.rename_table, self.right_select.rename_table
         for org_fuzzy_map in self.join_mapping:
@@ -404,6 +529,7 @@ class AggColl:
     output_type: Optional[str] = None
 
     def __init__(self, old_name: str, agg: str, new_name: str = None, output_type: str = None):
+        """Initializes an aggregation column with its source, function, and new name."""
         self.old_name = str(old_name)
         if agg != 'groupby':
             self.new_name = new_name if new_name is not None else self.old_name + "_" + agg
@@ -414,6 +540,7 @@ class AggColl:
 
     @property
     def agg_func(self):
+        """Returns the corresponding Polars aggregation function from the `agg` string."""
         if self.agg == 'groupby':
             return self.agg
         elif self.agg == 'concat':
@@ -448,6 +575,7 @@ class GroupByInput:
 
 @dataclass
 class PivotInput:
+    """Defines the settings for a pivot (long-to-wide) operation."""
     index_columns: List[str]
     pivot_column: str
     value_col: str
@@ -455,9 +583,11 @@ class PivotInput:
 
     @property
     def grouped_columns(self) -> List[str]:
+        """Returns the list of columns to be used for the initial grouping stage of the pivot."""
         return self.index_columns + [self.pivot_column]
 
     def get_group_by_input(self) -> GroupByInput:
+        """Constructs the `GroupByInput` needed for the pre-aggregation step of the pivot."""
         group_by_cols = [AggColl(c, 'groupby') for c in self.grouped_columns]
         agg_cols = [AggColl(self.value_col, agg=aggregation, new_name=aggregation) for aggregation in self.aggregations]
         return GroupByInput(group_by_cols+agg_cols)
@@ -465,22 +595,25 @@ class PivotInput:
     def get_index_columns(self) -> List[pl.col]:
         return [pl.col(c) for c in self.index_columns]
 
-    def get_pivot_column(self) -> pl.col:
+    def get_pivot_column(self) -> pl.Expr:
+        """Returns the pivot column as a Polars column expression."""
         return pl.col(self.pivot_column)
 
     def get_values_expr(self) -> pl.Expr:
+        """Creates the struct expression used to gather the values for pivoting."""
         return pl.struct([pl.col(c) for c in self.aggregations]).alias('vals')
 
 
-
 @dataclass
 class SortByInput:
+    """Defines a single sort condition on a column, including the direction."""
     column: str
     how: str = 'asc'
 
 
 @dataclass
 class RecordIdInput:
+    """Defines settings for adding a record ID (row number) column to the data."""
     output_column_name: str = 'record_id'
     offset: int = 1
     group_by: Optional[bool] = False
@@ -489,6 +622,7 @@ class RecordIdInput:
 
 @dataclass
 class TextToRowsInput:
+    """Defines settings for splitting a text column into multiple rows based on a delimiter."""
     column_to_split: str
     output_column_name: Optional[str] = None
     split_by_fixed_value: Optional[bool] = True
@@ -498,12 +632,14 @@ class TextToRowsInput:
 
 @dataclass
 class UnpivotInput:
+    """Defines settings for an unpivot (wide-to-long) operation."""
     index_columns: Optional[List[str]] = field(default_factory=list)
     value_columns: Optional[List[str]] = field(default_factory=list)
     data_type_selector: Optional[Literal['float', 'all', 'date', 'numeric', 'string']] = None
     data_type_selector_mode: Optional[Literal['data_type', 'column']] = 'column'
 
     def __post_init__(self):
+        """Ensures that list attributes are initialized correctly if they are None."""
         if self.index_columns is None:
             self.index_columns = []
         if self.value_columns is None:
@@ -512,7 +648,8 @@ class UnpivotInput:
             self.data_type_selector_mode = 'column'
 
     @property
-    def data_type_selector_expr(self) -> Callable:
+    def data_type_selector_expr(self) -> Optional[Callable]:
+        """Returns a Polars selector function based on the `data_type_selector` string."""
         if self.data_type_selector_mode == 'data_type':
             if self.data_type_selector is not None:
                 try:
@@ -525,17 +662,20 @@ class UnpivotInput:
 
 @dataclass
 class UnionInput:
+    """Defines settings for a union (concatenation) operation."""
     mode: Literal['selective', 'relaxed'] = 'relaxed'
 
 
 @dataclass
 class UniqueInput:
+    """Defines settings for a uniqueness operation, specifying columns and which row to keep."""
     columns: Optional[List[str]] = None
     strategy: Literal["first", "last", "any", "none"] = "any"
 
 
 @dataclass
 class GraphSolverInput:
+    """Defines settings for a graph-solving operation (e.g., finding connected components)."""
     col_from: str
     col_to: str
     output_column_name: Optional[str] = 'graph_group'
@@ -543,5 +683,5 @@ class GraphSolverInput:
 
 @dataclass
 class PolarsCodeInput:
+    """A simple container for a string of user-provided Polars code to be executed."""
     polars_code: str
-

flowfile_core/utils/utils.py

@@ -1,8 +1,47 @@
 import re
-
+from itertools import chain
+from typing import List, Dict
 
 def camel_case_to_snake_case(text: str) -> str:
     # Use a regular expression to find capital letters and replace them with _ followed by the lowercase letter
     transformed_text = re.sub(r'(?<!^)(?=[A-Z])', '_', text).lower()
     return transformed_text
 
+
+def ensure_similarity_dicts(datas: List[Dict], respect_order: bool = True):
+    all_cols = (data.keys() for data in datas)
+    if not respect_order:
+        unique_cols = set(chain(*all_cols))
+    else:
+        col_store = set()
+        unique_cols = list()
+        for row in all_cols:
+            for col in row:
+                if col not in col_store:
+                    unique_cols.append(col)
+                    col_store.update((col,))
+    output = []
+    for data in datas:
+        new_record = dict()
+        for col in unique_cols:
+            val = data.get(col)
+            new_record[col] = val
+        output.append(new_record)
+    return output
+
+
+def convert_to_string(v):
+    try:
+        return str(v)
+    except:
+        return None
+
+
+def standardize_col_dtype(vals):
+    types = set(type(val) for val in vals)
+    if len(types) == 1:
+        return vals
+    elif int in types and float in types:
+        return vals
+    else:
+        return [convert_to_string(v) for v in vals]

flowfile_core/utils/validate_setup.py

@@ -0,0 +1,41 @@
+"""This script runs on run time and checks if all the nodes that are created have a function in the flow_graph as well
+as have a component in flowfile_frontend"""
+
+from flowfile_core.schemas import input_schema
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.configs.node_store.nodes import nodes_list, NodeTemplate
+import inspect
+
+
+def check_if_node_has_add_function_in_flow_graph(node: NodeTemplate):
+    func_name = "add_" + node.item
+    if not hasattr(FlowGraph, func_name):
+        raise ValueError(
+            f"Node {node.name} ({node.item}) does not have a corresponding function in FlowGraph: {func_name}"
+            "Check if the function is implemented in flow_graph.py or if the node item is correct."
+        )
+
+
+def check_if_node_has_input_schema_definition(node: NodeTemplate):
+    if "node"+node.item.replace("_","") not in {k.lower() for k in inspect.getmodule(input_schema).__dict__.keys()}:
+        raise ValueError(
+            f"Node {node.name} ({node.item}) does not have a corresponding input schema definition in input_schema.py."
+            "Check if the schema is implemented or if the node item is correct."
+        )
+
+
+def validate_setup():
+    """
+    Validates the setup by checking if all nodes in the nodes_list have a corresponding function in FlowGraph
+    and a corresponding input schema definition in input_schema.py.
+    Raises ValueError if any node is missing either.
+    """
+    for node in nodes_list:
+        check_if_node_has_add_function_in_flow_graph(node)
+        check_if_node_has_input_schema_definition(node)
+
+    print("All nodes have corresponding functions in FlowGraph and input schema definitions.")
+
+
+if __name__ == "__main__":
+    validate_setup()

flowfile_frame/__init__.py

@@ -31,7 +31,15 @@ from flowfile_frame.series import Series
 
 # File I/O
 from flowfile_frame.flow_frame_methods import (  # noqa: F401
-    read_csv, read_parquet, from_dict, concat,  scan_csv, scan_parquet)
+    read_csv, read_parquet, from_dict, concat,  scan_csv, scan_parquet, scan_json_from_cloud_storage,
+    scan_parquet_from_cloud_storage,
+    scan_csv_from_cloud_storage,
+    scan_delta)
+
+from flowfile_frame.cloud_storage.secret_manager import (del_cloud_storage_connection,
+                                                         create_cloud_storage_connection,
+                                                         get_all_available_cloud_storage_connections,
+                                                         create_cloud_storage_connection_if_not_exists)
 
 from polars.datatypes import (  # noqa: F401
     # Integer types

flowfile_frame/cloud_storage/frame_helpers.py

@@ -0,0 +1,39 @@
+from typing import Optional, Literal
+
+from polars._typing import (CsvEncoding)
+
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.schemas import input_schema, cloud_storage_schemas
+from flowfile_frame.cloud_storage.secret_manager import get_current_user_id
+from flowfile_frame.utils import generate_node_id
+
+
+def add_write_ff_to_cloud_storage(path: str,
+                                  flow_graph: Optional[FlowGraph],
+                                  depends_on_node_id: int,
+                                  *,
+                                  connection_name: Optional[str] = None,
+                                  write_mode: Literal["overwrite", "append"] = "overwrite",
+                                  file_format: Literal["csv", "parquet", "json", "delta"] = "parquet",
+                                  csv_delimiter: str = ";",
+                                  csv_encoding: CsvEncoding = "utf8",
+                                  parquet_compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy",
+                                  description: Optional[str] = None
+                              ) -> int:
+    node_id = generate_node_id()
+    flow_id = flow_graph.flow_id
+    settings = input_schema.NodeCloudStorageWriter(
+        flow_id=flow_id,
+        node_id=node_id,
+        cloud_storage_settings=cloud_storage_schemas.CloudStorageWriteSettings(resource_path=path,
+                                                                               connection_name=connection_name,
+                                                                               file_format=file_format,
+                                                                               write_mode=write_mode,
+                                                                               csv_delimiter=csv_delimiter,
+                                                                               csv_encoding=csv_encoding,
+                                                                               parquet_compression=parquet_compression),
+        user_id=get_current_user_id(),
+        depending_on_id=depends_on_node_id,
+        description=description)
+    flow_graph.add_cloud_storage_writer(settings)
+    return node_id