Flowfile 0.3.5__py3-none-any.whl → 0.3.6__py3-none-any.whl

flowfile_core/schemas/cloud_storage_schemas.py

@@ -0,0 +1,215 @@
+"""Cloud storage connection schemas for S3, ADLS, and other cloud providers."""
+
+from typing import Optional, Literal
+import polars as pl
+import base64
+
+from pydantic import BaseModel, SecretStr, field_validator, Field
+
+from flowfile_core.secret_manager.secret_manager import encrypt_secret
+
+CloudStorageType = Literal["s3", "adls", "gcs"]
+AuthMethod = Literal["access_key", "iam_role", "service_principal", "managed_identity", "sas_token", "aws-cli", "env_vars"]
+
+
+def encrypt_for_worker(secret_value: SecretStr|None) -> str|None:
+    """
+    Encrypts a secret value for use in worker contexts.
+    This is a placeholder function that simulates encryption.
+    In practice, you would use a secure encryption method.
+    """
+    if secret_value is not None:
+        return encrypt_secret(secret_value.get_secret_value())
+
+
+class AuthSettingsInput(BaseModel):
+    """
+    The information needed for the user to provide the details that are needed to provide how to connect to the
+     Cloud provider
+    """
+    storage_type: CloudStorageType
+    auth_method: AuthMethod
+    connection_name: Optional[str] = "None"  # This is the reference to the item we will fetch that contains the data
+
+
+class FullCloudStorageConnectionWorkerInterface(AuthSettingsInput):
+    """Internal model with decrypted secrets"""
+
+    # AWS S3
+    aws_region: Optional[str] = None
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    aws_role_arn: Optional[str] = None
+    aws_allow_unsafe_html: Optional[bool] = None
+    aws_session_token: Optional[str] = None
+
+    # Azure ADLS
+    azure_account_name: Optional[str] = None
+    azure_account_key: Optional[str] = None
+    azure_tenant_id: Optional[str] = None
+    azure_client_id: Optional[str] = None
+    azure_client_secret: Optional[str] = None
+
+    # Common
+    endpoint_url: Optional[str] = None
+    verify_ssl: bool = True
+
+
+class FullCloudStorageConnection(AuthSettingsInput):
+    """Internal model with decrypted secrets"""
+
+    # AWS S3
+    aws_region: Optional[str] = None
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[SecretStr] = None
+    aws_role_arn: Optional[str] = None
+    aws_allow_unsafe_html: Optional[bool] = None
+    aws_session_token: Optional[SecretStr] = None
+
+    # Azure ADLS
+    azure_account_name: Optional[str] = None
+    azure_account_key: Optional[SecretStr] = None
+    azure_tenant_id: Optional[str] = None
+    azure_client_id: Optional[str] = None
+    azure_client_secret: Optional[SecretStr] = None
+
+    # Common
+    endpoint_url: Optional[str] = None
+    verify_ssl: bool = True
+
+    def get_worker_interface(self) -> "FullCloudStorageConnectionWorkerInterface":
+        """
+        Convert to a public interface model without secrets.
+        """
+        return FullCloudStorageConnectionWorkerInterface(
+            storage_type=self.storage_type,
+            auth_method=self.auth_method,
+            connection_name=self.connection_name,
+            aws_allow_unsafe_html=self.aws_allow_unsafe_html,
+            aws_secret_access_key=encrypt_for_worker(self.aws_secret_access_key),
+            aws_region=self.aws_region,
+            aws_access_key_id=self.aws_access_key_id,
+            aws_role_arn=self.aws_role_arn,
+            aws_session_token=encrypt_for_worker(self.aws_session_token),
+            azure_account_name=self.azure_account_name,
+            azure_tenant_id=self.azure_tenant_id,
+            azure_account_key=encrypt_for_worker(self.azure_account_key),
+            azure_client_id=self.azure_client_id,
+            azure_client_secret=encrypt_for_worker(self.azure_client_secret),
+            endpoint_url=self.endpoint_url,
+            verify_ssl=self.verify_ssl
+        )
+
+
+class FullCloudStorageConnectionInterface(AuthSettingsInput):
+    """API response model - no secrets exposed"""
+
+    # Public fields only
+    aws_allow_unsafe_html: Optional[bool] = None
+    aws_region: Optional[str] = None
+    aws_access_key_id: Optional[str] = None
+    aws_role_arn: Optional[str] = None
+    azure_account_name: Optional[str] = None
+    azure_tenant_id: Optional[str] = None
+    azure_client_id: Optional[str] = None
+    endpoint_url: Optional[str] = None
+    verify_ssl: bool = True
+
+
+class CloudStorageSettings(BaseModel):
+    """Settings for cloud storage nodes in the visual designer"""
+
+    auth_mode: AuthMethod = "auto"
+    connection_name: Optional[str] = None  # Required only for 'reference' mode
+    resource_path: str  # s3://bucket/path/to/file.csv
+
+    @field_validator("auth_mode", mode="after")
+    def validate_auth_requirements(cls, v, values):
+        data = values.data
+        if v == "reference" and not data.get("connection_name"):
+            raise ValueError("connection_name required when using reference mode")
+        return v
+
+
+class CloudStorageReadSettings(CloudStorageSettings):
+    """Settings for reading from cloud storage"""
+
+    scan_mode: Literal["single_file", "directory"] = "single_file"
+    file_format: Literal["csv", "parquet", "json", "delta", "iceberg"] = "parquet"
+    # CSV specific options
+    csv_has_header: Optional[bool] = True
+    csv_delimiter: Optional[str] = ","
+    csv_encoding: Optional[str] = "utf8"
+    # Deltalake specific settings
+    delta_version: Optional[int] = None
+
+
+class CloudStorageReadSettingsInternal(BaseModel):
+    read_settings: CloudStorageReadSettings
+    connection: FullCloudStorageConnection
+
+
+class WriteSettingsWorkerInterface(BaseModel):
+    """Settings for writing to cloud storage"""
+    resource_path: str  # s3://bucket/path/to/file.csv
+
+    write_mode: Literal["overwrite", "append"] = "overwrite"
+    file_format: Literal["csv", "parquet", "json", "delta"] = "parquet"
+
+    parquet_compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy"
+
+    csv_delimiter: str = ","
+    csv_encoding: str = "utf8"
+
+
+class CloudStorageWriteSettings(CloudStorageSettings, WriteSettingsWorkerInterface):
+    """Settings for writing to cloud storage"""
+    pass
+
+    def get_write_setting_worker_interface(self) -> WriteSettingsWorkerInterface:
+        """
+        Convert to a worker interface model without secrets.
+        """
+        return WriteSettingsWorkerInterface(
+            resource_path=self.resource_path,
+            write_mode=self.write_mode,
+            file_format=self.file_format,
+            parquet_compression=self.parquet_compression,
+            csv_delimiter=self.csv_delimiter,
+            csv_encoding=self.csv_encoding
+        )
+
+
+class CloudStorageWriteSettingsInternal(BaseModel):
+    write_settings: CloudStorageWriteSettings
+    connection: FullCloudStorageConnection
+
+
+class CloudStorageWriteSettingsWorkerInterface(BaseModel):
+    """Settings for writing to cloud storage in worker context"""
+    operation: str
+    write_settings: WriteSettingsWorkerInterface
+    connection: FullCloudStorageConnectionWorkerInterface
+    flowfile_flow_id: int = 1
+    flowfile_node_id: int | str = -1
+
+
+def get_cloud_storage_write_settings_worker_interface(
+        write_settings: CloudStorageWriteSettings,
+        connection: FullCloudStorageConnection,
+        lf: pl.LazyFrame,
+        flowfile_flow_id: int = 1,
+        flowfile_node_id: int | str = -1,
+        ) -> CloudStorageWriteSettingsWorkerInterface:
+    """
+    Convert to a worker interface model with hashed secrets.
+    """
+    operation = base64.b64encode(lf.serialize()).decode()
+
+    return CloudStorageWriteSettingsWorkerInterface(
+        operation=operation,
+        write_settings=write_settings.get_write_setting_worker_interface(),
+        connection=connection.get_worker_interface(),
+        flowfile_flow_id=flowfile_flow_id,  # Default value, can be overridden
+        flowfile_node_id=flowfile_node_id  # Default value, can be overridden
+    )

flowfile_core/schemas/input_schema.py

@@ -3,8 +3,11 @@ from flowfile_core.schemas import transform_schema
 from pathlib import Path
 import os
 from flowfile_core.schemas.analysis_schemas import graphic_walker_schemas as gs_schemas
-from flowfile_core.schemas.external_sources.airbyte_schemas import AirbyteConfig
+from flowfile_core.schemas.cloud_storage_schemas import CloudStorageReadSettings, CloudStorageWriteSettings
+from flowfile_core.schemas.schemas import SecretRef
+from flowfile_core.utils.utils import ensure_similarity_dicts, standardize_col_dtype
 from pydantic import BaseModel, Field, model_validator, SecretStr, ConfigDict
+import polars as pl
 
 
 OutputConnectionClass = Literal['output-0', 'output-1', 'output-2', 'output-3', 'output-4',
@@ -33,7 +36,7 @@ class RemoveItemsInput(BaseModel):
 
 class MinimalFieldInfo(BaseModel):
     name: str
-    data_type: str
+    data_type: str = "String"
 
 
 class ReceivedTableBase(BaseModel):
@@ -250,11 +253,29 @@ class NodeDatasource(NodeBase):
 
 class RawData(BaseModel):
     columns: List[MinimalFieldInfo] = None
-    data: List[List]  # List of list where each inner list is a column of data. This ensures more efficient storage
+    data: List[List]
+
+    @classmethod
+    def from_columns(cls, columns: List[str], data: List[List]):
+        return cls(columns=[MinimalFieldInfo(name=column) for column in columns], data=data)
+
+    @classmethod
+    def from_pylist(cls, pylist: List[dict]):
+        if len(pylist) == 0:
+            return cls(columns=[], data=[])
+        pylist = ensure_similarity_dicts(pylist)
+        values = [standardize_col_dtype([vv for vv in c]) for c in
+                  zip(*(r.values() for r in pylist))]
+
+        data_types = (pl.DataType.from_python(type(next((v for v in column_values), None))) for column_values in values)
+        columns = [MinimalFieldInfo(name=c, data_type=str(next(data_types))) for c in pylist[0].keys()]
+        return cls(columns=columns, data=values)
+
+    def to_pylist(self):
+        return [{c.name: self.data[ci][ri] for ci, c in enumerate(self.columns)} for ri in range(len(self.data[0]))]
 
 
 class NodeManualInput(NodeBase):
-    raw_data: Optional[List] = None
     raw_data_format: Optional[RawData] = None
 
 
@@ -265,7 +286,7 @@ class NodeRead(NodeBase):
 class DatabaseConnection(BaseModel):
     database_type: str = "postgresql"  # Database type (postgresql, mysql, etc.)
     username: Optional[str] = None
-    password_ref: Optional[str] = None
+    password_ref: Optional[SecretRef] = None
     host: Optional[str] = None
     port: Optional[int] = None
     database: Optional[str] = None
@@ -338,6 +359,17 @@ class NodeDatabaseWriter(NodeSingleInput):
     database_write_settings: DatabaseWriteSettings
 
 
+class NodeCloudStorageReader(NodeBase):
+    """Cloud storage source node"""
+    cloud_storage_settings: CloudStorageReadSettings
+    fields: Optional[List[MinimalFieldInfo]] = None
+
+
+class NodeCloudStorageWriter(NodeSingleInput):
+    """Cloud storage destination node"""
+    cloud_storage_settings: CloudStorageWriteSettings
+
+
 class ExternalSource(BaseModel):
     orientation: str = 'row'
     fields: Optional[List[MinimalFieldInfo]] = None
@@ -349,11 +381,6 @@ class SampleUsers(ExternalSource):
     size: int = 100
 
 
-class AirbyteReader(AirbyteConfig):
-    class_name: Optional[str] = "airbyte_reader"
-    fields: Optional[List[MinimalFieldInfo]] = None
-
-
 class AccessToken(BaseModel):
     user_id: str
     access_token: SecretStr = None
@@ -364,11 +391,6 @@ class NodeExternalSource(NodeBase):
     source_settings: SampleUsers
 
 
-class NodeAirbyteReader(NodeExternalSource):
-    identifier: str = 'airbyte'
-    source_settings: AirbyteReader
-
-
 class NodeFormula(NodeSingleInput):
     function: transform_schema.FunctionInput = None
 

flowfile_core/schemas/schemas.py

@@ -1,6 +1,6 @@
-from pydantic import BaseModel, field_validator, ConfigDict
-from typing import List, Dict, Tuple, Iterable, Optional, Any, Literal
+from typing import List, Dict, Tuple, Optional, Any, Literal, Annotated
 
+from pydantic import BaseModel, field_validator, ConfigDict, Field, StringConstraints
 
 ExecutionModeLiteral = Literal['Development', 'Performance']
 ExecutionLocationsLiteral = Literal['auto', 'local', 'remote']
@@ -104,3 +104,8 @@ class NodeDefault(BaseModel):
     node_type: NodeTypeLiteral
     transform_type: TransformTypeLiteral
     has_default_settings: Optional[Any] = None
+
+
+# Define SecretRef here if not in a common location
+SecretRef = Annotated[str, StringConstraints(min_length=1, max_length=100),
+                      Field(description="An ID referencing an encrypted secret.")]

flowfile_core/schemas/transform_schema.py

@@ -4,6 +4,8 @@ import polars as pl
 from polars import selectors
 from copy import deepcopy
 
+from typing import NamedTuple
+
 
 def get_func_type_mapping(func: str):
     if func in ["mean", "avg", "median", "std", "var"]:
@@ -20,10 +22,30 @@ def string_concat(*column: str):
     return pl.col(column).cast(pl.Utf8).str.concat(delimiter=',')
 
 
-JoinStrategy = Literal['inner', 'left', 'right', 'full', 'semi', 'anti', 'cross']
+SideLit = Literal["left", "right"]
+JoinStrategy = Literal['inner', 'left', 'right', 'full', 'semi', 'anti', 'cross', 'outer']
 FuzzyTypeLiteral = Literal['levenshtein', 'jaro', 'jaro_winkler', 'hamming', 'damerau_levenshtein', 'indel']
 
 
+def construct_join_key_name(side: SideLit, column_name: str) -> str:
+    return "_FLOWFILE_JOIN_KEY_" + side.upper() + "_" + column_name
+
+
+class JoinKeyRename(NamedTuple):
+    original_name: str
+    temp_name: str
+
+
+class JoinKeyRenameResponse(NamedTuple):
+    side: SideLit
+    join_key_renames: List[JoinKeyRename]
+
+
+class FullJoinKeyResponse(NamedTuple):
+    left: JoinKeyRenameResponse
+    right: JoinKeyRenameResponse
+
+
 @dataclass
 class SelectInput:
     # __slots__ = ['old_name', 'new_name', 'keep', 'data_type', 'data_type_change', 'join_key']
@@ -108,11 +130,11 @@ class SelectInputs:
 
     @property
     def new_cols(self) -> Set:
-        return set(v.new_name for v in self.renames if v.keep or v.join_key)
+        return set(v.new_name for v in self.renames if v.keep)
 
     @property
     def rename_table(self):
-        return {v.old_name: v.new_name for v in self.renames if (v.keep or v.join_key) and v.is_available}
+        return {v.old_name: v.new_name for v in self.renames if v.is_available}
 
     def get_select_cols(self, include_join_key: bool = True):
         return [v.old_name for v in self.renames if v.keep or (v.join_key and include_join_key)]
@@ -126,6 +148,11 @@ class SelectInputs:
     def remove_select_input(self, old_key: str):
         self.renames = [rename for rename in self.renames if rename.old_name != old_key]
 
+    def unselect_field(self, old_key: str):
+        for rename in self.renames:
+            if old_key == rename.old_name:
+                rename.keep = False
+
     @classmethod
     def create_from_list(cls, col_list: str):
         return cls([SelectInput(c) for c in col_list])
@@ -134,13 +161,42 @@ class SelectInputs:
     def create_from_pl_df(cls, df: pl.DataFrame | pl.LazyFrame):
         return cls([SelectInput(c) for c in df.columns])
 
+    def get_select_input_on_old_name(self, old_name: str) -> SelectInput | None:
+        return next((v for v in self.renames if v.old_name == old_name), None)
+
+    def get_select_input_on_new_name(self, old_name: str) -> SelectInput | None:
+        return next((v for v in self.renames if v.new_name == old_name), None)
+
+
+class JoinInputs(SelectInputs):
+
+    def __init__(self, renames: List[SelectInput]):
+        self.renames = renames
+
+    @property
+    def join_key_selects(self) -> List[SelectInput]:
+        return [v for v in self.renames if v.join_key]
+
+    def get_join_key_renames(self, side: SideLit, filter_drop: bool = False) -> JoinKeyRenameResponse:
+        return JoinKeyRenameResponse(
+            side,
+            [JoinKeyRename(jk.new_name,
+                           construct_join_key_name(side, jk.new_name))
+             for jk in self.join_key_selects if jk.keep or not filter_drop]
+        )
+
+    def get_join_key_rename_mapping(self, side: SideLit) -> Dict[str, str]:
+        return {jkr[0]: jkr[1] for jkr in self.get_join_key_renames(side)[1]}
+
 
 @dataclass
 class JoinMap:
+    # __slots__ = "left_col", "right_col"
     left_col: str
     right_col: str
 
 
+
 @dataclass
 class FuzzyMap(JoinMap):
     threshold_score: Optional[float] = 80.0
@@ -168,19 +224,21 @@ class FuzzyMap(JoinMap):
 
 class JoinSelectMixin:
     """Mixin for common join selection functionality"""
+    left_select: JoinInputs = None
+    right_select: JoinInputs = None
 
     @staticmethod
-    def parse_select(select: List[SelectInput] | List[str] | List[Dict]) -> SelectInputs:
+    def parse_select(select: List[SelectInput] | List[str] | List[Dict]) -> JoinInputs | None:
         if all(isinstance(c, SelectInput) for c in select):
-            return SelectInputs(select)
+            return JoinInputs(select)
         elif all(isinstance(c, dict) for c in select):
-            return SelectInputs([SelectInput(**c) for c in select])
+            return JoinInputs([SelectInput(**c.__dict__) for c in select])
         elif isinstance(select, dict):
             renames = select.get('renames')
             if renames:
-                return SelectInputs([SelectInput(**c) for c in renames])
+                return JoinInputs([SelectInput(**c) for c in renames])
         elif all(isinstance(c, str) for c in select):
-            return SelectInputs([SelectInput(s, s) for s in select])
+            return JoinInputs([SelectInput(s, s) for s in select])
 
     def auto_generate_new_col_name(self, old_col_name: str, side: str) -> str:
         current_names = self.left_select.new_cols & self.right_select.new_cols
@@ -223,8 +281,8 @@ class CrossJoinInput(JoinSelectMixin):
 @dataclass
 class JoinInput(JoinSelectMixin):
     join_mapping: List[JoinMap]
-    left_select: SelectInputs = None
-    right_select: SelectInputs = None
+    left_select: JoinInputs = None
+    right_select: JoinInputs = None
     how: JoinStrategy = 'inner'
 
     @staticmethod
@@ -254,9 +312,26 @@ class JoinInput(JoinSelectMixin):
         self.join_mapping = self.parse_join_mapping(join_mapping)
         self.left_select = self.parse_select(left_select)
         self.right_select = self.parse_select(right_select)
+        self.set_join_keys()
+        self.how = how
+
+    def set_join_keys(self):
         [setattr(v, "join_key", v.old_name in self._left_join_keys) for v in self.left_select.renames]
         [setattr(v, "join_key", v.old_name in self._right_join_keys) for v in self.right_select.renames]
-        self.how = how
+
+    def get_join_key_renames(self, filter_drop: bool = False) -> FullJoinKeyResponse:
+        return FullJoinKeyResponse(self.left_select.get_join_key_renames(side="left", filter_drop=filter_drop),
+                                   self.right_select.get_join_key_renames(side="right", filter_drop=filter_drop))
+
+    def get_names_for_table_rename(self) -> List[JoinMap]:
+        new_mappings: List[JoinMap] = []
+        left_rename_table, right_rename_table = self.left_select.rename_table, self.right_select.rename_table
+        for join_map in self.join_mapping:
+            new_mappings.append(JoinMap(left_rename_table.get(join_map.left_col, join_map.left_col),
+                                        right_rename_table.get(join_map.right_col, join_map.right_col)
+                                        )
+                                )
+        return new_mappings
 
     @property
     def _left_join_keys(self) -> Set:
@@ -268,22 +343,21 @@ class JoinInput(JoinSelectMixin):
 
     @property
     def left_join_keys(self) -> List:
-        return [self.left_select.rename_table.get(jm.left_col) for jm in self.join_mapping]
+        return [jm.left_col for jm in self.used_join_mapping]
 
     @property
     def right_join_keys(self) -> List:
-        return [self.right_select.rename_table.get(jm.right_col, jm.right_col) for jm in self.join_mapping]
+        return [jm.right_col for jm in self.used_join_mapping]
 
     @property
     def overlapping_records(self):
         if self.how in ('left', 'right', 'inner'):
-            #  Never consider join keys as overlapping records since they will be dropped after the join
-            return ((self.left_select.new_cols & self.right_select.new_cols) -
-                    (set(self.left_join_keys) & set(self.right_join_keys)))
+            return self.left_select.new_cols & self.right_select.new_cols
         else:
             return self.left_select.new_cols & self.right_select.new_cols
 
     def auto_rename(self):
+        self.set_join_keys()
         overlapping_records = self.overlapping_records
         while len(overlapping_records) > 0:
             for right_col in self.right_select.renames:
@@ -292,13 +366,15 @@ class JoinInput(JoinSelectMixin):
             overlapping_records = self.overlapping_records
 
     @property
-    def join_mappings(self):
-        new_mappings = []
+    def used_join_mapping(self):
+        new_mappings: List[JoinMap] = []
         left_rename_table, right_rename_table = self.left_select.rename_table, self.right_select.rename_table
+        left_join_rename_mapping: Dict[str, str] = self.left_select.get_join_key_rename_mapping("left")
+        right_join_rename_mapping: Dict[str, str] = self.right_select.get_join_key_rename_mapping("right")
         for join_map in self.join_mapping:
             # del self.right_select.rename_table, self.left_select.rename_table
-            new_mappings.append(JoinMap(left_rename_table.get(join_map.left_col),
-                                        right_rename_table.get(join_map.right_col)
+            new_mappings.append(JoinMap(left_join_rename_mapping.get(left_rename_table.get(join_map.left_col, join_map.left_col)),
+                                        right_join_rename_mapping.get(right_rename_table.get(join_map.right_col, join_map.right_col))
                                         )
                                 )
         return new_mappings
@@ -332,7 +408,7 @@ class FuzzyMatchInput(JoinInput):
         return fuzz_mapping
 
     def __init__(self, join_mapping: List[FuzzyMap] | Tuple[str, str] | str, left_select: List[SelectInput] | List[str],
-                 right_select: List[SelectInput] | List[str], aggregate_output: bool = False, how: str = 'inner'):
+                 right_select: List[SelectInput] | List[str], aggregate_output: bool = False, how: JoinStrategy = 'inner'):
         self.join_mapping = self.parse_fuzz_mapping(join_mapping)
         self.left_select = self.parse_select(left_select)
         self.right_select = self.parse_select(right_select)
@@ -472,7 +548,6 @@ class PivotInput:
         return pl.struct([pl.col(c) for c in self.aggregations]).alias('vals')
 
 
-
 @dataclass
 class SortByInput:
     column: str

flowfile_core/utils/utils.py

@@ -1,8 +1,47 @@
 import re
-
+from itertools import chain
+from typing import List, Dict
 
 def camel_case_to_snake_case(text: str) -> str:
     # Use a regular expression to find capital letters and replace them with _ followed by the lowercase letter
     transformed_text = re.sub(r'(?<!^)(?=[A-Z])', '_', text).lower()
     return transformed_text
 
+
+def ensure_similarity_dicts(datas: List[Dict], respect_order: bool = True):
+    all_cols = (data.keys() for data in datas)
+    if not respect_order:
+        unique_cols = set(chain(*all_cols))
+    else:
+        col_store = set()
+        unique_cols = list()
+        for row in all_cols:
+            for col in row:
+                if col not in col_store:
+                    unique_cols.append(col)
+                    col_store.update((col,))
+    output = []
+    for data in datas:
+        new_record = dict()
+        for col in unique_cols:
+            val = data.get(col)
+            new_record[col] = val
+        output.append(new_record)
+    return output
+
+
+def convert_to_string(v):
+    try:
+        return str(v)
+    except:
+        return None
+
+
+def standardize_col_dtype(vals):
+    types = set(type(val) for val in vals)
+    if len(types) == 1:
+        return vals
+    elif int in types and float in types:
+        return vals
+    else:
+        return [convert_to_string(v) for v in vals]

flowfile_core/utils/validate_setup.py

@@ -0,0 +1,41 @@
+"""This script runs on run time and checks if all the nodes that are created have a function in the flow_graph as well
+as have a component in flowfile_frontend"""
+
+from flowfile_core.schemas import input_schema
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.configs.node_store.nodes import nodes_list, NodeTemplate
+import inspect
+
+
+def check_if_node_has_add_function_in_flow_graph(node: NodeTemplate):
+    func_name = "add_" + node.item
+    if not hasattr(FlowGraph, func_name):
+        raise ValueError(
+            f"Node {node.name} ({node.item}) does not have a corresponding function in FlowGraph: {func_name}"
+            "Check if the function is implemented in flow_graph.py or if the node item is correct."
+        )
+
+
+def check_if_node_has_input_schema_definition(node: NodeTemplate):
+    if "node"+node.item.replace("_","") not in {k.lower() for k in inspect.getmodule(input_schema).__dict__.keys()}:
+        raise ValueError(
+            f"Node {node.name} ({node.item}) does not have a corresponding input schema definition in input_schema.py."
+            "Check if the schema is implemented or if the node item is correct."
+        )
+
+
+def validate_setup():
+    """
+    Validates the setup by checking if all nodes in the nodes_list have a corresponding function in FlowGraph
+    and a corresponding input schema definition in input_schema.py.
+    Raises ValueError if any node is missing either.
+    """
+    for node in nodes_list:
+        check_if_node_has_add_function_in_flow_graph(node)
+        check_if_node_has_input_schema_definition(node)
+
+    print("All nodes have corresponding functions in FlowGraph and input schema definitions.")
+
+
+if __name__ == "__main__":
+    validate_setup()