Flowfile 0.3.4.1__py3-none-any.whl → 0.3.6__py3-none-any.whl

flowfile_core/flowfile/flow_node/flow_node.py

@@ -13,7 +13,7 @@ from flowfile_core.configs.node_store import nodes as node_interface
 from flowfile_core.flowfile.setting_generator import setting_generator, setting_updator
 from time import sleep
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations import (
-    ExternalDfFetcher, ExternalSampler, results_exists, get_external_df_result, ExternalDatabaseFetcher, ExternalDatabaseWriter)
+    ExternalDfFetcher, ExternalSampler, results_exists, get_external_df_result, ExternalDatabaseFetcher, ExternalDatabaseWriter, ExternalCloudWriter)
 from flowfile_core.flowfile.flow_node.models import (NodeStepSettings, NodeStepInputs, NodeSchemaInformation,
                                                      NodeStepStats, NodeResults)
 from flowfile_core.flowfile.flow_node.schema_callback import SingleExecutionFuture
@@ -31,13 +31,14 @@ class FlowNode:
     results: NodeResults
     node_information: Optional[schemas.NodeInformation] = None
     leads_to_nodes: List["FlowNode"] = []  # list with target flows, after execution the step will trigger those step(s)
+    user_provided_schema_callback: Optional[Callable] = None  # user provided callback function for schema calculation
     _setting_input: Any = None
     _hash: Optional[str] = None  # host this for caching results
     _function: Callable = None  # the function that needs to be executed when triggered
     _schema_callback: Optional[SingleExecutionFuture] = None  # Function that calculates the schema without executing
     _state_needs_reset: bool = False
-    _fetch_cached_df: Optional[ExternalDfFetcher | ExternalDatabaseFetcher | ExternalDatabaseWriter] = None
-    _cache_progress: Optional[ExternalDfFetcher | ExternalDatabaseFetcher | ExternalDatabaseWriter] = None
+    _fetch_cached_df: Optional[ExternalDfFetcher | ExternalDatabaseFetcher | ExternalDatabaseWriter | ExternalCloudWriter] = None
+    _cache_progress: Optional[ExternalDfFetcher | ExternalDatabaseFetcher | ExternalDatabaseWriter | ExternalCloudWriter] = None
 
     def post_init(self):
         self.node_inputs = NodeStepInputs()
@@ -60,8 +61,29 @@ class FlowNode:
     def state_needs_reset(self, v: bool):
         self._state_needs_reset = v
 
+    @staticmethod
+    def create_schema_callback_from_function(f: Callable) -> Callable[[], List[FlowfileColumn]]:
+        """
+        Create a schema callback from a function.
+        :param f: Function that returns the schema
+        :return: Callable that returns the schema
+        """
+        def schema_callback() -> List[FlowfileColumn]:
+            try:
+                logger.info('Executing the schema callback function based on the node function')
+                return f().schema
+            except Exception as e:
+                logger.warning(f'Error with the schema callback: {e}')
+                return []
+        return schema_callback
+
     @property
-    def schema_callback(self):
+    def schema_callback(self) -> SingleExecutionFuture:
+        if self._schema_callback is None:
+            if self.user_provided_schema_callback is not None:
+                self.schema_callback = self.user_provided_schema_callback
+            elif self.is_start:
+                self.schema_callback = self.create_schema_callback_from_function(self._function)
         return self._schema_callback
 
     @schema_callback.setter
@@ -76,7 +98,6 @@ class FlowNode:
             return []
 
         self._schema_callback = SingleExecutionFuture(f, error_callback)
-        self._schema_callback.start()
 
     @property
     def is_start(self) -> bool:
@@ -133,13 +154,13 @@ class FlowNode:
                     pos_y: float = 0,
                     schema_callback: Callable = None,
                     ):
-
-        self.schema_callback = schema_callback
+        self.user_provided_schema_callback = schema_callback
         self.node_information.y_position = pos_y
         self.node_information.x_position = pos_x
         self.node_information.setting_input = setting_input
         self.name = self.node_type if name is None else name
         self._function = function
+
         self.node_schema.input_columns = [] if input_columns is None else input_columns
         self.node_schema.output_columns = [] if output_schema is None else output_schema
         self.node_schema.drop_columns = [] if drop_columns is None else drop_columns
@@ -147,7 +168,6 @@ class FlowNode:
         if hasattr(setting_input, 'cache_results'):
             self.node_settings.cache_results = setting_input.cache_results
 
-        self.setting_input = setting_input
         self.results.errors = None
         self.add_lead_to_in_depend_source()
         _ = self.hash
@@ -155,6 +175,7 @@ class FlowNode:
         if self.node_template is None:
             raise Exception(f'Node template {self.node_type} not found')
         self.node_default = node_interface.node_defaults.get(self.node_type)
+        self.setting_input = setting_input  # wait until the end so that the hash is calculated correctly
 
     @property
     def name(self):
@@ -171,11 +192,17 @@ class FlowNode:
 
     @setting_input.setter
     def setting_input(self, setting_input: Any):
+        is_manual_input = (self.node_type == 'manual_input' and
+                           isinstance(setting_input, input_schema.NodeManualInput) and
+                           isinstance(self._setting_input, input_schema.NodeManualInput)
+                           )
+        if is_manual_input:
+            _ = self.hash
         self._setting_input = setting_input
         self.set_node_information()
-        if self.node_type == 'manual_input' and isinstance(self._setting_input, input_schema.NodeManualInput):
-            if self.hash != self.calculate_hash(setting_input) or not self.node_stats.has_run:
-                self.function = self.function.__class__(setting_input.raw_data_format)
+        if is_manual_input:
+            if self.hash != self.calculate_hash(setting_input) or not self.node_stats.has_run_with_current_setup:
+                self.function = FlowDataEngine(setting_input.raw_data_format)
                 self.reset()
                 self.get_predicted_schema()
         elif self._setting_input is not None:
@@ -291,7 +318,7 @@ class FlowNode:
         Method to get a predicted schema based on the columns that are dropped and added
         :return:
         """
-        if self.node_schema.predicted_schema is not None and not force:
+        if self.node_schema.predicted_schema and not force:
             return self.node_schema.predicted_schema
         if self.schema_callback is not None and (self.node_schema.predicted_schema is None or force):
             self.print('Getting the data from a schema callback')
@@ -299,7 +326,7 @@ class FlowNode:
                 # Force the schema callback to reset, so that it will be executed again
                 self.schema_callback.reset()
             schema = self.schema_callback()
-            if schema is not None:
+            if schema is not None and len(schema) > 0:
                 self.print('Calculating the schema based on the schema callback')
                 self.node_schema.predicted_schema = schema
                 return self.node_schema.predicted_schema
@@ -320,14 +347,14 @@ class FlowNode:
     def print(self, v: Any):
         logger.info(f'{self.node_type}, node_id: {self.node_id}: {v}')
 
-    def get_resulting_data(self) -> FlowDataEngine:
+    def get_resulting_data(self) -> FlowDataEngine | None:
         if self.is_setup:
             if self.results.resulting_data is None and self.results.errors is None:
                 self.print('getting resulting data')
                 try:
                     if isinstance(self.function, FlowDataEngine):
                         fl: FlowDataEngine = self.function
-                    elif self.node_type in ('external_source', 'airbyte_reader'):
+                    elif self.node_type == 'external_source':
                         fl: FlowDataEngine = self.function()
                         fl.collect_external()
                         self.node_settings.streamable = False
@@ -342,11 +369,12 @@ class FlowNode:
                 except Exception as e:
                     self.results.resulting_data = FlowDataEngine()
                     self.results.errors = str(e)
-                    self.node_stats.has_run = False
+                    self.node_stats.has_run_with_current_setup = False
+                    self.node_stats.has_completed_last_run = False
                     raise e
             return self.results.resulting_data
 
-    def _predicted_data_getter(self) -> FlowDataEngine|None:
+    def _predicted_data_getter(self) -> FlowDataEngine | None:
         try:
             fl = self._function(*[v.get_predicted_resulting_data() for v in self.all_inputs])
             return fl
@@ -365,6 +393,7 @@ class FlowNode:
     def get_predicted_resulting_data(self) -> FlowDataEngine:
         if self.needs_run(False) and self.schema_callback is not None or self.node_schema.result_schema is not None:
             self.print('Getting data based on the schema')
+
             _s = self.schema_callback() if self.node_schema.result_schema is None else self.node_schema.result_schema
             return FlowDataEngine.create_from_schema(_s)
         else:
@@ -425,7 +454,7 @@ class FlowNode:
             return False
         flow_logger = logger if node_logger is None else node_logger
         cache_result_exists = results_exists(self.hash)
-        if not self.node_stats.has_run:
+        if not self.node_stats.has_run_with_current_setup:
             flow_logger.info('Node has not run, needs to run')
             return True
         if self.node_settings.cache_results and cache_result_exists:
@@ -449,17 +478,18 @@ class FlowNode:
                                                    wait_on_completion=True, node_id=self.node_id, flow_id=flow_id)
                 self.store_example_data_generator(external_sampler)
                 if self.results.errors is None and not self.node_stats.is_canceled:
-                    self.node_stats.has_run = True
+                    self.node_stats.has_run_with_current_setup = True
             self.node_schema.result_schema = resulting_data.schema
 
         except Exception as e:
             logger.warning(f"Error with step {self.__name__}")
             logger.error(str(e))
             self.results.errors = str(e)
-            self.node_stats.has_run = False
+            self.node_stats.has_run_with_current_setup = False
+            self.node_stats.has_completed_last_run = False
             raise e
 
-        if self.node_stats.has_run:
+        if self.node_stats.has_run_with_current_setup:
             for step in self.leads_to_nodes:
                 if not self.node_settings.streamable:
                     step.node_settings.streamable = self.node_settings.streamable
@@ -477,7 +507,7 @@ class FlowNode:
                 node_logger.warning('Failed to read the cache, rerunning the code')
         if self.node_type == 'output':
             self.results.resulting_data = self.get_resulting_data()
-            self.node_stats.has_run = True
+            self.node_stats.has_run_with_current_setup = True
             return
         try:
             self.get_resulting_data()
@@ -498,7 +528,7 @@ class FlowNode:
                 )
                 if not performance_mode:
                     self.store_example_data_generator(external_df_fetcher)
-                    self.node_stats.has_run = True
+                    self.node_stats.has_run_with_current_setup = True
 
             except Exception as e:
                 node_logger.error('Error with external process')
@@ -541,7 +571,8 @@ class FlowNode:
         # node_logger = flow_logger.get_node_logger(self.node_id)
         if reset_cache:
             self.remove_cache()
-            self.node_stats.has_run = False
+            self.node_stats.has_run_with_current_setup = False
+            self.node_stats.has_completed_last_run = False
         if self.is_setup:
             node_logger.info(f'Starting to run {self.__name__}')
             if self.needs_run(performance_mode, node_logger, run_location):
@@ -572,7 +603,6 @@ class FlowNode:
                                           performance_mode=performance_mode, retry=False,
                                           node_logger=node_logger)
                     else:
-                        self.node_stats.has_run = False
                         self.results.errors = str(e)
                         node_logger.error(f'Error with running the node: {e}')
 
@@ -596,15 +626,19 @@ class FlowNode:
         needs_reset = self.needs_reset() or deep
         if needs_reset:
             logger.info(f'{self.node_id}: Node needs reset')
-            self.node_stats.has_run = False
+            self.node_stats.has_run_with_current_setup = False
             self.results.reset()
-            if self.schema_callback:
-                self.schema_callback.reset()
+            if self.is_correct:
+                self._schema_callback = None  # Ensure the schema callback is reset
+                if self.schema_callback:
+                    logger.info(f'{self.node_id}: Resetting the schema callback')
+                    self.schema_callback.start()
             self.node_schema.result_schema = None
             self.node_schema.predicted_schema = None
             self._hash = None
             self.node_information.is_setup = None
             self.evaluate_nodes()
+            _ = self.hash  # Recalculate the hash after reset
 
     def delete_lead_to_node(self, node_id: int) -> bool:
         logger.info(f'Deleting lead to node: {node_id}')
@@ -682,10 +716,11 @@ class FlowNode:
 
     def get_table_example(self, include_data: bool = False) -> TableExample | None:
         self.print('Getting a table example')
-        if self.node_type == 'output':
-            self.print('getting the table example')
-            return self.main_input[0].get_table_example(include_data)
-        if self.is_setup and include_data:
+        if self.is_setup and include_data and self.node_stats.has_completed_last_run:
+            if self.node_template.node_group == 'output':
+                self.print('getting the table example')
+                return self.main_input[0].get_table_example(include_data)
+
             logger.info('getting the table example since the node has run')
             example_data_getter = self.results.example_data_generator
             if example_data_getter is not None:
@@ -720,7 +755,7 @@ class FlowNode:
     def get_node_data(self, flow_id: int, include_example: bool = False) -> NodeData:
         node = NodeData(flow_id=flow_id,
                         node_id=self.node_id,
-                        has_run=self.node_stats.has_run,
+                        has_run=self.node_stats.has_run_with_current_setup,
                         setting_input=self.setting_input,
                         flow_type=self.node_type)
         if self.main_input:

flowfile_core/flowfile/flow_node/models.py

@@ -21,10 +21,39 @@ class NodeStepPromise:
 
 class NodeStepStats:
     error: str = None
-    has_run: bool = False
+    _has_run_with_current_setup: bool = False
+    has_completed_last_run: bool = False
     active: bool = True
     is_canceled: bool = False
 
+    def __init__(self, error: str = None,
+                 has_run_with_current_setup: bool = False,
+                 has_completed_last_run: bool = False,
+                 active: bool = True,
+                 is_canceled: bool = False):
+        self.error = error
+        self._has_run_with_current_setup = has_run_with_current_setup
+        self.has_completed_last_run = has_completed_last_run
+        self.active = active
+        self.is_canceled = is_canceled
+
+    def __repr__(self):
+        return (f"NodeStepStats(error={self.error}, has_run_with_current_setup={self.has_run_with_current_setup}, "
+                f"has_completed_last_run={self.has_completed_last_run}, "
+                f"active={self.active}, is_canceled={self.is_canceled})")
+
+    @property
+    def has_run_with_current_setup(self) -> bool:
+        return self._has_run_with_current_setup
+
+    @has_run_with_current_setup.setter
+    def has_run_with_current_setup(self, value: bool):
+        if value:
+            self._has_run_with_current_setup = True
+            self.has_completed_last_run = True
+        else:
+            self._has_run_with_current_setup = False
+
 
 class NodeStepSettings:
     cache_results: bool = False
@@ -66,8 +95,8 @@ class NodeStepInputs:
 
 
 class NodeSchemaInformation:
-    result_schema: Optional[List[FlowfileColumn]] = []  # resulting schema of the function
-    predicted_schema: Optional[List[FlowfileColumn]] = []  # predicted resulting schema of the function
+    result_schema: Optional[List[FlowfileColumn]] = None  # resulting schema of the function
+    predicted_schema: Optional[List[FlowfileColumn]] = None  # predicted resulting schema of the function
     input_columns: List[str] = []  # columns that are needed for the function
     drop_columns: List[str] = []  # columns that will not be available after the function
     output_columns: List[FlowfileColumn] = []  # columns that will be added with the function

flowfile_core/flowfile/flow_node/schema_callback.py

@@ -14,6 +14,7 @@ class SingleExecutionFuture(Generic[T]):
     func: Callable[[], T]
     on_error: Optional[Callable[[Exception], Any]]
     result_value: Optional[T]
+    has_run_at_least_once: bool = False  # Indicates if the function has been run at least once
 
     def __init__(
         self,
@@ -26,6 +27,7 @@ class SingleExecutionFuture(Generic[T]):
         self.func = func
         self.on_error = on_error
         self.result_value = None
+        self.has_run_at_least_once = False
 
     def start(self) -> None:
         """Start the function execution if not already started."""
@@ -35,8 +37,7 @@ class SingleExecutionFuture(Generic[T]):
 
     def cleanup(self) -> None:
         """Clean up resources by clearing the future and shutting down the executor."""
-        # if self.future:
-        #     self.future = None
+        self.has_run_at_least_once = True
         self.executor.shutdown(wait=False)
 
     def __call__(self) -> Optional[T]:

flowfile_core/flowfile/sources/external_sources/__init__.py

@@ -1,3 +1 @@
 from flowfile_core.flowfile.sources.external_sources import custom_external_sources
-# from flowfile.sources.external_sources.custom_external_sources.external_source import check_for_key_vault_existence
-# from flowfile.sources.external_sources.airbyte_sources.airbyte import AirbyteSource

flowfile_core/flowfile/sources/external_sources/factory.py

@@ -1,22 +1,19 @@
 from flowfile_core.flowfile.sources.external_sources.custom_external_sources.external_source import CustomExternalSource
-from flowfile_core.flowfile.sources.external_sources.airbyte_sources.airbyte import AirbyteSource
 
 
-def data_source_factory(source_type: str, **kwargs) -> CustomExternalSource | AirbyteSource:
+def data_source_factory(source_type: str, **kwargs) -> CustomExternalSource:
     """
-    Factory function to generate either CustomExternalSource or AirbyteSource.
+    Factory function to generate either CustomExternalSource .
 
     Args:
-        source_type (str): The type of source to create ("custom" or "airbyte").
+        source_type (str): The type of source to create ("custom").
         **kwargs: The keyword arguments required for the specific source type.
 
     Returns:
-        Union[CustomExternalSource, AirbyteSource]: An instance of the selected data source type.
+        Union[CustomExternalSource]: An instance of the selected data source type.
     """
     if source_type == "custom":
         return CustomExternalSource(**kwargs)
-    elif source_type == "airbyte":
-        return AirbyteSource(**kwargs)
     else:
         raise ValueError(f"Unknown source type: {source_type}")
 

flowfile_core/flowfile/utils.py

@@ -1,16 +1,12 @@
 import os
 import hashlib
 import json
-import polars as pl
 import shutil
 
 from datetime import datetime, date, time
 from typing import List
 from decimal import Decimal
 
-from flowfile_core.flowfile.flow_data_engine.utils import standardize_col_dtype
-from flowfile_core.schemas import input_schema
-
 
 def generate_sha256_hash(data: bytes):
     sha256 = hashlib.sha256()
@@ -61,7 +57,7 @@ def get_hash(val):
     if hasattr(val, 'overridden_hash') and val.overridden_hash():
         val = hash(val)
     elif hasattr(val, '__dict__'):
-        val = {k: v for k, v in val.__dict__.items() if k not in {'pos_x', 'pos_y'}}
+        val = {k: v for k, v in val.__dict__.items() if k not in {'pos_x', 'pos_y', 'description'}}
     elif hasattr(val, 'json'):
         pass
     return generate_sha256_hash(json_dumps(val).encode('utf-8'))
@@ -117,21 +113,3 @@ def batch_generator(input_list: List, batch_size: int = 10000):
             yield input_list
             input_list = []
             run = False
-
-
-def _handle_raw_data(node_manual_input: input_schema.NodeManualInput):
-    """Ensure compatibility with the new typed raw data and the old dict form data type"""
-    if (not (hasattr(node_manual_input, "raw_data_format") and node_manual_input.raw_data_format)
-            and (hasattr(node_manual_input, 'raw_data') and node_manual_input.raw_data)):
-        values = [standardize_col_dtype([vv for vv in c]) for c in zip(*(r.values()
-                                                                         for r in node_manual_input.raw_data))]
-        data_types = (pl.DataType.from_python(type(next((v for v in column_values), None))) for column_values in values)
-        _columns = [input_schema.MinimalFieldInfo(name=c, data_type=str(next(data_types))) for c in
-                    node_manual_input.raw_data[0].keys()]
-
-        node_manual_input.raw_data_format = input_schema.RawData(columns=_columns, data=values)
-    elif ((hasattr(node_manual_input, "raw_data_format") and node_manual_input.raw_data_format)
-          and not (hasattr(node_manual_input, 'raw_data') and node_manual_input.raw_data)):
-        node_manual_input.raw_data = [{c.name: node_manual_input.raw_data_format.data[ci][ri] for ci, c in
-                                       enumerate(node_manual_input.raw_data_format.columns)}
-                                      for ri in range(len(node_manual_input.raw_data_format.data[0]))]

flowfile_core/main.py

@@ -8,14 +8,14 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
 from flowfile_core import ServerRun
-from flowfile_core.configs.settings import (SERVER_HOST, SERVER_PORT, WORKER_HOST, WORKER_PORT, WORKER_URL,
-                                            OFFLOAD_TO_WORKER)
+from flowfile_core.configs.settings import (SERVER_HOST, SERVER_PORT, WORKER_HOST, WORKER_PORT, WORKER_URL,)
 
 from flowfile_core.routes.auth import router as auth_router
 from flowfile_core.routes.secrets import router as secrets_router
 from flowfile_core.routes.routes import router
 from flowfile_core.routes.public import router as public_router
 from flowfile_core.routes.logs import router as logs_router
+from flowfile_core.routes.cloud_connections import router as cloud_connections_router
 
 from flowfile_core.configs.flow_logger import clear_all_flow_logs
 
@@ -72,6 +72,7 @@ app.include_router(router)
 app.include_router(logs_router, tags=["logs"])
 app.include_router(auth_router, prefix="/auth", tags=["auth"])
 app.include_router(secrets_router, prefix="/secrets", tags=["secrets"])
+app.include_router(cloud_connections_router, prefix="/cloud_connections", tags=["cloud_connections"])
 
 
 @app.post("/shutdown")

flowfile_core/routes/cloud_connections.py

@@ -0,0 +1,81 @@
+from typing import List
+
+from fastapi import HTTPException, Depends, APIRouter
+from sqlalchemy.orm import Session
+
+# Core modules
+from flowfile_core.auth.jwt import get_current_active_user
+from flowfile_core.configs import logger
+from flowfile_core.database.connection import get_db
+from flowfile_core.flowfile.database_connection_manager.db_connections import (store_cloud_connection,
+                                                                               get_cloud_connection_schema,
+                                                                               get_all_cloud_connections_interface,
+                                                                               delete_cloud_connection)
+# Schema and models
+from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection, FullCloudStorageConnectionInterface
+
+# External dependencies
+# File handling
+router = APIRouter()
+
+
+@router.post("/cloud_connection", tags=['cloud_connections'])
+def create_cloud_storage_connection(input_connection: FullCloudStorageConnection,
+                                    current_user=Depends(get_current_active_user),
+                                    db: Session = Depends(get_db)
+                                    ):
+    """
+    Create a new cloud storage connection.
+    Parameters
+    ----------
+    input_connection: FullCloudStorageConnection schema containing connection details
+    current_user: User obtained from Depends(get_current_active_user)
+    db: Session obtained from Depends(get_db)
+    Returns
+    -------
+    Dict with a success message
+    """
+    logger.info(f'Create cloud connection {input_connection.connection_name}')
+    try:
+        store_cloud_connection(db, input_connection, current_user.id)
+    except ValueError:
+        raise HTTPException(422, 'Connection name already exists')
+    except Exception as e:
+        logger.error(e)
+        raise HTTPException(422, str(e))
+    return {"message": "Cloud connection created successfully"}
+
+
+@router.delete('/cloud_connection', tags=['cloud_connections'])
+def delete_cloud_connection_with_connection_name(connection_name: str,
+                                                 current_user=Depends(get_current_active_user),
+                                                 db: Session = Depends(get_db)
+                                                 ):
+    """
+    Delete a cloud connection.
+    """
+    logger.info(f'Deleting cloud connection {connection_name}')
+    cloud_storage_connection = get_cloud_connection_schema(db, connection_name, current_user.id)
+    if cloud_storage_connection is None:
+        raise HTTPException(404, 'Cloud connection connection not found')
+    delete_cloud_connection(db, connection_name, current_user.id)
+    return {"message": "Cloud connection deleted successfully"}
+
+
+@router.get('/cloud_connections', tags=['cloud_connection'],
+            response_model=List[FullCloudStorageConnectionInterface])
+def get_cloud_connections(
+        db: Session = Depends(get_db),
+        current_user=Depends(get_current_active_user)) -> List[FullCloudStorageConnectionInterface]:
+    """
+    Get all cloud storage connections for the current user.
+    Parameters
+    ----------
+    db: Session obtained from Depends(get_db)
+    current_user: User obtained from Depends(get_current_active_user)
+
+    Returns
+    -------
+    List[FullCloudStorageConnectionInterface]
+    """
+    return get_all_cloud_connections_interface(db, current_user.id)

flowfile_core/routes/logs.py

@@ -86,7 +86,6 @@ async def stream_log_file(
                 line = await file.readline()
                 if line:
                     formatted_message = await format_sse_message(line.strip())
-                    logger.info(f'Yielding line: {line.strip()}')
                     yield formatted_message
                     last_active = time.monotonic()  # Reset idle timer on activity
                 else:

flowfile_core/routes/routes.py

@@ -27,12 +27,7 @@ from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_
 from flowfile_core.flowfile.analytics.analytics_processor import AnalyticsProcessor
 from flowfile_core.flowfile.extensions import get_instant_func_results
 # Flow handling
-from flowfile_core.flowfile.sources.external_sources.airbyte_sources.models import AirbyteConfigTemplate
-# Airbyte
-from flowfile_core.flowfile.sources.external_sources.airbyte_sources.settings import (
-    airbyte_config_handler,
-    AirbyteHandler
-)
+
 from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import create_sql_source_from_db_settings
 from flowfile_core.run_lock import get_flow_run_lock
 # Schema and models
@@ -49,7 +44,6 @@ from flowfile_core.database.connection import get_db
 
 
 
-# Router setup
 router = APIRouter(dependencies=[Depends(get_current_active_user)])
 
 # Initialize services
@@ -57,9 +51,11 @@ file_explorer = FileExplorer('/app/shared' if IS_RUNNING_IN_DOCKER else None)
 
 
 def get_node_model(setting_name_ref: str):
+    logger.info("Getting node model for: " + setting_name_ref)
     for ref_name, ref in inspect.getmodule(input_schema).__dict__.items():
         if ref_name.lower() == setting_name_ref:
             return ref
+    logger.error(f"Could not find node model for: {setting_name_ref}")
 
 
 @router.post("/upload/")
@@ -384,38 +380,6 @@ def close_flow(flow_id: int) -> None:
     flow_file_handler.delete_flow(flow_id)
 
 
-@router.get('/airbyte/available_connectors', tags=['airbyte'])
-def get_available_connectors():
-    return airbyte_config_handler.available_connectors
-
-
-@router.get('/airbyte/available_configs', tags=['airbyte'])
-def get_available_configs() -> List[str]:
-    """
-    Get the available configurations for the airbyte connectors
-    Returns: List of available configurations
-    """
-    return airbyte_config_handler.available_configs
-
-
-@router.get('/airbyte/config_template', tags=['airbyte'], response_model=AirbyteConfigTemplate)
-def get_config_spec(connector_name: str):
-    a = airbyte_config_handler.get_config('source-' + connector_name)
-    return a
-
-
-@router.post('/airbyte/set_airbyte_configs_for_streams', tags=['airbyte'])
-def set_airbyte_configs_for_streams(airbyte_config: input_schema.AirbyteConfig):
-    logger.info('Setting airbyte config, update_style = ')
-    logger.info(f'Setting config for {airbyte_config.source_name}')
-    logger.debug(f'Config: {airbyte_config.mapped_config_spec}')
-    airbyte_handler = AirbyteHandler(airbyte_config=airbyte_config)
-    try:
-        _ = airbyte_handler.get_available_streams()
-    except Exception as e:
-        raise HTTPException(404, str(e))
-
-
 @router.post('/update_settings/', tags=['transform'])
 def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_user=Depends(get_current_active_user)):
     input_data['user_id'] = current_user.id