Flowfile 0.2.2__py3-none-any.whl

flowfile_core/flowfile/setting_generator/settings.py

@@ -0,0 +1,176 @@
+
+from flowfile_core.schemas import input_schema, transform_schema
+from typing import Callable, Iterable
+from functools import wraps
+from flowfile_core.schemas.output_model import NodeData
+from flowfile_core.flowfile.setting_generator.setting_generator import SettingGenerator, SettingUpdator
+
+setting_generator = SettingGenerator()
+setting_updator = SettingUpdator()
+
+
+def setting_generator_method(f: callable) -> Callable:
+    @wraps(f)
+    def inner(node_data: NodeData) -> NodeData:
+        if node_data.setting_input is None or isinstance(node_data.setting_input, input_schema.NodePromise):
+            f(node_data)
+        return node_data
+    setting_generator.add_setting_generator_func(inner)
+    return inner
+
+
+def setting_updator_method(f: callable) -> Callable:
+    @wraps(f)
+    def inner(node_data: NodeData) -> NodeData:
+        if node_data.setting_input is not None and not isinstance(node_data.setting_input, input_schema.NodePromise):
+            f(node_data)
+        return node_data
+
+    setting_updator.add_setting_updator_func(inner)
+    return inner
+
+
+@setting_generator_method
+def join(node_data: "NodeData") -> NodeData:
+    if node_data.right_input and node_data.main_input:
+        overlapping_cols = list(set(node_data.main_input.columns) & set(node_data.right_input.columns))
+        if len(overlapping_cols) > 0:
+            join_key = overlapping_cols[0]
+        else:
+            join_key = ''
+        ji = transform_schema.JoinInput(join_mapping=join_key,
+                                        left_select=node_data.main_input.columns,
+                                        right_select=node_data.right_input.columns
+                                        )
+        ji.auto_rename()
+        node_data.setting_input = input_schema.NodeJoin(flow_id=node_data.flow_id,
+                                                        node_id=node_data.node_id,
+                                                        join_input=ji)
+    return node_data
+
+
+@setting_generator_method
+def cross_join(node_data: "NodeData") -> NodeData:
+    if node_data.right_input and node_data.main_input:
+        ji = transform_schema.CrossJoinInput(left_select=node_data.main_input.columns,
+                                             right_select=node_data.right_input.columns)
+        ji.auto_rename()
+        print(ji)
+        node_data.setting_input = input_schema.NodeCrossJoin(flow_id=node_data.flow_id,
+                                                             node_id=node_data.node_id,
+                                                             cross_join_input=ji)
+    return node_data
+
+
+@setting_generator_method
+def filter(node_data: "NodeData") -> NodeData:
+    if node_data.main_input:
+        fi = transform_schema.FilterInput(basic_filter=transform_schema.BasicFilter(), filter_type='advanced')
+        node_data.setting_input = input_schema.NodeFilter(flow_id=node_data.flow_id,
+                                                          node_id=node_data.node_id,
+                                                          filter_input=fi)
+    return node_data
+
+
+@setting_updator_method
+def join(node_data: NodeData):
+    if node_data.right_input and node_data.main_input:
+        setting_input: input_schema.NodeJoin = node_data.setting_input
+        left_columns = set(node_data.main_input.columns)
+        right_columns = set(node_data.right_input.columns)
+        left_select = setting_input.join_input.left_select
+        right_select = setting_input.join_input.right_select
+        for ls in left_select.renames:
+            if ls.old_name not in right_columns:
+                left_select.remove_select_input(ls.old_name)
+        for rs in right_select.renames:
+            if rs.old_name not in right_columns:
+                right_select.remove_select_input(rs.old_name)
+        existing_columns_right = set(r.old_name for r in right_select.renames if r.is_available)
+        existing_columns_left = set(r.old_name for r in left_select.renames if r.is_available)
+        missing_incoming_left_columns = [ilc for ilc in left_columns if ilc not in existing_columns_left]
+        missing_incoming_right_columns = [irc for irc in right_columns if irc not in existing_columns_right]
+        if not hasattr(setting_input, 'auto_keep_left'):
+            setting_input.auto_keep_left = False
+        if not hasattr(setting_input, 'auto_keep_right'):
+            setting_input.auto_keep_right = False
+        for milc in missing_incoming_left_columns:
+            select_input = transform_schema.SelectInput(old_name=milc, keep=setting_input.auto_keep_left)
+            setting_input.join_input.add_new_select_column(select_input, 'left')
+        for mirc in missing_incoming_right_columns:
+            select_input = transform_schema.SelectInput(old_name=mirc, keep=setting_input.auto_keep_right)
+            setting_input.join_input.add_new_select_column(select_input, 'right')
+    return node_data
+
+
+@setting_updator_method
+def cross_join(node_data: NodeData):
+    if node_data.right_input and node_data.main_input:
+        setting_input: input_schema.NodeCrossJoin = node_data.setting_input
+        left_columns = set(node_data.main_input.columns)
+        right_columns = set(node_data.right_input.columns)
+        left_select = setting_input.cross_join_input.left_select
+        right_select = setting_input.cross_join_input.right_select
+        for ls in left_select.renames:
+            if ls.old_name not in right_columns:
+                left_select.remove_select_input(ls.old_name)
+        for rs in right_select.renames:
+            if rs.old_name not in right_columns:
+                right_select.remove_select_input(rs.old_name)
+        existing_columns_right = set(r.old_name for r in right_select.renames if r.is_available)
+        existing_columns_left = set(r.old_name for r in left_select.renames if r.is_available)
+        missing_incoming_left_columns = [ilc for ilc in left_columns if ilc not in existing_columns_left]
+        missing_incoming_right_columns = [irc for irc in right_columns if irc not in existing_columns_right]
+        if not hasattr(setting_input, 'auto_keep_left'):
+            setting_input.auto_keep_left = False
+        if not hasattr(setting_input, 'auto_keep_right'):
+            setting_input.auto_keep_right = False
+        for milc in missing_incoming_left_columns:
+            select_input = transform_schema.SelectInput(old_name=milc, keep=setting_input.auto_keep_left)
+            setting_input.cross_join_input.add_new_select_column(select_input, 'left')
+        for mirc in missing_incoming_right_columns:
+            select_input = transform_schema.SelectInput(old_name=mirc, keep=setting_input.auto_keep_right)
+            setting_input.cross_join_input.add_new_select_column(select_input, 'right')
+    return node_data
+
+
+def check_if_fuzzy_match_is_valid(left_columns: Iterable[str], right_columns: Iterable[str],
+                                  fuzzy_map: transform_schema.FuzzyMap) -> bool:
+    if fuzzy_map.left_col not in left_columns:
+        return False
+    if fuzzy_map.right_col not in right_columns:
+        return False
+    return True
+
+
+@setting_updator_method
+def fuzzy_match(node_data: NodeData):
+    if node_data.right_input and node_data.main_input:
+        setting_input: input_schema.NodeFuzzyMatch = node_data.setting_input
+        left_columns = set(node_data.main_input.columns)
+        right_columns = set(node_data.right_input.columns)
+        left_select = setting_input.join_input.left_select
+        right_select = setting_input.join_input.right_select
+        for fuzzy_map in setting_input.join_input.join_mapping:
+            fuzzy_map.valid = check_if_fuzzy_match_is_valid(left_columns, right_columns, fuzzy_map)
+        for ls in left_select.renames:
+            if ls.old_name not in right_columns:
+                left_select.remove_select_input(ls.old_name)
+        for rs in right_select.renames:
+            if rs.old_name not in right_columns:
+                right_select.remove_select_input(rs.old_name)
+        existing_columns_right = set(r.old_name for r in right_select.renames if r.is_available)
+        existing_columns_left = set(r.old_name for r in left_select.renames if r.is_available)
+        missing_incoming_left_columns = [ilc for ilc in left_columns if ilc not in existing_columns_left]
+        missing_incoming_right_columns = [irc for irc in right_columns if irc not in existing_columns_right]
+        if not hasattr(setting_input, 'auto_keep_left'):
+            setting_input.auto_keep_left = False
+        if not hasattr(setting_input, 'auto_keep_right'):
+            setting_input.auto_keep_right = False
+        for milc in missing_incoming_left_columns:
+            select_input = transform_schema.SelectInput(old_name=milc, keep=setting_input.auto_keep_left)
+            setting_input.join_input.add_new_select_column(select_input, 'left')
+        for mirc in missing_incoming_right_columns:
+            select_input = transform_schema.SelectInput(old_name=mirc, keep=setting_input.auto_keep_right)
+            setting_input.join_input.add_new_select_column(select_input, 'right')
+    return node_data

flowfile_core/flowfile/sources/external_sources/__init__.py

@@ -0,0 +1,3 @@
+from flowfile_core.flowfile.sources.external_sources import custom_external_sources
+# from flowfile.sources.external_sources.custom_external_sources.external_source import check_for_key_vault_existence
+# from flowfile.sources.external_sources.airbyte_sources.airbyte import AirbyteSource

flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py

@@ -0,0 +1,159 @@
+import os
+from ast import literal_eval
+import polars as pl
+from typing import Any, Dict, Generator, List, Optional
+from flowfile_core.configs import logger
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.flowfile.sources.external_sources.base_class import ExternalDataSource
+from flowfile_core.flowfile.sources.external_sources.airbyte_sources.models import (
+    AirbyteProperty, JsonSchema, AirbyteResponse, AirbyteSettings
+)
+
+
+class LazyAirbyteImporter:
+    """Lazy importer for airbyte module."""
+    _airbyte = None
+
+    @classmethod
+    def get_airbyte(cls):
+        if cls._airbyte is None:
+            logger.info("Importing airbyte module")
+            import airbyte as ab
+            cls._airbyte = ab
+        return cls._airbyte
+
+
+class AirbyteSource(ExternalDataSource):
+    stream: str
+    source_name: str
+    cache_store: Optional['airbyte.results.ReadResult'] = None
+    _type: str
+    is_collected: bool
+    _airbyte_response: Optional[AirbyteResponse] = None
+    _airbyte_module = None
+    _enforce_full_refresh: Optional[bool] = True
+    version: Optional[str] = None
+
+    def __init__(self, airbyte_settings: AirbyteSettings):
+        self.is_collected = False
+        self._airbyte_response = None
+        self.stream = airbyte_settings.stream
+        self.source_name = airbyte_settings.source_name
+        self._enforce_full_refresh = airbyte_settings.enforce_full_refresh
+        if hasattr(airbyte_settings, 'version'):
+            self.version = airbyte_settings.version
+
+        # Handle config
+        if airbyte_settings.config_ref and not airbyte_settings.config:
+            logger.info(f"Getting config from {airbyte_settings.config_ref}")
+            config = literal_eval(os.environ.get(airbyte_settings.config_ref))
+        else:
+            logger.info(f"Using provided config")
+            config = airbyte_settings.config
+
+        if config is None:
+            raise ValueError("Config must be provided")
+
+        self.config = config
+        self._type = 'airbyte'
+        self.read_result = None
+
+        # Only load source if fields aren't provided
+        if not airbyte_settings.fields:
+            self.load_source(airbyte_settings)
+        else:
+            logger.info('Using provided schema')
+            self.schema = [
+                FlowfileColumn.from_input(column_name=col.name, data_type=col.data_type)
+                for col in airbyte_settings.fields
+            ]
+
+    def load_source(self, airbyte_settings: AirbyteSettings):
+        logger.info(f"Loading source {self.source_name}")
+        if airbyte_settings.fields is not None and len(airbyte_settings.fields) > 0:
+            logger.info('Using provided schema')
+            self.schema = [
+                FlowfileColumn.from_input(column_name=col.name, data_type=col.data_type)
+                for col in airbyte_settings.fields
+            ]
+        else:
+            logger.info('Using airbyte schema')
+            logger.info(f"Loading source {self.source_name}")
+            _ = self.airbyte_response
+
+    @property
+    def airbyte_response(self) -> AirbyteResponse:
+        if self._airbyte_response is None:
+            # Lazy import airbyte
+            ab = LazyAirbyteImporter.get_airbyte()
+
+            source = ab.get_source(
+                name=self.source_name,
+                config=self.config,
+                streams=self.stream,
+                docker_image=True,
+                version=self.version
+            )
+
+            try:
+                source.check()
+            except Exception:
+                logger.warning('Source check failed, trying to continue')
+
+            logger.info(f'Source check passed, starting to load data for {self.stream}')
+
+            json_schema = source.get_stream_json_schema(self.stream)['properties']
+            properties = [
+                AirbyteProperty(name=name, json_schema=JsonSchema(**schema))
+                for name, schema in json_schema.items()
+            ]
+
+            logger.info(f"Loaded source {self.source_name}")
+            self._airbyte_response = AirbyteResponse(properties=properties, source=source)
+            self.schema = self.parse_schema(self._airbyte_response)
+
+        return self._airbyte_response
+
+    def get_initial_data(self):
+        return []
+
+    def get_iter(self) -> Generator[Dict[str, Any], None, None]:
+        logger.warning('Getting data in iteration, this is suboptimal')
+        data = self.data_getter()
+        for row in data:
+            yield row
+        self.is_collected = True
+
+    def get_sample(self, n: int = 10000):
+        logger.warning('Getting sample in iteration, this is suboptimal')
+        data = self.get_iter()
+        for i in range(n):
+            try:
+                yield next(data)
+            except StopIteration:
+                break
+
+    @staticmethod
+    def parse_schema(airbyte_response: AirbyteResponse) -> List[FlowfileColumn]:
+        return airbyte_response.get_flow_file_columns()
+
+    def get_df(self):
+        if self.read_result is None:
+            self.read_result = self.airbyte_response.source.read()
+
+        df = self.read_result[self.stream].to_pandas()
+        drop_cols = [c for c in df.columns if c.startswith('_airbyte')]
+        df.drop(drop_cols, axis=1, inplace=True)
+        return df
+
+    def get_pl_df(self) -> pl.DataFrame:
+        self.is_collected = True
+        return pl.from_pandas(self.get_df())
+
+    def data_getter(self) -> List[Dict]:
+        return self.get_df().to_dict(orient='records')
+
+    @classmethod
+    def create_from_frontend_input(cls, config: Any, stream_name: str, source_name: str):
+        # Implementation details to be added
+        pass

flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py

@@ -0,0 +1,172 @@
+from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union
+from pydantic import BaseModel, field_validator, ConfigDict
+import polars as pl
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import type_to_polars_str
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.schemas.input_schema import MinimalFieldInfo
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
+from flowfile_core.configs import logger
+
+# Use TYPE_CHECKING to avoid circular imports
+if TYPE_CHECKING:
+    from airbyte import Source
+else:
+    Source = Any
+
+
+class LazyAirbyteSource:
+    """Lazy wrapper for airbyte Source class."""
+    _source_class = None
+
+    @classmethod
+    def get_source_class(cls):
+        if cls._source_class is None:
+            logger.info("Importing airbyte Source class")
+            from airbyte import Source
+            cls._source_class = Source
+        return cls._source_class
+
+
+class JsonSchema(BaseModel):
+    type: Optional[Union[str, List[str]]]
+    airbyte_type: Optional[Union[str, List[str]]] = None
+    format: Optional[str] = None
+
+    def get_pl_type(self) -> pl.DataType:
+        if self.format:
+            format_mapping = {
+                'date-time': 'datetime',
+                'date': 'date',
+                'time': 'time'
+            }
+            dtype = format_mapping.get(self.format, 'string')
+        else:
+            type_mapping = {
+                'string': 'string',
+                'boolean': 'bool',
+                'integer': 'int',
+                'number': 'float',
+                'array': 'string',
+                'object': 'string'
+            }
+            if isinstance(self.type, list) and len(self.type) >= 1:
+                _type_mappings = (type_mapping.get(t) for t in self.type)
+                dtype = next((t for t in _type_mappings if t is not None), self.type[0])
+            elif isinstance(self.type, list) and len(self.type) == 0:
+                dtype = 'string'
+            else:
+                dtype = type_mapping.get(self.type[0] if isinstance(self.type, list) else self.type, 'string')
+        return type_to_polars_str(dtype)
+
+
+class AirbyteProperty(BaseModel):
+    name: str
+    json_schema: JsonSchema
+
+    def get_pl_type(self) -> PlType:
+        return PlType(column_name=self.name, pl_datatype=self.json_schema.get_pl_type())
+
+
+class AirbyteResponse(BaseModel):
+    source: Any  # Using Any to avoid direct Source import
+    properties: list[AirbyteProperty]
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @field_validator('source')
+    @classmethod
+    def validate_source(cls, v: Any) -> Any:
+        source_class = LazyAirbyteSource.get_source_class()
+        if not isinstance(v, source_class):
+            raise ValueError(f"Source must be an instance of airbyte.Source, got {type(v)}")
+        return v
+
+    def get_flow_file_columns(self) -> List[FlowfileColumn]:
+        return [
+            FlowfileColumn.create_from_polars_type(c.get_pl_type(), col_index=i)
+            for i, c in enumerate(self.properties)
+        ]
+
+
+class GenericProperties(BaseModel):
+    type: str
+    title: Optional[str] = None
+    description: Optional[str] = None
+    order: Optional[int] = None
+    required: Optional[List[str]] = None
+    airbyte_secret: Optional[bool] = None
+    pattern: Optional[str] = None
+    pattern_descriptor: Optional[str] = None
+    format: Optional[str] = None
+    examples: Optional[List[Any]] = None
+    enum: Optional[List[str]] = None
+    minimum: Optional[float] = None
+    maximum: Optional[float] = None
+    items: Optional[Any] = None
+    properties: Optional[Dict[str, Any]] = None
+
+    @field_validator('items', 'properties')
+    @classmethod
+    def validate_nested(cls, value: Any) -> Any:
+        if isinstance(value, dict):
+            if 'type' in value:
+                return GenericProperties(**value)
+            return {k: GenericProperties(**v) if isinstance(v, dict) else v for k, v in value.items()}
+        return value
+
+
+class GenericSchema(BaseModel):
+    title: str
+    type: str
+    required: Optional[List[str]] = None
+    additionalProperties: Optional[bool] = None
+    properties: Dict[str, GenericProperties]
+
+
+class FieldProperty(BaseModel):
+    title: Optional[str] = None
+    type: str
+    key: str
+    description: Optional[str] = None
+    airbyte_secret: Optional[bool] = None
+    input_value: Optional[str] = None
+    default: Any
+
+
+class OverallFieldProperty(BaseModel):
+    title: Optional[str] = None
+    type: str
+    key: str
+    required: bool
+    properties: List[FieldProperty]
+    items: Optional[List[FieldProperty]]
+    isOpen: bool
+    description: Optional[str] = None
+    input_value: Optional[str] = None
+    airbyte_secret: Optional[bool] = None
+    default: Any
+
+
+class AirbyteConfigTemplate(BaseModel):
+    source_name: str
+    docs_url: Optional[str] = None
+    config_spec: Dict
+    available_streams: Optional[List[str]] = None
+
+
+class AirbyteSettings(BaseModel):
+    source_name: str
+    stream: str
+    config_ref: Optional[str] = None
+    config: Optional[Dict] = None
+    fields: Optional[List[MinimalFieldInfo]] = None
+    enforce_full_refresh: Optional[bool] = True
+    flowfile_flow_id: int
+    flowfile_node_id: int
+    version: Optional[str] = None
+
+
+def get_source_instance(*args, **kwargs) -> 'Source':
+    """Helper function to get a Source instance with lazy loading."""
+    source_class = LazyAirbyteSource.get_source_class()
+    return source_class(*args, **kwargs)

flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py

@@ -0,0 +1,173 @@
+from typing import List, Dict, Optional, Any, Type
+from flowfile_core.configs import logger
+from flowfile_core.flowfile.sources.external_sources.airbyte_sources.models import AirbyteConfigTemplate, \
+    AirbyteSettings
+from flowfile_core.schemas.external_sources.airbyte_schemas import AirbyteConfig
+from flowfile_core.flowfile.connection_manager import connection_manager
+from flowfile_core.flowfile.connection_manager._connection_manager import Connection
+
+
+class LazyAirbyteImporter:
+    """Lazy importer for airbyte module."""
+    _airbyte = None
+
+    @classmethod
+    def get_airbyte(cls):
+        if cls._airbyte is None:
+            logger.info("Importing airbyte module")
+            import airbyte as ab
+            cls._airbyte = ab
+        return cls._airbyte
+
+
+class AirbyteConfigHandler:
+    _available_connectors: Optional[List[str]] = None
+    configs: Dict[str, AirbyteConfigTemplate]
+    _airbyte = None
+
+    def __init__(self):
+        self.configs = {}
+        self._available_connectors = None
+
+    @property
+    def airbyte(self):
+        """Lazy load airbyte module when needed."""
+        if self._airbyte is None:
+            self._airbyte = LazyAirbyteImporter.get_airbyte()
+        return self._airbyte
+
+    @property
+    def available_connectors(self) -> List[str]:
+        if self._available_connectors is None:
+            self._available_connectors = [
+                c.replace('source-', '')
+                for c in self.airbyte.get_available_connectors()
+                if c.startswith('source-')
+            ]
+        return self._available_connectors
+
+    @property
+    def available_configs(self) -> List[str]:
+        return list(self.configs.keys())
+
+    def get_config(self, config_name: str) -> AirbyteConfigTemplate:
+        """Get configuration for a specific source."""
+        logger.info(f"Getting config for {config_name}")
+
+        if config_name not in self.configs:
+            try:
+                source = self.airbyte.get_source(
+                    name=config_name,
+                    install_if_missing=True,
+                    docker_image=True
+                )
+                logger.info(f"Got source {config_name}")
+
+                self.configs[config_name] = AirbyteConfigTemplate(
+                    config_spec=source.config_spec,
+                    docs_url=source.docs_url,
+                    source_name=config_name
+                )
+            except Exception as e:
+                logger.error(f"Error getting config for {config_name}: {str(e)}")
+                raise
+
+        return self.configs[config_name]
+
+    def get_available_streams(self, config_name: str, config_settings: Any) -> List[str]:
+        """Get available streams for a specific configuration."""
+        if config_name not in self.configs:
+            raise ValueError(f"Config {config_name} not found")
+
+        logger.info(f'Getting available streams for {config_name}')
+
+        try:
+            source = self.airbyte.get_source(
+                name=config_name,
+                install_if_missing=True,
+                config=config_settings,
+                docker_image=True
+            )
+            streams = source.get_available_streams()
+            if len(streams) == 0 or streams is None:
+                raise ValueError(f"No streams found for {config_name}")
+            self.configs[config_name].available_streams = streams
+            return self.configs[config_name].available_streams
+
+        except Exception as e:
+            logger.error(f"Error getting streams for {config_name}: {str(e)}")
+            raise
+
+
+# Create singleton instance
+airbyte_config_handler = AirbyteConfigHandler()
+
+
+class AirbyteHandler:
+    """Handler for Airbyte configurations and connections."""
+    config: AirbyteConfig
+
+    def __init__(self, airbyte_config: AirbyteConfig):
+        self.config = airbyte_config
+
+    def set_airbyte_config(self, airbyte_config_in: AirbyteConfig) -> AirbyteConfig:
+        """Update the current configuration."""
+        self.config.mapped_config_spec = airbyte_config_in.mapped_config_spec
+        self.config.parsed_config = airbyte_config_in.parsed_config
+        return airbyte_config_in
+
+    def get_available_streams(self) -> List[str]:
+        """Get available streams for the current configuration."""
+        config_template = airbyte_config_handler.configs.get(self.config.full_source_name)
+
+        if not config_template:
+            logger.warning(
+                f"Config {self.config.source_name} not found, trying to recreate the config"
+            )
+            try:
+                _ = airbyte_config_handler.get_config(self.config.full_source_name)
+                logger.info(f"Config {self.config.source_name} recreated")
+            except Exception as e:
+                logger.error(f"Error recreating config: {str(e)}")
+                raise
+
+        return airbyte_config_handler.get_available_streams(
+            self.config.full_source_name,
+            self.config.mapped_config_spec
+        )
+
+    def save_connection(self, connection_name: str) -> None:
+        """Save the current configuration as a connection."""
+        connection = Connection(
+            group=self.config.source_name,
+            name=connection_name,
+            config_setting=self.config,
+            type='airbyte'
+        )
+
+        connection_manager.add_connection(
+            self.config.source_name,
+            connection_name=connection_name,
+            connection=connection
+        )
+
+
+def airbyte_settings_from_config(airbyte_config: AirbyteConfig, flow_id: int, node_id: int|str) -> AirbyteSettings:
+    """Create AirbyteSettings from an AirbyteConfig."""
+    if airbyte_config.config_mode == 'key_vault':
+        connection = connection_manager.get_connection(
+            connection_group=airbyte_config.source_name,
+            connection_name=airbyte_config.connection_name
+        )
+        config = connection.config_setting.mapped_config_spec
+    else:
+        config = airbyte_config.mapped_config_spec
+
+    return AirbyteSettings(
+        source_name=airbyte_config.full_source_name,
+        stream=airbyte_config.selected_stream,
+        config=config,
+        flowfile_flow_id=flow_id,
+        flowfile_node_id=node_id,
+        version=airbyte_config.version
+    )