airbyte-cdk 6.38.0.dev0__py3-none-any.whl → 6.38.2__py3-none-any.whl

airbyte_cdk/sources/embedded/base_integration.py

@@ -1,61 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from abc import ABC, abstractmethod
-from typing import Generic, Iterable, Optional, TypeVar
-
-from airbyte_cdk.connector import TConfig
-from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type
-from airbyte_cdk.sources.embedded.catalog import (
-    create_configured_catalog,
-    get_stream,
-    get_stream_names,
-)
-from airbyte_cdk.sources.embedded.runner import SourceRunner
-from airbyte_cdk.sources.embedded.tools import get_defined_id
-from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
-
-TOutput = TypeVar("TOutput")
-
-
-class BaseEmbeddedIntegration(ABC, Generic[TConfig, TOutput]):
-    def __init__(self, runner: SourceRunner[TConfig], config: TConfig):
-        check_config_against_spec_or_exit(config, runner.spec())
-
-        self.source = runner
-        self.config = config
-
-        self.last_state: Optional[AirbyteStateMessage] = None
-
-    @abstractmethod
-    def _handle_record(self, record: AirbyteRecordMessage, id: Optional[str]) -> Optional[TOutput]:
-        """
-        Turn an Airbyte record into the appropriate output type for the integration.
-        """
-        pass
-
-    def _load_data(
-        self, stream_name: str, state: Optional[AirbyteStateMessage] = None
-    ) -> Iterable[TOutput]:
-        catalog = self.source.discover(self.config)
-        stream = get_stream(catalog, stream_name)
-        if not stream:
-            raise ValueError(
-                f"Stream {stream_name} not found, the following streams are available: {', '.join(get_stream_names(catalog))}"
-            )
-        if SyncMode.incremental not in stream.supported_sync_modes:
-            configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.full_refresh)
-        else:
-            configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.incremental)
-
-        for message in self.source.read(self.config, configured_catalog, state):
-            if message.type == Type.RECORD:
-                output = self._handle_record(
-                    message.record,
-                    get_defined_id(stream, message.record.data),  # type: ignore[union-attr, arg-type]
-                )
-                if output:
-                    yield output
-            elif message.type is Type.STATE and message.state:
-                self.last_state = message.state

airbyte_cdk/sources/embedded/catalog.py

@@ -1,57 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import List, Optional
-
-from airbyte_cdk.models import (
-    AirbyteCatalog,
-    AirbyteStream,
-    ConfiguredAirbyteCatalog,
-    ConfiguredAirbyteStream,
-    DestinationSyncMode,
-    SyncMode,
-)
-from airbyte_cdk.sources.embedded.tools import get_first
-
-
-def get_stream(catalog: AirbyteCatalog, stream_name: str) -> Optional[AirbyteStream]:
-    return get_first(catalog.streams, lambda s: s.name == stream_name)
-
-
-def get_stream_names(catalog: AirbyteCatalog) -> List[str]:
-    return [stream.name for stream in catalog.streams]
-
-
-def to_configured_stream(
-    stream: AirbyteStream,
-    sync_mode: SyncMode = SyncMode.full_refresh,
-    destination_sync_mode: DestinationSyncMode = DestinationSyncMode.append,
-    cursor_field: Optional[List[str]] = None,
-    primary_key: Optional[List[List[str]]] = None,
-) -> ConfiguredAirbyteStream:
-    return ConfiguredAirbyteStream(
-        stream=stream,
-        sync_mode=sync_mode,
-        destination_sync_mode=destination_sync_mode,
-        cursor_field=cursor_field,
-        primary_key=primary_key,
-    )
-
-
-def to_configured_catalog(
-    configured_streams: List[ConfiguredAirbyteStream],
-) -> ConfiguredAirbyteCatalog:
-    return ConfiguredAirbyteCatalog(streams=configured_streams)
-
-
-def create_configured_catalog(
-    stream: AirbyteStream, sync_mode: SyncMode = SyncMode.full_refresh
-) -> ConfiguredAirbyteCatalog:
-    configured_streams = [
-        to_configured_stream(
-            stream, sync_mode=sync_mode, primary_key=stream.source_defined_primary_key
-        )
-    ]
-
-    return to_configured_catalog(configured_streams)

airbyte_cdk/sources/embedded/runner.py

@@ -1,57 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import logging
-from abc import ABC, abstractmethod
-from typing import Generic, Iterable, Optional
-
-from airbyte_cdk.connector import TConfig
-from airbyte_cdk.models import (
-    AirbyteCatalog,
-    AirbyteMessage,
-    AirbyteStateMessage,
-    ConfiguredAirbyteCatalog,
-    ConnectorSpecification,
-)
-from airbyte_cdk.sources.source import Source
-
-
-class SourceRunner(ABC, Generic[TConfig]):
-    @abstractmethod
-    def spec(self) -> ConnectorSpecification:
-        pass
-
-    @abstractmethod
-    def discover(self, config: TConfig) -> AirbyteCatalog:
-        pass
-
-    @abstractmethod
-    def read(
-        self,
-        config: TConfig,
-        catalog: ConfiguredAirbyteCatalog,
-        state: Optional[AirbyteStateMessage],
-    ) -> Iterable[AirbyteMessage]:
-        pass
-
-
-class CDKRunner(SourceRunner[TConfig]):
-    def __init__(self, source: Source, name: str):
-        self._source = source
-        self._logger = logging.getLogger(name)
-
-    def spec(self) -> ConnectorSpecification:
-        return self._source.spec(self._logger)
-
-    def discover(self, config: TConfig) -> AirbyteCatalog:
-        return self._source.discover(self._logger, config)
-
-    def read(
-        self,
-        config: TConfig,
-        catalog: ConfiguredAirbyteCatalog,
-        state: Optional[AirbyteStateMessage],
-    ) -> Iterable[AirbyteMessage]:
-        return self._source.read(self._logger, config, catalog, state=[state] if state else [])

airbyte_cdk/sources/embedded/tools.py

@@ -1,27 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Any, Callable, Dict, Iterable, Optional
-
-import dpath
-
-from airbyte_cdk.models import AirbyteStream
-
-
-def get_first(
-    iterable: Iterable[Any], predicate: Callable[[Any], bool] = lambda m: True
-) -> Optional[Any]:
-    return next(filter(predicate, iterable), None)
-
-
-def get_defined_id(stream: AirbyteStream, data: Dict[str, Any]) -> Optional[str]:
-    if not stream.source_defined_primary_key:
-        return None
-    primary_key = []
-    for key in stream.source_defined_primary_key:
-        try:
-            primary_key.append(str(dpath.get(data, key)))
-        except KeyError:
-            primary_key.append("__not_found__")
-    return "_".join(primary_key)