airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py

@@ -1,64 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Mapping, Type
-
-from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream
-from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
-from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
-from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
-from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
-from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
-from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
-from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
-from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
-from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter
-from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler
-from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler
-from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import HttpResponseFilter
-from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester
-from airbyte_cdk.sources.declarative.requesters.paginators.default_paginator import RequestOption
-from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination
-from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
-from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import (
-    InterpolatedRequestOptionsProvider,
-)
-from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider
-from airbyte_cdk.sources.declarative.requesters.requester import Requester
-from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
-from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
-from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
-from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
-from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer
-from airbyte_cdk.sources.declarative.stream_slicers.substream_slicer import ParentStreamConfig
-from airbyte_cdk.sources.streams.core import Stream
-
-"""
-DEFAULT_IMPLEMENTATIONS_REGISTRY contains a mapping of interface -> subclass
-enabling the factory to instantiate a reasonable default class when no type or classname is specified
-"""
-DEFAULT_IMPLEMENTATIONS_REGISTRY: Mapping[Type, Type] = {
-    ConnectionChecker: CheckStream,
-    Decoder: JsonDecoder,
-    ErrorHandler: DefaultErrorHandler,
-    HttpResponseFilter: HttpResponseFilter,
-    HttpSelector: RecordSelector,
-    InterpolatedBoolean: InterpolatedBoolean,
-    InterpolatedRequestOptionsProvider: InterpolatedRequestOptionsProvider,
-    InterpolatedString: InterpolatedString,
-    MinMaxDatetime: MinMaxDatetime,
-    Paginator: NoPagination,
-    ParentStreamConfig: ParentStreamConfig,
-    RecordExtractor: DpathExtractor,
-    RequestOption: RequestOption,
-    RequestOptionsProvider: InterpolatedRequestOptionsProvider,
-    Requester: HttpRequester,
-    Retriever: SimpleRetriever,
-    SchemaLoader: DefaultSchemaLoader,
-    Stream: DeclarativeStream,
-    StreamSlicer: SinglePartitionRouter,
-}

airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py

@@ -1,16 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from enum import Enum
-
-
-class ResponseAction(Enum):
-    """
-    Response statuses for non retriable responses
-    """
-
-    SUCCESS = "SUCCESS"  # "Request was successful"
-    FAIL = "FAIL"  # "Request failed unexpectedly"
-    IGNORE = "IGNORE"  # "Request failed but can be ignored"
-    RETRY = "RETRY"  # Request failed and should be retried

airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py

@@ -1,68 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Final, Optional, Union
-
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
-
-
-class ResponseStatus:
-    """
-    ResponseAction amended with backoff time if an action is RETRY
-    """
-
-    def __init__(self, response_action: Union[ResponseAction, str], retry_in: Optional[float] = None, error_message: str = ""):
-        """
-        :param response_action: response action to execute
-        :param retry_in: backoff time (if action is RETRY)
-        :param error_message: the error to be displayed back to the customer
-        """
-        if isinstance(response_action, str):
-            response_action = ResponseAction[response_action]
-        if retry_in and response_action != ResponseAction.RETRY:
-            raise ValueError(f"Unexpected backoff time ({retry_in} for non-retryable response action {response_action}")
-        self._retry_in = retry_in
-        self._action = response_action
-        self._error_message = error_message
-
-    @property
-    def action(self):
-        """The ResponseAction to execute when a response matches the filter"""
-        return self._action
-
-    @property
-    def retry_in(self) -> Optional[float]:
-        """How long to backoff before retrying a response. None if no wait required."""
-        return self._retry_in
-
-    @property
-    def error_message(self) -> str:
-        """The message to be displayed when an error response is received"""
-        return self._error_message
-
-    @classmethod
-    def retry(cls, retry_in: Optional[float]) -> "ResponseStatus":
-        """
-        Returns a ResponseStatus defining how long to backoff before retrying
-
-        :param retry_in: how long to backoff before retrying. None if no wait required
-        :return: A response status defining how long to backoff before retrying
-        """
-        return ResponseStatus(ResponseAction.RETRY, retry_in)
-
-    def __eq__(self, other):
-        if not other:
-            return not self
-        return self.action == other.action and self.retry_in == other.retry_in
-
-    def __hash__(self):
-        return hash([self.action, self.retry_in])
-
-
-"""Response is successful. No need to retry"""
-SUCCESS: Final[ResponseStatus] = ResponseStatus(ResponseAction.SUCCESS)
-"""Response is unsuccessful. The failure needs to be handled"""
-FAIL: Final[ResponseStatus] = ResponseStatus(ResponseAction.FAIL)
-"""Response is unsuccessful, but can be ignored. No need to retry"""
-IGNORE: Final[ResponseStatus] = ResponseStatus(ResponseAction.IGNORE)

airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py

@@ -1,114 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import itertools
-from collections import ChainMap
-from dataclasses import InitVar, dataclass
-from typing import Any, Iterable, List, Mapping, Optional
-
-from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer
-from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState
-
-
-@dataclass
-class CartesianProductStreamSlicer(StreamSlicer):
-    """
-    Stream slicers that iterates over the cartesian product of input stream slicers
-    Given 2 stream slicers with the following slices:
-    A: [{"i": 0}, {"i": 1}, {"i": 2}]
-    B: [{"s": "hello"}, {"s": "world"}]
-    the resulting stream slices are
-    [
-        {"i": 0, "s": "hello"},
-        {"i": 0, "s": "world"},
-        {"i": 1, "s": "hello"},
-        {"i": 1, "s": "world"},
-        {"i": 2, "s": "hello"},
-        {"i": 2, "s": "world"},
-    ]
-
-    Attributes:
-        stream_slicers (List[StreamSlicer]): Underlying stream slicers. The RequestOptions (e.g: Request headers, parameters, etc..) returned by this slicer are the combination of the RequestOptions of its input slicers. If there are conflicts e.g: two slicers define the same header or request param, the conflict is resolved by taking the value from the first slicer, where ordering is determined by the order in which slicers were input to this composite slicer.
-    """
-
-    stream_slicers: List[StreamSlicer]
-    parameters: InitVar[Mapping[str, Any]]
-
-    def get_request_params(
-        self,
-        *,
-        stream_state: Optional[StreamState] = None,
-        stream_slice: Optional[StreamSlice] = None,
-        next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> Mapping[str, Any]:
-        return dict(
-            ChainMap(
-                *[  # type: ignore # ChainMap expects a MutableMapping[Never, Never] for reasons
-                    s.get_request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token)
-                    for s in self.stream_slicers
-                ]
-            )
-        )
-
-    def get_request_headers(
-        self,
-        *,
-        stream_state: Optional[StreamState] = None,
-        stream_slice: Optional[StreamSlice] = None,
-        next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> Mapping[str, Any]:
-        return dict(
-            ChainMap(
-                *[  # type: ignore # ChainMap expects a MutableMapping[Never, Never] for reasons
-                    s.get_request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token)
-                    for s in self.stream_slicers
-                ]
-            )
-        )
-
-    def get_request_body_data(
-        self,
-        *,
-        stream_state: Optional[StreamState] = None,
-        stream_slice: Optional[StreamSlice] = None,
-        next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> Mapping[str, Any]:
-        return dict(
-            ChainMap(
-                *[  # type: ignore # ChainMap expects a MutableMapping[Never, Never] for reasons
-                    s.get_request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token)
-                    for s in self.stream_slicers
-                ]
-            )
-        )
-
-    def get_request_body_json(
-        self,
-        *,
-        stream_state: Optional[StreamState] = None,
-        stream_slice: Optional[StreamSlice] = None,
-        next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> Mapping[str, Any]:
-        return dict(
-            ChainMap(
-                *[  # type: ignore # ChainMap expects a MutableMapping[Never, Never] for reasons
-                    s.get_request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token)
-                    for s in self.stream_slicers
-                ]
-            )
-        )
-
-    def stream_slices(self) -> Iterable[StreamSlice]:
-        sub_slices = (s.stream_slices() for s in self.stream_slicers)
-        product = itertools.product(*sub_slices)
-        for stream_slice_tuple in product:
-            partition = dict(ChainMap(*[s.partition for s in stream_slice_tuple]))
-            cursor_slices = [s.cursor_slice for s in stream_slice_tuple if s.cursor_slice]
-            if len(cursor_slices) > 1:
-                raise ValueError(f"There should only be a single cursor slice. Found {cursor_slices}")
-            if cursor_slices:
-                cursor_slice = cursor_slices[0]
-            else:
-                cursor_slice = {}
-            yield StreamSlice(partition=partition, cursor_slice=cursor_slice)

airbyte_cdk/sources/deprecated/base_source.py

@@ -1,94 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import copy
-import logging
-from datetime import datetime
-from typing import Any, Iterable, Mapping, MutableMapping, Type
-
-from airbyte_cdk.models import (
-    AirbyteCatalog,
-    AirbyteConnectionStatus,
-    AirbyteMessage,
-    AirbyteRecordMessage,
-    AirbyteStateMessage,
-    ConfiguredAirbyteCatalog,
-    ConfiguredAirbyteStream,
-    Status,
-    SyncMode,
-)
-from airbyte_cdk.models import Type as MessageType
-from airbyte_cdk.sources.source import Source
-
-from .client import BaseClient
-
-
-class BaseSource(Source):
-    """Base source that designed to work with clients derived from BaseClient"""
-
-    client_class: Type[BaseClient]
-
-    @property
-    def name(self) -> str:
-        """Source name"""
-        return self.__class__.__name__
-
-    def _get_client(self, config: Mapping):
-        """Construct client"""
-        return self.client_class(**config)
-
-    def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
-        """Discover streams"""
-        client = self._get_client(config)
-
-        return AirbyteCatalog(streams=[stream for stream in client.streams])
-
-    def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
-        """Check connection"""
-        client = self._get_client(config)
-        alive, error = client.health_check()
-        if not alive:
-            return AirbyteConnectionStatus(status=Status.FAILED, message=str(error))
-
-        return AirbyteConnectionStatus(status=Status.SUCCEEDED)
-
-    def read(
-        self, logger: logging.Logger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] = None
-    ) -> Iterable[AirbyteMessage]:
-        state = state or {}
-        client = self._get_client(config)
-
-        logger.info(f"Starting syncing {self.name}")
-        total_state = copy.deepcopy(state)
-        for configured_stream in catalog.streams:
-            try:
-                yield from self._read_stream(logger=logger, client=client, configured_stream=configured_stream, state=total_state)
-
-            except Exception:
-                logger.exception(f"Encountered an exception while reading stream {self.name}")
-                raise
-
-        logger.info(f"Finished syncing {self.name}")
-
-    def _read_stream(
-        self, logger: logging.Logger, client: BaseClient, configured_stream: ConfiguredAirbyteStream, state: MutableMapping[str, Any]
-    ):
-        stream_name = configured_stream.stream.name
-        use_incremental = configured_stream.sync_mode == SyncMode.incremental and client.stream_has_state(stream_name)
-
-        if use_incremental and state.get(stream_name):
-            logger.info(f"Set state of {stream_name} stream to {state.get(stream_name)}")
-            client.set_stream_state(stream_name, state.get(stream_name))
-
-        logger.info(f"Syncing {stream_name} stream")
-        for record in client.read_stream(configured_stream.stream):
-            now = int(datetime.now().timestamp()) * 1000
-            message = AirbyteRecordMessage(stream=stream_name, data=record, emitted_at=now)
-            yield AirbyteMessage(type=MessageType.RECORD, record=message)
-
-        if use_incremental and client.get_stream_state(stream_name):
-            state[stream_name] = client.get_stream_state(stream_name)
-            # output state object only together with other stream states
-            yield AirbyteMessage(type=MessageType.STATE, state=AirbyteStateMessage(data=state))

airbyte_cdk/sources/deprecated/client.py

@@ -1,99 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import inspect
-from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Generator, List, Mapping, Tuple
-
-from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, SyncMode
-from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
-
-
-def package_name_from_class(cls: object) -> str:
-    """Find the package name given a class name"""
-    module: Any = inspect.getmodule(cls)
-    return module.__name__.split(".")[0]
-
-
-class StreamStateMixin:
-    def get_stream_state(self, name: str) -> Any:
-        """Get state of stream with corresponding name"""
-        raise NotImplementedError
-
-    def set_stream_state(self, name: str, state: Any):
-        """Set state of stream with corresponding name"""
-        raise NotImplementedError
-
-    def stream_has_state(self, name: str) -> bool:
-        """Tell if stream supports incremental sync"""
-        return False
-
-
-class BaseClient(StreamStateMixin, ABC):
-    """Base client for API"""
-
-    schema_loader_class = ResourceSchemaLoader
-
-    def __init__(self, **kwargs):
-        package_name = package_name_from_class(self.__class__)
-        self._schema_loader = self.schema_loader_class(package_name)
-        self._stream_methods = self._enumerate_methods()
-
-    def _enumerate_methods(self) -> Mapping[str, Callable]:
-        """Detect available streams and return mapping"""
-        prefix = "stream__"
-        mapping = {}
-        methods = inspect.getmembers(self.__class__, predicate=inspect.isfunction)
-        for name, method in methods:
-            if name.startswith(prefix):
-                mapping[name[len(prefix) :]] = getattr(self, name)
-
-        return mapping
-
-    @staticmethod
-    def _get_fields_from_stream(stream: AirbyteStream) -> List[str]:
-        return list(stream.json_schema.get("properties", {}).keys())
-
-    def _get_stream_method(self, name: str) -> Callable:
-        method = self._stream_methods.get(name)
-        if not method:
-            raise ValueError(f"Client does not know how to read stream `{name}`")
-        return method
-
-    def read_stream(self, stream: AirbyteStream) -> Generator[Dict[str, Any], None, None]:
-        """Yield records from stream"""
-        method = self._get_stream_method(stream.name)
-        fields = self._get_fields_from_stream(stream)
-
-        for message in method(fields=fields):
-            yield dict(message)
-
-    @property
-    def streams(self) -> Generator[AirbyteStream, None, None]:
-        """List of available streams"""
-        for name, method in self._stream_methods.items():
-            supported_sync_modes = [SyncMode.full_refresh]
-            source_defined_cursor = False
-            if self.stream_has_state(name):
-                supported_sync_modes += [SyncMode.incremental]
-                source_defined_cursor = True
-
-            yield AirbyteStream(
-                name=name,
-                json_schema=self._schema_loader.get_schema(name),
-                supported_sync_modes=supported_sync_modes,
-                source_defined_cursor=source_defined_cursor,
-            )
-
-    @abstractmethod
-    def health_check(self) -> Tuple[bool, str]:
-        """Check if service is up and running"""
-
-
-def configured_catalog_from_client(client: BaseClient) -> ConfiguredAirbyteCatalog:
-    """Helper to generate configured catalog for testing"""
-    catalog = ConfiguredAirbyteCatalog(streams=[ConfiguredAirbyteStream(stream=stream) for stream in client.streams])
-
-    return catalog

airbyte_cdk/sources/singer/__init__.py

@@ -1,8 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#
-
-from .singer_helpers import SingerHelper, SyncModeInfo
-from .source import ConfigContainer, SingerSource
-
-__all__ = ["ConfigContainer", "SingerSource", "SyncModeInfo", "SingerHelper"]