airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

unit_tests/sources/fixtures/source_test_fixture.py

@@ -1,153 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-import logging
-from abc import ABC
-from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union
-
-import requests
-from airbyte_cdk.models import (
-    AirbyteStream,
-    ConfiguredAirbyteCatalog,
-    ConfiguredAirbyteStream,
-    ConnectorSpecification,
-    DestinationSyncMode,
-    SyncMode,
-)
-from airbyte_cdk.sources import AbstractSource
-from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.http import HttpStream
-from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator
-from requests.auth import AuthBase
-
-
-class SourceTestFixture(AbstractSource):
-    """
-    This is a concrete implementation of a Source connector that provides implementations of all the methods needed to run sync
-    operations. For simplicity, it also overrides functions that read from files in favor of returning the data directly avoiding
-    the need to load static files (ex. spec.yaml, config.json, configured_catalog.json) into the unit-test package.
-    """
-
-    def __init__(self, streams: Optional[List[Stream]] = None, authenticator: Optional[AuthBase] = None):
-        self._streams = streams
-        self._authenticator = authenticator
-
-    def spec(self, logger: logging.Logger) -> ConnectorSpecification:
-        return ConnectorSpecification(
-            connectionSpecification={
-                "$schema": "http://json-schema.org/draft-07/schema#",
-                "title": "Test Fixture Spec",
-                "type": "object",
-                "required": ["api_token"],
-                "properties": {
-                    "api_token": {
-                        "type": "string",
-                        "title": "API token",
-                        "description": "The token used to authenticate requests to the API.",
-                        "airbyte_secret": True,
-                    }
-                },
-            }
-        )
-
-    def read_config(self, config_path: str) -> Mapping[str, Any]:
-        return {"api_token": "just_some_token"}
-
-    @classmethod
-    def read_catalog(cls, catalog_path: str) -> ConfiguredAirbyteCatalog:
-        return ConfiguredAirbyteCatalog(
-            streams=[
-                ConfiguredAirbyteStream(
-                    stream=AirbyteStream(
-                        name="http_test_stream",
-                        json_schema={},
-                        supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental],
-                        default_cursor_field=["updated_at"],
-                        source_defined_cursor=True,
-                        source_defined_primary_key=[["id"]],
-                    ),
-                    sync_mode=SyncMode.full_refresh,
-                    destination_sync_mode=DestinationSyncMode.overwrite,
-                )
-            ]
-        )
-
-    def check_connection(self, *args, **kwargs) -> Tuple[bool, Optional[Any]]:
-        return True, ""
-
-    def streams(self, *args, **kwargs) -> List[Stream]:
-        return [HttpTestStream(authenticator=self._authenticator)]
-
-
-class HttpTestStream(HttpStream, ABC):
-    url_base = "https://airbyte.com/api/v1/"
-
-    def supports_incremental(self):
-        return True
-
-    @property
-    def availability_strategy(self):
-        return None
-
-    def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
-        return "id"
-
-    def path(
-        self,
-        *,
-        stream_state: Mapping[str, Any] = None,
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> str:
-        return "cast"
-
-    def parse_response(
-        self,
-        response: requests.Response,
-        *,
-        stream_state: Mapping[str, Any],
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> Iterable[Mapping]:
-        body = response.json() or {}
-        return body["records"]
-
-    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-        return None
-
-    def get_json_schema(self) -> Mapping[str, Any]:
-        return {}
-
-
-def fixture_mock_send(self, request, **kwargs) -> requests.Response:
-    """
-    Helper method that can be used by a test to patch the Session.send() function and mock the outbound send operation to provide
-    faster and more reliable responses compared to actual API requests
-    """
-    response = requests.Response()
-    response.request = request
-    response.status_code = 200
-    response.headers = {"header": "value"}
-    response_body = {
-        "records": [
-            {"id": 1, "name": "Celine Song", "position": "director"},
-            {"id": 2, "name": "Shabier Kirchner", "position": "cinematographer"},
-            {"id": 3, "name": "Christopher Bear", "position": "composer"},
-            {"id": 4, "name": "Daniel Rossen", "position": "composer"},
-        ]
-    }
-    response._content = json.dumps(response_body).encode("utf-8")
-    return response
-
-
-class SourceFixtureOauthAuthenticator(Oauth2Authenticator):
-    """
-    Test OAuth authenticator that only overrides the request and response aspect of the authenticator flow
-    """
-
-    def refresh_access_token(self) -> Tuple[str, int]:
-        response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), params={})
-        response.raise_for_status()
-        return "some_access_token", 1800  # Mock oauth response values to be used during the data retrieval step

unit_tests/sources/message/test_repository.py

@@ -1,153 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from unittest.mock import Mock
-
-import pytest
-from airbyte_cdk.models import AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, Level, OrchestratorType, Type
-from airbyte_cdk.sources.message import (
-    InMemoryMessageRepository,
-    LogAppenderMessageRepositoryDecorator,
-    MessageRepository,
-    NoopMessageRepository,
-)
-from pydantic.error_wrappers import ValidationError
-
-A_CONTROL = AirbyteControlMessage(
-    type=OrchestratorType.CONNECTOR_CONFIG,
-    emitted_at=0,
-    connectorConfig=AirbyteControlConnectorConfigMessage(config={"a config": "value"}),
-)
-ANY_MESSAGE = AirbyteMessage(
-    type=Type.CONTROL,
-    control=AirbyteControlMessage(
-        type=OrchestratorType.CONNECTOR_CONFIG,
-        emitted_at=0,
-        connectorConfig=AirbyteControlConnectorConfigMessage(config={"any message": "value"}),
-    ),
-)
-ANOTHER_CONTROL = AirbyteControlMessage(
-    type=OrchestratorType.CONNECTOR_CONFIG,
-    emitted_at=0,
-    connectorConfig=AirbyteControlConnectorConfigMessage(config={"another config": "another value"}),
-)
-UNKNOWN_LEVEL = "potato"
-
-
-class TestInMemoryMessageRepository:
-    def test_given_no_messages_when_consume_queue_then_return_empty(self):
-        repo = InMemoryMessageRepository()
-        messages = list(repo.consume_queue())
-        assert messages == []
-
-    def test_given_messages_when_consume_queue_then_return_messages(self):
-        repo = InMemoryMessageRepository()
-        first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL)
-        repo.emit_message(first_message)
-        second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL)
-        repo.emit_message(second_message)
-
-        messages = repo.consume_queue()
-
-        assert list(messages) == [first_message, second_message]
-
-    def test_given_message_is_consumed_when_consume_queue_then_remove_message_from_queue(self):
-        repo = InMemoryMessageRepository()
-        first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL)
-        repo.emit_message(first_message)
-        second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL)
-        repo.emit_message(second_message)
-
-        message_generator = repo.consume_queue()
-        consumed_message = next(message_generator)
-        assert consumed_message == first_message
-
-        second_message_generator = repo.consume_queue()
-        assert list(second_message_generator) == [second_message]
-
-    def test_given_log_level_is_severe_enough_when_log_message_then_allow_message_to_be_consumed(self):
-        repo = InMemoryMessageRepository(Level.DEBUG)
-        repo.log_message(Level.INFO, lambda: {"message": "this is a log message"})
-        assert list(repo.consume_queue())
-
-    def test_given_log_level_is_severe_enough_when_log_message_then_filter_secrets(self, mocker):
-        filtered_message = "a filtered message"
-        mocker.patch("airbyte_cdk.sources.message.repository.filter_secrets", return_value=filtered_message)
-        repo = InMemoryMessageRepository(Level.DEBUG)
-
-        repo.log_message(Level.INFO, lambda: {"message": "this is a log message"})
-
-        assert list(repo.consume_queue())[0].log.message == filtered_message
-
-    def test_given_log_level_not_severe_enough_when_log_message_then_do_not_allow_message_to_be_consumed(self):
-        repo = InMemoryMessageRepository(Level.ERROR)
-        repo.log_message(Level.INFO, lambda: {"message": "this is a log message"})
-        assert not list(repo.consume_queue())
-
-    def test_given_unknown_log_level_as_threshold_when_log_message_then_allow_message_to_be_consumed(self):
-        repo = InMemoryMessageRepository(UNKNOWN_LEVEL)
-        repo.log_message(Level.DEBUG, lambda: {"message": "this is a log message"})
-        assert list(repo.consume_queue())
-
-    def test_given_unknown_log_level_for_log_when_log_message_then_raise_error(self):
-        """
-        Pydantic will fail if the log level is unknown but on our side, we should try to log at least
-        """
-        repo = InMemoryMessageRepository(Level.ERROR)
-        with pytest.raises(ValidationError):
-            repo.log_message(UNKNOWN_LEVEL, lambda: {"message": "this is a log message"})
-
-
-class TestNoopMessageRepository:
-    def test_given_message_emitted_when_consume_queue_then_return_empty(self):
-        repo = NoopMessageRepository()
-        repo.emit_message(AirbyteMessage(type=Type.CONTROL, control=A_CONTROL))
-        repo.log_message(Level.INFO, lambda: {"message": "this is a log message"})
-
-        assert not list(repo.consume_queue())
-
-
-class TestLogAppenderMessageRepositoryDecorator:
-
-    _DICT_TO_APPEND = {"airbyte_cdk": {"stream": {"is_substream": False}}}
-
-    @pytest.fixture()
-    def decorated(self):
-        return Mock(spec=MessageRepository)
-
-    def test_when_emit_message_then_delegate_call(self, decorated):
-        repo = LogAppenderMessageRepositoryDecorator(self._DICT_TO_APPEND, decorated, Level.DEBUG)
-        repo.emit_message(ANY_MESSAGE)
-        decorated.emit_message.assert_called_once_with(ANY_MESSAGE)
-
-    def test_when_log_message_then_append(self, decorated):
-        repo = LogAppenderMessageRepositoryDecorator({"a": {"dict_to_append": "appended value"}}, decorated, Level.DEBUG)
-        repo.log_message(Level.INFO, lambda: {"a": {"original": "original value"}})
-        assert decorated.log_message.call_args_list[0].args[1]() == {
-            "a": {"dict_to_append": "appended value", "original": "original value"}
-        }
-
-    def test_given_value_clash_when_log_message_then_overwrite_value(self, decorated):
-        repo = LogAppenderMessageRepositoryDecorator({"clash": "appended value"}, decorated, Level.DEBUG)
-        repo.log_message(Level.INFO, lambda: {"clash": "original value"})
-        assert decorated.log_message.call_args_list[0].args[1]() == {"clash": "appended value"}
-
-    def test_given_log_level_is_severe_enough_when_log_message_then_allow_message_to_be_consumed(self, decorated):
-        repo = LogAppenderMessageRepositoryDecorator(self._DICT_TO_APPEND, decorated, Level.DEBUG)
-        repo.log_message(Level.INFO, lambda: {})
-        assert decorated.log_message.call_count == 1
-
-    def test_given_log_level_not_severe_enough_when_log_message_then_do_not_allow_message_to_be_consumed(self, decorated):
-        repo = LogAppenderMessageRepositoryDecorator(self._DICT_TO_APPEND, decorated, Level.ERROR)
-        repo.log_message(Level.INFO, lambda: {})
-        assert decorated.log_message.call_count == 0
-
-    def test_when_consume_queue_then_return_delegate_queue(self, decorated):
-        repo = LogAppenderMessageRepositoryDecorator(self._DICT_TO_APPEND, decorated, Level.DEBUG)
-        queue = [ANY_MESSAGE, ANY_MESSAGE, ANY_MESSAGE]
-        decorated.consume_queue.return_value = iter(queue)
-
-        result = list(repo.consume_queue())
-
-        assert result == queue

unit_tests/sources/streams/concurrent/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/streams/concurrent/scenarios/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py

@@ -1,250 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
-from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ConcurrencyCompatibleStateType
-from airbyte_cdk.test.state_builder import StateBuilder
-from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder
-from unit_tests.sources.streams.concurrent.scenarios.stream_facade_builder import StreamFacadeSourceBuilder
-from unit_tests.sources.streams.concurrent.scenarios.utils import MockStream
-
-_NO_SLICE_BOUNDARIES = None
-_NO_INPUT_STATE = []
-test_incremental_stream_without_slice_boundaries_no_input_state = (
-    TestScenarioBuilder()
-    .set_name("test_incremental_stream_without_slice_boundaries_no_input_state")
-    .set_config({})
-    .set_source_builder(
-        StreamFacadeSourceBuilder()
-        .set_streams(
-            [
-                MockStream(
-                    [
-                        ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]),
-                        ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]),
-                    ],
-                    "stream1",
-                    json_schema={
-                        "type": "object",
-                        "properties": {
-                            "id": {"type": ["null", "string"]},
-                        },
-                    },
-                )
-            ]
-        )
-        .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES)
-        .set_input_state(_NO_INPUT_STATE)
-    )
-    .set_expected_read_error(ValueError, "test exception")
-    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
-    .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=_NO_INPUT_STATE))
-    .build()
-)
-
-
-test_incremental_stream_with_slice_boundaries_no_input_state = (
-    TestScenarioBuilder()
-    .set_name("test_incremental_stream_with_slice_boundaries_no_input_state")
-    .set_config({})
-    .set_source_builder(
-        StreamFacadeSourceBuilder()
-        .set_streams(
-            [
-                MockStream(
-                    [
-                        ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]),
-                        ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]),
-                    ],
-                    "stream1",
-                    json_schema={
-                        "type": "object",
-                        "properties": {
-                            "id": {"type": ["null", "string"]},
-                        },
-                    },
-                )
-            ]
-        )
-        .set_incremental(CursorField("cursor_field"), ("from", "to"))
-        .set_input_state(_NO_INPUT_STATE)
-    )
-    .set_expected_records(
-        [
-            {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"},
-            {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"},
-            {"cursor_field": 1},
-            {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"},
-            {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"},
-            {"cursor_field": 2},
-            {"cursor_field": 2},  # see Cursor.ensure_at_least_one_state_emitted
-        ]
-    )
-    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
-    .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=_NO_INPUT_STATE))
-    .build()
-)
-
-
-LEGACY_STATE = StateBuilder().with_stream_state("stream1", {"cursor_field": 0}).build()
-test_incremental_stream_without_slice_boundaries_with_legacy_state = (
-    TestScenarioBuilder()
-    .set_name("test_incremental_stream_without_slice_boundaries_with_legacy_state")
-    .set_config({})
-    .set_source_builder(
-        StreamFacadeSourceBuilder()
-        .set_streams(
-            [
-                MockStream(
-                    [
-                        ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]),
-                        ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]),
-                    ],
-                    "stream1",
-                    json_schema={
-                        "type": "object",
-                        "properties": {
-                            "id": {"type": ["null", "string"]},
-                        },
-                    },
-                )
-            ]
-        )
-        .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES)
-        .set_input_state(LEGACY_STATE)
-    )
-    .set_expected_read_error(ValueError, "test exception")
-    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
-    .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=LEGACY_STATE))
-    .build()
-)
-
-
-test_incremental_stream_with_slice_boundaries_with_legacy_state = (
-    TestScenarioBuilder()
-    .set_name("test_incremental_stream_with_slice_boundaries_with_legacy_state")
-    .set_config({})
-    .set_source_builder(
-        StreamFacadeSourceBuilder()
-        .set_streams(
-            [
-                MockStream(
-                    [
-                        ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]),
-                        ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]),
-                    ],
-                    "stream1",
-                    json_schema={
-                        "type": "object",
-                        "properties": {
-                            "id": {"type": ["null", "string"]},
-                        },
-                    },
-                )
-            ]
-        )
-        .set_incremental(CursorField("cursor_field"), ("from", "to"))
-        .set_input_state(LEGACY_STATE)
-    )
-    .set_expected_records(
-        [
-            {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"},
-            {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"},
-            {"cursor_field": 1},
-            {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"},
-            {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"},
-            {"cursor_field": 2},
-            {"cursor_field": 2},  # see Cursor.ensure_at_least_one_state_emitted
-        ]
-    )
-    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
-    .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=LEGACY_STATE))
-    .build()
-)
-
-
-CONCURRENT_STATE = (
-    StateBuilder()
-    .with_stream_state(
-        "stream1",
-        {
-            "slices": [{"start": 0, "end": 0}],
-            "state_type": ConcurrencyCompatibleStateType.date_range.value,
-        },
-    )
-    .build()
-)
-test_incremental_stream_without_slice_boundaries_with_concurrent_state = (
-    TestScenarioBuilder()
-    .set_name("test_incremental_stream_without_slice_boundaries_with_concurrent_state")
-    .set_config({})
-    .set_source_builder(
-        StreamFacadeSourceBuilder()
-        .set_streams(
-            [
-                MockStream(
-                    [
-                        ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]),
-                        ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]),
-                    ],
-                    "stream1",
-                    json_schema={
-                        "type": "object",
-                        "properties": {
-                            "id": {"type": ["null", "string"]},
-                        },
-                    },
-                )
-            ]
-        )
-        .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES)
-        .set_input_state(CONCURRENT_STATE)
-    )
-    .set_expected_read_error(ValueError, "test exception")
-    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
-    .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=CONCURRENT_STATE))
-    .build()
-)
-
-
-test_incremental_stream_with_slice_boundaries_with_concurrent_state = (
-    TestScenarioBuilder()
-    .set_name("test_incremental_stream_with_slice_boundaries_with_concurrent_state")
-    .set_config({})
-    .set_source_builder(
-        StreamFacadeSourceBuilder()
-        .set_streams(
-            [
-                MockStream(
-                    [
-                        ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]),
-                        ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]),
-                    ],
-                    "stream1",
-                    json_schema={
-                        "type": "object",
-                        "properties": {
-                            "id": {"type": ["null", "string"]},
-                        },
-                    },
-                )
-            ]
-        )
-        .set_incremental(CursorField("cursor_field"), ("from", "to"))
-        .set_input_state(CONCURRENT_STATE)
-    )
-    .set_expected_records(
-        [
-            {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"},
-            {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"},
-            {"cursor_field": 1},
-            {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"},
-            {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"},
-            {"cursor_field": 2},
-            {"cursor_field": 2},  # see Cursor.ensure_at_least_one_state_emitted
-        ]
-    )
-    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
-    .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=CONCURRENT_STATE))
-    .build()
-)