airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

unit_tests/connector_builder/utils.py

@@ -1,27 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Any, Mapping
-
-from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog
-
-
-def create_configured_catalog_dict(stream_name: str) -> Mapping[str, Any]:
-    return {
-        "streams": [
-            {
-                "stream": {
-                    "name": stream_name,
-                    "json_schema": {},
-                    "supported_sync_modes": ["full_refresh", "incremental"],
-                },
-                "sync_mode": "full_refresh",
-                "destination_sync_mode": "overwrite",
-            }
-        ]
-    }
-
-
-def create_configured_catalog(stream_name: str) -> ConfiguredAirbyteCatalog:
-    return ConfiguredAirbyteCatalog.parse_obj(create_configured_catalog_dict(stream_name))

unit_tests/destinations/test_destination.py

@@ -1,243 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import argparse
-import io
-import json
-from os import PathLike
-from typing import Any, Dict, Iterable, List, Mapping, Union
-from unittest.mock import ANY
-
-import pytest
-from airbyte_cdk.destinations import Destination
-from airbyte_cdk.destinations import destination as destination_module
-from airbyte_cdk.models import (
-    AirbyteCatalog,
-    AirbyteConnectionStatus,
-    AirbyteMessage,
-    AirbyteRecordMessage,
-    AirbyteStateMessage,
-    AirbyteStream,
-    ConfiguredAirbyteCatalog,
-    ConfiguredAirbyteStream,
-    ConnectorSpecification,
-    DestinationSyncMode,
-    Status,
-    SyncMode,
-    Type,
-)
-
-
-@pytest.fixture(name="destination")
-def destination_fixture(mocker) -> Destination:
-    # Wipe the internal list of abstract methods to allow instantiating the abstract class without implementing its abstract methods
-    mocker.patch("airbyte_cdk.destinations.Destination.__abstractmethods__", set())
-    # Mypy yells at us because we're init'ing an abstract class
-    return Destination()  # type: ignore
-
-
-class TestArgParsing:
-    @pytest.mark.parametrize(
-        ("arg_list", "expected_output"),
-        [
-            (["spec"], {"command": "spec"}),
-            (["check", "--config", "bogus_path/"], {"command": "check", "config": "bogus_path/"}),
-            (
-                ["write", "--config", "config_path1", "--catalog", "catalog_path1"],
-                {"command": "write", "config": "config_path1", "catalog": "catalog_path1"},
-            ),
-        ],
-    )
-    def test_successful_parse(self, arg_list: List[str], expected_output: Mapping[str, Any], destination: Destination):
-        parsed_args = vars(destination.parse_args(arg_list))
-        assert (
-            parsed_args == expected_output
-        ), f"Expected parsing {arg_list} to return parsed args {expected_output} but instead found {parsed_args}"
-
-    @pytest.mark.parametrize(
-        ("arg_list"),
-        [
-            # Invalid commands
-            ([]),
-            (["not-a-real-command"]),
-            ([""]),
-            # Incorrect parameters
-            (["spec", "--config", "path"]),
-            (["check"]),
-            (["check", "--catalog", "path"]),
-            (["check", "path"]),
-        ],
-    )
-    def test_failed_parse(self, arg_list: List[str], destination: Destination):
-        # We use BaseException because it encompasses SystemExit (raised by failed parsing) and other exceptions (raised by additional semantic
-        # checks)
-        with pytest.raises(BaseException):
-            destination.parse_args(arg_list)
-
-
-def _state(state: Dict[str, Any]) -> AirbyteStateMessage:
-    return AirbyteStateMessage(data=state)
-
-
-def _record(stream: str, data: Dict[str, Any]) -> AirbyteRecordMessage:
-    return AirbyteRecordMessage(stream=stream, data=data, emitted_at=0)
-
-
-def _spec(schema: Dict[str, Any]) -> ConnectorSpecification:
-    return ConnectorSpecification(connectionSpecification=schema)
-
-
-def write_file(path: PathLike, content: Union[str, Mapping]):
-    content = json.dumps(content) if isinstance(content, Mapping) else content
-    with open(path, "w") as f:
-        f.write(content)
-
-
-def _wrapped(
-    msg: Union[AirbyteRecordMessage, AirbyteStateMessage, AirbyteCatalog, ConnectorSpecification, AirbyteConnectionStatus]
-) -> AirbyteMessage:
-    if isinstance(msg, AirbyteRecordMessage):
-        return AirbyteMessage(type=Type.RECORD, record=msg)
-    elif isinstance(msg, AirbyteStateMessage):
-        return AirbyteMessage(type=Type.STATE, state=msg)
-    elif isinstance(msg, AirbyteCatalog):
-        return AirbyteMessage(type=Type.CATALOG, catalog=msg)
-    elif isinstance(msg, AirbyteConnectionStatus):
-        return AirbyteMessage(type=Type.CONNECTION_STATUS, connectionStatus=msg)
-    elif isinstance(msg, ConnectorSpecification):
-        return AirbyteMessage(type=Type.SPEC, spec=msg)
-    else:
-        raise Exception(f"Invalid Airbyte Message: {msg}")
-
-
-class OrderedIterableMatcher(Iterable):
-    """
-    A class whose purpose is to verify equality of one iterable object against another
-    in an ordered fashion
-    """
-
-    def attempt_consume(self, iterator):
-        try:
-            return next(iterator)
-        except StopIteration:
-            return None
-
-    def __iter__(self):
-        return iter(self.iterable)
-
-    def __init__(self, iterable: Iterable):
-        self.iterable = iterable
-
-    def __eq__(self, other):
-        if not isinstance(other, Iterable):
-            return False
-
-        return list(self) == list(other)
-
-
-class TestRun:
-    def test_run_initializes_exception_handler(self, mocker, destination: Destination):
-        mocker.patch.object(destination_module, "init_uncaught_exception_handler")
-        mocker.patch.object(destination, "parse_args")
-        mocker.patch.object(destination, "run_cmd")
-        destination.run(["dummy"])
-        destination_module.init_uncaught_exception_handler.assert_called_once_with(destination_module.logger)
-
-    def test_run_spec(self, mocker, destination: Destination):
-        args = {"command": "spec"}
-        parsed_args = argparse.Namespace(**args)
-
-        expected_spec = ConnectorSpecification(connectionSpecification={"json_schema": {"prop": "value"}})
-        mocker.patch.object(destination, "spec", return_value=expected_spec, autospec=True)
-
-        spec_message = next(iter(destination.run_cmd(parsed_args)))
-
-        # Mypy doesn't understand magicmock so it thinks spec doesn't have assert_called_once attr
-        destination.spec.assert_called_once()  # type: ignore
-
-        # verify the output of spec was returned
-        assert _wrapped(expected_spec) == spec_message
-
-    def test_run_check(self, mocker, destination: Destination, tmp_path):
-        file_path = tmp_path / "config.json"
-        dummy_config = {"user": "sherif"}
-        write_file(file_path, dummy_config)
-        args = {"command": "check", "config": file_path}
-
-        parsed_args = argparse.Namespace(**args)
-        destination.run_cmd(parsed_args)
-        spec_msg = ConnectorSpecification(connectionSpecification={})
-        mocker.patch.object(destination, "spec", return_value=spec_msg)
-        validate_mock = mocker.patch("airbyte_cdk.destinations.destination.check_config_against_spec_or_exit")
-        expected_check_result = AirbyteConnectionStatus(status=Status.SUCCEEDED)
-        mocker.patch.object(destination, "check", return_value=expected_check_result, autospec=True)
-
-        returned_check_result = next(iter(destination.run_cmd(parsed_args)))
-        # verify method call with the correct params
-        # Affirm to Mypy that this is indeed a method on this mock
-        destination.check.assert_called_once()  # type: ignore
-        # Affirm to Mypy that this is indeed a method on this mock
-        destination.check.assert_called_with(logger=ANY, config=dummy_config)  # type: ignore
-        # Check if config validation has been called
-        validate_mock.assert_called_with(dummy_config, spec_msg)
-
-        # verify output was correct
-        assert _wrapped(expected_check_result) == returned_check_result
-
-    def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch):
-        config_path, dummy_config = tmp_path / "config.json", {"user": "sherif"}
-        write_file(config_path, dummy_config)
-
-        dummy_catalog = ConfiguredAirbyteCatalog(
-            streams=[
-                ConfiguredAirbyteStream(
-                    stream=AirbyteStream(name="mystream", json_schema={"type": "object"}, supported_sync_modes=[SyncMode.full_refresh]),
-                    sync_mode=SyncMode.full_refresh,
-                    destination_sync_mode=DestinationSyncMode.overwrite,
-                )
-            ]
-        )
-        catalog_path = tmp_path / "catalog.json"
-        write_file(catalog_path, dummy_catalog.json(exclude_unset=True))
-
-        args = {"command": "write", "config": config_path, "catalog": catalog_path}
-        parsed_args = argparse.Namespace(**args)
-
-        expected_write_result = [_wrapped(_state({"k1": "v1"})), _wrapped(_state({"k2": "v2"}))]
-        mocker.patch.object(
-            destination, "write", return_value=iter(expected_write_result), autospec=True  # convert to iterator to mimic real usage
-        )
-        spec_msg = ConnectorSpecification(connectionSpecification={})
-        mocker.patch.object(destination, "spec", return_value=spec_msg)
-        validate_mock = mocker.patch("airbyte_cdk.destinations.destination.check_config_against_spec_or_exit")
-        # mock input is a record followed by some state messages
-        mocked_input: List[AirbyteMessage] = [_wrapped(_record("s1", {"k1": "v1"})), *expected_write_result]
-        mocked_stdin_string = "\n".join([record.json(exclude_unset=True) for record in mocked_input])
-        mocked_stdin_string += "\n add this non-serializable string to verify the destination does not break on malformed input"
-        mocked_stdin = io.TextIOWrapper(io.BytesIO(bytes(mocked_stdin_string, "utf-8")))
-
-        monkeypatch.setattr("sys.stdin", mocked_stdin)
-
-        returned_write_result = list(destination.run_cmd(parsed_args))
-        # verify method call with the correct params
-        # Affirm to Mypy that call_count is indeed a method on this mock
-        destination.write.assert_called_once()  # type: ignore
-        # Affirm to Mypy that call_count is indeed a method on this mock
-        destination.write.assert_called_with(  # type: ignore
-            config=dummy_config,
-            configured_catalog=dummy_catalog,
-            # Stdin is internally consumed as a generator so we use a custom matcher
-            # that iterates over two iterables to check equality
-            input_messages=OrderedIterableMatcher(mocked_input),
-        )
-        # Check if config validation has been called
-        validate_mock.assert_called_with(dummy_config, spec_msg)
-
-        # verify output was correct
-        assert expected_write_result == returned_write_result
-
-    @pytest.mark.parametrize("args", [{}, {"command": "fake"}])
-    def test_run_cmd_with_incorrect_args_fails(self, args, destination: Destination):
-        with pytest.raises(Exception):
-            list(destination.run_cmd(parsed_args=argparse.Namespace(**args)))

unit_tests/singer/test_singer_helpers.py

@@ -1,56 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import copy
-
-from airbyte_cdk.sources.singer import SingerHelper
-
-basic_singer_catalog = {
-    "streams": [
-        {
-            "type": "SCHEMA",
-            "stream": "users",
-            "schema": {
-                "properties": {
-                    "id": {"type": "integer"},
-                    "name": {"type": "string"},
-                    "updated_at": {"type": "string", "format": "date-time"},
-                }
-            },
-            "key_properties": ["id"],
-            "bookmark_properties": ["updated_at"],
-        }
-    ]
-}
-
-
-def test_singer_catalog_to_airbyte_catalog():
-    airbyte_catalog = SingerHelper.singer_catalog_to_airbyte_catalog(
-        singer_catalog=basic_singer_catalog, sync_mode_overrides={}, primary_key_overrides={}
-    )
-
-    user_stream = airbyte_catalog.streams[0]
-    assert user_stream.source_defined_primary_key == [["id"]]
-
-
-def test_singer_catalog_to_airbyte_catalog_composite_pk():
-    singer_catalog = copy.deepcopy(basic_singer_catalog)
-    singer_catalog["streams"][0]["key_properties"] = ["id", "name"]
-
-    airbyte_catalog = SingerHelper.singer_catalog_to_airbyte_catalog(
-        singer_catalog=singer_catalog, sync_mode_overrides={}, primary_key_overrides={}
-    )
-
-    user_stream = airbyte_catalog.streams[0]
-    assert user_stream.source_defined_primary_key == [["id"], ["name"]]
-
-
-def test_singer_catalog_to_airbyte_catalog_pk_override():
-    airbyte_catalog = SingerHelper.singer_catalog_to_airbyte_catalog(
-        singer_catalog=basic_singer_catalog, sync_mode_overrides={}, primary_key_overrides={"users": ["name"]}
-    )
-
-    user_stream = airbyte_catalog.streams[0]
-    assert user_stream.source_defined_primary_key == [["name"]]

unit_tests/singer/test_singer_source.py

@@ -1,112 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import copy
-import logging
-from unittest.mock import patch
-
-from airbyte_cdk.models.airbyte_protocol import SyncMode
-from airbyte_cdk.sources.singer import SingerHelper, SyncModeInfo
-from airbyte_cdk.sources.singer.source import BaseSingerSource, ConfigContainer
-
-logger = logging.getLogger("airbyte")
-
-
-class TetsBaseSinger(BaseSingerSource):
-    tap_cmd = ""
-
-
-USER_STREAM = {
-    "type": "SCHEMA",
-    "stream": "users",
-    "schema": {
-        "properties": {"id": {"type": "integer"}, "name": {"type": "string"}, "updated_at": {"type": "string", "format": "date-time"}}
-    },
-    "key_properties": ["id"],
-    "bookmark_properties": ["updated_at"],
-}
-
-ROLES_STREAM = {
-    "type": "SCHEMA",
-    "stream": "roles",
-    "schema": {
-        "properties": {
-            "name": {"type": "string"},
-        }
-    },
-    "key_properties": ["name"],
-    "bookmark_properties": ["updated_at"],
-    "metadata": [
-        {
-            "metadata": {
-                "inclusion": "available",
-                "table-key-properties": ["id"],
-                "selected": True,
-                "valid-replication-keys": ["name"],
-                "schema-name": "roles",
-            },
-            "breadcrumb": [],
-        }
-    ],
-}
-
-basic_singer_catalog = {"streams": [USER_STREAM, ROLES_STREAM]}
-
-
-@patch.object(SingerHelper, "_read_singer_catalog", return_value=basic_singer_catalog)
-def test_singer_discover_single_pk(mock_read_catalog):
-    airbyte_catalog = TetsBaseSinger().discover(logger, ConfigContainer({}, ""))
-    _user_stream = airbyte_catalog.streams[0]
-    _roles_stream = airbyte_catalog.streams[1]
-    assert _user_stream.source_defined_primary_key == [["id"]]
-    assert _roles_stream.json_schema == ROLES_STREAM["schema"]
-    assert _user_stream.json_schema == USER_STREAM["schema"]
-
-
-def test_singer_discover_with_composite_pk():
-    singer_catalog_composite_pk = copy.deepcopy(basic_singer_catalog)
-    singer_catalog_composite_pk["streams"][0]["key_properties"] = ["id", "name"]
-    with patch.object(SingerHelper, "_read_singer_catalog", return_value=singer_catalog_composite_pk):
-        airbyte_catalog = TetsBaseSinger().discover(logger, ConfigContainer({}, ""))
-
-    _user_stream = airbyte_catalog.streams[0]
-    _roles_stream = airbyte_catalog.streams[1]
-    assert _user_stream.source_defined_primary_key == [["id"], ["name"]]
-    assert _roles_stream.json_schema == ROLES_STREAM["schema"]
-    assert _user_stream.json_schema == USER_STREAM["schema"]
-
-
-@patch.object(BaseSingerSource, "get_primary_key_overrides", return_value={"users": ["updated_at"]})
-@patch.object(SingerHelper, "_read_singer_catalog", return_value=basic_singer_catalog)
-def test_singer_discover_pk_overrides(mock_pk_override, mock_read_catalog):
-    airbyte_catalog = TetsBaseSinger().discover(logger, ConfigContainer({}, ""))
-    _user_stream = airbyte_catalog.streams[0]
-    _roles_stream = airbyte_catalog.streams[1]
-    assert _user_stream.source_defined_primary_key == [["updated_at"]]
-    assert _roles_stream.json_schema == ROLES_STREAM["schema"]
-    assert _user_stream.json_schema == USER_STREAM["schema"]
-
-
-@patch.object(SingerHelper, "_read_singer_catalog", return_value=basic_singer_catalog)
-def test_singer_discover_metadata(mock_read_catalog):
-    airbyte_catalog = TetsBaseSinger().discover(logger, ConfigContainer({}, ""))
-    _user_stream = airbyte_catalog.streams[0]
-    _roles_stream = airbyte_catalog.streams[1]
-
-    # assert _user_stream.supported_sync_modes is None
-    assert _user_stream.default_cursor_field is None
-    assert _roles_stream.supported_sync_modes == [SyncMode.incremental]
-    assert _roles_stream.default_cursor_field == ["name"]
-
-
-@patch.object(SingerHelper, "_read_singer_catalog", return_value=basic_singer_catalog)
-def test_singer_discover_sync_mode_overrides(mock_read_catalog):
-    sync_mode_override = SyncModeInfo(supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], default_cursor_field=["name"])
-    with patch.object(BaseSingerSource, "get_sync_mode_overrides", return_value={"roles": sync_mode_override}):
-        airbyte_catalog = TetsBaseSinger().discover(logger, ConfigContainer({}, ""))
-
-    _roles_stream = airbyte_catalog.streams[1]
-    assert _roles_stream.supported_sync_modes == sync_mode_override.supported_sync_modes
-    assert _roles_stream.default_cursor_field == sync_mode_override.default_cursor_field

unit_tests/sources/concurrent_source/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py

@@ -1,106 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-import logging
-from typing import Any, List, Mapping, Optional, Tuple
-from unittest.mock import Mock
-
-import freezegun
-from airbyte_cdk.models import (
-    AirbyteMessage,
-    AirbyteRecordMessage,
-    AirbyteStream,
-    ConfiguredAirbyteCatalog,
-    ConfiguredAirbyteStream,
-    DestinationSyncMode,
-    SyncMode,
-)
-from airbyte_cdk.models import Type as MessageType
-from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
-from airbyte_cdk.sources.message import InMemoryMessageRepository
-from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
-from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
-
-
-class _MockSource(ConcurrentSourceAdapter):
-    def __init__(self, concurrent_source, _streams_to_is_concurrent, logger):
-        super().__init__(concurrent_source)
-        self._streams_to_is_concurrent = _streams_to_is_concurrent
-        self._logger = logger
-
-    message_repository = InMemoryMessageRepository()
-
-    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
-        raise NotImplementedError
-
-    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        return [
-            StreamFacade.create_from_stream(s, self, self._logger, None, FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository())) if is_concurrent else s
-            for s, is_concurrent in self._streams_to_is_concurrent.items()
-        ]
-
-
-@freezegun.freeze_time("2020-01-01T00:00:00")
-def test_concurrent_source_adapter():
-    concurrent_source = Mock()
-    message_from_concurrent_stream = AirbyteMessage(
-        type=MessageType.RECORD,
-        record=AirbyteRecordMessage(
-            stream="s2",
-            data={"data": 2},
-            emitted_at=1577836800000,
-        ),
-    )
-    concurrent_source.read.return_value = iter([message_from_concurrent_stream])
-    regular_stream = _mock_stream("s1", [{"data": 1}])
-    concurrent_stream = _mock_stream("s2", [])
-    unavailable_stream = _mock_stream("s3", [{"data": 3}], False)
-    concurrent_stream.name = "s2"
-    logger = Mock()
-    adapter = _MockSource(concurrent_source, {regular_stream: False, concurrent_stream: True, unavailable_stream: False}, logger)
-
-    messages = list(adapter.read(logger, {}, _configured_catalog([regular_stream, concurrent_stream, unavailable_stream])))
-    records = [m for m in messages if m.type == MessageType.RECORD]
-
-    expected_records = [
-        message_from_concurrent_stream,
-        AirbyteMessage(
-            type=MessageType.RECORD,
-            record=AirbyteRecordMessage(
-                stream="s1",
-                data={"data": 1},
-                emitted_at=1577836800000,
-            ),
-        ),
-    ]
-
-    assert records == expected_records
-
-
-def _mock_stream(name: str, data=[], available: bool = True):
-    s = Mock()
-    s.name = name
-    s.namespace = None
-    s.as_airbyte_stream.return_value = AirbyteStream(
-        name=name,
-        json_schema={},
-        supported_sync_modes=[SyncMode.full_refresh],
-    )
-    s.check_availability.return_value = (True, None) if available else (False, "not available")
-    s.read.return_value = iter(data)
-    s.primary_key = None
-    return s
-
-
-def _configured_catalog(streams: List[Stream]):
-    return ConfiguredAirbyteCatalog(
-        streams=[
-            ConfiguredAirbyteStream(
-                stream=stream.as_airbyte_stream(),
-                sync_mode=SyncMode.full_refresh,
-                destination_sync_mode=DestinationSyncMode.overwrite,
-            )
-            for stream in streams
-        ]
-    )

unit_tests/sources/declarative/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/declarative/auth/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#