airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

unit_tests/sources/declarative/auth/test_token_auth.py

@@ -1,200 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import logging
-
-import pytest
-import requests
-from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator
-from airbyte_cdk.sources.declarative.auth.token_provider import InterpolatedStringTokenProvider
-from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType
-from requests import Response
-
-LOGGER = logging.getLogger(__name__)
-
-resp = Response()
-config = {"username": "user", "password": "password", "header": "header"}
-parameters = {"username": "user", "password": "password", "header": "header"}
-
-
-@pytest.mark.parametrize(
-    "test_name, token, expected_header_value",
-    [
-        ("test_static_token", "test-token", "Bearer test-token"),
-        ("test_token_from_config", "{{ config.username }}", "Bearer user"),
-        ("test_token_from_parameters", "{{ parameters.username }}", "Bearer user"),
-    ],
-)
-def test_bearer_token_authenticator(test_name, token, expected_header_value):
-    """
-    Should match passed in token, no matter how many times token is retrieved.
-    """
-    token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters)
-    token_auth = BearerAuthenticator(token_provider, config, parameters=parameters)
-    header1 = token_auth.get_auth_header()
-    header2 = token_auth.get_auth_header()
-
-    prepared_request = requests.PreparedRequest()
-    prepared_request.headers = {}
-    token_auth(prepared_request)
-
-    assert {"Authorization": expected_header_value} == prepared_request.headers
-    assert {"Authorization": expected_header_value} == header1
-    assert {"Authorization": expected_header_value} == header2
-
-
-@pytest.mark.parametrize(
-    "test_name, username, password, expected_header_value",
-    [
-        ("test_static_creds", "user", "password", "Basic dXNlcjpwYXNzd29yZA=="),
-        ("test_creds_from_config", "{{ config.username }}", "{{ config.password }}", "Basic dXNlcjpwYXNzd29yZA=="),
-        ("test_creds_from_parameters", "{{ parameters.username }}", "{{ parameters.password }}", "Basic dXNlcjpwYXNzd29yZA=="),
-    ],
-)
-def test_basic_authenticator(test_name, username, password, expected_header_value):
-    """
-    Should match passed in token, no matter how many times token is retrieved.
-    """
-    token_auth = BasicHttpAuthenticator(username=username, password=password, config=config, parameters=parameters)
-    header1 = token_auth.get_auth_header()
-    header2 = token_auth.get_auth_header()
-
-    prepared_request = requests.PreparedRequest()
-    prepared_request.headers = {}
-    token_auth(prepared_request)
-
-    assert {"Authorization": expected_header_value} == prepared_request.headers
-    assert {"Authorization": expected_header_value} == header1
-    assert {"Authorization": expected_header_value} == header2
-
-
-@pytest.mark.parametrize(
-    "test_name, header, token, expected_header, expected_header_value",
-    [
-        ("test_static_token", "Authorization", "test-token", "Authorization", "test-token"),
-        ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user"),
-        ("test_token_from_parameters", "{{ parameters.header }}", "{{ parameters.username }}", "header", "user"),
-    ],
-)
-def test_api_key_authenticator(test_name, header, token, expected_header, expected_header_value):
-    """
-    Should match passed in token, no matter how many times token is retrieved.
-    """
-    token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters)
-    token_auth = ApiKeyAuthenticator(
-        request_option=RequestOption(inject_into=RequestOptionType.header, field_name=header, parameters=parameters),
-        token_provider=token_provider,
-        config=config,
-        parameters=parameters,
-    )
-    header1 = token_auth.get_auth_header()
-    header2 = token_auth.get_auth_header()
-
-    prepared_request = requests.PreparedRequest()
-    prepared_request.headers = {}
-    token_auth(prepared_request)
-
-    assert {expected_header: expected_header_value} == prepared_request.headers
-    assert {expected_header: expected_header_value} == header1
-    assert {expected_header: expected_header_value} == header2
-
-
-@pytest.mark.parametrize(
-    "test_name, field_name, token, expected_field_name, expected_field_value, inject_type, validation_fn",
-    [
-        (
-            "test_static_token",
-            "Authorization",
-            "test-token",
-            "Authorization",
-            "test-token",
-            RequestOptionType.request_parameter,
-            "get_request_params",
-        ),
-        (
-            "test_token_from_config",
-            "{{ config.header }}",
-            "{{ config.username }}",
-            "header",
-            "user",
-            RequestOptionType.request_parameter,
-            "get_request_params",
-        ),
-        (
-            "test_token_from_parameters",
-            "{{ parameters.header }}",
-            "{{ parameters.username }}",
-            "header",
-            "user",
-            RequestOptionType.request_parameter,
-            "get_request_params",
-        ),
-        (
-            "test_static_token",
-            "Authorization",
-            "test-token",
-            "Authorization",
-            "test-token",
-            RequestOptionType.body_data,
-            "get_request_body_data",
-        ),
-        (
-            "test_token_from_config",
-            "{{ config.header }}",
-            "{{ config.username }}",
-            "header",
-            "user",
-            RequestOptionType.body_data,
-            "get_request_body_data",
-        ),
-        (
-            "test_token_from_parameters",
-            "{{ parameters.header }}",
-            "{{ parameters.username }}",
-            "header",
-            "user",
-            RequestOptionType.body_data,
-            "get_request_body_data",
-        ),
-        (
-            "test_static_token",
-            "Authorization",
-            "test-token",
-            "Authorization",
-            "test-token",
-            RequestOptionType.body_json,
-            "get_request_body_json",
-        ),
-        (
-            "test_token_from_config",
-            "{{ config.header }}",
-            "{{ config.username }}",
-            "header",
-            "user",
-            RequestOptionType.body_json,
-            "get_request_body_json",
-        ),
-        (
-            "test_token_from_parameters",
-            "{{ parameters.header }}",
-            "{{ parameters.username }}",
-            "header",
-            "user",
-            RequestOptionType.body_json,
-            "get_request_body_json",
-        ),
-    ],
-)
-def test_api_key_authenticator_inject(test_name, field_name, token, expected_field_name, expected_field_value, inject_type, validation_fn):
-    """
-    Should match passed in token, no matter how many times token is retrieved.
-    """
-    token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters)
-    token_auth = ApiKeyAuthenticator(
-        request_option=RequestOption(inject_into=inject_type, field_name=field_name, parameters=parameters),
-        token_provider=token_provider,
-        config=config,
-        parameters=parameters,
-    )
-    assert {expected_field_name: expected_field_value} == getattr(token_auth, validation_fn)()

unit_tests/sources/declarative/auth/test_token_provider.py

@@ -1,73 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from unittest.mock import MagicMock
-
-import pendulum
-import pytest
-from airbyte_cdk.sources.declarative.auth.token_provider import InterpolatedStringTokenProvider, SessionTokenProvider
-from airbyte_cdk.sources.declarative.exceptions import ReadException
-from isodate import parse_duration
-
-
-def create_session_token_provider():
-    login_requester = MagicMock()
-    login_response = MagicMock()
-    login_response.json.return_value = {"nested": {"token": "my_token"}}
-    login_requester.send_request.return_value = login_response
-
-    return SessionTokenProvider(
-        login_requester=login_requester,
-        session_token_path=["nested", "token"],
-        expiration_duration=parse_duration("PT1H"),
-        parameters={"test": "test"},
-    )
-
-
-def test_interpolated_string_token_provider():
-    provider = InterpolatedStringTokenProvider(
-        config={"config_key": "val"}, api_token="{{ config.config_key }}-{{ parameters.test }}", parameters={"test": "test"}
-    )
-    assert provider.get_token() == "val-test"
-
-
-def test_session_token_provider():
-    provider = create_session_token_provider()
-    assert provider.get_token() == "my_token"
-
-
-def test_session_token_provider_cache():
-    provider = create_session_token_provider()
-    provider.get_token()
-    assert provider.get_token() == "my_token"
-    assert provider.login_requester.send_request.call_count == 1
-
-
-def test_session_token_provider_cache_expiration():
-    with pendulum.test(pendulum.datetime(2001, 5, 21, 12)):
-        provider = create_session_token_provider()
-        provider.get_token()
-
-    provider.login_requester.send_request.return_value.json.return_value = {"nested": {"token": "updated_token"}}
-
-    with pendulum.test(pendulum.datetime(2001, 5, 21, 14)):
-        assert provider.get_token() == "updated_token"
-
-    assert provider.login_requester.send_request.call_count == 2
-
-
-def test_session_token_provider_no_cache():
-    provider = create_session_token_provider()
-    provider.expiration_duration = None
-    provider.get_token()
-    assert provider.login_requester.send_request.call_count == 1
-    provider.get_token()
-    assert provider.login_requester.send_request.call_count == 2
-
-
-def test_session_token_provider_ignored_response():
-    provider = create_session_token_provider()
-    provider.login_requester.send_request.return_value = None
-    with pytest.raises(ReadException):
-        provider.get_token()

unit_tests/sources/declarative/checks/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/declarative/checks/test_check_stream.py

@@ -1,146 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import logging
-from typing import Any, Iterable, Mapping, Optional
-from unittest.mock import MagicMock
-
-import pytest
-import requests
-from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream
-from airbyte_cdk.sources.streams.http import HttpStream
-from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
-
-logger = logging.getLogger("test")
-config = dict()
-
-stream_names = ["s1"]
-record = MagicMock()
-
-
-@pytest.mark.parametrize(
-    "test_name, record, streams_to_check, stream_slice, expectation",
-    [
-        ("test_success_check", record, stream_names, {}, (True, None)),
-        ("test_success_check_stream_slice", record, stream_names, {"slice": "slice_value"}, (True, None)),
-        ("test_fail_check", None, stream_names, {}, (True, None)),
-        ("test_try_to_check_invalid stream", record, ["invalid_stream_name"], {}, None),
-    ],
-)
-@pytest.mark.parametrize("slices_as_list", [True, False])
-def test_check_stream_with_slices_as_list(test_name, record, streams_to_check, stream_slice, expectation, slices_as_list):
-    stream = MagicMock()
-    stream.name = "s1"
-    stream.availability_strategy = None
-    if slices_as_list:
-        stream.stream_slices.return_value = [stream_slice]
-    else:
-        stream.stream_slices.return_value = iter([stream_slice])
-
-    stream.read_records.side_effect = mock_read_records({frozenset(stream_slice): iter([record])})
-
-    source = MagicMock()
-    source.streams.return_value = [stream]
-
-    check_stream = CheckStream(streams_to_check, parameters={})
-
-    if expectation:
-        actual = check_stream.check_connection(source, logger, config)
-        assert actual == expectation
-    else:
-        with pytest.raises(ValueError):
-            check_stream.check_connection(source, logger, config)
-
-
-def mock_read_records(responses, default_response=None, **kwargs):
-    return lambda stream_slice, sync_mode: responses[frozenset(stream_slice)] if frozenset(stream_slice) in responses else default_response
-
-
-def test_check_empty_stream():
-    stream = MagicMock()
-    stream.name = "s1"
-    stream.availability_strategy = None
-    stream.read_records.return_value = iter([])
-    stream.stream_slices.return_value = iter([None])
-
-    source = MagicMock()
-    source.streams.return_value = [stream]
-
-    check_stream = CheckStream(["s1"], parameters={})
-    stream_is_available, reason = check_stream.check_connection(source, logger, config)
-    assert stream_is_available
-
-
-def test_check_stream_with_no_stream_slices_aborts():
-    stream = MagicMock()
-    stream.name = "s1"
-    stream.availability_strategy = None
-    stream.stream_slices.return_value = iter([])
-
-    source = MagicMock()
-    source.streams.return_value = [stream]
-
-    check_stream = CheckStream(["s1"], parameters={})
-    stream_is_available, reason = check_stream.check_connection(source, logger, config)
-    assert not stream_is_available
-    assert "no stream slices were found, likely because the parent stream is empty" in reason
-
-
-@pytest.mark.parametrize(
-    "test_name, response_code, available_expectation, expected_messages",
-    [
-        ("test_stream_unavailable_unhandled_error", 404, False, ["Unable to connect to stream mock_http_stream", "404 Client Error"]),
-        (
-            "test_stream_unavailable_handled_error",
-            403,
-            False,
-            [
-                "Unable to read mock_http_stream stream",
-                "This is most likely due to insufficient permissions on the credentials in use.",
-            ],
-        ),
-        ("test_stream_available", 200, True, []),
-    ],
-)
-def test_check_http_stream_via_availability_strategy(mocker, test_name, response_code, available_expectation, expected_messages):
-    class MockHttpStream(HttpStream):
-        url_base = "https://test_base_url.com"
-        primary_key = ""
-
-        def __init__(self, **kwargs):
-            super().__init__(**kwargs)
-            self.resp_counter = 1
-
-        def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-            return None
-
-        def path(self, **kwargs) -> str:
-            return ""
-
-        def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-            stub_resp = {"data": self.resp_counter}
-            self.resp_counter += 1
-            yield stub_resp
-
-        pass
-
-    http_stream = MockHttpStream()
-    assert isinstance(http_stream, HttpStream)
-    assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy)
-
-    source = MagicMock()
-    source.streams.return_value = [http_stream]
-
-    check_stream = CheckStream(stream_names=["mock_http_stream"], parameters={})
-
-    req = requests.Response()
-    req.status_code = response_code
-    mocker.patch.object(requests.Session, "send", return_value=req)
-
-    logger = logging.getLogger(f"airbyte.{getattr(source, 'name', '')}")
-    stream_is_available, reason = check_stream.check_connection(source, logger, config)
-
-    assert stream_is_available == available_expectation
-    for message in expected_messages:
-        assert message in reason

unit_tests/sources/declarative/decoders/test_json_decoder.py

@@ -1,16 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-import requests
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
-
-
-@pytest.mark.parametrize(
-    "response_body, expected_json", (("", {}), ('{"healthcheck": {"status": "ok"}}', {"healthcheck": {"status": "ok"}}))
-)
-def test_json_decoder(requests_mock, response_body, expected_json):
-    requests_mock.register_uri("GET", "https://airbyte.io/", text=response_body)
-    response = requests.get("https://airbyte.io/")
-    assert JsonDecoder(parameters={}).decode(response) == expected_json

unit_tests/sources/declarative/external_component.py

@@ -1,13 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from airbyte_cdk.sources.declarative.requesters import HttpRequester
-
-
-class SampleCustomComponent(HttpRequester):
-    """
-    A test class used to validate manifests that rely on custom defined Python components
-    """
-
-    pass

unit_tests/sources/declarative/extractors/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/declarative/extractors/test_dpath_extractor.py

@@ -1,55 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-
-import pytest
-import requests
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
-from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
-
-config = {"field": "record_array"}
-parameters = {"parameters_field": "record_array"}
-
-decoder = JsonDecoder(parameters={})
-
-
-@pytest.mark.parametrize(
-    "test_name, field_path, body, expected_records",
-    [
-        ("test_extract_from_array", ["data"], {"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]),
-        ("test_extract_single_record", ["data"], {"data": {"id": 1}}, [{"id": 1}]),
-        ("test_extract_single_record_from_root", [], {"id": 1}, [{"id": 1}]),
-        ("test_extract_from_root_array", [], [{"id": 1}, {"id": 2}], [{"id": 1}, {"id": 2}]),
-        ("test_nested_field", ["data", "records"], {"data": {"records": [{"id": 1}, {"id": 2}]}}, [{"id": 1}, {"id": 2}]),
-        ("test_field_in_config", ["{{ config['field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]),
-        (
-            "test_field_in_parameters",
-            ["{{ parameters['parameters_field'] }}"],
-            {"record_array": [{"id": 1}, {"id": 2}]},
-            [{"id": 1}, {"id": 2}],
-        ),
-        ("test_field_does_not_exist", ["record"], {"id": 1}, []),
-        ("test_nested_list", ["list", "*", "item"], {"list": [{"item": {"id": "1"}}]}, [{"id": "1"}]),
-        (
-            "test_complex_nested_list",
-            ["data", "*", "list", "data2", "*"],
-            {"data": [{"list": {"data2": [{"id": 1}, {"id": 2}]}}, {"list": {"data2": [{"id": 3}, {"id": 4}]}}]},
-            [{"id": 1}, {"id": 2}, {"id": 3}, {"id": 4}],
-        ),
-    ],
-)
-def test_dpath_extractor(test_name, field_path, body, expected_records):
-    extractor = DpathExtractor(field_path=field_path, config=config, decoder=decoder, parameters=parameters)
-
-    response = create_response(body)
-    actual_records = extractor.extract_records(response)
-
-    assert actual_records == expected_records
-
-
-def create_response(body):
-    response = requests.Response()
-    response._content = json.dumps(body).encode("utf-8")
-    return response

unit_tests/sources/declarative/extractors/test_record_filter.py

@@ -1,55 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
-
-
-@pytest.mark.parametrize(
-    "test_name, filter_template, records, expected_records",
-    [
-        (
-            "test_using_state_filter",
-            "{{ record['created_at'] > stream_state['created_at'] }}",
-            [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}],
-            [{"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}],
-        ),
-        (
-            "test_with_slice_filter",
-            "{{ record['last_seen'] >= stream_slice['last_seen'] }}",
-            [{"id": 1, "last_seen": "06-06-21"}, {"id": 2, "last_seen": "06-07-21"}, {"id": 3, "last_seen": "06-10-21"}],
-            [{"id": 3, "last_seen": "06-10-21"}],
-        ),
-        (
-            "test_with_next_page_token_filter",
-            "{{ record['id'] >= next_page_token['last_seen_id'] }}",
-            [{"id": 11}, {"id": 12}, {"id": 13}, {"id": 14}, {"id": 15}],
-            [{"id": 14}, {"id": 15}],
-        ),
-        (
-            "test_missing_filter_fields_return_no_results",
-            "{{ record['id'] >= next_page_token['path_to_nowhere'] }}",
-            [{"id": 11}, {"id": 12}, {"id": 13}, {"id": 14}, {"id": 15}],
-            [],
-        ),
-        (
-            "test_using_parameters_filter",
-            "{{ record['created_at'] > parameters['created_at'] }}",
-            [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}],
-            [{"id": 3, "created_at": "06-08-21"}],
-        ),
-    ],
-)
-def test_record_filter(test_name, filter_template, records, expected_records):
-    config = {"response_override": "stop_if_you_see_me"}
-    parameters = {"created_at": "06-07-21"}
-    stream_state = {"created_at": "06-06-21"}
-    stream_slice = {"last_seen": "06-10-21"}
-    next_page_token = {"last_seen_id": 14}
-    record_filter = RecordFilter(config=config, condition=filter_template, parameters=parameters)
-
-    actual_records = record_filter.filter_records(
-        records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
-    )
-    assert actual_records == expected_records