airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4107__py3-none-any.whl

unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py

@@ -1,121 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-
-import pytest
-import requests
-from airbyte_cdk.sources.declarative.requesters.error_handlers import HttpResponseFilter
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus
-
-
-@pytest.mark.parametrize(
-    "action, http_codes, predicate, error_contains, back_off, error_message, response, expected_response_status",
-    [
-        pytest.param(
-            ResponseAction.FAIL,
-            {501, 503},
-            "",
-            "",
-            None,
-            "custom error message",
-            {"status_code": 503},
-            ResponseStatus(response_action=ResponseAction.FAIL, error_message="custom error message"),
-            id="test_http_code_matches",
-        ),
-        pytest.param(
-            ResponseAction.IGNORE,
-            {403},
-            "",
-            "",
-            None,
-            "",
-            {"status_code": 403},
-            ResponseStatus(response_action=ResponseAction.IGNORE),
-            id="test_http_code_matches_ignore_action",
-        ),
-        pytest.param(
-            ResponseAction.RETRY,
-            {429},
-            "",
-            "",
-            30,
-            "",
-            {"status_code": 429},
-            ResponseStatus(response_action=ResponseAction.RETRY, retry_in=30),
-            id="test_http_code_matches_retry_action",
-        ),
-        pytest.param(
-            ResponseAction.FAIL,
-            {},
-            '{{ response.the_body == "do_i_match" }}',
-            "",
-            None,
-            "error message was: {{ response.failure }}",
-            {"status_code": 404, "json": {"the_body": "do_i_match", "failure": "i failed you"}},
-            ResponseStatus(response_action=ResponseAction.FAIL, error_message="error message was: i failed you"),
-            id="test_predicate_matches_json",
-        ),
-        pytest.param(
-            ResponseAction.FAIL,
-            {},
-            '{{ headers.the_key == "header_match" }}',
-            "",
-            None,
-            "error from header: {{ headers.warning }}",
-            {"status_code": 404, "headers": {"the_key": "header_match", "warning": "this failed"}},
-            ResponseStatus(response_action=ResponseAction.FAIL, error_message="error from header: this failed"),
-            id="test_predicate_matches_headers",
-        ),
-        pytest.param(
-            ResponseAction.FAIL,
-            {},
-            None,
-            "DENIED",
-            None,
-            "",
-            {"status_code": 403, "json": {"error": "REQUEST_DENIED"}},
-            ResponseStatus(response_action=ResponseAction.FAIL),
-            id="test_predicate_matches_headers",
-        ),
-        pytest.param(
-            ResponseAction.FAIL,
-            {400, 404},
-            '{{ headers.error == "invalid_input" or response.reason == "bad request"}}',
-            "",
-            None,
-            "",
-            {"status_code": 403, "headers": {"error": "authentication_error"}, "json": {"reason": "permission denied"}},
-            None,
-            id="test_response_does_not_match_filter",
-        ),
-    ],
-)
-def test_matches(requests_mock, action, http_codes, predicate, error_contains, back_off, error_message, response, expected_response_status):
-    requests_mock.register_uri(
-        "GET",
-        "https://airbyte.io/",
-        text=response.get("json") and json.dumps(response.get("json")),
-        headers=response.get("headers") or {},
-        status_code=response.get("status_code"),
-    )
-    response = requests.get("https://airbyte.io/")
-    response_filter = HttpResponseFilter(
-        action=action,
-        config={},
-        parameters={},
-        http_codes=http_codes,
-        predicate=predicate,
-        error_message_contains=error_contains,
-        error_message=error_message,
-    )
-
-    actual_response_status = response_filter.matches(response, backoff_time=back_off or 10)
-    if expected_response_status:
-        assert actual_response_status.action == expected_response_status.action
-        assert actual_response_status.retry_in == expected_response_status.retry_in
-        assert actual_response_status.error_message == expected_response_status.error_message
-    else:
-        assert actual_response_status is None

unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py

@@ -1,44 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus
-
-
-@pytest.mark.parametrize(
-    "test_name, response_action, retry_in, error_message, expected_action, expected_backoff, expected_message",
-    [
-        ("test_fail_with_backoff", ResponseAction.FAIL, 10, "", None, None, ""),
-        (
-            "test_success_no_backoff_error_message",
-            ResponseAction.FAIL,
-            None,
-            "custom error message",
-            ResponseAction.FAIL,
-            None,
-            "custom error message",
-        ),
-        ("test_ignore_with_backoff", ResponseAction.IGNORE, 10, "", None, None, ""),
-        ("test_success_no_backoff", ResponseAction.IGNORE, None, "", ResponseAction.IGNORE, None, ""),
-        ("test_success_with_backoff", ResponseAction.SUCCESS, 10, "", None, None, ""),
-        ("test_success_no_backoff", ResponseAction.SUCCESS, None, "", ResponseAction.SUCCESS, None, ""),
-        ("test_retry_with_backoff", ResponseAction.RETRY, 10, "", ResponseAction.RETRY, 10, ""),
-        ("test_retry_no_backoff", ResponseAction.RETRY, None, "", ResponseAction.RETRY, None, ""),
-    ],
-)
-def test_response_status(test_name, response_action, retry_in, error_message, expected_action, expected_backoff, expected_message):
-    if expected_action or expected_backoff or expected_message:
-        response_status = ResponseStatus(response_action, retry_in, error_message)
-        assert (
-            response_status.action == expected_action
-            and response_status.retry_in == expected_backoff
-            and response_status.error_message == expected_message
-        )
-    else:
-        try:
-            ResponseStatus(response_action, retry_in)
-            assert False
-        except ValueError:
-            pass

unit_tests/sources/declarative/requesters/paginators/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py

@@ -1,64 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-
-import pytest
-import requests
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
-from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import CursorPaginationStrategy
-
-
-@pytest.mark.parametrize(
-    "test_name, template_string, stop_condition, expected_token, page_size",
-    [
-        ("test_static_token", "token", None, "token", None),
-        ("test_static_token_with_page_size", "token", None, "token", 5),
-        ("test_token_from_config", "{{ config.config_key }}", None, "config_value", None),
-        ("test_token_from_last_record", "{{ last_records[-1].id }}", None, 1, None),
-        ("test_token_from_response", "{{ response._metadata.content }}", None, "content_value", None),
-        ("test_token_from_parameters", "{{ parameters.key }}", None, "value", None),
-        ("test_token_not_found", "{{ response.invalid_key }}", None, None, None),
-        ("test_static_token_with_stop_condition_false", "token", InterpolatedBoolean("{{False}}", parameters={}), "token", None),
-        ("test_static_token_with_stop_condition_true", "token", InterpolatedBoolean("{{True}}", parameters={}), None, None),
-        (
-            "test_token_from_header",
-            "{{ headers.next }}",
-            InterpolatedBoolean("{{ not headers.has_more }}", parameters={}),
-            "ready_to_go",
-            None,
-        ),
-        (
-            "test_token_from_response_header_links",
-            "{{ headers.link.next.url }}",
-            InterpolatedBoolean("{{ not headers.link.next.url }}", parameters={}),
-            "https://adventure.io/api/v1/records?page=2&per_page=100",
-            None,
-        ),
-    ],
-)
-def test_cursor_pagination_strategy(test_name, template_string, stop_condition, expected_token, page_size):
-    decoder = JsonDecoder(parameters={})
-    config = {"config_key": "config_value"}
-    parameters = {"key": "value"}
-    strategy = CursorPaginationStrategy(
-        page_size=page_size,
-        cursor_value=template_string,
-        config=config,
-        stop_condition=stop_condition,
-        decoder=decoder,
-        parameters=parameters,
-    )
-
-    response = requests.Response()
-    link_str = '<https://adventure.io/api/v1/records?page=2&per_page=100>; rel="next"'
-    response.headers = {"has_more": True, "next": "ready_to_go", "link": link_str}
-    response_body = {"_metadata": {"content": "content_value"}, "accounts": [], "end": 99, "total": 200, "characters": {}}
-    response._content = json.dumps(response_body).encode("utf-8")
-    last_records = [{"id": 0, "more_records": True}, {"id": 1, "more_records": True}]
-
-    token = strategy.next_page_token(response, last_records)
-    assert expected_token == token
-    assert page_size == strategy.get_page_size()

unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py

@@ -1,313 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-from unittest.mock import MagicMock
-
-import pytest
-import requests
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
-from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-from airbyte_cdk.sources.declarative.requesters.paginators.default_paginator import (
-    DefaultPaginator,
-    PaginatorTestReadDecorator,
-    RequestOption,
-    RequestOptionType,
-)
-from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import CursorPaginationStrategy
-from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import OffsetIncrement
-from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
-
-
-@pytest.mark.parametrize(
-    "page_token_request_option, stop_condition, expected_updated_path, expected_request_params, expected_headers, expected_body_data, expected_body_json, last_records, expected_next_page_token, limit",
-    [
-        (
-            RequestPath(parameters={}),
-            None,
-            "/next_url",
-            {"limit": 2},
-            {},
-            {},
-            {},
-            [{"id": 0}, {"id": 1}],
-            {"next_page_token": "https://airbyte.io/next_url"},
-            2,
-        ),
-        (
-            RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from", parameters={}),
-            None,
-            None,
-            {"limit": 2, "from": "https://airbyte.io/next_url"},
-            {},
-            {},
-            {},
-            [{"id": 0}, {"id": 1}],
-            {"next_page_token": "https://airbyte.io/next_url"},
-            2,
-        ),
-        (
-            RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from", parameters={}),
-            InterpolatedBoolean(condition="{{True}}", parameters={}),
-            None,
-            {"limit": 2},
-            {},
-            {},
-            {},
-            [{"id": 0}, {"id": 1}],
-            None,
-            2,
-        ),
-        (
-            RequestOption(inject_into=RequestOptionType.header, field_name="from", parameters={}),
-            None,
-            None,
-            {"limit": 2},
-            {"from": "https://airbyte.io/next_url"},
-            {},
-            {},
-            [{"id": 0}, {"id": 1}],
-            {"next_page_token": "https://airbyte.io/next_url"},
-            2,
-        ),
-        (
-            RequestOption(inject_into=RequestOptionType.body_data, field_name="from", parameters={}),
-            None,
-            None,
-            {"limit": 2},
-            {},
-            {"from": "https://airbyte.io/next_url"},
-            {},
-            [{"id": 0}, {"id": 1}],
-            {"next_page_token": "https://airbyte.io/next_url"},
-            2,
-        ),
-        (
-            RequestOption(inject_into=RequestOptionType.body_json, field_name="from", parameters={}),
-            None,
-            None,
-            {"limit": 2},
-            {},
-            {},
-            {"from": "https://airbyte.io/next_url"},
-            [{"id": 0}, {"id": 1}],
-            {"next_page_token": "https://airbyte.io/next_url"},
-            2,
-        ),
-    ],
-    ids=[
-        "test_default_paginator_path",
-        "test_default_paginator_request_param",
-        "test_default_paginator_no_token",
-        "test_default_paginator_cursor_header",
-        "test_default_paginator_cursor_body_data",
-        "test_default_paginator_cursor_body_json",
-    ],
-)
-def test_default_paginator_with_cursor(
-    page_token_request_option,
-    stop_condition,
-    expected_updated_path,
-    expected_request_params,
-    expected_headers,
-    expected_body_data,
-    expected_body_json,
-    last_records,
-    expected_next_page_token,
-    limit,
-):
-    page_size_request_option = RequestOption(
-        inject_into=RequestOptionType.request_parameter, field_name="{{parameters['page_limit']}}", parameters={"page_limit": "limit"}
-    )
-    cursor_value = "{{ response.next }}"
-    url_base = "https://airbyte.io"
-    config = {}
-    parameters = {}
-    strategy = CursorPaginationStrategy(
-        page_size=limit,
-        cursor_value=cursor_value,
-        stop_condition=stop_condition,
-        decoder=JsonDecoder(parameters={}),
-        config=config,
-        parameters=parameters,
-    )
-    paginator = DefaultPaginator(
-        page_size_option=page_size_request_option,
-        page_token_option=page_token_request_option,
-        pagination_strategy=strategy,
-        config=config,
-        url_base=url_base,
-        parameters={},
-    )
-
-    response = requests.Response()
-    response.headers = {"A_HEADER": "HEADER_VALUE"}
-    response_body = {"next": "https://airbyte.io/next_url"}
-    response._content = json.dumps(response_body).encode("utf-8")
-
-    actual_next_page_token = paginator.next_page_token(response, last_records)
-    actual_next_path = paginator.path()
-    actual_request_params = paginator.get_request_params()
-    actual_headers = paginator.get_request_headers()
-    actual_body_data = paginator.get_request_body_data()
-    actual_body_json = paginator.get_request_body_json()
-    assert actual_next_page_token == expected_next_page_token
-    assert actual_next_path == expected_updated_path
-    assert actual_request_params == expected_request_params
-    assert actual_headers == expected_headers
-    assert actual_body_data == expected_body_data
-    assert actual_body_json == expected_body_json
-
-
-@pytest.mark.parametrize(
-    "field_name_page_size_interpolation, field_name_page_token_interpolation, expected_request_params",
-    [
-        (
-            "{{parameters['page_limit']}}",
-            "{{parameters['page_token']}}",
-            {"parameters_limit": 50, "parameters_token": "https://airbyte.io/next_url"},
-        ),
-        ("{{config['page_limit']}}", "{{config['page_token']}}", {"config_limit": 50, "config_token": "https://airbyte.io/next_url"}),
-    ],
-    ids=[
-        "parameters_interpolation",
-        "config_interpolation",
-    ],
-)
-def test_paginator_request_param_interpolation(
-    field_name_page_size_interpolation: str, field_name_page_token_interpolation: str, expected_request_params: dict
-):
-    config = {"page_limit": "config_limit", "page_token": "config_token"}
-    parameters = {"page_limit": "parameters_limit", "page_token": "parameters_token"}
-    page_size_request_option = RequestOption(
-        inject_into=RequestOptionType.request_parameter,
-        field_name=field_name_page_size_interpolation,
-        parameters=parameters,
-    )
-    cursor_value = "{{ response.next }}"
-    url_base = "https://airbyte.io"
-    limit = 50
-    strategy = CursorPaginationStrategy(
-        page_size=limit,
-        cursor_value=cursor_value,
-        stop_condition=None,
-        decoder=JsonDecoder(parameters={}),
-        config=config,
-        parameters=parameters,
-    )
-    paginator = DefaultPaginator(
-        page_size_option=page_size_request_option,
-        page_token_option=RequestOption(
-            inject_into=RequestOptionType.request_parameter, field_name=field_name_page_token_interpolation, parameters=parameters
-        ),
-        pagination_strategy=strategy,
-        config=config,
-        url_base=url_base,
-        parameters=parameters,
-    )
-    response = requests.Response()
-    response.headers = {"A_HEADER": "HEADER_VALUE"}
-    response_body = {"next": "https://airbyte.io/next_url"}
-    response._content = json.dumps(response_body).encode("utf-8")
-    last_records = [{"id": 0}, {"id": 1}]
-    paginator.next_page_token(response, last_records)
-    actual_request_params = paginator.get_request_params()
-    assert actual_request_params == expected_request_params
-
-
-def test_page_size_option_cannot_be_set_if_strategy_has_no_limit():
-    page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="page_size", parameters={})
-    page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", parameters={})
-    cursor_value = "{{ response.next }}"
-    url_base = "https://airbyte.io"
-    config = {}
-    parameters = {}
-    strategy = CursorPaginationStrategy(page_size=None, cursor_value=cursor_value, config=config, parameters=parameters)
-    try:
-        DefaultPaginator(
-            page_size_option=page_size_request_option,
-            page_token_option=page_token_request_option,
-            pagination_strategy=strategy,
-            config=config,
-            url_base=url_base,
-            parameters={},
-        )
-        assert False
-    except ValueError:
-        pass
-
-
-@pytest.mark.parametrize(
-    "test_name, inject_on_first_request",
-    [
-        pytest.param("test_reset_inject_on_first_request", True),
-        pytest.param("test_reset_no_inject_on_first_request", False),
-    ],
-)
-def test_reset(test_name, inject_on_first_request):
-    page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", parameters={})
-    page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", parameters={})
-    url_base = "https://airbyte.io"
-    config = {}
-    strategy = OffsetIncrement(config={}, page_size=2, inject_on_first_request=inject_on_first_request, parameters={})
-    paginator = DefaultPaginator(
-        strategy, config, url_base, parameters={}, page_size_option=page_size_request_option, page_token_option=page_token_request_option
-    )
-    initial_request_parameters = paginator.get_request_params()
-    paginator.next_page_token(MagicMock(), [{"first key": "first value"}, {"second key": "second value"}])
-    request_parameters_for_second_request = paginator.get_request_params()
-    paginator.reset()
-    request_parameters_after_reset = paginator.get_request_params()
-    assert initial_request_parameters == request_parameters_after_reset
-    assert request_parameters_for_second_request != request_parameters_after_reset
-
-
-def test_initial_token_with_offset_pagination():
-    page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", parameters={})
-    page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", parameters={})
-    url_base = "https://airbyte.io"
-    config = {}
-    strategy = OffsetIncrement(config={}, page_size=2, parameters={}, inject_on_first_request=True)
-    paginator = DefaultPaginator(
-        strategy, config, url_base, parameters={}, page_size_option=page_size_request_option, page_token_option=page_token_request_option
-    )
-    initial_request_parameters = paginator.get_request_params()
-
-    assert initial_request_parameters == {"limit": 2, "offset": 0}
-
-
-def test_limit_page_fetched():
-    maximum_number_of_pages = 5
-    number_of_next_performed = maximum_number_of_pages - 1
-    paginator = PaginatorTestReadDecorator(
-        DefaultPaginator(
-            page_size_option=MagicMock(),
-            page_token_option=MagicMock(),
-            pagination_strategy=MagicMock(),
-            config=MagicMock(),
-            url_base=MagicMock(),
-            parameters={},
-        ),
-        maximum_number_of_pages,
-    )
-
-    for _ in range(number_of_next_performed):
-        last_token = paginator.next_page_token(MagicMock(), MagicMock())
-        assert last_token
-
-    assert not paginator.next_page_token(MagicMock(), MagicMock())
-
-
-def test_paginator_with_page_option_no_page_size():
-    pagination_strategy = OffsetIncrement(config={}, page_size=None, parameters={})
-
-    with pytest.raises(ValueError):
-        DefaultPaginator(
-            page_size_option=MagicMock(),
-            page_token_option=RequestOption("limit", RequestOptionType.request_parameter, parameters={}),
-            pagination_strategy=pagination_strategy,
-            config=MagicMock(),
-            url_base=MagicMock(),
-            parameters={},
-        ),

unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py

@@ -1,12 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import requests
-from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination
-
-
-def test():
-    paginator = NoPagination(parameters={})
-    next_page_token = paginator.next_page_token(requests.Response(), [])
-    assert next_page_token == {}

unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py

@@ -1,58 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-from typing import Any, Optional
-
-import pytest
-import requests
-from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import OffsetIncrement
-
-
-@pytest.mark.parametrize(
-    "page_size, parameters, last_records, expected_next_page_token, expected_offset",
-    [
-        pytest.param("2", {}, [{"id": 0}, {"id": 1}], 2, 2, id="test_same_page_size"),
-        pytest.param(2, {}, [{"id": 0}, {"id": 1}], 2, 2, id="test_same_page_size"),
-        pytest.param("{{ parameters['page_size'] }}", {"page_size": 3}, [{"id": 0}, {"id": 1}], None, 0, id="test_larger_page_size"),
-        pytest.param(None, {}, [], None, 0, id="test_stop_if_no_records"),
-        pytest.param("{{ response['page_metadata']['limit'] }}", {}, [{"id": 0}, {"id": 1}], None, 0, id="test_page_size_from_response"),
-    ],
-)
-def test_offset_increment_paginator_strategy(page_size, parameters, last_records, expected_next_page_token, expected_offset):
-    paginator_strategy = OffsetIncrement(page_size=page_size, parameters=parameters, config={})
-    assert paginator_strategy._offset == 0
-
-    response = requests.Response()
-
-    response.headers = {"A_HEADER": "HEADER_VALUE"}
-    response_body = {"next": "https://airbyte.io/next_url", "page_metadata": {"limit": 5}}
-    response._content = json.dumps(response_body).encode("utf-8")
-
-    next_page_token = paginator_strategy.next_page_token(response, last_records)
-    assert expected_next_page_token == next_page_token
-    assert expected_offset == paginator_strategy._offset
-
-    paginator_strategy.reset()
-    assert 0 == paginator_strategy._offset
-
-
-def test_offset_increment_paginator_strategy_rises():
-    paginator_strategy = OffsetIncrement(page_size="{{ parameters['page_size'] }}", parameters={"page_size": "invalid value"}, config={})
-    with pytest.raises(Exception) as exc:
-        paginator_strategy.get_page_size()
-    assert str(exc.value) == "invalid value is of type <class 'str'>. Expected <class 'int'>"
-
-
-@pytest.mark.parametrize(
-    "inject_on_first_request, expected_initial_token",
-    [
-        pytest.param(True, 0, id="test_with_inject_offset"),
-        pytest.param(False, None, id="test_without_inject_offset"),
-    ],
-)
-def test_offset_increment_paginator_strategy_initial_token(inject_on_first_request: bool, expected_initial_token: Optional[Any]):
-    paginator_strategy = OffsetIncrement(page_size=20, parameters={}, config={}, inject_on_first_request=inject_on_first_request)
-
-    assert paginator_strategy.initial_token == expected_initial_token