airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

unit_tests/sources/declarative/stream_slicers/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py

@@ -1,225 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest as pytest
-from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
-from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
-from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter
-from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType
-from airbyte_cdk.sources.declarative.stream_slicers.cartesian_product_stream_slicer import CartesianProductStreamSlicer
-from airbyte_cdk.sources.declarative.types import StreamSlice
-
-
-@pytest.mark.parametrize(
-    "test_name, stream_slicers, expected_slices",
-    [
-        (
-            "test_single_stream_slicer",
-            [ListPartitionRouter(values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={})],
-            [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}),
-             StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}),
-             StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})],
-        ),
-        (
-            "test_two_stream_slicers",
-            [
-                ListPartitionRouter(values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={}),
-                ListPartitionRouter(values=["A", "B"], cursor_field="letter", config={}, parameters={}),
-            ],
-            [
-                StreamSlice(partition={"owner_resource": "customer", "letter": "A"}, cursor_slice={}),
-                StreamSlice(partition={"owner_resource": "customer", "letter": "B"}, cursor_slice={}),
-                StreamSlice(partition={"owner_resource": "store", "letter": "A"}, cursor_slice={}),
-                StreamSlice(partition={"owner_resource": "store", "letter": "B"}, cursor_slice={}),
-                StreamSlice(partition={"owner_resource": "subscription", "letter": "A"}, cursor_slice={}),
-                StreamSlice(partition={"owner_resource": "subscription", "letter": "B"}, cursor_slice={}),
-            ],
-        ),
-        (
-                "test_singledatetime",
-                [
-                    DatetimeBasedCursor(
-                        start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}),
-                        end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}),
-                        step="P1D",
-                        cursor_field=InterpolatedString.create("", parameters={}),
-                        datetime_format="%Y-%m-%d",
-                        cursor_granularity="P1D",
-                        config={},
-                        parameters={},
-                    ),
-                ],
-                [
-                    StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}),
-                    StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}),
-                    StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}),
-                ],
-        ),
-        (
-            "test_list_and_datetime",
-            [
-                ListPartitionRouter(values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={}),
-                DatetimeBasedCursor(
-                    start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}),
-                    end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}),
-                    step="P1D",
-                    cursor_field=InterpolatedString.create("", parameters={}),
-                    datetime_format="%Y-%m-%d",
-                    cursor_granularity="P1D",
-                    config={},
-                    parameters={},
-                ),
-            ],
-            [
-                StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}),
-                StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}),
-                StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}),
-                StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}),
-                StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}),
-                StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}),
-                StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}),
-                StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}),
-                StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}),
-            ],
-        ),
-    ],
-)
-def test_substream_slicer(test_name, stream_slicers, expected_slices):
-    slicer = CartesianProductStreamSlicer(stream_slicers=stream_slicers, parameters={})
-    slices = [s for s in slicer.stream_slices()]
-    assert slices == expected_slices
-
-
-def test_stream_slices_raises_exception_if_multiple_cursor_slice_components():
-    stream_slicers = [
-        DatetimeBasedCursor(
-            start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}),
-            end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}),
-            step="P1D",
-            cursor_field=InterpolatedString.create("", parameters={}),
-            datetime_format="%Y-%m-%d",
-            cursor_granularity="P1D",
-            config={},
-            parameters={},
-        ),
-        DatetimeBasedCursor(
-            start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}),
-            end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}),
-            step="P1D",
-            cursor_field=InterpolatedString.create("", parameters={}),
-            datetime_format="%Y-%m-%d",
-            cursor_granularity="P1D",
-            config={},
-            parameters={},
-        ),
-    ]
-    slicer = CartesianProductStreamSlicer(stream_slicers=stream_slicers, parameters={})
-    with pytest.raises(ValueError):
-        list(slicer.stream_slices())
-
-
-@pytest.mark.parametrize(
-    "test_name, stream_1_request_option, stream_2_request_option, expected_req_params, expected_headers,expected_body_json, expected_body_data",
-    [
-        (
-            "test_param_header",
-            RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="owner"),
-            RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="repo"),
-            {"owner": "customer"},
-            {"repo": "airbyte"},
-            {},
-            {},
-        ),
-        (
-            "test_header_header",
-            RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="owner"),
-            RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="repo"),
-            {},
-            {"owner": "customer", "repo": "airbyte"},
-            {},
-            {},
-        ),
-        (
-            "test_body_data",
-            RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="owner"),
-            RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="repo"),
-            {},
-            {},
-            {},
-            {"owner": "customer", "repo": "airbyte"},
-        ),
-        (
-            "test_body_json",
-            RequestOption(inject_into=RequestOptionType.body_json, parameters={}, field_name="owner"),
-            RequestOption(inject_into=RequestOptionType.body_json, parameters={}, field_name="repo"),
-            {},
-            {},
-            {"owner": "customer", "repo": "airbyte"},
-            {},
-        ),
-    ],
-)
-def test_request_option(
-    test_name,
-    stream_1_request_option,
-    stream_2_request_option,
-    expected_req_params,
-    expected_headers,
-    expected_body_json,
-    expected_body_data,
-):
-    slicer = CartesianProductStreamSlicer(
-        stream_slicers=[
-            ListPartitionRouter(
-                values=["customer", "store", "subscription"],
-                cursor_field="owner_resource",
-                config={},
-                request_option=stream_1_request_option,
-                parameters={},
-            ),
-            ListPartitionRouter(
-                values=["airbyte", "airbyte-cloud"],
-                cursor_field="repository",
-                config={},
-                request_option=stream_2_request_option,
-                parameters={},
-            ),
-        ],
-        parameters={},
-    )
-    stream_slice = {"owner_resource": "customer", "repository": "airbyte"}
-
-    assert expected_req_params == slicer.get_request_params(stream_slice=stream_slice)
-    assert expected_headers == slicer.get_request_headers(stream_slice=stream_slice)
-    assert expected_body_json == slicer.get_request_body_json(stream_slice=stream_slice)
-    assert expected_body_data == slicer.get_request_body_data(stream_slice=stream_slice)
-
-
-def test_request_option_before_updating_cursor():
-    stream_1_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="owner")
-    stream_2_request_option = RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="repo")
-    slicer = CartesianProductStreamSlicer(
-        stream_slicers=[
-            ListPartitionRouter(
-                values=["customer", "store", "subscription"],
-                cursor_field="owner_resource",
-                config={},
-                request_option=stream_1_request_option,
-                parameters={},
-            ),
-            ListPartitionRouter(
-                values=["airbyte", "airbyte-cloud"],
-                cursor_field="repository",
-                config={},
-                request_option=stream_2_request_option,
-                parameters={},
-            ),
-        ],
-        parameters={},
-    )
-    assert {} == slicer.get_request_params()
-    assert {} == slicer.get_request_headers()
-    assert {} == slicer.get_request_body_json()
-    assert {} == slicer.get_request_body_data()

unit_tests/sources/declarative/test_create_partial.py

@@ -1,83 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.sources.declarative.create_partial import _key_is_unset_or_identical, create
-from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-
-
-class AClass:
-    def __init__(self, parameter, another_param, parameters):
-        self.parameter = parameter
-        self.another_param = another_param
-        self.parameters = parameters
-
-
-class OuterClass:
-    def __init__(self, name, some_field, inner_param):
-        self.name = name
-        self.some_field = some_field
-        self.inner_param = inner_param
-
-
-class OuterOuterClass:
-    def __init__(self, name, param, inner_class):
-        self.name = name
-        self.param = param
-        self.inner_class = inner_class
-
-
-def test_pass_parameter_to_create_function():
-    object = create(AClass, parameter="A")(another_param="B")
-    assert object.parameter == "A"
-    assert object.another_param == "B"
-
-
-def test_parameter_not_overwritten_by_parameters():
-    object = create(AClass, parameter="A", another_param="B", **{"$parameters": {"parameter": "C"}})()
-    assert object.parameter == "A"
-    assert object.another_param == "B"
-
-
-def test_overwrite_param():
-    object = create(AClass, parameter="A", another_param="B")(parameter="C")
-    assert object.parameter == "C"
-    assert object.another_param == "B"
-
-
-def test_string_interpolation():
-    s = "{{ next_page_token['next_page_url'] }}"
-    partial = create(InterpolatedString, string=s)
-    interpolated_string = partial()
-    assert interpolated_string.string == s
-
-
-def test_string_interpolation_through_parameters():
-    s = "{{ parameters['name'] }}"
-    parameters = {"name": "airbyte"}
-    partial = create(InterpolatedString, string=s, **parameters)
-    interpolated_string = partial()
-    assert interpolated_string.eval({}) == "airbyte"
-
-
-def test_string_interpolation_through_parameters_keyword():
-    s = "{{ parameters['name'] }}"
-    parameters = {"$parameters": {"name": "airbyte"}}
-    partial = create(InterpolatedString, string=s, **parameters)
-    interpolated_string = partial()
-    assert interpolated_string.eval({}) == "airbyte"
-
-
-@pytest.mark.parametrize(
-    "test_name, key, value, expected_result",
-    [
-        ("test", "key", "value", True),
-        ("test", "key", "a_different_value", False),
-        ("test", "a_different_key", "value", True),
-    ],
-)
-def test_key_is_unset_or_identical(test_name, key, value, expected_result):
-    mapping = {"key": "value"}
-    result = _key_is_unset_or_identical(key, value, mapping)
-    assert expected_result == result

unit_tests/sources/declarative/test_declarative_stream.py

@@ -1,103 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from unittest.mock import MagicMock
-
-import pytest
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteTraceMessage, Level, SyncMode, TraceType, Type
-from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
-from airbyte_cdk.sources.declarative.types import StreamSlice
-
-SLICE_NOT_CONSIDERED_FOR_EQUALITY = {}
-
-_name = "stream"
-_primary_key = "pk"
-_cursor_field = "created_at"
-_json_schema = {"name": {"type": "string"}}
-
-
-def test_declarative_stream():
-    schema_loader = _schema_loader()
-
-    state = MagicMock()
-    records = [
-        {"pk": 1234, "field": "value"},
-        {"pk": 4567, "field": "different_value"},
-        AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="This is a log  message")),
-        AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345)),
-    ]
-    stream_slices = [
-        StreamSlice(partition={}, cursor_slice={"date": "2021-01-01"}),
-        StreamSlice(partition={}, cursor_slice={"date": "2021-01-02"}),
-        StreamSlice(partition={}, cursor_slice={"date": "2021-01-03"}),
-    ]
-
-    retriever = MagicMock()
-    retriever.state = state
-    retriever.read_records.return_value = records
-    retriever.stream_slices.return_value = stream_slices
-
-    config = {"api_key": "open_sesame"}
-
-    stream = DeclarativeStream(
-        name=_name,
-        primary_key=_primary_key,
-        stream_cursor_field="{{ parameters['cursor_field'] }}",
-        schema_loader=schema_loader,
-        retriever=retriever,
-        config=config,
-        parameters={"cursor_field": "created_at"},
-    )
-
-    assert stream.name == _name
-    assert stream.get_json_schema() == _json_schema
-    assert stream.state == state
-    input_slice = stream_slices[0]
-    assert list(stream.read_records(SyncMode.full_refresh, _cursor_field, input_slice, state)) == records
-    assert stream.primary_key == _primary_key
-    assert stream.cursor_field == _cursor_field
-    assert stream.stream_slices(sync_mode=SyncMode.incremental, cursor_field=_cursor_field, stream_state=None) == stream_slices
-
-
-def test_read_records_raises_exception_if_stream_slice_is_not_per_partition_stream_slice():
-    schema_loader = _schema_loader()
-
-    retriever = MagicMock()
-    retriever.state = MagicMock()
-    retriever.read_records.return_value = []
-    stream_slice = {"date": "2021-01-01"}
-    retriever.stream_slices.return_value = [stream_slice]
-
-    stream = DeclarativeStream(
-        name=_name,
-        primary_key=_primary_key,
-        stream_cursor_field="{{ parameters['cursor_field'] }}",
-        schema_loader=schema_loader,
-        retriever=retriever,
-        config={},
-        parameters={"cursor_field": "created_at"},
-    )
-
-    with pytest.raises(ValueError):
-        list(stream.read_records(SyncMode.full_refresh, _cursor_field, stream_slice, MagicMock()))
-
-
-def test_state_checkpoint_interval():
-    stream = DeclarativeStream(
-        name="any name",
-        primary_key="any primary key",
-        stream_cursor_field="{{ parameters['cursor_field'] }}",
-        schema_loader=MagicMock(),
-        retriever=MagicMock(),
-        config={},
-        parameters={},
-    )
-
-    assert stream.state_checkpoint_interval is None
-
-
-def _schema_loader():
-    schema_loader = MagicMock()
-    schema_loader.get_json_schema.return_value = _json_schema
-    return schema_loader