airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

unit_tests/sources/streams/concurrent/test_cursor.py

@@ -1,139 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-from typing import Any, Mapping, Optional
-from unittest import TestCase
-from unittest.mock import Mock
-
-import pytest
-from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
-from airbyte_cdk.sources.message import MessageRepository
-from airbyte_cdk.sources.streams.concurrent.cursor import Comparable, ConcurrentCursor, CursorField
-from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
-from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter
-
-_A_STREAM_NAME = "a stream name"
-_A_STREAM_NAMESPACE = "a stream namespace"
-_A_CURSOR_FIELD_KEY = "a_cursor_field_key"
-_NO_PARTITION_IDENTIFIER = None
-_NO_SLICE = None
-_NO_SLICE_BOUNDARIES = None
-_LOWER_SLICE_BOUNDARY_FIELD = "lower_boundary"
-_UPPER_SLICE_BOUNDARY_FIELD = "upper_boundary"
-_SLICE_BOUNDARY_FIELDS = (_LOWER_SLICE_BOUNDARY_FIELD, _UPPER_SLICE_BOUNDARY_FIELD)
-_A_VERY_HIGH_CURSOR_VALUE = 1000000000
-
-
-def _partition(_slice: Optional[Mapping[str, Any]]) -> Partition:
-    partition = Mock(spec=Partition)
-    partition.to_slice.return_value = _slice
-    return partition
-
-
-def _record(cursor_value: Comparable) -> Record:
-    return Record(data={_A_CURSOR_FIELD_KEY: cursor_value}, stream_name=_A_STREAM_NAME)
-
-
-class ConcurrentCursorTest(TestCase):
-    def setUp(self) -> None:
-        self._message_repository = Mock(spec=MessageRepository)
-        self._state_manager = Mock(spec=ConnectorStateManager)
-        self._state_converter = EpochValueConcurrentStreamStateConverter()
-
-    def _cursor_with_slice_boundary_fields(self) -> ConcurrentCursor:
-        return ConcurrentCursor(
-            _A_STREAM_NAME,
-            _A_STREAM_NAMESPACE,
-            {},
-            self._message_repository,
-            self._state_manager,
-            self._state_converter,
-            CursorField(_A_CURSOR_FIELD_KEY),
-            _SLICE_BOUNDARY_FIELDS,
-            None,
-        )
-
-    def _cursor_without_slice_boundary_fields(self) -> ConcurrentCursor:
-        return ConcurrentCursor(
-            _A_STREAM_NAME,
-            _A_STREAM_NAMESPACE,
-            {},
-            self._message_repository,
-            self._state_manager,
-            self._state_converter,
-            CursorField(_A_CURSOR_FIELD_KEY),
-            None,
-            None,
-        )
-
-    def test_given_boundary_fields_when_close_partition_then_emit_state(self) -> None:
-        cursor = self._cursor_with_slice_boundary_fields()
-        cursor.close_partition(
-            _partition(
-                {_LOWER_SLICE_BOUNDARY_FIELD: 12, _UPPER_SLICE_BOUNDARY_FIELD: 30},
-            )
-        )
-
-        self._message_repository.emit_message.assert_called_once_with(self._state_manager.create_state_message.return_value)
-        self._state_manager.update_state_for_stream.assert_called_once_with(
-            _A_STREAM_NAME,
-            _A_STREAM_NAMESPACE,
-            {_A_CURSOR_FIELD_KEY: 0},  # State message is updated to the legacy format before being emitted
-        )
-
-    def test_given_boundary_fields_when_close_partition_then_emit_updated_state(self) -> None:
-        self._cursor_with_slice_boundary_fields().close_partition(
-            _partition(
-                {_LOWER_SLICE_BOUNDARY_FIELD: 0, _UPPER_SLICE_BOUNDARY_FIELD: 30},
-            )
-        )
-
-        self._message_repository.emit_message.assert_called_once_with(self._state_manager.create_state_message.return_value)
-        self._state_manager.update_state_for_stream.assert_called_once_with(
-            _A_STREAM_NAME,
-            _A_STREAM_NAMESPACE,
-            {_A_CURSOR_FIELD_KEY: 30},  # State message is updated to the legacy format before being emitted
-        )
-
-    def test_given_boundary_fields_and_record_observed_when_close_partition_then_ignore_records(self) -> None:
-        cursor = self._cursor_with_slice_boundary_fields()
-        cursor.observe(_record(_A_VERY_HIGH_CURSOR_VALUE))
-
-        cursor.close_partition(_partition({_LOWER_SLICE_BOUNDARY_FIELD: 12, _UPPER_SLICE_BOUNDARY_FIELD: 30}))
-
-        assert self._state_manager.update_state_for_stream.call_args_list[0].args[2][_A_CURSOR_FIELD_KEY] != _A_VERY_HIGH_CURSOR_VALUE
-
-    def test_given_no_boundary_fields_when_close_partition_then_emit_state(self) -> None:
-        cursor = self._cursor_without_slice_boundary_fields()
-        cursor.observe(_record(10))
-        cursor.close_partition(_partition(_NO_SLICE))
-
-        self._state_manager.update_state_for_stream.assert_called_once_with(
-            _A_STREAM_NAME,
-            _A_STREAM_NAMESPACE,
-            {"a_cursor_field_key": 10},
-        )
-
-    def test_given_no_boundary_fields_when_close_multiple_partitions_then_raise_exception(self) -> None:
-        cursor = self._cursor_without_slice_boundary_fields()
-        cursor.observe(_record(10))
-        cursor.close_partition(_partition(_NO_SLICE))
-
-        with pytest.raises(ValueError):
-            cursor.close_partition(_partition(_NO_SLICE))
-
-    def test_given_no_records_observed_when_close_partition_then_do_not_emit_state(self) -> None:
-        cursor = self._cursor_without_slice_boundary_fields()
-        cursor.close_partition(_partition(_NO_SLICE))
-        assert self._message_repository.emit_message.call_count == 0
-
-    def test_given_slice_boundaries_and_no_slice_when_close_partition_then_raise_error(self) -> None:
-        cursor = self._cursor_with_slice_boundary_fields()
-        with pytest.raises(KeyError):
-            cursor.close_partition(_partition(_NO_SLICE))
-
-    def test_given_slice_boundaries_not_matching_slice_when_close_partition_then_raise_error(self) -> None:
-        cursor = self._cursor_with_slice_boundary_fields()
-        with pytest.raises(KeyError):
-            cursor.close_partition(_partition({"not_matching_key": "value"}))

unit_tests/sources/streams/concurrent/test_datetime_state_converter.py

@@ -1,369 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from datetime import datetime, timezone
-
-import pytest
-from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
-from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ConcurrencyCompatibleStateType
-from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
-    EpochValueConcurrentStreamStateConverter,
-    IsoMillisConcurrentStreamStateConverter,
-)
-
-
-@pytest.mark.parametrize(
-    "converter, input_state, is_compatible",
-    [
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            {"state_type": "date-range"},
-            True,
-            id="no-input-state-is-compatible-epoch",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            {
-                "created_at": "2022_05_22",
-                "state_type": ConcurrencyCompatibleStateType.date_range.value,
-            },
-            True,
-            id="input-state-with-date_range-is-compatible-epoch",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            {
-                "created_at": "2022_05_22",
-                "state_type": "fake",
-            },
-            False,
-            id="input-state-with-fake-state-type-is-not-compatible-epoch",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            {
-                "created_at": "2022_05_22",
-            },
-            False,
-            id="input-state-without-state_type-is-not-compatible-epoch",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            {"state_type": "date-range"},
-            True,
-            id="no-input-state-is-compatible-isomillis",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            {
-                "created_at": "2022_05_22",
-                "state_type": ConcurrencyCompatibleStateType.date_range.value,
-            },
-            True,
-            id="input-state-with-date_range-is-compatible-isomillis",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            {
-                "created_at": "2022_05_22",
-                "state_type": "fake",
-            },
-            False,
-            id="input-state-with-fake-state-type-is-not-compatible-isomillis",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            {
-                "created_at": "2022_05_22",
-            },
-            False,
-            id="input-state-without-state_type-is-not-compatible-isomillis",
-        ),
-    ],
-)
-def test_concurrent_stream_state_converter_is_state_message_compatible(converter, input_state, is_compatible):
-    assert converter.is_state_message_compatible(input_state) == is_compatible
-
-
-@pytest.mark.parametrize(
-    "converter,start,state,expected_start",
-    [
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            None,
-            {},
-            EpochValueConcurrentStreamStateConverter().zero_value,
-            id="epoch-converter-no-state-no-start-start-is-zero-value",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            1617030403,
-            {},
-            datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-            id="epoch-converter-no-state-with-start-start-is-start",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            None,
-            {"created_at": 1617030404},
-            datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc),
-            id="epoch-converter-state-without-start-start-is-from-state",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            1617030404,
-            {"created_at": 1617030403},
-            datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc),
-            id="epoch-converter-state-before-start-start-is-start",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            1617030403,
-            {"created_at": 1617030404},
-            datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc),
-            id="epoch-converter-state-after-start-start-is-from-state",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            None,
-            {},
-            IsoMillisConcurrentStreamStateConverter().zero_value,
-            id="isomillis-converter-no-state-no-start-start-is-zero-value",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            "2021-08-22T05:03:27.000Z",
-            {},
-            datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc),
-            id="isomillis-converter-no-state-with-start-start-is-start",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            None,
-            {"created_at": "2021-08-22T05:03:27.000Z"},
-            datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc),
-            id="isomillis-converter-state-without-start-start-is-from-state",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            "2022-08-22T05:03:27.000Z",
-            {"created_at": "2021-08-22T05:03:27.000Z"},
-            datetime(2022, 8, 22, 5, 3, 27, tzinfo=timezone.utc),
-            id="isomillis-converter-state-before-start-start-is-start",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            "2022-08-22T05:03:27.000Z",
-            {"created_at": "2023-08-22T05:03:27.000Z"},
-            datetime(2023, 8, 22, 5, 3, 27, tzinfo=timezone.utc),
-            id="isomillis-converter-state-after-start-start-is-from-state",
-        ),
-    ],
-)
-def test_get_sync_start(converter, start, state, expected_start):
-    assert converter._get_sync_start(CursorField("created_at"), state, start) == expected_start
-
-
-@pytest.mark.parametrize(
-    "converter, start, sequential_state, expected_output_state",
-    [
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            0,
-            {},
-            {
-                "legacy": {},
-                "slices": [
-                    {
-                        "start": EpochValueConcurrentStreamStateConverter().zero_value,
-                        "end": EpochValueConcurrentStreamStateConverter().zero_value,
-                    }
-                ],
-                "state_type": "date-range",
-            },
-            id="empty-input-state-epoch",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            1617030403,
-            {"created": 1617030403},
-            {
-                "state_type": "date-range",
-                "slices": [
-                    {
-                        "start": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                        "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    }
-                ],
-                "legacy": {"created": 1617030403},
-            },
-            id="with-input-state-epoch",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            "2020-01-01T00:00:00.000Z",
-            {"created": "2021-08-22T05:03:27.000Z"},
-            {
-                "state_type": "date-range",
-                "slices": [
-                    {
-                        "start": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc),
-                        "end": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc),
-                    }
-                ],
-                "legacy": {"created": "2021-08-22T05:03:27.000Z"},
-            },
-            id="with-input-state-isomillis",
-        ),
-    ],
-)
-def test_convert_from_sequential_state(converter, start, sequential_state, expected_output_state):
-    comparison_format = "%Y-%m-%dT%H:%M:%S.%f"
-    if expected_output_state["slices"]:
-        _, conversion = converter.convert_from_sequential_state(CursorField("created"), sequential_state, start)
-        assert conversion["state_type"] == expected_output_state["state_type"]
-        assert conversion["legacy"] == expected_output_state["legacy"]
-        for actual, expected in zip(conversion["slices"], expected_output_state["slices"]):
-            assert actual["start"].strftime(comparison_format) == expected["start"].strftime(comparison_format)
-            assert actual["end"].strftime(comparison_format) == expected["end"].strftime(comparison_format)
-    else:
-        _, conversion = converter.convert_from_sequential_state(CursorField("created"), sequential_state, start)
-        assert conversion == expected_output_state
-
-
-@pytest.mark.parametrize(
-    "converter, concurrent_state, expected_output_state",
-    [
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            {
-                "state_type": "date-range",
-                "slices": [
-                    {
-                        "start": datetime(1970, 1, 3, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    }
-                ],
-            },
-            {"created": 1617030403},
-            id="epoch-single-slice",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            {
-                "state_type": "date-range",
-                "slices": [
-                    {
-                        "start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    },
-                    {
-                        "start": datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2022, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    },
-                ],
-            },
-            {"created": 1648566403},
-            id="epoch-overlapping-slices",
-        ),
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            {
-                "state_type": "date-range",
-                "slices": [
-                    {
-                        "start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    },
-                    {
-                        "start": datetime(2022, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2023, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    },
-                ],
-            },
-            {"created": 1617030403},
-            id="epoch-multiple-slices",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            {
-                "state_type": "date-range",
-                "slices": [
-                    {
-                        "start": datetime(1970, 1, 3, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    }
-                ],
-            },
-            {"created": "2021-03-29T15:06:43.000Z"},
-            id="isomillis-single-slice",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            {
-                "state_type": "date-range",
-                "slices": [
-                    {
-                        "start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    },
-                    {
-                        "start": datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2022, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    },
-                ],
-            },
-            {"created": "2022-03-29T15:06:43.000Z"},
-            id="isomillis-overlapping-slices",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            {
-                "state_type": "date-range",
-                "slices": [
-                    {
-                        "start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    },
-                    {
-                        "start": datetime(2022, 1, 1, 0, 0, 0, tzinfo=timezone.utc),
-                        "end": datetime(2023, 3, 29, 15, 6, 43, tzinfo=timezone.utc),
-                    },
-                ],
-            },
-            {"created": "2021-03-29T15:06:43.000Z"},
-            id="isomillis-multiple-slices",
-        ),
-    ],
-)
-def test_convert_to_sequential_state(converter, concurrent_state, expected_output_state):
-    assert converter.convert_to_sequential_state(CursorField("created"), concurrent_state) == expected_output_state
-
-
-@pytest.mark.parametrize(
-    "converter, concurrent_state, expected_output_state",
-    [
-        pytest.param(
-            EpochValueConcurrentStreamStateConverter(),
-            {
-                "state_type": ConcurrencyCompatibleStateType.date_range.value,
-                "start": EpochValueConcurrentStreamStateConverter().zero_value,
-            },
-            {"created": 0},
-            id="empty-slices-epoch",
-        ),
-        pytest.param(
-            IsoMillisConcurrentStreamStateConverter(),
-            {
-                "state_type": ConcurrencyCompatibleStateType.date_range.value,
-                "start": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc),
-            },
-            {"created": "2021-08-22T05:03:27.000Z"},
-            id="empty-slices-isomillis",
-        ),
-    ],
-)
-def test_convert_to_sequential_state_no_slices_returns_legacy_state(converter, concurrent_state, expected_output_state):
-    with pytest.raises(RuntimeError):
-        converter.convert_to_sequential_state(CursorField("created"), concurrent_state)