airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py

@@ -1,332 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from unittest.mock import patch
-
-from airbyte_cdk.logger import init_logger
-from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type
-from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import StreamSlice
-from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
-from airbyte_cdk.sources.declarative.types import Record
-
-CURSOR_FIELD = "cursor_field"
-SYNC_MODE = SyncMode.incremental
-
-
-class ManifestBuilder:
-    def __init__(self):
-        self._incremental_sync = {}
-        self._partition_router = {}
-        self._substream_partition_router = {}
-
-    def with_list_partition_router(self, stream_name, cursor_field, partitions):
-        self._partition_router[stream_name] = {
-            "type": "ListPartitionRouter",
-            "cursor_field": cursor_field,
-            "values": partitions,
-        }
-        return self
-
-    def with_substream_partition_router(self, stream_name):
-        self._substream_partition_router[stream_name] = {
-            "type": "SubstreamPartitionRouter",
-            "parent_stream_configs": [
-                {
-                    "type": "ParentStreamConfig",
-                    "stream": "#/definitions/Rates",
-                    "parent_key": "id",
-                    "partition_field": "parent_id",
-
-                }
-            ]
-        }
-        return self
-
-    def with_incremental_sync(self, stream_name, start_datetime, end_datetime, datetime_format, cursor_field, step, cursor_granularity):
-        self._incremental_sync[stream_name] = {
-            "type": "DatetimeBasedCursor",
-            "start_datetime": start_datetime,
-            "end_datetime": end_datetime,
-            "datetime_format": datetime_format,
-            "cursor_field": cursor_field,
-            "step": step,
-            "cursor_granularity": cursor_granularity,
-        }
-        return self
-
-    def build(self):
-        manifest = {
-            "version": "0.34.2",
-            "type": "DeclarativeSource",
-            "check": {"type": "CheckStream", "stream_names": ["Rates"]},
-            "definitions": {
-                "AnotherStream": {
-                    "type": "DeclarativeStream",
-                    "name": "AnotherStream",
-                    "primary_key": [],
-                    "schema_loader": {
-                        "type": "InlineSchemaLoader",
-                        "schema": {"$schema": "http://json-schema.org/schema#", "properties": {"id": {"type": "string"}}, "type": "object"},
-                    },
-                    "retriever": {
-                        "type": "SimpleRetriever",
-                        "requester": {
-                            "type": "HttpRequester",
-                            "url_base": "https://api.apilayer.com",
-                            "path": "/exchangerates_data/latest",
-                            "http_method": "GET",
-                        },
-                        "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": []}},
-                    },
-                },
-                "Rates": {
-                    "type": "DeclarativeStream",
-                    "name": "Rates",
-                    "primary_key": [],
-                    "schema_loader": {
-                        "type": "InlineSchemaLoader",
-                        "schema": {"$schema": "http://json-schema.org/schema#", "properties": {}, "type": "object"},
-                    },
-                    "retriever": {
-                        "type": "SimpleRetriever",
-                        "requester": {
-                            "type": "HttpRequester",
-                            "url_base": "https://api.apilayer.com",
-                            "path": "/exchangerates_data/latest",
-                            "http_method": "GET",
-                        },
-                        "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": []}},
-                    },
-                },
-            },
-            "streams": [
-                {"$ref": "#/definitions/Rates"},
-                {"$ref": "#/definitions/AnotherStream"}
-            ],
-            "spec": {
-                "connection_specification": {
-                    "$schema": "http://json-schema.org/draft-07/schema#",
-                    "type": "object",
-                    "required": [],
-                    "properties": {},
-                    "additionalProperties": True,
-                },
-                "documentation_url": "https://example.org",
-                "type": "Spec",
-            },
-        }
-        for stream_name, incremental_sync_definition in self._incremental_sync.items():
-            manifest["definitions"][stream_name]["incremental_sync"] = incremental_sync_definition
-        for stream_name, partition_router_definition in self._partition_router.items():
-            manifest["definitions"][stream_name]["retriever"]["partition_router"] = partition_router_definition
-        for stream_name, partition_router_definition in self._substream_partition_router.items():
-            manifest["definitions"][stream_name]["retriever"]["partition_router"] = partition_router_definition
-        return manifest
-
-
-def test_given_state_for_only_some_partition_when_stream_slices_then_create_slices_using_state_or_start_from_start_datetime():
-    source = ManifestDeclarativeSource(
-        source_config=ManifestBuilder()
-        .with_list_partition_router("Rates", "partition_field", ["1", "2"])
-        .with_incremental_sync(
-            "Rates",
-            start_datetime="2022-01-01",
-            end_datetime="2022-02-28",
-            datetime_format="%Y-%m-%d",
-            cursor_field=CURSOR_FIELD,
-            step="P1M",
-            cursor_granularity="P1D",
-        )
-        .build()
-    )
-    stream_instance = source.streams({})[0]
-    stream_instance.state = {
-        "states": [
-            {
-                "partition": {"partition_field": "1"},
-                "cursor": {CURSOR_FIELD: "2022-02-01"},
-            }
-        ]
-    }
-
-    slices = stream_instance.stream_slices(
-        sync_mode=SYNC_MODE,
-        stream_state={},
-    )
-
-    assert list(slices) == [
-        {"partition_field": "1", "start_time": "2022-02-01", "end_time": "2022-02-28"},
-        {"partition_field": "2", "start_time": "2022-01-01", "end_time": "2022-01-31"},
-        {"partition_field": "2", "start_time": "2022-02-01", "end_time": "2022-02-28"},
-    ]
-
-
-def test_given_record_for_partition_when_read_then_update_state():
-    source = ManifestDeclarativeSource(
-        source_config=ManifestBuilder()
-        .with_list_partition_router("Rates", "partition_field", ["1", "2"])
-        .with_incremental_sync(
-            "Rates",
-            start_datetime="2022-01-01",
-            end_datetime="2022-02-28",
-            datetime_format="%Y-%m-%d",
-            cursor_field=CURSOR_FIELD,
-            step="P1M",
-            cursor_granularity="P1D",
-        )
-        .build()
-    )
-    stream_instance = source.streams({})[0]
-    list(stream_instance.stream_slices(sync_mode=SYNC_MODE))
-
-    stream_slice = StreamSlice(partition={"partition_field": "1"},
-                               cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"})
-    with patch.object(
-            SimpleRetriever, "_read_pages",
-            side_effect=[[Record({"a record key": "a record value", CURSOR_FIELD: "2022-01-15"}, stream_slice)]]
-    ):
-        list(
-            stream_instance.read_records(
-                sync_mode=SYNC_MODE,
-                stream_slice=stream_slice,
-                stream_state={"states": []},
-                cursor_field=CURSOR_FIELD,
-            )
-        )
-
-    assert stream_instance.state == {
-        "states": [
-            {
-                "partition": {"partition_field": "1"},
-                "cursor": {CURSOR_FIELD: "2022-01-15"},
-            }
-        ]
-    }
-
-
-def test_substream_without_input_state():
-    test_source = ManifestDeclarativeSource(
-        source_config=ManifestBuilder()
-        .with_substream_partition_router("AnotherStream")
-        .with_incremental_sync(
-            "Rates",
-            start_datetime="2022-01-01",
-            end_datetime="2022-02-28",
-            datetime_format="%Y-%m-%d",
-            cursor_field=CURSOR_FIELD,
-            step="P1M",
-            cursor_granularity="P1D",
-        )
-        .with_incremental_sync(
-            "AnotherStream",
-            start_datetime="2022-01-01",
-            end_datetime="2022-02-28",
-            datetime_format="%Y-%m-%d",
-            cursor_field=CURSOR_FIELD,
-            step="P1M",
-            cursor_granularity="P1D",
-        )
-        .build()
-    )
-
-    stream_instance = test_source.streams({})[1]
-
-    stream_slice = StreamSlice(partition={"parent_id": "1"},
-                               cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"})
-
-    with patch.object(
-            SimpleRetriever, "_read_pages", side_effect=[[Record({"id": "1", CURSOR_FIELD: "2022-01-15"}, stream_slice)],
-                                                         [Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, stream_slice)]]
-    ):
-        slices = list(stream_instance.stream_slices(sync_mode=SYNC_MODE))
-        assert list(slices) == [
-            StreamSlice(partition={"parent_id": "1", "parent_slice": {}, },
-                        cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}),
-            StreamSlice(partition={"parent_id": "1", "parent_slice": {}, },
-                        cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}),
-            StreamSlice(partition={"parent_id": "2", "parent_slice": {}, },
-                        cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}),
-            StreamSlice(partition={"parent_id": "2", "parent_slice": {}, },
-                        cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}),
-        ]
-
-
-def test_substream_with_legacy_input_state():
-    source = ManifestDeclarativeSource(
-        source_config=ManifestBuilder()
-        .with_substream_partition_router("AnotherStream")
-        .with_incremental_sync(
-            "Rates",
-            start_datetime="2022-01-01",
-            end_datetime="2022-02-28",
-            datetime_format="%Y-%m-%d",
-            cursor_field=CURSOR_FIELD,
-            step="P1M",
-            cursor_granularity="P1D",
-        )
-        .with_incremental_sync(
-            "AnotherStream",
-            start_datetime="2022-01-01",
-            end_datetime="2022-02-28",
-            datetime_format="%Y-%m-%d",
-            cursor_field=CURSOR_FIELD,
-            step="P1M",
-            cursor_granularity="P1D",
-        )
-        .build()
-    )
-
-    stream_instance = source.streams({})[1]
-
-    input_state = {
-        "states": [
-            {
-                "partition": {"item_id": "an_item_id",
-                              "parent_slice": {"end_time": "1629640663", "start_time": "1626962264"},
-                              },
-                "cursor": {
-                    "updated_at": "1709058818"
-                }
-            }
-        ]
-    }
-    stream_instance.state = input_state
-
-    stream_slice = StreamSlice(partition={"parent_id": "1"},
-                               cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"})
-
-    logger = init_logger("airbyte")
-    configured_catalog = ConfiguredAirbyteCatalog(
-        streams=[
-            {
-                "stream": {"name": "AnotherStream", "json_schema": {}, "supported_sync_modes": ["incremental"]},
-                "sync_mode": "incremental",
-                "destination_sync_mode": "overwrite",
-            },
-        ]
-    )
-
-    with patch.object(
-            SimpleRetriever, "_read_pages", side_effect=[
-                [Record({"id": "1", CURSOR_FIELD: "2022-01-15"}, stream_slice)],
-                [Record({"parent_id": "1", CURSOR_FIELD: "2022-01-15"}, stream_slice)],
-                [Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, stream_slice)],
-                [Record({"parent_id": "2", CURSOR_FIELD: "2022-01-15"}, stream_slice)]
-            ]
-    ):
-        messages = list(source.read(logger, {}, configured_catalog, input_state))
-
-        output_state_message = [message for message in messages if message.type == Type.STATE][0]
-
-        expected_state = {"states": [
-            {
-                "cursor": {
-                    CURSOR_FIELD: "2022-01-15"
-                },
-                "partition": {"parent_id": "1", "parent_slice": {}}
-            }
-        ]}
-
-        assert output_state_message.state.stream.stream_state == expected_state

unit_tests/sources/declarative/interpolation/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#

unit_tests/sources/declarative/interpolation/test_filters.py

@@ -1,80 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-import base64
-import hashlib
-
-import pytest
-from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-
-interpolation = JinjaInterpolation()
-
-
-def test_hash_md5_no_salt():
-    input_string = "abcd"
-    s = "{{ '%s' | hash('md5') }}" % input_string
-    filter_hash = interpolation.eval(s, config={})
-
-    # compute expected hash calling hashlib directly
-    hash_obj = hashlib.md5()
-    hash_obj.update(str(input_string).encode("utf-8"))
-    hashlib_computed_hash = hash_obj.hexdigest()
-
-    assert filter_hash == hashlib_computed_hash
-
-
-def test_hash_md5_on_numeric_value():
-    input_value = 123.456
-    s = "{{ %f | hash('md5') }}" % input_value
-    filter_hash = interpolation.eval(s, config={})
-
-    # compute expected hash calling hashlib directly
-    hash_obj = hashlib.md5()
-    hash_obj.update(str(input_value).encode("utf-8"))
-    hashlib_computed_hash = hash_obj.hexdigest()
-
-    assert filter_hash == hashlib_computed_hash
-
-
-def test_hash_md5_with_salt():
-    input_string = "test_input_string"
-    input_salt = "test_input_salt"
-
-    s = "{{ '%s' | hash('md5', '%s' ) }}" % (input_string, input_salt)
-    filter_hash = interpolation.eval(s, config={})
-
-    # compute expected value calling hashlib directly
-    hash_obj = hashlib.md5()
-    hash_obj.update(str(input_string + input_salt).encode("utf-8"))
-    hashlib_computed_hash = hash_obj.hexdigest()
-
-    assert filter_hash == hashlib_computed_hash
-
-
-@pytest.mark.parametrize(
-    "input_string",
-    ["test_input_client_id", "some_client_secret_1", "12345", "775.78"],
-)
-def test_base64encode(input_string: str):
-    s = "{{ '%s' | base64encode }}" % input_string
-    filter_base64encode = interpolation.eval(s, config={})
-
-    # compute expected base64encode calling base64 library directly
-    base64_obj = base64.b64encode(input_string.encode("utf-8")).decode()
-
-    assert filter_base64encode == base64_obj
-
-
-@pytest.mark.parametrize(
-    "input_string, expected_string",
-    [
-        ("aW5wdXRfc3RyaW5n", "input_string"),
-        ("YWlyYnl0ZQ==", "airbyte"),
-        ("cGFzc3dvcmQ=", "password"),
-    ],
-)
-def test_base64decode(input_string: str, expected_string: str):
-    s = "{{ '%s' | base64decode }}" % input_string
-    filter_base64decode = interpolation.eval(s, config={})
-
-    assert filter_base64decode == expected_string

unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py

@@ -1,40 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-
-config = {
-    "parent": {"key_with_true": True},
-    "string_key": "compare_me",
-    "zero_value": 0,
-    "empty_array": [],
-    "non_empty_array": [1],
-    "empty_dict": {},
-    "empty_tuple": (),
-}
-
-
-@pytest.mark.parametrize(
-    "test_name, template, expected_result",
-    [
-        ("test_interpolated_true_value", "{{ config['parent']['key_with_true'] }}", True),
-        ("test_interpolated_true_comparison", "{{ config['string_key'] == \"compare_me\" }}", True),
-        ("test_interpolated_false_condition", "{{ config['string_key'] == \"witness_me\" }}", False),
-        ("test_path_has_value_returns_true", "{{ config['string_key'] }}", True),
-        ("test_zero_is_false", "{{ config['zero_value'] }}", False),
-        ("test_empty_array_is_false", "{{ config['empty_array'] }}", False),
-        ("test_empty_dict_is_false", "{{ config['empty_dict'] }}", False),
-        ("test_empty_tuple_is_false", "{{ config['empty_tuple'] }}", False),
-        ("test_lowercase_false", '{{ "false" }}', False),
-        ("test_False", "{{ False }}", False),
-        ("test_True", "{{ True }}", True),
-        ("test_value_in_array", "{{ 1 in config['non_empty_array'] }}", True),
-        ("test_value_not_in_array", "{{ 2 in config['non_empty_array'] }}", False),
-        ("test_interpolation_using_parameters", "{{ parameters['from_parameters'] == \"come_find_me\" }}", True),
-    ],
-)
-def test_interpolated_boolean(test_name, template, expected_result):
-    interpolated_bool = InterpolatedBoolean(condition=template, parameters={"from_parameters": "come_find_me"})
-    assert interpolated_bool.eval(config) == expected_result

unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py

@@ -1,35 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
-
-
-@pytest.mark.parametrize(
-    "test_name, key, expected_value",
-    [
-        ("test_field_value", "field", "value"),
-        ("test_number", "number", 100),
-        ("test_field_to_interpolate_from_config", "field_to_interpolate_from_config", "VALUE_FROM_CONFIG"),
-        ("test_field_to_interpolate_from_kwargs", "field_to_interpolate_from_kwargs", "VALUE_FROM_KWARGS"),
-        ("test_field_to_interpolate_from_parameters", "field_to_interpolate_from_parameters", "VALUE_FROM_PARAMETERS"),
-        ("test_key_is_interpolated", "key", "VALUE"),
-    ],
-)
-def test(test_name, key, expected_value):
-    d = {
-        "field": "value",
-        "number": 100,
-        "field_to_interpolate_from_config": "{{ config['c'] }}",
-        "field_to_interpolate_from_kwargs": "{{ kwargs['a'] }}",
-        "field_to_interpolate_from_parameters": "{{ parameters['b'] }}",
-        "{{ parameters.k }}": "VALUE",
-    }
-    config = {"c": "VALUE_FROM_CONFIG"}
-    kwargs = {"a": "VALUE_FROM_KWARGS"}
-    mapping = InterpolatedMapping(mapping=d, parameters={"b": "VALUE_FROM_PARAMETERS", "k": "key"})
-
-    interpolated = mapping.eval(config, **{"kwargs": kwargs})
-
-    assert interpolated[key] == expected_value

unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py

@@ -1,45 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import dpath.util
-import pytest
-from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import InterpolatedNestedMapping
-
-
-@pytest.mark.parametrize(
-    "test_name, path, expected_value",
-    [
-        ("test_field_value", "nested/field", "value"),
-        ("test_number", "nested/number", 100),
-        ("test_interpolated_number", "nested/nested_array/1/value", 5),
-        ("test_interpolated_boolean", "nested/nested_array/2/value", True),
-        ("test_field_to_interpolate_from_config", "nested/config_value", "VALUE_FROM_CONFIG"),
-        ("test_field_to_interpolate_from_kwargs", "nested/kwargs_value", "VALUE_FROM_KWARGS"),
-        ("test_field_to_interpolate_from_parameters", "nested/parameters_value", "VALUE_FROM_PARAMETERS"),
-        ("test_key_is_interpolated", "nested/nested_array/0/key", "VALUE"),
-    ],
-)
-def test(test_name, path, expected_value):
-    d = {
-        "nested": {
-            "field": "value",
-            "number": 100,
-            "nested_array": [
-                {"{{ parameters.k }}": "VALUE"},
-                {"value": "{{ config['num_value'] | int + 2 }}"},
-                {"value": "{{ True }}"},
-            ],
-            "config_value": "{{ config['c'] }}",
-            "parameters_value": "{{ parameters['b'] }}",
-            "kwargs_value": "{{ kwargs['a'] }}",
-        }
-    }
-
-    config = {"c": "VALUE_FROM_CONFIG", "num_value": 3}
-    kwargs = {"a": "VALUE_FROM_KWARGS"}
-    mapping = InterpolatedNestedMapping(mapping=d, parameters={"b": "VALUE_FROM_PARAMETERS", "k": "key"})
-
-    interpolated = mapping.eval(config, **{"kwargs": kwargs})
-
-    assert dpath.util.get(interpolated, path) == expected_value

unit_tests/sources/declarative/interpolation/test_interpolated_string.py

@@ -1,25 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-
-config = {"field": "value"}
-parameters = {"hello": "world"}
-kwargs = {"c": "airbyte"}
-
-
-@pytest.mark.parametrize(
-    "test_name, input_string, expected_value",
-    [
-        ("test_static_value", "HELLO WORLD", "HELLO WORLD"),
-        ("test_eval_from_parameters", "{{ parameters['hello'] }}", "world"),
-        ("test_eval_from_config", "{{ config['field'] }}", "value"),
-        ("test_eval_from_kwargs", "{{ kwargs['c'] }}", "airbyte"),
-        ("test_eval_from_kwargs", "{{ kwargs['c'] }}", "airbyte"),
-    ],
-)
-def test_interpolated_string(test_name, input_string, expected_value):
-    s = InterpolatedString.create(input_string, parameters=parameters)
-    assert s.eval(config, **{"kwargs": kwargs}) == expected_value