airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py

@@ -1,406 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer
-
-
-@pytest.mark.parametrize(
-    "component, expected_component",
-    [
-        pytest.param(
-            {"type": "DeclarativeSource", "streams": [{"type": "DeclarativeStream", "retriever": {}, "schema_loader": {}}]},
-            {
-                "type": "DeclarativeSource",
-                "streams": [
-                    {
-                        "type": "DeclarativeStream",
-                        "retriever": {"type": "SimpleRetriever"},
-                        "schema_loader": {"type": "JsonFileSchemaLoader"},
-                    }
-                ],
-            },
-            id="test_declarative_stream",
-        ),
-        pytest.param(
-            {
-                "type": "DeclarativeStream",
-                "retriever": {"type": "SimpleRetriever", "paginator": {}, "record_selector": {}, "requester": {}},
-            },
-            {
-                "type": "DeclarativeStream",
-                "retriever": {
-                    "type": "SimpleRetriever",
-                    "paginator": {"type": "NoPagination"},
-                    "record_selector": {"type": "RecordSelector"},
-                    "requester": {"type": "HttpRequester"},
-                },
-            },
-            id="test_simple_retriever",
-        ),
-        pytest.param(
-            {"type": "DeclarativeStream", "requester": {"type": "HttpRequester", "error_handler": {}}},
-            {
-                "type": "DeclarativeStream",
-                "requester": {
-                    "type": "HttpRequester",
-                    "error_handler": {"type": "DefaultErrorHandler"},
-                },
-            },
-            id="test_http_requester",
-        ),
-        pytest.param(
-            {"type": "SimpleRetriever", "paginator": {"type": "DefaultPaginator", "page_size_option": {}, "page_token_option": {}}},
-            {
-                "type": "SimpleRetriever",
-                "paginator": {
-                    "type": "DefaultPaginator",
-                    "page_size_option": {"type": "RequestOption"},
-                    "page_token_option": {},
-                },
-            },
-            id="test_default_paginator",
-        ),
-        pytest.param(
-            {"type": "SimpleRetriever", "partition_router": {"type": "SubstreamPartitionRouter", "parent_stream_configs": [{}, {}, {}]}},
-            {
-                "type": "SimpleRetriever",
-                "partition_router": {
-                    "type": "SubstreamPartitionRouter",
-                    "parent_stream_configs": [
-                        {"type": "ParentStreamConfig"},
-                        {"type": "ParentStreamConfig"},
-                        {"type": "ParentStreamConfig"},
-                    ],
-                },
-            },
-            id="test_substream_slicer",
-        ),
-    ],
-)
-def test_find_default_types(component, expected_component):
-    transformer = ManifestComponentTransformer()
-    actual_component = transformer.propagate_types_and_parameters("", component, {})
-
-    assert actual_component == expected_component
-
-
-@pytest.mark.parametrize(
-    "component, expected_component",
-    [
-        pytest.param(
-            {
-                "type": "SimpleRetriever",
-                "requester": {"type": "HttpRequester", "authenticator": {"class_name": "source_greenhouse.components.NewAuthenticator"}},
-            },
-            {
-                "type": "SimpleRetriever",
-                "requester": {
-                    "type": "HttpRequester",
-                    "authenticator": {"type": "CustomAuthenticator", "class_name": "source_greenhouse.components.NewAuthenticator"},
-                },
-            },
-            id="test_custom_authenticator",
-        ),
-        pytest.param(
-            {
-                "type": "SimpleRetriever",
-                "record_selector": {
-                    "type": "RecordSelector",
-                    "extractor": {"class_name": "source_greenhouse.components.NewRecordExtractor"},
-                },
-            },
-            {
-                "type": "SimpleRetriever",
-                "record_selector": {
-                    "type": "RecordSelector",
-                    "extractor": {"type": "CustomRecordExtractor", "class_name": "source_greenhouse.components.NewRecordExtractor"},
-                },
-            },
-            id="test_custom_extractor",
-        ),
-    ],
-)
-def test_transform_custom_components(component, expected_component):
-    transformer = ManifestComponentTransformer()
-    actual_component = transformer.propagate_types_and_parameters("", component, {})
-
-    assert actual_component == expected_component
-
-
-def test_propagate_parameters_to_all_components():
-    component = {
-        "type": "DeclarativeSource",
-        "streams": [
-            {
-                "type": "DeclarativeStream",
-                "$parameters": {"name": "roasters", "primary_key": "id"},
-                "retriever": {
-                    "type": "SimpleRetriever",
-                    "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": []}},
-                    "requester": {
-                        "type": "HttpRequester",
-                        "name": '{{ parameters["name"] }}',
-                        "url_base": "https://coffee.example.io/v1/",
-                        "http_method": "GET",
-                    },
-                },
-            }
-        ],
-    }
-
-    expected_component = {
-        "type": "DeclarativeSource",
-        "streams": [
-            {
-                "type": "DeclarativeStream",
-                "retriever": {
-                    "type": "SimpleRetriever",
-                    "name": "roasters",
-                    "primary_key": "id",
-                    "record_selector": {
-                        "type": "RecordSelector",
-                        "extractor": {
-                            "type": "DpathExtractor",
-                            "field_path": [],
-                            "name": "roasters",
-                            "primary_key": "id",
-                            "$parameters": {"name": "roasters", "primary_key": "id"},
-                        },
-                        "name": "roasters",
-                        "primary_key": "id",
-                        "$parameters": {"name": "roasters", "primary_key": "id"},
-                    },
-                    "requester": {
-                        "type": "HttpRequester",
-                        "name": '{{ parameters["name"] }}',
-                        "url_base": "https://coffee.example.io/v1/",
-                        "http_method": "GET",
-                        "primary_key": "id",
-                        "$parameters": {"name": "roasters", "primary_key": "id"},
-                    },
-                    "$parameters": {"name": "roasters", "primary_key": "id"},
-                },
-                "name": "roasters",
-                "primary_key": "id",
-                "$parameters": {"name": "roasters", "primary_key": "id"},
-            }
-        ],
-    }
-
-    transformer = ManifestComponentTransformer()
-    actual_component = transformer.propagate_types_and_parameters("", component, {})
-
-    assert actual_component == expected_component
-
-
-def test_component_parameters_take_precedence_over_parent_parameters():
-    component = {
-        "type": "DeclarativeStream",
-        "retriever": {
-            "type": "SimpleRetriever",
-            "requester": {
-                "type": "HttpRequester",
-                "name": "high_priority",
-                "url_base": "https://coffee.example.io/v1/",
-                "http_method": "GET",
-                "primary_key": "id",
-                "$parameters": {
-                    "name": "high_priority",
-                },
-            },
-            "$parameters": {
-                "name": "low_priority",
-            },
-        },
-    }
-
-    expected_component = {
-        "type": "DeclarativeStream",
-        "retriever": {
-            "type": "SimpleRetriever",
-            "name": "low_priority",
-            "requester": {
-                "type": "HttpRequester",
-                "name": "high_priority",
-                "url_base": "https://coffee.example.io/v1/",
-                "http_method": "GET",
-                "primary_key": "id",
-                "$parameters": {
-                    "name": "high_priority",
-                },
-            },
-            "$parameters": {
-                "name": "low_priority",
-            },
-        },
-    }
-
-    transformer = ManifestComponentTransformer()
-    actual_component = transformer.propagate_types_and_parameters("", component, {})
-
-    assert actual_component == expected_component
-
-
-def test_do_not_propagate_parameters_that_have_the_same_field_name():
-    component = {
-        "type": "DeclarativeStream",
-        "streams": [
-            {
-                "type": "DeclarativeStream",
-                "$parameters": {
-                    "name": "roasters",
-                    "primary_key": "id",
-                    "schema_loader": {"type": "JsonFileSchemaLoader", "file_path": './source_coffee/schemas/{{ parameters["name"] }}.json'},
-                },
-            }
-        ],
-    }
-
-    expected_component = {
-        "type": "DeclarativeStream",
-        "streams": [
-            {
-                "type": "DeclarativeStream",
-                "name": "roasters",
-                "primary_key": "id",
-                "schema_loader": {
-                    "type": "JsonFileSchemaLoader",
-                    "file_path": './source_coffee/schemas/{{ parameters["name"] }}.json',
-                    "name": "roasters",
-                    "primary_key": "id",
-                    "$parameters": {
-                        "name": "roasters",
-                        "primary_key": "id",
-                    },
-                },
-                "$parameters": {
-                    "name": "roasters",
-                    "primary_key": "id",
-                    "schema_loader": {"type": "JsonFileSchemaLoader", "file_path": './source_coffee/schemas/{{ parameters["name"] }}.json'},
-                },
-            }
-        ],
-    }
-
-    transformer = ManifestComponentTransformer()
-    actual_component = transformer.propagate_types_and_parameters("", component, {})
-
-    assert actual_component == expected_component
-
-
-def test_ignore_empty_parameters():
-    component = {
-        "type": "DeclarativeStream",
-        "retriever": {
-            "type": "SimpleRetriever",
-            "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": []}},
-        },
-    }
-
-    transformer = ManifestComponentTransformer()
-    actual_component = transformer.propagate_types_and_parameters("", component, {})
-
-    assert actual_component == component
-
-
-def test_only_propagate_parameters_to_components():
-    component = {
-        "type": "ParentComponent",
-        "component_with_object_properties": {
-            "type": "TestComponent",
-            "subcomponent": {
-                "type": "TestSubComponent",
-                "some_field": "high_priority",
-                "$parameters": {
-                    "some_option": "already",
-                },
-            },
-            "dictionary_field": {"details": "should_not_contain_parameters", "other": "no_parameters_as_fields"},
-            "$parameters": {
-                "included": "not!",
-            },
-        },
-    }
-
-    expected_component = {
-        "type": "ParentComponent",
-        "component_with_object_properties": {
-            "type": "TestComponent",
-            "subcomponent": {
-                "type": "TestSubComponent",
-                "some_field": "high_priority",
-                "some_option": "already",
-                "included": "not!",
-                "$parameters": {"some_option": "already", "included": "not!"},
-            },
-            "dictionary_field": {"details": "should_not_contain_parameters", "other": "no_parameters_as_fields"},
-            "included": "not!",
-            "$parameters": {
-                "included": "not!",
-            },
-        },
-    }
-
-    transformer = ManifestComponentTransformer()
-    actual_component = transformer.propagate_types_and_parameters("", component, {})
-
-    assert actual_component == expected_component
-
-
-def test_do_not_propagate_parameters_on_json_schema_object():
-    component = {
-        "type": "DeclarativeStream",
-        "streams": [
-            {
-                "type": "DeclarativeStream",
-                "schema_loader": {
-                    "type": "InlineSchemaLoader",
-                    "schema": {
-                        "type": "object",
-                        "$schema": "http://json-schema.org/schema#",
-                        "properties": {"id": {"type": "string"}},
-                    },
-                },
-                "$parameters": {
-                    "name": "roasters",
-                    "primary_key": "id",
-                },
-            }
-        ],
-    }
-
-    expected_component = {
-        "type": "DeclarativeStream",
-        "streams": [
-            {
-                "type": "DeclarativeStream",
-                "name": "roasters",
-                "primary_key": "id",
-                "schema_loader": {
-                    "type": "InlineSchemaLoader",
-                    "name": "roasters",
-                    "primary_key": "id",
-                    "schema": {
-                        "type": "object",
-                        "$schema": "http://json-schema.org/schema#",
-                        "properties": {"id": {"type": "string"}},
-                    },
-                    "$parameters": {
-                        "name": "roasters",
-                        "primary_key": "id",
-                    },
-                },
-                "$parameters": {
-                    "name": "roasters",
-                    "primary_key": "id",
-                },
-            }
-        ],
-    }
-
-    transformer = ManifestComponentTransformer()
-    actual_component = transformer.propagate_types_and_parameters("", component, {})
-
-    assert actual_component == expected_component

unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py

@@ -1,139 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.sources.declarative.parsers.custom_exceptions import CircularReferenceException, UndefinedReferenceException
-from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver, _parse_path
-
-resolver = ManifestReferenceResolver()
-
-
-# @
-def test_refer():
-    content = {"limit": 50, "limit_ref": "#/limit"}
-    config = resolver.preprocess_manifest(content)
-    assert config["limit_ref"] == 50
-
-
-def test_refer_to_inner():
-    content = {"dict": {"limit": 50}, "limit_ref": "#/dict/limit"}
-    config = resolver.preprocess_manifest(content)
-    assert config["limit_ref"] == 50
-
-
-def test_refer_to_non_existant_struct():
-    content = {"dict": {"limit": 50}, "limit_ref": "#/not_dict"}
-    with pytest.raises(UndefinedReferenceException):
-        resolver.preprocess_manifest(content)
-
-
-def test_refer_in_dict():
-    content = {"limit": 50, "offset_request_parameters": {"offset": "{{ next_page_token['offset'] }}", "limit": "#/limit"}}
-    config = resolver.preprocess_manifest(content)
-    assert config["offset_request_parameters"]["offset"] == "{{ next_page_token['offset'] }}"
-    assert config["offset_request_parameters"]["limit"] == 50
-
-
-def test_refer_to_dict():
-    content = {
-        "limit": 50,
-        "offset_request_parameters": {"offset": "{{ next_page_token['offset'] }}", "limit": "#/limit"},
-        "offset_pagination_request_parameters": {
-            "class": "InterpolatedRequestParameterProvider",
-            "request_parameters": "#/offset_request_parameters",
-        },
-    }
-    config = resolver.preprocess_manifest(content)
-    assert config["limit"] == 50
-    assert config["offset_request_parameters"]["limit"] == 50
-    assert len(config["offset_pagination_request_parameters"]) == 2
-    assert config["offset_pagination_request_parameters"]["request_parameters"]["limit"] == 50
-    assert config["offset_pagination_request_parameters"]["request_parameters"]["offset"] == "{{ next_page_token['offset'] }}"
-
-
-def test_refer_and_overwrite():
-    content = {
-        "limit": 50,
-        "custom_limit": 25,
-        "offset_request_parameters": {"offset": "{{ next_page_token['offset'] }}", "limit": "#/limit"},
-        "custom_request_parameters": {"$ref": "#/offset_request_parameters", "limit": "#/custom_limit"},
-    }
-    config = resolver.preprocess_manifest(content)
-    assert config["offset_request_parameters"]["limit"] == 50
-    assert config["custom_request_parameters"]["limit"] == 25
-
-    assert config["offset_request_parameters"]["offset"] == "{{ next_page_token['offset'] }}"
-    assert config["custom_request_parameters"]["offset"] == "{{ next_page_token['offset'] }}"
-
-
-def test_collision():
-    content = {
-        "example": {
-            "nested": {"path": "first one", "more_nested": {"value": "found it!"}},
-            "nested/path": "uh oh",
-        },
-        "reference_to_nested_path": {"$ref": "#/example/nested/path"},
-        "reference_to_nested_nested_value": {"$ref": "#/example/nested/more_nested/value"},
-    }
-    config = resolver.preprocess_manifest(content)
-    assert config["example"]["nested"]["path"] == "first one"
-    assert config["example"]["nested/path"] == "uh oh"
-    assert config["reference_to_nested_path"] == "uh oh"
-    assert config["example"]["nested"]["more_nested"]["value"] == "found it!"
-    assert config["reference_to_nested_nested_value"] == "found it!"
-
-
-def test_internal_collision():
-    content = {
-        "example": {
-            "nested": {"path": {"internal": "uh oh"}, "path/internal": "found it!"},
-        },
-        "reference": {"$ref": "#/example/nested/path/internal"},
-    }
-    config = resolver.preprocess_manifest(content)
-    assert config["example"]["nested"]["path"]["internal"] == "uh oh"
-    assert config["example"]["nested"]["path/internal"] == "found it!"
-    assert config["reference"] == "found it!"
-
-
-def test_parse_path():
-    assert _parse_path("foo/bar") == ("foo", "bar")
-    assert _parse_path("foo/7/8/bar") == ("foo", "7/8/bar")
-    assert _parse_path("7/8/bar") == (7, "8/bar")
-    assert _parse_path("8/bar") == (8, "bar")
-    assert _parse_path("8foo/bar") == ("8foo", "bar")
-
-
-def test_list():
-    content = {"list": ["A", "B"], "elem_ref": "#/list/0"}
-    config = resolver.preprocess_manifest(content)
-    elem_ref = config["elem_ref"]
-    assert elem_ref == "A"
-
-
-def test_nested_list():
-    content = {"list": [["A"], ["B"]], "elem_ref": "#/list/1/0"}
-    config = resolver.preprocess_manifest(content)
-    elem_ref = config["elem_ref"]
-    assert elem_ref == "B"
-
-
-def test_list_of_dicts():
-    content = {"list": [{"A": "a"}, {"B": "b"}], "elem_ref": "#/list/1/B"}
-    config = resolver.preprocess_manifest(content)
-    elem_ref = config["elem_ref"]
-    assert elem_ref == "b"
-
-
-def test_multiple_levels_of_indexing():
-    content = {"list": [{"A": ["a1", "a2"]}, {"B": ["b1", "b2"]}], "elem_ref": "#/list/1/B/0"}
-    config = resolver.preprocess_manifest(content)
-    elem_ref = config["elem_ref"]
-    assert elem_ref == "b1"
-
-
-def test_circular_reference():
-    content = {"elem_ref1": "#/elem_ref2", "elem_ref2": "#/elem_ref1"}
-    with pytest.raises(CircularReferenceException):
-        resolver.preprocess_manifest(content)