airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev1__py3-none-any.whl

unit_tests/utils/test_schema_inferrer.py

@@ -1,202 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import List, Mapping
-
-import pytest
-from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage
-from airbyte_cdk.utils.schema_inferrer import SchemaInferrer
-
-NOW = 1234567
-
-
-@pytest.mark.parametrize(
-    "input_records,expected_schemas",
-    [
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": "abc"}},
-                {"stream": "my_stream", "data": {"field_A": "def"}},
-            ],
-            {"my_stream": {"field_A": {"type": "string"}}},
-            id="test_basic",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": 1.0}},
-                {"stream": "my_stream", "data": {"field_A": "abc"}},
-            ],
-            {"my_stream": {"field_A": {"type": ["number", "string"]}}},
-            id="test_deriving_schema_refine",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"obj": {"data": [1.0, 2.0, 3.0]}}},
-                {"stream": "my_stream", "data": {"obj": {"other_key": "xyz"}}},
-            ],
-            {
-                "my_stream": {
-                    "obj": {
-                        "type": "object",
-                        "properties": {
-                            "data": {"type": "array", "items": {"type": "number"}},
-                            "other_key": {"type": "string"},
-                        },
-                    }
-                }
-            },
-            id="test_derive_schema_for_nested_structures",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": 1}},
-                {"stream": "my_stream", "data": {"field_A": 2}},
-            ],
-            {"my_stream": {"field_A": {"type": "number"}}},
-            id="test_integer_number",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": None}},
-            ],
-            {"my_stream": {}},
-            id="test_null",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": None}},
-                {"stream": "my_stream", "data": {"field_A": "abc"}},
-            ],
-            {"my_stream": {"field_A": {"type": ["null", "string"]}}},
-            id="test_null_optional",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": None}},
-                {"stream": "my_stream", "data": {"field_A": {"nested": "abc"}}},
-            ],
-            {"my_stream": {"field_A": {"type": ["object", "null"], "properties": {"nested": {"type": "string"}}}}},
-            id="test_any_of",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": None}},
-                {"stream": "my_stream", "data": {"field_A": {"nested": "abc", "nully": None}}},
-            ],
-            {"my_stream": {"field_A": {"type": ["object", "null"], "properties": {"nested": {"type": "string"}}}}},
-            id="test_any_of_with_null",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": None}},
-                {"stream": "my_stream", "data": {"field_A": {"nested": "abc", "nully": None}}},
-                {"stream": "my_stream", "data": {"field_A": {"nested": "abc", "nully": "a string"}}},
-            ],
-            {
-                "my_stream": {
-                    "field_A": {
-                        "type": ["object", "null"],
-                        "properties": {"nested": {"type": "string"}, "nully": {"type": ["null", "string"]}},
-                    }
-                }
-            },
-            id="test_any_of_with_null_union",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": {"nested": "abc", "nully": "a string"}}},
-                {"stream": "my_stream", "data": {"field_A": None}},
-                {"stream": "my_stream", "data": {"field_A": {"nested": "abc", "nully": None}}},
-            ],
-            {
-                "my_stream": {
-                    "field_A": {
-                        "type": ["object", "null"],
-                        "properties": {"nested": {"type": "string"}, "nully": {"type": ["null", "string"]}},
-                    }
-                }
-            },
-            id="test_any_of_with_null_union_changed_order",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": "abc", "nested": {"field_B": None}}},
-            ],
-            {"my_stream": {"field_A": {"type": "string"}, "nested": {"type": "object", "properties": {}}}},
-            id="test_nested_null",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": "abc", "nested": [{"field_B": None, "field_C": "abc"}]}},
-            ],
-            {
-                "my_stream": {
-                    "field_A": {"type": "string"},
-                    "nested": {"type": "array", "items": {"type": "object", "properties": {"field_C": {"type": "string"}}}},
-                }
-            },
-            id="test_array_nested_null",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": "abc", "nested": None}},
-                {"stream": "my_stream", "data": {"field_A": "abc", "nested": [{"field_B": None, "field_C": "abc"}]}},
-            ],
-            {
-                "my_stream": {
-                    "field_A": {"type": "string"},
-                    "nested": {"type": ["array", "null"], "items": {"type": "object", "properties": {"field_C": {"type": "string"}}}},
-                }
-            },
-            id="test_array_top_level_null",
-        ),
-        pytest.param(
-            [
-                {"stream": "my_stream", "data": {"field_A": None}},
-                {"stream": "my_stream", "data": {"field_A": "abc"}},
-            ],
-            {"my_stream": {"field_A": {"type": ["null", "string"]}}},
-            id="test_null_string",
-        ),
-    ],
-)
-def test_schema_derivation(input_records: List, expected_schemas: Mapping):
-    inferrer = SchemaInferrer()
-    for record in input_records:
-        inferrer.accumulate(AirbyteRecordMessage(stream=record["stream"], data=record["data"], emitted_at=NOW))
-
-    for stream_name, expected_schema in expected_schemas.items():
-        assert inferrer.get_inferred_schemas()[stream_name] == {
-            "$schema": "http://json-schema.org/schema#",
-            "type": "object",
-            "properties": expected_schema,
-        }
-
-
-def test_deriving_schema_multiple_streams():
-    inferrer = SchemaInferrer()
-    inferrer.accumulate(AirbyteRecordMessage(stream="my_stream", data={"field_A": 1.0}, emitted_at=NOW))
-    inferrer.accumulate(AirbyteRecordMessage(stream="my_stream2", data={"field_A": "abc"}, emitted_at=NOW))
-    inferred_schemas = inferrer.get_inferred_schemas()
-    assert inferred_schemas["my_stream"] == {
-        "$schema": "http://json-schema.org/schema#",
-        "type": "object",
-        "properties": {"field_A": {"type": "number"}},
-    }
-    assert inferred_schemas["my_stream2"] == {
-        "$schema": "http://json-schema.org/schema#",
-        "type": "object",
-        "properties": {"field_A": {"type": "string"}},
-    }
-
-
-def test_get_individual_schema():
-    inferrer = SchemaInferrer()
-    inferrer.accumulate(AirbyteRecordMessage(stream="my_stream", data={"field_A": 1.0}, emitted_at=NOW))
-    assert inferrer.get_stream_schema("my_stream") == {
-        "$schema": "http://json-schema.org/schema#",
-        "type": "object",
-        "properties": {"field_A": {"type": "number"}},
-    }
-    assert inferrer.get_stream_schema("another_stream") is None

unit_tests/utils/test_secret_utils.py

@@ -1,135 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pytest
-from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets, filter_secrets, get_secret_paths, get_secrets, update_secrets
-
-SECRET_STRING_KEY = "secret_key1"
-SECRET_STRING_VALUE = "secret_value"
-SECRET_STRING_2_KEY = "secret_key2"
-SECRET_STRING_2_VALUE = "second_secret_val"
-SECRET_INT_KEY = "secret_int"
-SECRET_INT_VALUE = 1337
-NOT_SECRET_KEY = "not_a_secret"
-NOT_SECRET_VALUE = "unimportant value"
-
-
-flat_spec_with_secret = {"properties": {SECRET_STRING_KEY: {"type": "string", "airbyte_secret": True}, NOT_SECRET_KEY: {"type": "string"}}}
-flat_config_with_secret = {SECRET_STRING_KEY: SECRET_STRING_VALUE, NOT_SECRET_KEY: NOT_SECRET_VALUE}
-
-flat_spec_with_secret_int = {
-    "properties": {SECRET_INT_KEY: {"type": "integer", "airbyte_secret": True}, NOT_SECRET_KEY: {"type": "string"}}
-}
-flat_config_with_secret_int = {SECRET_INT_KEY: SECRET_INT_VALUE, NOT_SECRET_KEY: NOT_SECRET_VALUE}
-
-flat_spec_without_secrets = {"properties": {NOT_SECRET_KEY: {"type": "string"}}}
-flat_config_without_secrets = {NOT_SECRET_KEY: NOT_SECRET_VALUE}
-
-spec_with_oneof_secrets = {
-    "properties": {
-        SECRET_STRING_KEY: {"type": "string", "airbyte_secret": True},
-        NOT_SECRET_KEY: {"type": "string"},
-        "credentials": {
-            "type": "object",
-            "oneOf": [
-                {
-                    "type": "object",
-                    "properties": {SECRET_STRING_2_KEY: {"type": "string", "airbyte_secret": True}, NOT_SECRET_KEY: {"type": "string"}},
-                },
-                {
-                    "type": "object",
-                    "properties": {SECRET_INT_KEY: {"type": "integer", "airbyte_secret": True}, NOT_SECRET_KEY: {"type": "string"}},
-                },
-            ],
-        },
-    }
-}
-config_with_oneof_secrets_1 = {
-    SECRET_STRING_KEY: SECRET_STRING_VALUE,
-    NOT_SECRET_KEY: NOT_SECRET_VALUE,
-    "credentials": {SECRET_STRING_2_KEY: SECRET_STRING_2_VALUE},
-}
-config_with_oneof_secrets_2 = {
-    SECRET_STRING_KEY: SECRET_STRING_VALUE,
-    NOT_SECRET_KEY: NOT_SECRET_VALUE,
-    "credentials": {SECRET_INT_KEY: SECRET_INT_VALUE},
-}
-
-spec_with_nested_secrets = {
-    "properties": {
-        SECRET_STRING_KEY: {"type": "string", "airbyte_secret": True},
-        NOT_SECRET_KEY: {"type": "string"},
-        "credentials": {
-            "type": "object",
-            "properties": {
-                SECRET_STRING_2_KEY: {"type": "string", "airbyte_secret": True},
-                NOT_SECRET_KEY: {"type": "string"},
-                SECRET_INT_KEY: {"type": "integer", "airbyte_secret": True},
-            },
-        },
-    }
-}
-config_with_nested_secrets = {
-    SECRET_STRING_KEY: SECRET_STRING_VALUE,
-    NOT_SECRET_KEY: NOT_SECRET_VALUE,
-    "credentials": {SECRET_STRING_2_KEY: SECRET_STRING_2_VALUE, SECRET_INT_KEY: SECRET_INT_VALUE},
-}
-
-
-@pytest.mark.parametrize(
-    ["spec", "expected"],
-    [
-        (flat_spec_with_secret, [[SECRET_STRING_KEY]]),
-        (flat_spec_without_secrets, []),
-        (flat_spec_with_secret_int, [[SECRET_INT_KEY]]),
-        (spec_with_oneof_secrets, [[SECRET_STRING_KEY], ["credentials", SECRET_STRING_2_KEY], ["credentials", SECRET_INT_KEY]]),
-        (spec_with_nested_secrets, [[SECRET_STRING_KEY], ["credentials", SECRET_STRING_2_KEY], ["credentials", SECRET_INT_KEY]]),
-    ],
-)
-def test_get_secret_paths(spec, expected):
-    assert get_secret_paths(spec) == expected, f"Expected {spec} to yield secret paths {expected}"
-
-
-@pytest.mark.parametrize(
-    ["spec", "config", "expected"],
-    [
-        (flat_spec_with_secret, flat_config_with_secret, [SECRET_STRING_VALUE]),
-        (flat_spec_without_secrets, flat_config_without_secrets, []),
-        (flat_spec_with_secret_int, flat_config_with_secret_int, [SECRET_INT_VALUE]),
-        (spec_with_oneof_secrets, config_with_oneof_secrets_1, [SECRET_STRING_VALUE, SECRET_STRING_2_VALUE]),
-        (spec_with_oneof_secrets, config_with_oneof_secrets_2, [SECRET_STRING_VALUE, SECRET_INT_VALUE]),
-        (spec_with_nested_secrets, config_with_nested_secrets, [SECRET_STRING_VALUE, SECRET_STRING_2_VALUE, SECRET_INT_VALUE]),
-    ],
-)
-def test_get_secrets(spec, config, expected):
-    assert get_secrets(spec, config) == expected, f"Expected the spec {spec} and config {config} to produce {expected}"
-
-
-def test_secret_filtering():
-    sensitive_str = f"{SECRET_STRING_VALUE} {NOT_SECRET_VALUE} {SECRET_STRING_VALUE} {SECRET_STRING_2_VALUE}"
-
-    update_secrets([])
-    filtered = filter_secrets(sensitive_str)
-    assert filtered == sensitive_str
-
-    # the empty secret should not affect the result
-    update_secrets([""])
-    filtered = filter_secrets(sensitive_str)
-    assert filtered == sensitive_str
-
-    update_secrets([SECRET_STRING_VALUE, SECRET_STRING_2_VALUE])
-    filtered = filter_secrets(sensitive_str)
-    assert filtered == f"**** {NOT_SECRET_VALUE} **** ****"
-
-
-def test_secrets_added_are_filtered():
-    ADDED_SECRET = "only_a_secret_if_added"
-    sensitive_str = f"{ADDED_SECRET} {NOT_SECRET_VALUE}"
-
-    filtered = filter_secrets(sensitive_str)
-    assert filtered == sensitive_str
-
-    add_to_secrets(ADDED_SECRET)
-    filtered = filter_secrets(sensitive_str)
-    assert filtered == f"**** {NOT_SECRET_VALUE}"

unit_tests/utils/test_stream_status_utils.py

@@ -1,61 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from airbyte_cdk.models import AirbyteMessage, AirbyteStream, AirbyteStreamStatus, SyncMode, TraceType
-from airbyte_cdk.models import Type as MessageType
-from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message
-
-stream = AirbyteStream(name="name", namespace="namespace", json_schema={}, supported_sync_modes=[SyncMode.full_refresh])
-
-
-def test_started_as_message():
-    stream_status = AirbyteStreamStatus.STARTED
-    airbyte_message = stream_status_as_airbyte_message(stream, stream_status)
-
-    assert type(airbyte_message) == AirbyteMessage
-    assert airbyte_message.type == MessageType.TRACE
-    assert airbyte_message.trace.type == TraceType.STREAM_STATUS
-    assert airbyte_message.trace.emitted_at > 0
-    assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name
-    assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace
-    assert airbyte_message.trace.stream_status.status == stream_status
-
-
-def test_running_as_message():
-    stream_status = AirbyteStreamStatus.RUNNING
-    airbyte_message = stream_status_as_airbyte_message(stream, stream_status)
-
-    assert type(airbyte_message) == AirbyteMessage
-    assert airbyte_message.type == MessageType.TRACE
-    assert airbyte_message.trace.type == TraceType.STREAM_STATUS
-    assert airbyte_message.trace.emitted_at > 0
-    assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name
-    assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace
-    assert airbyte_message.trace.stream_status.status == stream_status
-
-
-def test_complete_as_message():
-    stream_status = AirbyteStreamStatus.COMPLETE
-    airbyte_message = stream_status_as_airbyte_message(stream, stream_status)
-
-    assert type(airbyte_message) == AirbyteMessage
-    assert airbyte_message.type == MessageType.TRACE
-    assert airbyte_message.trace.type == TraceType.STREAM_STATUS
-    assert airbyte_message.trace.emitted_at > 0
-    assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name
-    assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace
-    assert airbyte_message.trace.stream_status.status == stream_status
-
-
-def test_incomplete_failed_as_message():
-    stream_status = AirbyteStreamStatus.INCOMPLETE
-    airbyte_message = stream_status_as_airbyte_message(stream, stream_status)
-
-    assert type(airbyte_message) == AirbyteMessage
-    assert airbyte_message.type == MessageType.TRACE
-    assert airbyte_message.trace.type == TraceType.STREAM_STATUS
-    assert airbyte_message.trace.emitted_at > 0
-    assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name
-    assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace
-    assert airbyte_message.trace.stream_status.status == stream_status

unit_tests/utils/test_traced_exception.py

@@ -1,107 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-
-import pytest
-from airbyte_cdk.models.airbyte_protocol import (
-    AirbyteErrorTraceMessage,
-    AirbyteMessage,
-    AirbyteTraceMessage,
-    FailureType,
-    Status,
-    TraceType,
-)
-from airbyte_cdk.models.airbyte_protocol import Type as MessageType
-from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-
-
-@pytest.fixture
-def raised_exception():
-    try:
-        raise RuntimeError("an error has occurred")
-    except RuntimeError as e:
-        return e
-
-
-def test_build_from_existing_exception(raised_exception):
-    traced_exc = AirbyteTracedException.from_exception(raised_exception, message="my user-friendly message")
-    assert traced_exc.message == "my user-friendly message"
-    assert traced_exc.internal_message == "an error has occurred"
-    assert traced_exc.failure_type == FailureType.system_error
-    assert traced_exc._exception == raised_exception
-
-
-def test_exception_as_airbyte_message():
-    traced_exc = AirbyteTracedException("an internal message")
-    airbyte_message = traced_exc.as_airbyte_message()
-
-    assert type(airbyte_message) == AirbyteMessage
-    assert airbyte_message.type == MessageType.TRACE
-    assert airbyte_message.trace.type == TraceType.ERROR
-    assert airbyte_message.trace.emitted_at > 0
-    assert airbyte_message.trace.error.failure_type == FailureType.system_error
-    assert airbyte_message.trace.error.message == "Something went wrong in the connector. See the logs for more details."
-    assert airbyte_message.trace.error.internal_message == "an internal message"
-    assert airbyte_message.trace.error.stack_trace == "airbyte_cdk.utils.traced_exception.AirbyteTracedException: an internal message\n"
-
-
-def test_existing_exception_as_airbyte_message(raised_exception):
-    traced_exc = AirbyteTracedException.from_exception(raised_exception)
-    airbyte_message = traced_exc.as_airbyte_message()
-
-    assert type(airbyte_message) == AirbyteMessage
-    assert airbyte_message.type == MessageType.TRACE
-    assert airbyte_message.trace.type == TraceType.ERROR
-    assert airbyte_message.trace.error.message == "Something went wrong in the connector. See the logs for more details."
-    assert airbyte_message.trace.error.internal_message == "an error has occurred"
-    assert airbyte_message.trace.error.stack_trace.startswith("Traceback (most recent call last):")
-    assert airbyte_message.trace.error.stack_trace.endswith(
-        'raise RuntimeError("an error has occurred")\n' "RuntimeError: an error has occurred\n"
-    )
-
-
-def test_config_error_as_connection_status_message():
-    traced_exc = AirbyteTracedException("an internal message", message="Config validation error", failure_type=FailureType.config_error)
-    airbyte_message = traced_exc.as_connection_status_message()
-
-    assert type(airbyte_message) == AirbyteMessage
-    assert airbyte_message.type == MessageType.CONNECTION_STATUS
-    assert airbyte_message.connectionStatus.status == Status.FAILED
-    assert airbyte_message.connectionStatus.message == "Config validation error"
-
-
-def test_other_error_as_connection_status_message():
-    traced_exc = AirbyteTracedException("an internal message", failure_type=FailureType.system_error)
-    airbyte_message = traced_exc.as_connection_status_message()
-
-    assert airbyte_message is None
-
-
-def test_emit_message(capsys):
-    traced_exc = AirbyteTracedException(
-        internal_message="internal message", message="user-friendly message", exception=RuntimeError("oh no")
-    )
-
-    expected_message = AirbyteMessage(
-        type="TRACE",
-        trace=AirbyteTraceMessage(
-            type="ERROR",
-            emitted_at=0.0,
-            error=AirbyteErrorTraceMessage(
-                failure_type="system_error",
-                message="user-friendly message",
-                internal_message="internal message",
-                stack_trace="RuntimeError: oh no\n",
-            ),
-        ),
-    )
-
-    traced_exc.emit_message()
-
-    stdout = capsys.readouterr().out
-    printed_message = AirbyteMessage.parse_obj(json.loads(stdout))
-    printed_message.trace.emitted_at = 0.0
-
-    assert printed_message == expected_message