airbyte-cdk 6.45.4.post49.dev14495925594__py3-none-any.whl → 6.45.4.post72.dev14497997772__py3-none-any.whl

airbyte_cdk/test/standard_tests/connector_base.py

@@ -0,0 +1,148 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Base class for connector test suites."""
+
+from __future__ import annotations
+
+import abc
+import inspect
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from typing import cast
+
+import yaml
+from boltons.typeutils import classproperty
+
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    Type,
+)
+from airbyte_cdk.test import entrypoint_wrapper
+from airbyte_cdk.test.declarative.models import (
+    ConnectorTestScenario,
+)
+from airbyte_cdk.test.declarative.utils.job_runner import IConnector, run_test_job
+
+ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
+MANIFEST_YAML = "manifest.yaml"
+
+
+class ConnectorTestSuiteBase(abc.ABC):
+    """Base class for connector test suites."""
+
+    connector: type[IConnector] | Callable[[], IConnector] | None = None
+    """The connector class or a factory function that returns an scenario of IConnector."""
+
+    @classmethod
+    def get_test_class_dir(cls) -> Path:
+        """Get the file path that contains the class."""
+        module = sys.modules[cls.__module__]
+        # Get the directory containing the test file
+        return Path(inspect.getfile(module)).parent
+
+    @classmethod
+    def create_connector(
+        cls,
+        scenario: ConnectorTestScenario,
+    ) -> IConnector:
+        """Instantiate the connector class."""
+        connector = cls.connector  # type: ignore
+        if connector:
+            if callable(connector) or isinstance(connector, type):
+                # If the connector is a class or factory function, instantiate it:
+                return cast(IConnector, connector())  # type: ignore [redundant-cast]
+
+        # Otherwise, we can't instantiate the connector. Fail with a clear error message.
+        raise NotImplementedError(
+            "No connector class or connector factory function provided. "
+            "Please provide a class or factory function in `cls.connector`, or "
+            "override `cls.create_connector()` to define a custom initialization process."
+        )
+
+    # Test Definitions
+
+    def test_check(
+        self,
+        scenario: ConnectorTestScenario,
+    ) -> None:
+        """Run `connection` acceptance tests."""
+        result: entrypoint_wrapper.EntrypointOutput = run_test_job(
+            self.create_connector(scenario),
+            "check",
+            test_scenario=scenario,
+        )
+        conn_status_messages: list[AirbyteMessage] = [
+            msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
+        ]  # noqa: SLF001  # Non-public API
+        assert len(conn_status_messages) == 1, (
+            f"Expected exactly one CONNECTION_STATUS message. Got: {result._messages}"
+        )
+
+    @classmethod
+    def get_connector_root_dir(cls) -> Path:
+        """Get the root directory of the connector."""
+        for parent in cls.get_test_class_dir().parents:
+            if (parent / MANIFEST_YAML).exists():
+                return parent
+            if (parent / ACCEPTANCE_TEST_CONFIG).exists():
+                return parent
+            if parent.name == "airbyte_cdk":
+                break
+        # If we reach here, we didn't find the manifest file in any parent directory
+        # Check if the manifest file exists in the current directory
+        for parent in Path.cwd().parents:
+            if (parent / MANIFEST_YAML).exists():
+                return parent
+            if (parent / ACCEPTANCE_TEST_CONFIG).exists():
+                return parent
+            if parent.name == "airbyte_cdk":
+                break
+
+        raise FileNotFoundError(
+            "Could not find connector root directory relative to "
+            f"'{str(cls.get_test_class_dir())}' or '{str(Path.cwd())}'."
+        )
+
+    @classproperty
+    def acceptance_test_config_path(cls) -> Path:
+        """Get the path to the acceptance test config file."""
+        result = cls.get_connector_root_dir() / ACCEPTANCE_TEST_CONFIG
+        if result.exists():
+            return result
+
+        raise FileNotFoundError(f"Acceptance test config file not found at: {str(result)}")
+
+    @classmethod
+    def get_scenarios(
+        cls,
+    ) -> list[ConnectorTestScenario]:
+        """Get acceptance tests for a given category.
+
+        This has to be a separate function because pytest does not allow
+        parametrization of fixtures with arguments from the test class itself.
+        """
+        category = "connection"
+        all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text())
+        if "acceptance_tests" not in all_tests_config:
+            raise ValueError(
+                f"Acceptance tests config not found in {cls.acceptance_test_config_path}."
+                f" Found only: {str(all_tests_config)}."
+            )
+        if category not in all_tests_config["acceptance_tests"]:
+            return []
+        if "tests" not in all_tests_config["acceptance_tests"][category]:
+            raise ValueError(f"No tests found for category {category}")
+
+        tests_scenarios = [
+            ConnectorTestScenario.model_validate(test)
+            for test in all_tests_config["acceptance_tests"][category]["tests"]
+            if "iam_role" not in test["config_path"]
+        ]
+        connector_root = cls.get_connector_root_dir().absolute()
+        for test in tests_scenarios:
+            if test.config_path:
+                test.config_path = connector_root / test.config_path
+            if test.configured_catalog_path:
+                test.configured_catalog_path = connector_root / test.configured_catalog_path
+
+        return tests_scenarios

airbyte_cdk/test/standard_tests/declarative_sources.py

@@ -0,0 +1,92 @@
+import os
+from hashlib import md5
+from pathlib import Path
+from typing import Any, cast
+
+import yaml
+from boltons.typeutils import classproperty
+
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.test.declarative.models import ConnectorTestScenario
+from airbyte_cdk.test.declarative.utils.job_runner import IConnector
+from airbyte_cdk.test.standard_tests.connector_base import MANIFEST_YAML
+from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
+
+
+def md5_checksum(file_path: Path) -> str:
+    """Helper function to calculate the MD5 checksum of a file.
+
+    This is used to calculate the checksum of the `components.py` file, if it exists.
+    """
+    with open(file_path, "rb") as file:
+        return md5(file.read()).hexdigest()
+
+
+class DeclarativeSourceTestSuite(SourceTestSuiteBase):
+    """Declarative source test suite.
+
+    This inherits from the Python-based source test suite and implements the
+    `create_connector` method to create a declarative source object instead of
+    requiring a custom Python source object.
+
+    The class also automatically locates the `manifest.yaml` file and the
+    `components.py` file (if it exists) in the connector's directory.
+    """
+
+    @classproperty
+    def manifest_yaml_path(cls) -> Path:
+        """Get the path to the manifest.yaml file."""
+        result = cls.get_connector_root_dir() / MANIFEST_YAML
+        if result.exists():
+            return result
+
+        raise FileNotFoundError(
+            f"Manifest YAML file not found at {result}. "
+            "Please ensure that the test suite is run in the correct directory.",
+        )
+
+    @classproperty
+    def components_py_path(cls) -> Path | None:
+        """Get the path to the `components.py` file, if one exists.
+
+        If not `components.py` file exists, return None.
+        """
+        result = cls.get_connector_root_dir() / "components.py"
+        if result.exists():
+            return result
+
+        return None
+
+    @classmethod
+    def create_connector(
+        cls,
+        scenario: ConnectorTestScenario,
+    ) -> IConnector:
+        """Create a connector scenario for the test suite.
+
+        This overrides `create_connector` from the create a declarative source object
+        instead of requiring a custom python source object.
+
+        Subclasses should not need to override this method.
+        """
+        config: dict[str, Any] = scenario.get_config_dict()
+
+        manifest_dict = yaml.safe_load(cls.manifest_yaml_path.read_text())
+        if cls.components_py_path and cls.components_py_path.exists():
+            os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"
+            config["__injected_components_py"] = cls.components_py_path.read_text()
+            config["__injected_components_py_checksums"] = {
+                "md5": md5_checksum(cls.components_py_path),
+            }
+
+        return cast(
+            IConnector,
+            ConcurrentDeclarativeSource(
+                config=config,
+                catalog=None,
+                state=None,
+                source_config=manifest_dict,
+            ),
+        )

airbyte_cdk/test/standard_tests/destination_base.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Base class for destination test suites."""
+
+from airbyte_cdk.test.standard_tests.connector_base import ConnectorTestSuiteBase
+
+
+class DestinationTestSuiteBase(ConnectorTestSuiteBase):
+    """Base class for destination test suites.
+
+    This class provides a base set of functionality for testing destination connectors, and it
+    inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
+
+    TODO: As of now, this class does not add any additional functionality or tests specific to
+    destination connectors. However, it serves as a placeholder for future enhancements and
+    customizations that may be needed for destination connectors.
+    """

airbyte_cdk/test/standard_tests/pytest_hooks.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Pytest hooks for Airbyte CDK tests.
+
+These hooks are used to customize the behavior of pytest during test discovery and execution.
+
+To use these hooks within a connector, add the following lines to the connector's `conftest.py`
+file, or to another file that is imported during test discovery:
+
+```python
+pytest_plugins = [
+    "airbyte_cdk.test.standard_tests.pytest_hooks",
+]
+```
+"""
+
+import pytest
+
+
+def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
+    """
+    A helper for pytest_generate_tests hook.
+
+    If a test method (in a class subclassed from our base class)
+    declares an argument 'scenario', this function retrieves the
+    'scenarios' attribute from the test class and parametrizes that
+    test with the values from 'scenarios'.
+
+    ## Usage
+
+    ```python
+    from airbyte_cdk.test.standard_tests.connector_base import (
+        generate_tests,
+        ConnectorTestSuiteBase,
+    )
+
+    def pytest_generate_tests(metafunc):
+        generate_tests(metafunc)
+
+    class TestMyConnector(ConnectorTestSuiteBase):
+        ...
+
+    ```
+    """
+    # Check if the test function requires an 'scenario' argument
+    if "scenario" in metafunc.fixturenames:
+        # Retrieve the test class
+        test_class = metafunc.cls
+        if test_class is None:
+            return
+
+        # Get the 'scenarios' attribute from the class
+        scenarios_attr = getattr(test_class, "get_scenarios", None)
+        if scenarios_attr is None:
+            raise ValueError(
+                f"Test class {test_class} does not have a 'scenarios' attribute. "
+                "Please define the 'scenarios' attribute in the test class."
+            )
+
+        scenarios = test_class.get_scenarios()
+        ids = [str(scenario) for scenario in scenarios]
+        metafunc.parametrize("scenario", scenarios, ids=ids)

airbyte_cdk/test/standard_tests/source_base.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Base class for source test suites."""
+
+from dataclasses import asdict
+
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+    SyncMode,
+    Type,
+)
+from airbyte_cdk.test import entrypoint_wrapper
+from airbyte_cdk.test.declarative.models import (
+    ConnectorTestScenario,
+)
+from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
+from airbyte_cdk.test.standard_tests.connector_base import (
+    ConnectorTestSuiteBase,
+)
+
+
+class SourceTestSuiteBase(ConnectorTestSuiteBase):
+    """Base class for source test suites.
+
+    This class provides a base set of functionality for testing source connectors, and it
+    inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
+    """
+
+    def test_check(
+        self,
+        scenario: ConnectorTestScenario,
+    ) -> None:
+        """Run standard `check` tests on the connector.
+
+        Assert that the connector returns a single CONNECTION_STATUS message.
+        This test is designed to validate the connector's ability to establish a connection
+        and return its status with the expected message type.
+        """
+        result: entrypoint_wrapper.EntrypointOutput = run_test_job(
+            self.create_connector(scenario),
+            "check",
+            test_scenario=scenario,
+        )
+        conn_status_messages: list[AirbyteMessage] = [
+            msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
+        ]  # noqa: SLF001  # Non-public API
+        num_status_messages = len(conn_status_messages)
+        assert num_status_messages == 1, (
+            f"Expected exactly one CONNECTION_STATUS message. Got {num_status_messages}: \n"
+            + "\n".join([str(m) for m in result._messages])
+        )
+
+    def test_discover(
+        self,
+        scenario: ConnectorTestScenario,
+    ) -> None:
+        """Standard test for `discover`."""
+        run_test_job(
+            self.create_connector(scenario),
+            "discover",
+            test_scenario=scenario,
+        )
+
+    def test_basic_read(
+        self,
+        scenario: ConnectorTestScenario,
+    ) -> None:
+        """Run standard `read` test on the connector.
+
+        This test is designed to validate the connector's ability to read data
+        from the source and return records. It first runs a `discover` job to
+        obtain the catalog of streams, and then it runs a `read` job to fetch
+        records from those streams.
+        """
+        discover_result = run_test_job(
+            self.create_connector(scenario),
+            "discover",
+            test_scenario=scenario,
+        )
+        if scenario.expect_exception:
+            assert discover_result.errors, "Expected exception but got none."
+            return
+
+        configured_catalog = ConfiguredAirbyteCatalog(
+            streams=[
+                ConfiguredAirbyteStream(
+                    stream=stream,
+                    sync_mode=SyncMode.full_refresh,
+                    destination_sync_mode=DestinationSyncMode.append_dedup,
+                )
+                for stream in discover_result.catalog.catalog.streams  # type: ignore [reportOptionalMemberAccess, union-attr]
+            ]
+        )
+        result = run_test_job(
+            self.create_connector(scenario),
+            "read",
+            test_scenario=scenario,
+            catalog=configured_catalog,
+        )
+
+        if not result.records:
+            raise AssertionError("Expected records but got none.")  # noqa: TRY003
+
+    def test_fail_read_with_bad_catalog(
+        self,
+        scenario: ConnectorTestScenario,
+    ) -> None:
+        """Standard test for `read` when passed a bad catalog file."""
+        invalid_configured_catalog = ConfiguredAirbyteCatalog(
+            streams=[
+                # Create ConfiguredAirbyteStream which is deliberately invalid
+                # with regard to the Airbyte Protocol.
+                # This should cause the connector to fail.
+                ConfiguredAirbyteStream(
+                    stream=AirbyteStream(
+                        name="__AIRBYTE__stream_that_does_not_exist",
+                        json_schema={
+                            "type": "object",
+                            "properties": {"f1": {"type": "string"}},
+                        },
+                        supported_sync_modes=[SyncMode.full_refresh],
+                    ),
+                    sync_mode="INVALID",  # type: ignore [reportArgumentType]
+                    destination_sync_mode="INVALID",  # type: ignore [reportArgumentType]
+                )
+            ]
+        )
+        # Set expected status to "failed" to ensure the test fails if the connector.
+        scenario.status = "failed"
+        result: entrypoint_wrapper.EntrypointOutput = run_test_job(
+            self.create_connector(scenario),
+            "read",
+            test_scenario=scenario,
+            catalog=asdict(invalid_configured_catalog),
+        )
+        assert result.errors, "Expected errors but got none."
+        assert result.trace_messages, "Expected trace messages but got none."

{airbyte_cdk-6.45.4.post49.dev14495925594.dist-info → airbyte_cdk-6.45.4.post72.dev14497997772.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.4.post49.dev14495925594
+Version: 6.45.4.post72.dev14497997772
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -22,10 +22,11 @@ Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
 Requires-Dist: anyascii (>=0.3.2,<0.4.0)
 Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff
+Requires-Dist: boltons (>=25.0.0,<26.0.0)
 Requires-Dist: cachetools
 Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
 Requires-Dist: cryptography (>=44.0.0,<45.0.0)