airbyte-cdk 6.45.0.post6.dev14369631849__tar.gz → 6.45.0.post20.dev14369762306__tar.gz

{airbyte_cdk-6.45.0.post6.dev14369631849 → airbyte_cdk-6.45.0.post20.dev14369762306}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.0.post6.dev14369631849
+Version: 6.45.0.post20.dev14369762306
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Declarative tests framework.
+
+This module provides fixtures and utilities for testing Airbyte sources and destinations
+in a declarative way.
+"""

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/models/__init__.py

@@ -0,0 +1,7 @@
+from airbyte_cdk.test.declarative.models.scenario import (
+    ConnectorTestScenario,
+)
+
+__all__ = [
+    "ConnectorTestScenario",
+]

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/models/scenario.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Run acceptance tests in PyTest.
+
+These tests leverage the same `acceptance-test-config.yml` configuration files as the
+acceptance tests in CAT, but they run in PyTest instead of CAT. This allows us to run
+the acceptance tests in the same local environment as we are developing in, speeding
+up iteration cycles.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel
+
+
+class ConnectorTestScenario(BaseModel):
+    """Acceptance test instance, as a Pydantic model.
+
+    This class represents an acceptance test instance, which is a single test case
+    that can be run against a connector. It is used to deserialize and validate the
+    acceptance test configuration file.
+    """
+
+    class AcceptanceTestExpectRecords(BaseModel):
+        path: Path
+        exact_order: bool = False
+
+    class AcceptanceTestFileTypes(BaseModel):
+        skip_test: bool
+        bypass_reason: str
+
+    config_path: Path | None = None
+    config_dict: dict | None = None
+
+    id: str | None = None
+
+    configured_catalog_path: Path | None = None
+    timeout_seconds: int | None = None
+    expect_records: AcceptanceTestExpectRecords | None = None
+    file_types: AcceptanceTestFileTypes | None = None
+    status: Literal["succeed", "failed"] | None = None
+
+    def get_config_dict(self) -> dict:
+        """Return the config dictionary.
+
+        If a config dictionary has already been loaded, return it. Otherwise, load
+        Otherwise, load the config file and return the dictionary.
+        """
+        if self.config_dict:
+            return self.config_dict
+
+        if self.config_path:
+            return yaml.safe_load(self.config_path.read_text())
+
+        raise ValueError("No config dictionary or path provided.")
+
+    @property
+    def expect_exception(self) -> bool:
+        return self.status and self.status == "failed"
+
+    @property
+    def instance_name(self) -> str:
+        return self.config_path.stem
+
+    def __str__(self) -> str:
+        if self.id:
+            return f"'{self.id}' Test Scenario"
+        if self.config_path:
+            return f"'{self.config_path.name}' Test Scenario"
+
+        return f"'{hash(self)}' Test Scenario"

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/test_suites/__init__.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Declarative test suites.
+
+Here we have base classes for a robust set of declarative connector test suites.
+"""
+
+from airbyte_cdk.test.declarative.test_suites.connector_base import (
+    ConnectorTestScenario,
+    generate_tests,
+)
+from airbyte_cdk.test.declarative.test_suites.declarative_sources import (
+    DeclarativeSourceTestSuite,
+)
+from airbyte_cdk.test.declarative.test_suites.destination_base import DestinationTestSuiteBase
+from airbyte_cdk.test.declarative.test_suites.source_base import SourceTestSuiteBase
+
+__all__ = [
+    "ConnectorTestScenario",
+    "ConnectorTestSuiteBase",
+    "DeclarativeSourceTestSuite",
+    "DestinationTestSuiteBase",
+    "SourceTestSuiteBase",
+    "generate_tests",
+]

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/test_suites/connector_base.py

@@ -0,0 +1,197 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Base class for connector test suites."""
+
+from __future__ import annotations
+
+import abc
+import functools
+import inspect
+import sys
+from pathlib import Path
+from typing import Any, Callable, Literal
+
+import pytest
+import yaml
+from pydantic import BaseModel
+from typing_extensions import override
+
+from airbyte_cdk import Connector
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    Type,
+)
+from airbyte_cdk.test import entrypoint_wrapper
+from airbyte_cdk.test.declarative.models import (
+    ConnectorTestScenario,
+)
+from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
+
+ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
+
+
+class JavaClass(str):
+    """A string that represents a Java class."""
+
+
+class DockerImage(str):
+    """A string that represents a Docker image."""
+
+
+class RunnableConnector(abc.ABC):
+    """A connector that can be run in a test scenario."""
+
+    @abc.abstractmethod
+    def launch(cls, args: list[str] | None): ...
+
+
+def generate_tests(metafunc):
+    """
+    A helper for pytest_generate_tests hook.
+
+    If a test method (in a class subclassed from our base class)
+    declares an argument 'instance', this function retrieves the
+    'scenarios' attribute from the test class and parametrizes that
+    test with the values from 'scenarios'.
+
+    ## Usage
+
+    ```python
+    from airbyte_cdk.test.declarative.test_suites.connector_base import (
+        generate_tests,
+        ConnectorTestSuiteBase,
+    )
+
+    def pytest_generate_tests(metafunc):
+        generate_tests(metafunc)
+
+    class TestMyConnector(ConnectorTestSuiteBase):
+        ...
+
+    ```
+    """
+    # Check if the test function requires an 'instance' argument
+    if "instance" in metafunc.fixturenames:
+        # Retrieve the test class
+        test_class = metafunc.cls
+        if test_class is None:
+            raise ValueError("Expected a class here.")
+        # Get the 'scenarios' attribute from the class
+        scenarios_attr = getattr(test_class, "get_scenarios", None)
+        if scenarios_attr is None:
+            raise ValueError(
+                f"Test class {test_class} does not have a 'scenarios' attribute. "
+                "Please define the 'scenarios' attribute in the test class."
+            )
+
+        scenarios = test_class.get_scenarios()
+        ids = [str(scenario) for scenario in scenarios]
+        metafunc.parametrize("instance", scenarios, ids=ids)
+
+
+class ConnectorTestSuiteBase(abc.ABC):
+    """Base class for connector test suites."""
+
+    acceptance_test_file_path = Path("./acceptance-test-config.json")
+    """The path to the acceptance test config file.
+
+    By default, this is set to the `acceptance-test-config.json` file in
+    the root of the connector source directory.
+    """
+
+    connector: type[Connector] | Path | JavaClass | DockerImage | None = None
+    """The connector class or path to the connector to test."""
+
+    working_dir: Path | None = None
+    """The root directory of the connector source code."""
+
+    @classmethod
+    def create_connector(
+        cls, scenario: ConnectorTestScenario
+    ) -> Source | AbstractSource | ConcurrentDeclarativeSource | RunnableConnector:
+        """Instantiate the connector class."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def run_test_scenario(
+        self,
+        verb: Literal["read", "check", "discover"],
+        test_scenario: ConnectorTestScenario,
+        *,
+        catalog: dict | None = None,
+    ) -> entrypoint_wrapper.EntrypointOutput:
+        """Run a test job from provided CLI args and return the result."""
+        return run_test_job(
+            self.create_connector(test_scenario),
+            verb,
+            test_instance=test_scenario,
+            catalog=catalog,
+        )
+
+    # Test Definitions
+
+    def test_check(
+        self,
+        instance: ConnectorTestScenario,
+    ) -> None:
+        """Run `connection` acceptance tests."""
+        result = self.run_test_scenario(
+            "check",
+            test_scenario=instance,
+        )
+        conn_status_messages: list[AirbyteMessage] = [
+            msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
+        ]  # noqa: SLF001  # Non-public API
+        assert len(conn_status_messages) == 1, (
+            "Expected exactly one CONNECTION_STATUS message. Got: \n" + "\n".join(result._messages)
+        )
+
+    @classmethod
+    @property
+    def acceptance_test_config_path(self) -> Path:
+        """Get the path to the acceptance test config file.
+
+        Check vwd and parent directories of cwd for the config file, and return the first one found.
+
+        Give up if the config file is not found in any parent directory.
+        """
+        current_dir = Path.cwd()
+        for parent_dir in current_dir.parents:
+            config_path = parent_dir / ACCEPTANCE_TEST_CONFIG
+            if config_path.exists():
+                return config_path
+        raise FileNotFoundError(
+            f"Acceptance test config file not found in any parent directory from : {Path.cwd()}"
+        )
+
+    @classmethod
+    def get_scenarios(
+        cls,
+    ) -> list[ConnectorTestScenario]:
+        """Get acceptance tests for a given category.
+
+        This has to be a separate function because pytest does not allow
+        parametrization of fixtures with arguments from the test class itself.
+        """
+        category = "connection"
+        all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text())
+        if "acceptance_tests" not in all_tests_config:
+            raise ValueError(
+                f"Acceptance tests config not found in {cls.acceptance_test_config_path}."
+                f" Found only: {str(all_tests_config)}."
+            )
+        if category not in all_tests_config["acceptance_tests"]:
+            return []
+        if "tests" not in all_tests_config["acceptance_tests"][category]:
+            raise ValueError(f"No tests found for category {category}")
+
+        tests_scenarios = [
+            ConnectorTestScenario.model_validate(test)
+            for test in all_tests_config["acceptance_tests"][category]["tests"]
+            if "iam_role" not in test["config_path"]
+        ]
+        working_dir = cls.working_dir or Path()
+        for test in tests_scenarios:
+            if test.config_path:
+                test.config_path = working_dir / test.config_path
+            if test.configured_catalog_path:
+                test.configured_catalog_path = working_dir / test.configured_catalog_path
+        return tests_scenarios

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/test_suites/declarative_sources.py

@@ -0,0 +1,47 @@
+import os
+from hashlib import md5
+from pathlib import Path
+from typing import Any, cast
+
+import yaml
+
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.test.declarative.models import ConnectorTestScenario
+from airbyte_cdk.test.declarative.test_suites.source_base import (
+    SourceTestSuiteBase,
+)
+
+
+def md5_checksum(file_path: Path) -> str:
+    with open(file_path, "rb") as file:
+        return md5(file.read()).hexdigest()
+
+
+class DeclarativeSourceTestSuite(SourceTestSuiteBase):
+    manifest_path = Path("manifest.yaml")
+    components_py_path: Path | None = None
+
+    def create_connector(
+        self, connector_test: ConnectorTestScenario
+    ) -> ConcurrentDeclarativeSource:
+        config = connector_test.get_config_dict()
+        # catalog = connector_test.get_catalog()
+        # state = connector_test.get_state()
+        # source_config = connector_test.get_source_config()
+
+        manifest_dict = yaml.safe_load(self.manifest_path.read_text())
+        if self.components_py_path and self.components_py_path.exists():
+            os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"
+            config["__injected_components_py"] = self.components_py_path.read_text()
+            config["__injected_components_py_checksums"] = {
+                "md5": md5_checksum(self.components_py_path),
+            }
+
+        return ConcurrentDeclarativeSource(
+            config=config,
+            catalog=None,
+            state=None,
+            source_config=manifest_dict,
+        )

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/test_suites/destination_base.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Base class for destination test suites."""
+
+from airbyte_cdk.test.declarative.test_suites.connector_base import ConnectorTestSuiteBase
+
+
+class DestinationTestSuiteBase(ConnectorTestSuiteBase):
+    """Base class for destination test suites.
+
+    This class provides a base set of functionality for testing destination connectors, and it
+    inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
+    """

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/test_suites/source_base.py

@@ -0,0 +1,129 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Base class for source test suites."""
+
+from dataclasses import asdict
+from pathlib import Path
+
+import pytest
+
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+    SyncMode,
+    Type,
+)
+from airbyte_cdk.test import entrypoint_wrapper
+from airbyte_cdk.test.declarative.models import (
+    ConnectorTestScenario,
+)
+from airbyte_cdk.test.declarative.test_suites.connector_base import (
+    ConnectorTestSuiteBase,
+)
+from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
+
+
+class SourceTestSuiteBase(ConnectorTestSuiteBase):
+    """Base class for source test suites.
+
+    This class provides a base set of functionality for testing source connectors, and it
+    inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
+    """
+
+    def test_check(
+        self,
+        instance: ConnectorTestScenario,
+    ) -> None:
+        """Run `connection` acceptance tests."""
+        result: entrypoint_wrapper.EntrypointOutput = run_test_job(
+            self.create_connector(instance),
+            "check",
+            test_instance=instance,
+        )
+        conn_status_messages: list[AirbyteMessage] = [
+            msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
+        ]  # noqa: SLF001  # Non-public API
+        assert len(conn_status_messages) == 1, (
+            "Expected exactly one CONNECTION_STATUS message. Got: \n" + "\n".join(result._messages)
+        )
+
+    def test_basic_read(
+        self,
+        instance: ConnectorTestScenario,
+    ) -> None:
+        """Run acceptance tests."""
+        discover_result = run_test_job(
+            self.create_connector(instance),
+            "discover",
+            test_instance=instance,
+        )
+        if instance.expect_exception:
+            assert discover_result.errors, "Expected exception but got none."
+            return
+
+        configured_catalog = ConfiguredAirbyteCatalog(
+            streams=[
+                ConfiguredAirbyteStream(
+                    stream=stream,
+                    sync_mode=SyncMode.full_refresh,
+                    destination_sync_mode=DestinationSyncMode.append_dedup,
+                )
+                for stream in discover_result.catalog.catalog.streams
+            ]
+        )
+        result = run_test_job(
+            self.create_connector(instance),
+            "read",
+            test_instance=instance,
+            catalog=configured_catalog,
+        )
+
+        if not result.records:
+            raise AssertionError("Expected records but got none.")  # noqa: TRY003
+
+    def test_fail_with_bad_catalog(
+        self,
+        instance: ConnectorTestScenario,
+    ) -> None:
+        """Test that a bad catalog fails."""
+        invalid_configured_catalog = ConfiguredAirbyteCatalog(
+            streams=[
+                # Create ConfiguredAirbyteStream which is deliberately invalid
+                # with regard to the Airbyte Protocol.
+                # This should cause the connector to fail.
+                ConfiguredAirbyteStream(
+                    stream=AirbyteStream(
+                        name="__AIRBYTE__stream_that_does_not_exist",
+                        json_schema={
+                            "type": "object",
+                            "properties": {"f1": {"type": "string"}},
+                        },
+                        supported_sync_modes=[SyncMode.full_refresh],
+                    ),
+                    sync_mode="INVALID",
+                    destination_sync_mode="INVALID",
+                )
+            ]
+        )
+        # Set expected status to "failed" to ensure the test fails if the connector.
+        instance.status = "failed"
+        result = self.run_test_scenario(
+            "read",
+            test_scenario=instance,
+            catalog=asdict(invalid_configured_catalog),
+        )
+        assert result.errors, "Expected errors but got none."
+        assert result.trace_messages, "Expected trace messages but got none."
+
+    def test_discover(
+        self,
+        instance: ConnectorTestScenario,
+    ) -> None:
+        """Run acceptance tests."""
+        run_test_job(
+            self.create_connector(instance),
+            "check",
+            test_instance=instance,
+        )

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/declarative/utils/job_runner.py

@@ -0,0 +1,128 @@
+import tempfile
+import uuid
+from pathlib import Path
+from typing import Callable, Literal
+
+import orjson
+
+from airbyte_cdk import Connector
+from airbyte_cdk.models import (
+    Status,
+)
+from airbyte_cdk.sources.abstract_source import AbstractSource
+from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
+from airbyte_cdk.test import entrypoint_wrapper
+from airbyte_cdk.test.declarative.models import (
+    ConnectorTestScenario,
+)
+
+
+def run_test_job(
+    connector: Connector | type[Connector] | Callable[[], Connector],
+    verb: Literal["read", "check", "discover"],
+    test_instance: ConnectorTestScenario,
+    *,
+    catalog: dict | None = None,
+) -> entrypoint_wrapper.EntrypointOutput:
+    """Run a test job from provided CLI args and return the result."""
+    if not connector:
+        raise ValueError("Connector is required")
+
+    connector_obj: Connector
+    if isinstance(connector, type):
+        connector_obj = connector()
+    elif isinstance(connector, Connector):
+        connector_obj = connector
+    elif isinstance(connector, DeclarativeSource | AbstractSource):
+        connector_obj = connector
+    elif isinstance(connector, Callable):
+        try:
+            connector_obj = connector()
+        except Exception as ex:
+            if not test_instance.expect_exception:
+                raise
+
+            return entrypoint_wrapper.EntrypointOutput(
+                messages=[],
+                uncaught_exception=ex,
+            )
+    else:
+        raise ValueError(f"Invalid source type: {type(connector)}")
+
+    args = [verb]
+    if test_instance.config_path:
+        args += ["--config", str(test_instance.config_path)]
+    elif test_instance.config_dict:
+        config_path = (
+            Path(tempfile.gettempdir()) / "airbyte-test" / f"temp_config_{uuid.uuid4().hex}.json"
+        )
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+        config_path.write_text(orjson.dumps(test_instance.config_dict).decode())
+        args += ["--config", str(config_path)]
+
+    catalog_path: Path | None = None
+    if verb not in ["discover", "check"]:
+        # We need a catalog for read.
+        if catalog:
+            # Write the catalog to a temp json file and pass the path to the file as an argument.
+            catalog_path = (
+                Path(tempfile.gettempdir())
+                / "airbyte-test"
+                / f"temp_catalog_{uuid.uuid4().hex}.json"
+            )
+            catalog_path.parent.mkdir(parents=True, exist_ok=True)
+            catalog_path.write_text(orjson.dumps(catalog).decode())
+        elif test_instance.configured_catalog_path:
+            catalog_path = Path(test_instance.configured_catalog_path)
+
+        if catalog_path:
+            args += ["--catalog", str(catalog_path)]
+
+    # This is a bit of a hack because the source needs the catalog early.
+    # Because it *also* can fail, we have ot redundantly wrap it in a try/except block.
+
+    result: entrypoint_wrapper.EntrypointOutput = entrypoint_wrapper._run_command(  # noqa: SLF001  # Non-public API
+        source=connector_obj,
+        args=args,
+        expecting_exception=test_instance.expect_exception,
+    )
+    if result.errors and not test_instance.expect_exception:
+        raise AssertionError(
+            "\n\n".join(
+                [str(err.trace.error).replace("\\n", "\n") for err in result.errors],
+            )
+        )
+
+    if verb == "check":
+        # Check is expected to fail gracefully without an exception.
+        # Instead, we assert that we have a CONNECTION_STATUS message with
+        # a failure status.
+        assert not result.errors, "Expected no errors from check. Got:\n" + "\n".join(
+            [str(error) for error in result.errors]
+        )
+        assert len(result.connection_status_messages) == 1, (
+            "Expected exactly one CONNECTION_STATUS message. Got "
+            f"{len(result.connection_status_messages)}:\n"
+            + "\n".join(result.connection_status_messages)
+        )
+        if test_instance.expect_exception:
+            assert result.connection_status_messages[0].connectionStatus.status == Status.FAILED, (
+                "Expected CONNECTION_STATUS message to be FAILED. Got: \n"
+                + "\n".join([str(result.connection_status_messages)])
+            )
+        return result
+
+    # For all other verbs, we assert check that an exception is raised (or not).
+    if test_instance.expect_exception:
+        if not result.errors:
+            raise AssertionError("Expected exception but got none.")
+
+        return result
+    if result.errors:
+        raise AssertionError(
+            "\n\n".join(
+                [str(err.trace.error).replace("\\n", "\n") for err in result.errors],
+            )
+        )
+
+    return result

{airbyte_cdk-6.45.0.post6.dev14369631849 → airbyte_cdk-6.45.0.post20.dev14369762306}/airbyte_cdk/test/entrypoint_wrapper.py

@@ -82,6 +82,10 @@ class EntrypointOutput:
     def state_messages(self) -> List[AirbyteMessage]:
         return self._get_message_by_types([Type.STATE])
 
+    @property
+    def connection_status_messages(self) -> List[AirbyteMessage]:
+        return self._get_message_by_types([Type.CONNECTION_STATUS])
+
     @property
     def most_recent_state(self) -> Any:
         state_messages = self._get_message_by_types([Type.STATE])

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/fixtures/auto.py

@@ -0,0 +1,14 @@
+"""Auto-use fixtures for pytest.
+
+WARNING: Importing this module will automatically apply these fixtures. If you want to selectively
+enable fixtures in a different context, you can import directly from the `fixtures.general` module.
+
+
+Usage:
+
+```python
+from airbyte_cdk.test.fixtures import auto
+# OR
+from airbyte_cdk.test.fixtures.auto import *
+```
+"""

airbyte_cdk-6.45.0.post20.dev14369762306/airbyte_cdk/test/fixtures/general.py

@@ -0,0 +1,15 @@
+"""General fixtures for pytest.
+
+Usage:
+
+```python
+from airbyte_cdk.test.fixtures.general import *
+# OR:
+from airbyte_cdk.test.fixtures.general import connector_test_dir
+```
+"""
+
+
+@pytest.fixture
+def connector_test_dir():
+    return Path(__file__).parent