airbyte-cdk 6.57.2__tar.gz → 6.57.3.post1.dev16088268564__tar.gz

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.57.2
+Version: 6.57.3.post1.dev16088268564
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -67,7 +67,7 @@ except ImportError:
 
 TEST_FILE_TEMPLATE = '''
 # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-"""FAST Airbyte Standard Tests for the {connector_name} source."""
+"""FAST Airbyte Standard Tests for the {connector_name} connector."""
 
 #from airbyte_cdk.test.standard_tests import {base_class_name}
 from airbyte_cdk.test.standard_tests.util import create_connector_test_suite
@@ -81,11 +81,13 @@ TestSuite = create_connector_test_suite(
     connector_directory=Path(),
 )
 
+# Uncomment the following lines to create a custom test suite class:
+#
 # class TestSuite({base_class_name}):
-#     """Test suite for the {connector_name} source.
-
-#     This class inherits from SourceTestSuiteBase and implements all of the tests in the suite.
-
+#     """Test suite for the `{connector_name}` connector.
+#
+#     This class inherits from `{base_class_name}` and implements all of the tests in the suite.
+#
 #     As long as the class name starts with "Test", pytest will automatically discover and run the
 #     tests in this class.
 #     """

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/sources/streams/http/rate_limiting.py

@@ -111,7 +111,8 @@ def user_defined_backoff_handler(
                     f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}"
                 )
             retry_after = exc.backoff
-            logger.info(f"Retrying. Sleeping for {retry_after} seconds")
+            # include logging og the current time to help with debugging
+            logger.info(f"Retrying. Sleeping for {retry_after} seconds at {time.strftime('%Y-%m-%d %H:%M:%S')}")
             time.sleep(retry_after + 1)  # extra second to cover any fractions of second
 
     def log_give_up(details: Mapping[str, Any]) -> None:

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/sql/shared/catalog_providers.py

@@ -119,11 +119,21 @@ class CatalogProvider:
     def get_primary_keys(
         self,
         stream_name: str,
-    ) -> list[str]:
-        """Return the primary keys for the given stream."""
-        pks = self.get_configured_stream_info(stream_name).primary_key
+    ) -> list[str] | None:
+        """Return the primary key column names for the given stream.
+
+        We return `source_defined_primary_key` if set, or `primary_key` otherwise. If both are set,
+        we assume they should not should differ, since Airbyte data integrity constraints do not
+        permit overruling a source's pre-defined primary keys. If neither is set, we return `None`.
+
+        Returns:
+            A list of column names that constitute the primary key, or None if no primary key is defined.
+        """
+        configured_stream = self.get_configured_stream_info(stream_name)
+        pks = configured_stream.stream.source_defined_primary_key or configured_stream.primary_key
+
         if not pks:
-            return []
+            return None
 
         normalized_pks: list[list[str]] = [
             [LowerCaseNormalizer.normalize(c) for c in pk] for pk in pks

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/sql/shared/sql_processor.py

@@ -666,9 +666,13 @@ class SqlProcessorBase(abc.ABC):
         """
         nl = "\n"
         columns = {self._quote_identifier(c) for c in self._get_sql_column_definitions(stream_name)}
-        pk_columns = {
-            self._quote_identifier(c) for c in self.catalog_provider.get_primary_keys(stream_name)
-        }
+        primary_keys = self.catalog_provider.get_primary_keys(stream_name)
+        if not primary_keys:
+            raise exc.AirbyteInternalError(
+                message="Cannot merge tables without primary keys. Primary keys are required for merge operations.",
+                context={"stream_name": stream_name},
+            )
+        pk_columns = {self._quote_identifier(c) for c in primary_keys}
         non_pk_columns = columns - pk_columns
         join_clause = f"{nl} AND ".join(f"tmp.{pk_col} = final.{pk_col}" for pk_col in pk_columns)
         set_clause = f"{nl}  , ".join(f"{col} = tmp.{col}" for col in non_pk_columns)
@@ -725,6 +729,11 @@ class SqlProcessorBase(abc.ABC):
         final_table = self._get_table_by_name(final_table_name)
         temp_table = self._get_table_by_name(temp_table_name)
         pk_columns = self.catalog_provider.get_primary_keys(stream_name)
+        if not pk_columns:
+            raise exc.AirbyteInternalError(
+                message="Cannot merge tables without primary keys. Primary keys are required for merge operations.",
+                context={"stream_name": stream_name},
+            )
 
         columns_to_update: set[str] = self._get_sql_column_definitions(
             stream_name=stream_name

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/test/entrypoint_wrapper.py

@@ -21,6 +21,7 @@ import tempfile
 import traceback
 from collections import deque
 from collections.abc import Generator, Mapping
+from dataclasses import dataclass
 from io import StringIO
 from pathlib import Path
 from typing import Any, List, Literal, Optional, Union, final, overload
@@ -50,6 +51,21 @@ from airbyte_cdk.sources import Source
 from airbyte_cdk.test.models.scenario import ExpectedOutcome
 
 
+@dataclass
+class AirbyteEntrypointException(Exception):
+    """Exception raised for errors in the AirbyteEntrypoint execution.
+
+    Used to provide details of an Airbyte connector execution failure in the output
+    captured in an `EntrypointOutput` object. Use `EntrypointOutput.as_exception()` to
+    convert it to an exception.
+
+    Example Usage:
+        output = EntrypointOutput(...)
+        if output.errors:
+            raise output.as_exception()
+    """
+
+
 class EntrypointOutput:
     """A class to encapsulate the output of an Airbyte connector's execution.
 
@@ -67,6 +83,7 @@ class EntrypointOutput:
         messages: list[str] | None = None,
         uncaught_exception: Optional[BaseException] = None,
         *,
+        command: list[str] | None = None,
         message_file: Path | None = None,
     ) -> None:
         if messages is None and message_file is None:
@@ -74,6 +91,7 @@ class EntrypointOutput:
         if messages is not None and message_file is not None:
             raise ValueError("Only one of messages or message_file can be provided")
 
+        self._command = command
         self._messages: list[AirbyteMessage] | None = None
         self._message_file: Path | None = message_file
         if messages:
@@ -182,6 +200,39 @@ class EntrypointOutput:
     def errors(self) -> List[AirbyteMessage]:
         return self._get_trace_message_by_trace_type(TraceType.ERROR)
 
+    def get_formatted_error_message(self) -> str:
+        """Returns a human-readable error message with the contents.
+
+        If there are no errors, returns an empty string.
+        """
+        errors = self.errors
+        if not errors:
+            # If there are no errors, return an empty string.
+            return ""
+
+        result = "Failed to run airbyte command"
+        result += ": " + " ".join(self._command) if self._command else "."
+        result += "\n" + "\n".join(
+            [str(error.trace.error).replace("\\n", "\n") for error in errors if error.trace],
+        )
+        return result
+
+    def as_exception(self) -> AirbyteEntrypointException:
+        """Convert the output to an exception."""
+        return AirbyteEntrypointException(self.get_formatted_error_message())
+
+    def raise_if_errors(
+        self,
+    ) -> None:
+        """Raise an exception if there are errors in the output.
+
+        Otherwise, do nothing.
+        """
+        if not self.errors:
+            return None
+
+        raise self.as_exception()
+
     @property
     def catalog(self) -> AirbyteMessage:
         catalog = self.get_message_by_types([Type.CATALOG])

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/test/standard_tests/_job_runner.py

@@ -21,26 +21,6 @@ from airbyte_cdk.test.models import (
 )
 
 
-def _errors_to_str(
-    entrypoint_output: entrypoint_wrapper.EntrypointOutput,
-) -> str:
-    """Convert errors from entrypoint output to a string."""
-    if not entrypoint_output.errors:
-        # If there are no errors, return an empty string.
-        return ""
-
-    return "\n" + "\n".join(
-        [
-            str(error.trace.error).replace(
-                "\\n",
-                "\n",
-            )
-            for error in entrypoint_output.errors
-            if error.trace
-        ],
-    )
-
-
 @runtime_checkable
 class IConnector(Protocol):
     """A connector that can be run in a test scenario.
@@ -125,9 +105,7 @@ def run_test_job(
         expected_outcome=test_scenario.expected_outcome,
     )
     if result.errors and test_scenario.expected_outcome.expect_success():
-        raise AssertionError(
-            f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result)
-        )
+        raise result.as_exception()
 
     if verb == "check":
         # Check is expected to fail gracefully without an exception.
@@ -137,7 +115,7 @@ def run_test_job(
             "Expected exactly one CONNECTION_STATUS message. Got "
             f"{len(result.connection_status_messages)}:\n"
             + "\n".join([str(msg) for msg in result.connection_status_messages])
-            + _errors_to_str(result)
+            + result.get_formatted_error_message()
         )
         if test_scenario.expected_outcome.expect_exception():
             conn_status = result.connection_status_messages[0].connectionStatus
@@ -161,7 +139,8 @@ def run_test_job(
 
     if test_scenario.expected_outcome.expect_success():
         assert not result.errors, (
-            f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result)
+            f"Test job failed with {len(result.errors)} error(s): \n"
+            + result.get_formatted_error_message()
         )
 
     return result

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/test/standard_tests/connector_base.py

@@ -45,7 +45,7 @@ class ConnectorTestSuiteBase(DockerConnectorTestSuite):
         specific connector class to be tested.
         """
         connector_root = cls.get_connector_root_dir()
-        connector_name = connector_root.absolute().name
+        connector_name = cls.connector_name
 
         expected_module_name = connector_name.replace("-", "_").lower()
         expected_class_name = connector_name.replace("-", "_").title().replace("_", "")
@@ -59,7 +59,16 @@ class ConnectorTestSuiteBase(DockerConnectorTestSuite):
         try:
             module = importlib.import_module(expected_module_name)
         except ModuleNotFoundError as e:
-            raise ImportError(f"Could not import module '{expected_module_name}'.") from e
+            raise ImportError(
+                f"Could not import module '{expected_module_name}'. "
+                "Please ensure you are running from within the connector's virtual environment, "
+                "for instance by running `poetry run airbyte-cdk connector test` from the "
+                "connector directory. If the issue persists, check that the connector "
+                f"module matches the expected module name '{expected_module_name}' and that the "
+                f"connector class matches the expected class name '{expected_class_name}'. "
+                "Alternatively, you can run `airbyte-cdk image test` to run a subset of tests "
+                "against the connector's image."
+            ) from e
         finally:
             # Change back to the original working directory
             os.chdir(cwd_snapshot)

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/test/standard_tests/docker_base.py

@@ -11,7 +11,7 @@ import warnings
 from dataclasses import asdict
 from pathlib import Path
 from subprocess import CompletedProcess, SubprocessError
-from typing import Literal
+from typing import Literal, cast
 
 import orjson
 import pytest
@@ -25,19 +25,18 @@ from airbyte_cdk.models import (
     DestinationSyncMode,
     SyncMode,
 )
-from airbyte_cdk.models.airbyte_protocol_serializers import (
-    AirbyteCatalogSerializer,
-    AirbyteStreamSerializer,
-)
 from airbyte_cdk.models.connector_metadata import MetadataFile
 from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput
 from airbyte_cdk.test.models import ConnectorTestScenario
-from airbyte_cdk.test.utils.reading import catalog
 from airbyte_cdk.utils.connector_paths import (
     ACCEPTANCE_TEST_CONFIG,
     find_connector_root,
 )
-from airbyte_cdk.utils.docker import build_connector_image, run_docker_command
+from airbyte_cdk.utils.docker import (
+    build_connector_image,
+    run_docker_airbyte_command,
+    run_docker_command,
+)
 
 
 class DockerConnectorTestSuite:
@@ -55,6 +54,17 @@ class DockerConnectorTestSuite:
         """Get the root directory of the connector."""
         return find_connector_root([cls.get_test_class_dir(), Path.cwd()])
 
+    @classproperty
+    def connector_name(self) -> str:
+        """Get the name of the connector."""
+        connector_root = self.get_connector_root_dir()
+        return connector_root.absolute().name
+
+    @classmethod
+    def is_destination_connector(cls) -> bool:
+        """Check if the connector is a destination."""
+        return cast(str, cls.connector_name).startswith("destination-")
+
     @classproperty
     def acceptance_test_config_path(cls) -> Path:
         """Get the path to the acceptance test config file."""
@@ -145,23 +155,16 @@ class DockerConnectorTestSuite:
                 no_verify=False,
             )
 
-        try:
-            result: CompletedProcess[str] = run_docker_command(
-                [
-                    "docker",
-                    "run",
-                    "--rm",
-                    connector_image,
-                    "spec",
-                ],
-                check=True,  # Raise an error if the command fails
-                capture_stderr=True,
-                capture_stdout=True,
-            )
-        except SubprocessError as ex:
-            raise AssertionError(
-                f"Failed to run `spec` command in docker image {connector_image!r}. Error: {ex!s}"
-            ) from None
+        _ = run_docker_airbyte_command(
+            [
+                "docker",
+                "run",
+                "--rm",
+                connector_image,
+                "spec",
+            ],
+            raise_if_errors=True,
+        )
 
     @pytest.mark.skipif(
         shutil.which("docker") is None,
@@ -203,7 +206,7 @@ class DockerConnectorTestSuite:
         with scenario.with_temp_config_file(
             connector_root=connector_root,
         ) as temp_config_file:
-            _ = run_docker_command(
+            _ = run_docker_airbyte_command(
                 [
                     "docker",
                     "run",
@@ -215,9 +218,7 @@ class DockerConnectorTestSuite:
                     "--config",
                     container_config_path,
                 ],
-                check=True,  # Raise an error if the command fails
-                capture_stderr=True,
-                capture_stdout=True,
+                raise_if_errors=True,
             )
 
     @pytest.mark.skipif(
@@ -242,6 +243,9 @@ class DockerConnectorTestSuite:
             the local docker image cache using `docker image prune -a` command.
           - If the --connector-image arg is provided, it will be used instead of building the image.
         """
+        if self.is_destination_connector():
+            pytest.skip("Skipping read test for destination connector.")
+
         if scenario.expected_outcome.expect_exception():
             pytest.skip("Skipping (expected to fail).")
 
@@ -295,7 +299,7 @@ class DockerConnectorTestSuite:
             ) as temp_dir_str,
         ):
             temp_dir = Path(temp_dir_str)
-            discover_result = run_docker_command(
+            discover_result = run_docker_airbyte_command(
                 [
                     "docker",
                     "run",
@@ -307,20 +311,12 @@ class DockerConnectorTestSuite:
                     "--config",
                     container_config_path,
                 ],
-                check=True,  # Raise an error if the command fails
-                capture_stderr=True,
-                capture_stdout=True,
+                raise_if_errors=True,
             )
-            parsed_output = EntrypointOutput(messages=discover_result.stdout.splitlines())
-            try:
-                catalog_message = parsed_output.catalog  # Get catalog message
-                assert catalog_message.catalog is not None, "Catalog message missing catalog."
-                discovered_catalog: AirbyteCatalog = parsed_output.catalog.catalog
-            except Exception as ex:
-                raise AssertionError(
-                    f"Failed to load discovered catalog from {discover_result.stdout}. "
-                    f"Error: {ex!s}"
-                ) from None
+
+            catalog_message = discover_result.catalog  # Get catalog message
+            assert catalog_message.catalog is not None, "Catalog message missing catalog."
+            discovered_catalog: AirbyteCatalog = catalog_message.catalog
             if not discovered_catalog.streams:
                 raise ValueError(
                     f"Discovered catalog for connector '{connector_name}' is empty. "
@@ -355,7 +351,7 @@ class DockerConnectorTestSuite:
             configured_catalog_path.write_text(
                 orjson.dumps(asdict(configured_catalog)).decode("utf-8")
             )
-            read_result: CompletedProcess[str] = run_docker_command(
+            read_result: EntrypointOutput = run_docker_airbyte_command(
                 [
                     "docker",
                     "run",
@@ -371,18 +367,5 @@ class DockerConnectorTestSuite:
                     "--catalog",
                     container_catalog_path,
                 ],
-                check=False,
-                capture_stderr=True,
-                capture_stdout=True,
+                raise_if_errors=True,
             )
-            if read_result.returncode != 0:
-                raise AssertionError(
-                    f"Failed to run `read` command in docker image {connector_image!r}. "
-                    "\n-----------------"
-                    f"EXIT CODE: {read_result.returncode}\n"
-                    "STDERR:\n"
-                    f"{read_result.stderr}\n"
-                    f"STDOUT:\n"
-                    f"{read_result.stdout}\n"
-                    "\n-----------------"
-                ) from None

{airbyte_cdk-6.57.2 → airbyte_cdk-6.57.3.post1.dev16088268564}/airbyte_cdk/test/standard_tests/util.py

@@ -10,6 +10,7 @@ from airbyte_cdk.test.standard_tests.connector_base import ConnectorTestSuiteBas
 from airbyte_cdk.test.standard_tests.declarative_sources import (
     DeclarativeSourceTestSuite,
 )
+from airbyte_cdk.test.standard_tests.destination_base import DestinationTestSuiteBase
 from airbyte_cdk.test.standard_tests.docker_base import DockerConnectorTestSuite
 from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
 from airbyte_cdk.utils.connector_paths import (
@@ -17,15 +18,6 @@ from airbyte_cdk.utils.connector_paths import (
     find_connector_root_from_name,
 )
 
-TEST_CLASS_MAPPING: dict[
-    Literal["python", "manifest-only", "java"],
-    type[DockerConnectorTestSuite],
-] = {
-    "python": SourceTestSuiteBase,
-    "manifest-only": DeclarativeSourceTestSuite,
-    "java": DockerConnectorTestSuite,
-}
-
 
 def create_connector_test_suite(
     *,
@@ -46,7 +38,7 @@ def create_connector_test_suite(
         # By here, we know that connector_directory is not None
         # but connector_name is None. Set the connector_name.
         assert connector_directory is not None, "connector_directory should not be None here."
-        connector_name = connector_directory.name
+        connector_name = connector_directory.absolute().name
 
     metadata_yaml_path = connector_directory / METADATA_YAML
     if not metadata_yaml_path.exists():
@@ -55,17 +47,25 @@ def create_connector_test_suite(
         )
     metadata_dict: dict[str, Any] = yaml.safe_load(metadata_yaml_path.read_text())
     metadata_tags = metadata_dict["data"].get("tags", [])
-    for language_option in TEST_CLASS_MAPPING:
-        if f"language:{language_option}" in metadata_tags:
-            language = language_option
-            test_suite_class = TEST_CLASS_MAPPING[language]
-            break
-    else:
+    language_tags: list[str] = [tag for tag in metadata_tags if tag.startswith("language:")]
+    if not language_tags:
         raise ValueError(
-            f"Unsupported connector type. "
-            f"Supported language values are: {', '.join(TEST_CLASS_MAPPING.keys())}. "
+            f"Metadata YAML file '{metadata_yaml_path}' does not contain a 'language' tag. "
+            "Please ensure the metadata file is correctly configured."
             f"Found tags: {', '.join(metadata_tags)}"
         )
+    language = language_tags[0].split(":")[1]
+
+    if language == "java":
+        test_suite_class = DockerConnectorTestSuite
+    elif language == "manifest-only":
+        test_suite_class = DeclarativeSourceTestSuite
+    elif language == "python" and connector_name.startswith("source-"):
+        test_suite_class = SourceTestSuiteBase
+    elif language == "python" and connector_name.startswith("destination-"):
+        test_suite_class = DestinationTestSuiteBase
+    else:
+        raise ValueError(f"Unsupported language for connector '{connector_name}': {language}")
 
     subclass_overrides: dict[str, Any] = {
         "get_connector_root_dir": classmethod(lambda cls: connector_directory),