airbyte-cdk 6.45.10__py3-none-any.whl → 6.46.1__py3-none-any.whl

airbyte_cdk/test/standard_tests/connector_base.py

@@ -4,7 +4,9 @@
 from __future__ import annotations
 
 import abc
+import importlib
 import inspect
+import os
 import sys
 from collections.abc import Callable
 from pathlib import Path
@@ -22,17 +24,61 @@ from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job
 from airbyte_cdk.test.standard_tests.models import (
     ConnectorTestScenario,
 )
-
-ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
-MANIFEST_YAML = "manifest.yaml"
+from airbyte_cdk.test.standard_tests.test_resources import (
+    ACCEPTANCE_TEST_CONFIG,
+    find_connector_root,
+)
 
 
 class ConnectorTestSuiteBase(abc.ABC):
     """Base class for connector test suites."""
 
-    connector: type[IConnector] | Callable[[], IConnector] | None = None
+    connector: type[IConnector] | Callable[[], IConnector] | None  # type: ignore [reportRedeclaration]
     """The connector class or a factory function that returns an scenario of IConnector."""
 
+    @classproperty  # type: ignore [no-redef]
+    def connector(cls) -> type[IConnector] | Callable[[], IConnector] | None:
+        """Get the connector class for the test suite.
+
+        This assumes a python connector and should be overridden by subclasses to provide the
+        specific connector class to be tested.
+        """
+        connector_root = cls.get_connector_root_dir()
+        connector_name = connector_root.absolute().name
+
+        expected_module_name = connector_name.replace("-", "_").lower()
+        expected_class_name = connector_name.replace("-", "_").title().replace("_", "")
+
+        # dynamically import and get the connector class: <expected_module_name>.<expected_class_name>
+
+        cwd_snapshot = Path().absolute()
+        os.chdir(connector_root)
+
+        # Dynamically import the module
+        try:
+            module = importlib.import_module(expected_module_name)
+        except ModuleNotFoundError as e:
+            raise ImportError(f"Could not import module '{expected_module_name}'.") from e
+        finally:
+            # Change back to the original working directory
+            os.chdir(cwd_snapshot)
+
+        # Dynamically get the class from the module
+        try:
+            return cast(type[IConnector], getattr(module, expected_class_name))
+        except AttributeError as e:
+            # We did not find it based on our expectations, so let's check if we can find it
+            # with a case-insensitive match.
+            matching_class_name = next(
+                (name for name in dir(module) if name.lower() == expected_class_name.lower()),
+                None,
+            )
+            if not matching_class_name:
+                raise ImportError(
+                    f"Module '{expected_module_name}' does not have a class named '{expected_class_name}'."
+                ) from e
+            return cast(type[IConnector], getattr(module, matching_class_name))
+
     @classmethod
     def get_test_class_dir(cls) -> Path:
         """Get the file path that contains the class."""
@@ -81,27 +127,7 @@ class ConnectorTestSuiteBase(abc.ABC):
     @classmethod
     def get_connector_root_dir(cls) -> Path:
         """Get the root directory of the connector."""
-        for parent in cls.get_test_class_dir().parents:
-            if (parent / MANIFEST_YAML).exists():
-                return parent
-            if (parent / ACCEPTANCE_TEST_CONFIG).exists():
-                return parent
-            if parent.name == "airbyte_cdk":
-                break
-        # If we reach here, we didn't find the manifest file in any parent directory
-        # Check if the manifest file exists in the current directory
-        for parent in Path.cwd().parents:
-            if (parent / MANIFEST_YAML).exists():
-                return parent
-            if (parent / ACCEPTANCE_TEST_CONFIG).exists():
-                return parent
-            if parent.name == "airbyte_cdk":
-                break
-
-        raise FileNotFoundError(
-            "Could not find connector root directory relative to "
-            f"'{str(cls.get_test_class_dir())}' or '{str(Path.cwd())}'."
-        )
+        return find_connector_root([cls.get_test_class_dir(), Path.cwd()])
 
     @classproperty
     def acceptance_test_config_path(cls) -> Path:

airbyte_cdk/test/standard_tests/declarative_sources.py

@@ -10,9 +10,9 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
     ConcurrentDeclarativeSource,
 )
 from airbyte_cdk.test.standard_tests._job_runner import IConnector
-from airbyte_cdk.test.standard_tests.connector_base import MANIFEST_YAML
 from airbyte_cdk.test.standard_tests.models import ConnectorTestScenario
 from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
+from airbyte_cdk.test.standard_tests.test_resources import MANIFEST_YAML
 
 
 def md5_checksum(file_path: Path) -> str:
@@ -35,6 +35,8 @@ class DeclarativeSourceTestSuite(SourceTestSuiteBase):
     `components.py` file (if it exists) in the connector's directory.
     """
 
+    connector: type[IConnector] | None = None
+
     @classproperty
     def manifest_yaml_path(cls) -> Path:
         """Get the path to the manifest.yaml file."""

airbyte_cdk/test/standard_tests/test_resources.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Resources for Airbyte CDK tests."""
+
+from contextlib import suppress
+from pathlib import Path
+
+ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
+MANIFEST_YAML = "manifest.yaml"
+METADATA_YAML = "metadata.yaml"
+
+
+def find_connector_root(from_paths: list[Path]) -> Path:
+    """Find the root directory of the connector."""
+    for path in from_paths:
+        # If we reach here, we didn't find the manifest file in any parent directory
+        # Check if the manifest file exists in the current directory
+        for parent in [path, *path.parents]:
+            if (parent / METADATA_YAML).exists():
+                return parent
+            if (parent / MANIFEST_YAML).exists():
+                return parent
+            if (parent / ACCEPTANCE_TEST_CONFIG).exists():
+                return parent
+            if parent.name == "airbyte_cdk":
+                break
+
+    raise FileNotFoundError(
+        "Could not find connector root directory relative to the provided directories: "
+        f"'{str(from_paths)}'."
+    )
+
+
+def find_connector_root_from_name(connector_name: str) -> Path:
+    """Find the root directory of the connector from its name."""
+    with suppress(FileNotFoundError):
+        return find_connector_root([Path(connector_name)])
+
+    # If the connector name is not found, check if we are in the airbyte monorepo
+    # and try to find the connector root from the current directory.
+
+    cwd: Path = Path().absolute()
+
+    if "airbyte" not in cwd.parts:
+        raise FileNotFoundError(
+            "Could not find connector root directory relative and we are not in the airbyte repo. "
+            f"Current directory: {cwd} "
+        )
+
+    # Find the connector root from the current directory
+
+    airbyte_repo_root: Path
+    for parent in [cwd, *cwd.parents]:
+        if parent.name == "airbyte":
+            airbyte_repo_root = parent
+            break
+    else:
+        raise FileNotFoundError(
+            "Could not find connector root directory relative and we are not in the airbyte repo."
+        )
+
+    expected_connector_dir: Path = (
+        airbyte_repo_root / "airbyte-integrations" / "connectors" / connector_name
+    )
+    if not expected_connector_dir.exists():
+        raise FileNotFoundError(
+            f"Could not find connector directory '{expected_connector_dir}' relative to the airbyte repo."
+        )
+
+    return expected_connector_dir

airbyte_cdk/test/standard_tests/util.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Utility and factory functions for testing Airbyte connectors."""
+
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+
+from airbyte_cdk.test.standard_tests.connector_base import ConnectorTestSuiteBase
+from airbyte_cdk.test.standard_tests.declarative_sources import (
+    DeclarativeSourceTestSuite,
+)
+from airbyte_cdk.test.standard_tests.destination_base import DestinationTestSuiteBase
+from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
+from airbyte_cdk.test.standard_tests.test_resources import (
+    METADATA_YAML,
+    find_connector_root_from_name,
+)
+
+TEST_CLASS_MAPPING: dict[
+    Literal["python", "manifest-only", "declarative"], type[ConnectorTestSuiteBase]
+] = {
+    "python": SourceTestSuiteBase,
+    "manifest-only": DeclarativeSourceTestSuite,
+    # "declarative": DeclarativeSourceTestSuite,
+}
+
+
+def create_connector_test_suite(
+    *,
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+) -> type[ConnectorTestSuiteBase]:
+    """Get the test class for the specified connector name or path."""
+    if connector_name and connector_directory:
+        raise ValueError("Specify either `connector_name` or `connector_directory`, not both.")
+    if not connector_name and not connector_directory:
+        raise ValueError("Specify either `connector_name` or `connector_directory`.")
+
+    if connector_name:
+        connector_directory = find_connector_root_from_name(
+            connector_name,
+        )
+    else:
+        # By here, we know that connector_directory is not None
+        # but connector_name is None. Set the connector_name.
+        assert connector_directory is not None, "connector_directory should not be None here."
+        connector_name = connector_directory.name
+
+    metadata_yaml_path = connector_directory / METADATA_YAML
+    if not metadata_yaml_path.exists():
+        raise FileNotFoundError(
+            f"Could not find metadata YAML file '{metadata_yaml_path}' relative to the connector directory."
+        )
+    metadata_dict: dict[str, Any] = yaml.safe_load(metadata_yaml_path.read_text())
+    metadata_tags = metadata_dict["data"].get("tags", [])
+    for language_option in TEST_CLASS_MAPPING:
+        if f"language:{language_option}" in metadata_tags:
+            language = language_option
+            test_suite_class = TEST_CLASS_MAPPING[language]
+            break
+    else:
+        raise ValueError(
+            f"Unsupported connector type. "
+            f"Supported language values are: {', '.join(TEST_CLASS_MAPPING.keys())}. "
+            f"Found tags: {', '.join(metadata_tags)}"
+        )
+
+    subclass_overrides: dict[str, Any] = {
+        "get_connector_root_dir": lambda: connector_directory,
+    }
+
+    TestSuiteAuto = type(
+        "TestSuiteAuto",
+        (test_suite_class,),
+        subclass_overrides,
+    )
+
+    return TestSuiteAuto

airbyte_cdk/utils/docker.py

@@ -0,0 +1,337 @@
+"""Docker build utilities for Airbyte CDK."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+
+import click
+
+from airbyte_cdk.models.connector_metadata import ConnectorLanguage, MetadataFile
+from airbyte_cdk.utils.docker_image_templates import (
+    DOCKERIGNORE_TEMPLATE,
+    MANIFEST_ONLY_DOCKERFILE_TEMPLATE,
+    PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE,
+)
+
+
+@dataclass(kw_only=True)
+class ConnectorImageBuildError(Exception):
+    """Custom exception for Docker build errors."""
+
+    error_text: str
+    build_args: list[str]
+
+    def __str__(self) -> str:
+        return "\n".join(
+            [
+                f"ConnectorImageBuildError: Could not build image.",
+                f"Build args: {self.build_args}",
+                f"Error text: {self.error_text}",
+            ]
+        )
+
+
+logger = logging.getLogger(__name__)
+
+
+class ArchEnum(str, Enum):
+    """Enum for supported architectures."""
+
+    ARM64 = "arm64"
+    AMD64 = "amd64"
+
+
+def _build_image(
+    context_dir: Path,
+    dockerfile: Path,
+    metadata: MetadataFile,
+    tag: str,
+    arch: ArchEnum,
+    build_args: dict[str, str | None] | None = None,
+) -> str:
+    """Build a Docker image for the specified architecture.
+
+    Returns the tag of the built image.
+
+    Raises: ConnectorImageBuildError if the build fails.
+    """
+    docker_args: list[str] = [
+        "docker",
+        "build",
+        "--platform",
+        f"linux/{arch.value}",
+        "--file",
+        str(dockerfile),
+        "--label",
+        f"io.airbyte.version={metadata.data.dockerImageTag}",
+        "--label",
+        f"io.airbyte.name={metadata.data.dockerRepository}",
+    ]
+    if build_args:
+        for key, value in build_args.items():
+            if value is not None:
+                docker_args.append(f"--build-arg={key}={value}")
+            else:
+                docker_args.append(f"--build-arg={key}")
+    docker_args.extend(
+        [
+            "-t",
+            tag,
+            str(context_dir),
+        ]
+    )
+
+    print(f"Building image: {tag} ({arch})")
+    try:
+        run_docker_command(
+            docker_args,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        raise ConnectorImageBuildError(
+            error_text=e.stderr,
+            build_args=docker_args,
+        ) from e
+
+    return tag
+
+
+def _tag_image(
+    tag: str,
+    new_tags: list[str] | str,
+) -> None:
+    """Build a Docker image for the specified architecture.
+
+    Returns the tag of the built image.
+
+    Raises:
+        ConnectorImageBuildError: If the docker tag command fails.
+    """
+    if not isinstance(new_tags, list):
+        new_tags = [new_tags]
+
+    for new_tag in new_tags:
+        print(f"Tagging image '{tag}' as: {new_tag}")
+        docker_args = [
+            "docker",
+            "tag",
+            tag,
+            new_tag,
+        ]
+        try:
+            run_docker_command(
+                docker_args,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            raise ConnectorImageBuildError(
+                error_text=e.stderr,
+                build_args=docker_args,
+            ) from e
+
+
+def build_connector_image(
+    connector_name: str,
+    connector_directory: Path,
+    metadata: MetadataFile,
+    tag: str,
+    primary_arch: ArchEnum = ArchEnum.ARM64,  # Assume MacBook M series by default
+    no_verify: bool = False,
+) -> None:
+    """Build a connector Docker image.
+
+    This command builds a Docker image for a connector, using either
+    the connector's Dockerfile or a base image specified in the metadata.
+    The image is built for both AMD64 and ARM64 architectures.
+
+    Args:
+        connector_name: The name of the connector.
+        connector_directory: The directory containing the connector code.
+        metadata: The metadata of the connector.
+        tag: The tag to apply to the built image.
+        primary_arch: The primary architecture for the build (default: arm64). This
+            architecture will be used for the same-named tag. Both AMD64 and ARM64
+            images will be built, with the suffixes '-amd64' and '-arm64'.
+        no_verify: If True, skip verification of the built image.
+
+    Raises:
+        ValueError: If the connector build options are not defined in metadata.yaml.
+        ConnectorImageBuildError: If the image build or tag operation fails.
+    """
+    connector_kebab_name = connector_name
+    connector_snake_name = connector_kebab_name.replace("-", "_")
+
+    dockerfile_path = connector_directory / "build" / "docker" / "Dockerfile"
+    dockerignore_path = connector_directory / "build" / "docker" / "Dockerfile.dockerignore"
+
+    extra_build_script: str = ""
+    build_customization_path = connector_directory / "build_customization.py"
+    if build_customization_path.exists():
+        extra_build_script = str(build_customization_path)
+
+    dockerfile_path.parent.mkdir(parents=True, exist_ok=True)
+    if not metadata.data.connectorBuildOptions:
+        raise ValueError(
+            "Connector build options are not defined in metadata.yaml. "
+            "Please check the connector's metadata file."
+        )
+
+    base_image = metadata.data.connectorBuildOptions.baseImage
+
+    dockerfile_path.write_text(get_dockerfile_template(metadata))
+    dockerignore_path.write_text(DOCKERIGNORE_TEMPLATE)
+
+    build_args: dict[str, str | None] = {
+        "BASE_IMAGE": base_image,
+        "CONNECTOR_SNAKE_NAME": connector_snake_name,
+        "CONNECTOR_KEBAB_NAME": connector_kebab_name,
+        "EXTRA_BUILD_SCRIPT": extra_build_script,
+    }
+
+    base_tag = f"{metadata.data.dockerRepository}:{tag}"
+    arch_images: list[str] = []
+    for arch in [ArchEnum.AMD64, ArchEnum.ARM64]:
+        docker_tag = f"{base_tag}-{arch.value}"
+        docker_tag_parts = docker_tag.split("/")
+        if len(docker_tag_parts) > 2:
+            docker_tag = "/".join(docker_tag_parts[-1:])
+        arch_images.append(
+            _build_image(
+                context_dir=connector_directory,
+                dockerfile=dockerfile_path,
+                metadata=metadata,
+                tag=docker_tag,
+                arch=arch,
+                build_args=build_args,
+            )
+        )
+
+    _tag_image(
+        tag=f"{base_tag}-{primary_arch.value}",
+        new_tags=[base_tag],
+    )
+    if not no_verify:
+        if verify_connector_image(base_tag):
+            click.echo(f"Build completed successfully: {base_tag}")
+            sys.exit(0)
+        else:
+            click.echo(f"Built image failed verification: {base_tag}", err=True)
+            sys.exit(1)
+    else:
+        click.echo(f"Build completed successfully (without verification): {base_tag}")
+        sys.exit(0)
+
+
+def get_dockerfile_template(
+    metadata: MetadataFile,
+) -> str:
+    """Get the Dockerfile template for the connector.
+
+    Args:
+        metadata: The metadata of the connector.
+        connector_name: The name of the connector.
+
+    Returns:
+        The Dockerfile template as a string.
+    """
+    if metadata.data.language == ConnectorLanguage.PYTHON:
+        return PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE
+
+    if metadata.data.language == ConnectorLanguage.MANIFEST_ONLY:
+        return MANIFEST_ONLY_DOCKERFILE_TEMPLATE
+
+    if metadata.data.language == ConnectorLanguage.JAVA:
+        raise ValueError(
+            f"Java and Kotlin connectors are not yet supported. "
+            "Please use airbyte-ci or gradle to build your image."
+        )
+
+    raise ValueError(
+        f"Unsupported connector language: {metadata.data.language}. "
+        "Please check the connector's metadata file."
+    )
+
+
+def run_docker_command(
+    cmd: list[str],
+    *,
+    check: bool = True,
+    capture_output: bool = False,
+) -> subprocess.CompletedProcess[str]:
+    """Run a Docker command as a subprocess.
+
+    Args:
+        cmd: The command to run as a list of strings.
+        check: If True, raises an exception if the command fails. If False, the caller is
+            responsible for checking the return code.
+        capture_output: If True, captures stdout and stderr and returns to the caller.
+            If False, the output is printed to the console.
+
+    Raises:
+        subprocess.CalledProcessError: If the command fails and check is True.
+    """
+    print(f"Running command: {' '.join(cmd)}")
+
+    process = subprocess.run(
+        cmd,
+        text=True,
+        check=check,
+        # If capture_output=True, stderr and stdout are captured and returned to caller:
+        capture_output=capture_output,
+        env={**os.environ, "DOCKER_BUILDKIT": "1"},
+    )
+    return process
+
+
+def verify_docker_installation() -> bool:
+    """Verify Docker is installed and running."""
+    try:
+        run_docker_command(["docker", "--version"])
+        return True
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return False
+
+
+def verify_connector_image(
+    image_name: str,
+) -> bool:
+    """Verify the built image by running the spec command.
+
+    Args:
+        image_name: The full image name with tag.
+
+    Returns:
+        True if the spec command succeeds, False otherwise.
+    """
+    logger.info(f"Verifying image {image_name} with 'spec' command...")
+
+    cmd = ["docker", "run", "--rm", image_name, "spec"]
+
+    try:
+        result = run_docker_command(
+            cmd,
+            check=True,
+            capture_output=True,
+        )
+        # check that the output is valid JSON
+        if result.stdout:
+            try:
+                json.loads(result.stdout)
+            except json.JSONDecodeError:
+                logger.error("Invalid JSON output from spec command.")
+                return False
+        else:
+            logger.error("No output from spec command.")
+            return False
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Image verification failed: {e.stderr}")
+        return False
+
+    return True