airbyte-cdk 6.45.10__py3-none-any.whl → 6.46.0__py3-none-any.whl

airbyte_cdk/test/standard_tests/declarative_sources.py

@@ -10,9 +10,9 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
     ConcurrentDeclarativeSource,
 )
 from airbyte_cdk.test.standard_tests._job_runner import IConnector
-from airbyte_cdk.test.standard_tests.connector_base import MANIFEST_YAML
 from airbyte_cdk.test.standard_tests.models import ConnectorTestScenario
 from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
+from airbyte_cdk.test.standard_tests.test_resources import MANIFEST_YAML
 
 
 def md5_checksum(file_path: Path) -> str:
@@ -35,6 +35,8 @@ class DeclarativeSourceTestSuite(SourceTestSuiteBase):
     `components.py` file (if it exists) in the connector's directory.
     """
 
+    connector: type[IConnector] | None = None
+
     @classproperty
     def manifest_yaml_path(cls) -> Path:
         """Get the path to the manifest.yaml file."""

airbyte_cdk/test/standard_tests/test_resources.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Resources for Airbyte CDK tests."""
+
+from contextlib import suppress
+from pathlib import Path
+
+ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
+MANIFEST_YAML = "manifest.yaml"
+METADATA_YAML = "metadata.yaml"
+
+
+def find_connector_root(from_paths: list[Path]) -> Path:
+    """Find the root directory of the connector."""
+    for path in from_paths:
+        # If we reach here, we didn't find the manifest file in any parent directory
+        # Check if the manifest file exists in the current directory
+        for parent in [path, *path.parents]:
+            if (parent / METADATA_YAML).exists():
+                return parent
+            if (parent / MANIFEST_YAML).exists():
+                return parent
+            if (parent / ACCEPTANCE_TEST_CONFIG).exists():
+                return parent
+            if parent.name == "airbyte_cdk":
+                break
+
+    raise FileNotFoundError(
+        "Could not find connector root directory relative to the provided directories: "
+        f"'{str(from_paths)}'."
+    )
+
+
+def find_connector_root_from_name(connector_name: str) -> Path:
+    """Find the root directory of the connector from its name."""
+    with suppress(FileNotFoundError):
+        return find_connector_root([Path(connector_name)])
+
+    # If the connector name is not found, check if we are in the airbyte monorepo
+    # and try to find the connector root from the current directory.
+
+    cwd: Path = Path().absolute()
+
+    if "airbyte" not in cwd.parts:
+        raise FileNotFoundError(
+            "Could not find connector root directory relative and we are not in the airbyte repo. "
+            f"Current directory: {cwd} "
+        )
+
+    # Find the connector root from the current directory
+
+    airbyte_repo_root: Path
+    for parent in [cwd, *cwd.parents]:
+        if parent.name == "airbyte":
+            airbyte_repo_root = parent
+            break
+    else:
+        raise FileNotFoundError(
+            "Could not find connector root directory relative and we are not in the airbyte repo."
+        )
+
+    expected_connector_dir: Path = (
+        airbyte_repo_root / "airbyte-integrations" / "connectors" / connector_name
+    )
+    if not expected_connector_dir.exists():
+        raise FileNotFoundError(
+            f"Could not find connector directory '{expected_connector_dir}' relative to the airbyte repo."
+        )
+
+    return expected_connector_dir

airbyte_cdk/test/standard_tests/util.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Utility and factory functions for testing Airbyte connectors."""
+
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+
+from airbyte_cdk.test.standard_tests.connector_base import ConnectorTestSuiteBase
+from airbyte_cdk.test.standard_tests.declarative_sources import (
+    DeclarativeSourceTestSuite,
+)
+from airbyte_cdk.test.standard_tests.destination_base import DestinationTestSuiteBase
+from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
+from airbyte_cdk.test.standard_tests.test_resources import (
+    METADATA_YAML,
+    find_connector_root_from_name,
+)
+
+TEST_CLASS_MAPPING: dict[
+    Literal["python", "manifest-only", "declarative"], type[ConnectorTestSuiteBase]
+] = {
+    "python": SourceTestSuiteBase,
+    "manifest-only": DeclarativeSourceTestSuite,
+    # "declarative": DeclarativeSourceTestSuite,
+}
+
+
+def create_connector_test_suite(
+    *,
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+) -> type[ConnectorTestSuiteBase]:
+    """Get the test class for the specified connector name or path."""
+    if connector_name and connector_directory:
+        raise ValueError("Specify either `connector_name` or `connector_directory`, not both.")
+    if not connector_name and not connector_directory:
+        raise ValueError("Specify either `connector_name` or `connector_directory`.")
+
+    if connector_name:
+        connector_directory = find_connector_root_from_name(
+            connector_name,
+        )
+    else:
+        # By here, we know that connector_directory is not None
+        # but connector_name is None. Set the connector_name.
+        assert connector_directory is not None, "connector_directory should not be None here."
+        connector_name = connector_directory.name
+
+    metadata_yaml_path = connector_directory / METADATA_YAML
+    if not metadata_yaml_path.exists():
+        raise FileNotFoundError(
+            f"Could not find metadata YAML file '{metadata_yaml_path}' relative to the connector directory."
+        )
+    metadata_dict: dict[str, Any] = yaml.safe_load(metadata_yaml_path.read_text())
+    metadata_tags = metadata_dict["data"].get("tags", [])
+    for language_option in TEST_CLASS_MAPPING:
+        if f"language:{language_option}" in metadata_tags:
+            language = language_option
+            test_suite_class = TEST_CLASS_MAPPING[language]
+            break
+    else:
+        raise ValueError(
+            f"Unsupported connector type. "
+            f"Supported language values are: {', '.join(TEST_CLASS_MAPPING.keys())}. "
+            f"Found tags: {', '.join(metadata_tags)}"
+        )
+
+    subclass_overrides: dict[str, Any] = {
+        "get_connector_root_dir": lambda: connector_directory,
+    }
+
+    TestSuiteAuto = type(
+        "TestSuiteAuto",
+        (test_suite_class,),
+        subclass_overrides,
+    )
+
+    return TestSuiteAuto

airbyte_cdk/utils/docker.py

@@ -0,0 +1,337 @@
+"""Docker build utilities for Airbyte CDK."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+
+import click
+
+from airbyte_cdk.models.connector_metadata import ConnectorLanguage, MetadataFile
+from airbyte_cdk.utils.docker_image_templates import (
+    DOCKERIGNORE_TEMPLATE,
+    MANIFEST_ONLY_DOCKERFILE_TEMPLATE,
+    PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE,
+)
+
+
+@dataclass(kw_only=True)
+class ConnectorImageBuildError(Exception):
+    """Custom exception for Docker build errors."""
+
+    error_text: str
+    build_args: list[str]
+
+    def __str__(self) -> str:
+        return "\n".join(
+            [
+                f"ConnectorImageBuildError: Could not build image.",
+                f"Build args: {self.build_args}",
+                f"Error text: {self.error_text}",
+            ]
+        )
+
+
+logger = logging.getLogger(__name__)
+
+
+class ArchEnum(str, Enum):
+    """Enum for supported architectures."""
+
+    ARM64 = "arm64"
+    AMD64 = "amd64"
+
+
+def _build_image(
+    context_dir: Path,
+    dockerfile: Path,
+    metadata: MetadataFile,
+    tag: str,
+    arch: ArchEnum,
+    build_args: dict[str, str | None] | None = None,
+) -> str:
+    """Build a Docker image for the specified architecture.
+
+    Returns the tag of the built image.
+
+    Raises: ConnectorImageBuildError if the build fails.
+    """
+    docker_args: list[str] = [
+        "docker",
+        "build",
+        "--platform",
+        f"linux/{arch.value}",
+        "--file",
+        str(dockerfile),
+        "--label",
+        f"io.airbyte.version={metadata.data.dockerImageTag}",
+        "--label",
+        f"io.airbyte.name={metadata.data.dockerRepository}",
+    ]
+    if build_args:
+        for key, value in build_args.items():
+            if value is not None:
+                docker_args.append(f"--build-arg={key}={value}")
+            else:
+                docker_args.append(f"--build-arg={key}")
+    docker_args.extend(
+        [
+            "-t",
+            tag,
+            str(context_dir),
+        ]
+    )
+
+    print(f"Building image: {tag} ({arch})")
+    try:
+        run_docker_command(
+            docker_args,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        raise ConnectorImageBuildError(
+            error_text=e.stderr,
+            build_args=docker_args,
+        ) from e
+
+    return tag
+
+
+def _tag_image(
+    tag: str,
+    new_tags: list[str] | str,
+) -> None:
+    """Build a Docker image for the specified architecture.
+
+    Returns the tag of the built image.
+
+    Raises:
+        ConnectorImageBuildError: If the docker tag command fails.
+    """
+    if not isinstance(new_tags, list):
+        new_tags = [new_tags]
+
+    for new_tag in new_tags:
+        print(f"Tagging image '{tag}' as: {new_tag}")
+        docker_args = [
+            "docker",
+            "tag",
+            tag,
+            new_tag,
+        ]
+        try:
+            run_docker_command(
+                docker_args,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            raise ConnectorImageBuildError(
+                error_text=e.stderr,
+                build_args=docker_args,
+            ) from e
+
+
+def build_connector_image(
+    connector_name: str,
+    connector_directory: Path,
+    metadata: MetadataFile,
+    tag: str,
+    primary_arch: ArchEnum = ArchEnum.ARM64,  # Assume MacBook M series by default
+    no_verify: bool = False,
+) -> None:
+    """Build a connector Docker image.
+
+    This command builds a Docker image for a connector, using either
+    the connector's Dockerfile or a base image specified in the metadata.
+    The image is built for both AMD64 and ARM64 architectures.
+
+    Args:
+        connector_name: The name of the connector.
+        connector_directory: The directory containing the connector code.
+        metadata: The metadata of the connector.
+        tag: The tag to apply to the built image.
+        primary_arch: The primary architecture for the build (default: arm64). This
+            architecture will be used for the same-named tag. Both AMD64 and ARM64
+            images will be built, with the suffixes '-amd64' and '-arm64'.
+        no_verify: If True, skip verification of the built image.
+
+    Raises:
+        ValueError: If the connector build options are not defined in metadata.yaml.
+        ConnectorImageBuildError: If the image build or tag operation fails.
+    """
+    connector_kebab_name = connector_name
+    connector_snake_name = connector_kebab_name.replace("-", "_")
+
+    dockerfile_path = connector_directory / "build" / "docker" / "Dockerfile"
+    dockerignore_path = connector_directory / "build" / "docker" / "Dockerfile.dockerignore"
+
+    extra_build_script: str = ""
+    build_customization_path = connector_directory / "build_customization.py"
+    if build_customization_path.exists():
+        extra_build_script = str(build_customization_path)
+
+    dockerfile_path.parent.mkdir(parents=True, exist_ok=True)
+    if not metadata.data.connectorBuildOptions:
+        raise ValueError(
+            "Connector build options are not defined in metadata.yaml. "
+            "Please check the connector's metadata file."
+        )
+
+    base_image = metadata.data.connectorBuildOptions.baseImage
+
+    dockerfile_path.write_text(get_dockerfile_template(metadata))
+    dockerignore_path.write_text(DOCKERIGNORE_TEMPLATE)
+
+    build_args: dict[str, str | None] = {
+        "BASE_IMAGE": base_image,
+        "CONNECTOR_SNAKE_NAME": connector_snake_name,
+        "CONNECTOR_KEBAB_NAME": connector_kebab_name,
+        "EXTRA_BUILD_SCRIPT": extra_build_script,
+    }
+
+    base_tag = f"{metadata.data.dockerRepository}:{tag}"
+    arch_images: list[str] = []
+    for arch in [ArchEnum.AMD64, ArchEnum.ARM64]:
+        docker_tag = f"{base_tag}-{arch.value}"
+        docker_tag_parts = docker_tag.split("/")
+        if len(docker_tag_parts) > 2:
+            docker_tag = "/".join(docker_tag_parts[-1:])
+        arch_images.append(
+            _build_image(
+                context_dir=connector_directory,
+                dockerfile=dockerfile_path,
+                metadata=metadata,
+                tag=docker_tag,
+                arch=arch,
+                build_args=build_args,
+            )
+        )
+
+    _tag_image(
+        tag=f"{base_tag}-{primary_arch.value}",
+        new_tags=[base_tag],
+    )
+    if not no_verify:
+        if verify_connector_image(base_tag):
+            click.echo(f"Build completed successfully: {base_tag}")
+            sys.exit(0)
+        else:
+            click.echo(f"Built image failed verification: {base_tag}", err=True)
+            sys.exit(1)
+    else:
+        click.echo(f"Build completed successfully (without verification): {base_tag}")
+        sys.exit(0)
+
+
+def get_dockerfile_template(
+    metadata: MetadataFile,
+) -> str:
+    """Get the Dockerfile template for the connector.
+
+    Args:
+        metadata: The metadata of the connector.
+        connector_name: The name of the connector.
+
+    Returns:
+        The Dockerfile template as a string.
+    """
+    if metadata.data.language == ConnectorLanguage.PYTHON:
+        return PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE
+
+    if metadata.data.language == ConnectorLanguage.MANIFEST_ONLY:
+        return MANIFEST_ONLY_DOCKERFILE_TEMPLATE
+
+    if metadata.data.language == ConnectorLanguage.JAVA:
+        raise ValueError(
+            f"Java and Kotlin connectors are not yet supported. "
+            "Please use airbyte-ci or gradle to build your image."
+        )
+
+    raise ValueError(
+        f"Unsupported connector language: {metadata.data.language}. "
+        "Please check the connector's metadata file."
+    )
+
+
+def run_docker_command(
+    cmd: list[str],
+    *,
+    check: bool = True,
+    capture_output: bool = False,
+) -> subprocess.CompletedProcess[str]:
+    """Run a Docker command as a subprocess.
+
+    Args:
+        cmd: The command to run as a list of strings.
+        check: If True, raises an exception if the command fails. If False, the caller is
+            responsible for checking the return code.
+        capture_output: If True, captures stdout and stderr and returns to the caller.
+            If False, the output is printed to the console.
+
+    Raises:
+        subprocess.CalledProcessError: If the command fails and check is True.
+    """
+    print(f"Running command: {' '.join(cmd)}")
+
+    process = subprocess.run(
+        cmd,
+        text=True,
+        check=check,
+        # If capture_output=True, stderr and stdout are captured and returned to caller:
+        capture_output=capture_output,
+        env={**os.environ, "DOCKER_BUILDKIT": "1"},
+    )
+    return process
+
+
+def verify_docker_installation() -> bool:
+    """Verify Docker is installed and running."""
+    try:
+        run_docker_command(["docker", "--version"])
+        return True
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return False
+
+
+def verify_connector_image(
+    image_name: str,
+) -> bool:
+    """Verify the built image by running the spec command.
+
+    Args:
+        image_name: The full image name with tag.
+
+    Returns:
+        True if the spec command succeeds, False otherwise.
+    """
+    logger.info(f"Verifying image {image_name} with 'spec' command...")
+
+    cmd = ["docker", "run", "--rm", image_name, "spec"]
+
+    try:
+        result = run_docker_command(
+            cmd,
+            check=True,
+            capture_output=True,
+        )
+        # check that the output is valid JSON
+        if result.stdout:
+            try:
+                json.loads(result.stdout)
+            except json.JSONDecodeError:
+                logger.error("Invalid JSON output from spec command.")
+                return False
+        else:
+            logger.error("No output from spec command.")
+            return False
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Image verification failed: {e.stderr}")
+        return False
+
+    return True

airbyte_cdk/utils/docker_image_templates.py

@@ -0,0 +1,101 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""A collection of Dockerfile templates for building Airbyte connectors.
+
+The templates are designed to be used with the Airbyte CDK and can be customized
+for different connectors and architectures.
+
+These templates are used to generate connector images.
+"""
+
+##############################
+## GLOBAL DOCKERIGNORE FILE ##
+##############################
+
+DOCKERIGNORE_TEMPLATE: str = "\n".join(
+    [
+        "# This file is auto-generated. Do not edit.",
+        # "*,"
+        "build/",
+        ".venv/",
+        "secrets/",
+        "!setup.py",
+        "!pyproject.toml",
+        "!poetry.lock",
+        "!poetry.toml",
+        "!components.py",
+        "!requirements.txt",
+        "!README.md",
+        "!metadata.yaml",
+        "!build_customization.py",
+        "!source_*",
+        "!destination_*",
+    ]
+)
+
+###########################
+# PYTHON CONNECTOR IMAGE ##
+###########################
+
+PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE = """
+# syntax=docker/dockerfile:1
+# check=skip=all
+ARG BASE_IMAGE
+
+FROM ${BASE_IMAGE} AS builder
+ARG BASE_IMAGE
+ARG CONNECTOR_SNAKE_NAME
+ARG CONNECTOR_KEBAB_NAME
+ARG EXTRA_PREREQS_SCRIPT=""
+
+WORKDIR /airbyte/integration_code
+
+COPY . ./
+
+# Conditionally copy and execute the extra build script if provided
+RUN if [ -n "${EXTRA_PREREQS_SCRIPT}" ]; then \
+        cp ${EXTRA_PREREQS_SCRIPT} ./extra_prereqs_script && \
+        ./extra_prereqs_script; \
+    fi
+
+# TODO: Pre-install uv on the base image to speed up the build.
+#       (uv is still faster even with the extra step.)
+RUN pip install --no-cache-dir uv
+RUN python -m uv pip install --no-cache-dir .
+
+FROM ${BASE_IMAGE}
+ARG CONNECTOR_SNAKE_NAME
+ARG CONNECTOR_KEBAB_NAME
+ARG BASE_IMAGE
+
+WORKDIR /airbyte/integration_code
+
+COPY --from=builder /usr/local /usr/local
+COPY --chmod=755 <<EOT /entrypoint.sh
+#!/usr/bin/env bash
+set -e
+
+${CONNECTOR_KEBAB_NAME} "\$\@"
+EOT
+
+ENV AIRBYTE_ENTRYPOINT="/entrypoint.sh"
+ENTRYPOINT ["/entrypoint.sh"]
+"""
+
+##################################
+# MANIFEST-ONLY CONNECTOR IMAGE ##
+##################################
+
+MANIFEST_ONLY_DOCKERFILE_TEMPLATE = """
+ARG BASE_IMAGE
+ARG CONNECTOR_SNAKE_NAME
+ARG CONNECTOR_KEBAB_NAME
+
+FROM ${BASE_IMAGE}
+
+WORKDIR /airbyte/integration_code
+
+COPY . ./
+
+ENV AIRBYTE_ENTRYPOINT="python ./main.py"
+ENTRYPOINT ["python", "./main.py"]
+"""

{airbyte_cdk-6.45.10.dist-info → airbyte_cdk-6.46.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.10
+Version: 6.46.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: dev
 Provides-Extra: file-based
 Provides-Extra: sql
 Provides-Extra: vector-db-based
@@ -28,12 +29,14 @@ Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff
 Requires-Dist: boltons (>=25.0.0,<26.0.0)
 Requires-Dist: cachetools
+Requires-Dist: click (>=8.1.8,<9.0.0)
 Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
 Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
+Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
 Requires-Dist: jsonref (>=0.2,<0.3)
 Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
@@ -54,6 +57,7 @@ Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
 Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
 Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
+Requires-Dist: pytest (>=7,<8) ; extra == "dev"
 Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
 Requires-Dist: python-dateutil (>=2.9.0,<3.0.0)
 Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
@@ -62,6 +66,7 @@ Requires-Dist: pytz (==2024.2)
 Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
 Requires-Dist: requests
 Requires-Dist: requests_cache
+Requires-Dist: rich-click (>=1.8.8,<2.0.0)
 Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
 Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"