airbyte-cdk 6.45.10__tar.gz → 6.46.0__tar.gz

{airbyte_cdk-6.45.10 → airbyte_cdk-6.46.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.10
+Version: 6.46.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: dev
 Provides-Extra: file-based
 Provides-Extra: sql
 Provides-Extra: vector-db-based
@@ -28,12 +29,14 @@ Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff
 Requires-Dist: boltons (>=25.0.0,<26.0.0)
 Requires-Dist: cachetools
+Requires-Dist: click (>=8.1.8,<9.0.0)
 Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
 Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
+Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
 Requires-Dist: jsonref (>=0.2,<0.3)
 Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
@@ -54,6 +57,7 @@ Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
 Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
 Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
+Requires-Dist: pytest (>=7,<8) ; extra == "dev"
 Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
 Requires-Dist: python-dateutil (>=2.9.0,<3.0.0)
 Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
@@ -62,6 +66,7 @@ Requires-Dist: pytz (==2024.2)
 Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
 Requires-Dist: requests
 Requires-Dist: requests_cache
+Requires-Dist: rich-click (>=1.8.8,<2.0.0)
 Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
 Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"

airbyte_cdk-6.46.0/airbyte_cdk/cli/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""The `airbyte-cdk.cli` module provides command-line interfaces for the Airbyte CDK.
+
+As of now, it includes the following CLI entry points:
+
+- `airbyte-cdk`: Commands for working with connectors.
+- `source-declarative-manifest`: Directly invoke the declarative manifests connector.
+
+"""

airbyte_cdk-6.46.0/airbyte_cdk/cli/airbyte_cdk/__init__.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""CLI commands for `airbyte-cdk`.
+
+This CLI interface allows you to interact with your connector, including
+testing and running commands.
+
+**Basic Usage:**
+
+```bash
+airbyte-cdk --help
+airbyte-cdk --version
+airbyte-cdk connector --help
+airbyte-cdk manifest --help
+```
+
+**Running Statelessly:**
+
+You can run the latest version of this CLI, from any machine, using `pipx` or `uvx`:
+
+```bash
+# Run the latest version of the CLI:
+pipx run airbyte-cdk connector --help
+uvx airbyte-cdk connector --help
+
+# Run from a specific CDK version:
+pipx run airbyte-cdk==6.5.1 connector --help
+uvx airbyte-cdk==6.5.1 connector --help
+```
+
+**Running within your virtualenv:**
+
+You can also run from your connector's virtualenv:
+
+```bash
+poetry run airbyte-cdk connector --help
+```
+
+"""
+
+from typing import cast
+
+import rich_click as click
+
+from airbyte_cdk.cli.airbyte_cdk._connector import connector_cli_group
+from airbyte_cdk.cli.airbyte_cdk._image import image_cli_group
+from airbyte_cdk.cli.airbyte_cdk._manifest import manifest_cli_group
+from airbyte_cdk.cli.airbyte_cdk._secrets import secrets_cli_group
+from airbyte_cdk.cli.airbyte_cdk._version import print_version
+
+
+@click.group(
+    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown)
+    invoke_without_command=True,
+)
+@click.option(
+    "--version",
+    is_flag=True,
+    help="Show the version of the Airbyte CDK.",
+)
+@click.pass_context
+def cli(
+    ctx: click.Context,
+    version: bool,
+) -> None:
+    """Airbyte CDK CLI.
+
+    Help text is provided from the file-level docstring.
+    """
+    if version:
+        print_version(short=False)
+        ctx.exit()
+
+    if ctx.invoked_subcommand is None:
+        # If no subcommand is provided, show the help message.
+        click.echo(ctx.get_help())
+        ctx.exit()
+
+
+cli.add_command(connector_cli_group)
+cli.add_command(manifest_cli_group)
+cli.add_command(image_cli_group)
+cli.add_command(secrets_cli_group)
+
+
+if __name__ == "__main__":
+    cli()

airbyte_cdk-6.46.0/airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -0,0 +1,179 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""CLI command for `airbyte-cdk`."""
+
+USAGE = """CLI command for `airbyte-cdk`.
+
+This CLI interface allows you to interact with your connector, including
+testing and running commands.
+
+**Basic Usage:**
+
+```bash
+airbyte-cdk --help
+airbyte-cdk connector --help
+airbyte-cdk manifest --help
+```
+
+**Running Statelessly:**
+
+You can run the latest version of this CLI, from any machine, using `pipx` or `uvx`:
+
+```bash
+# Run the latest version of the CLI:
+pipx run airbyte-cdk connector --help
+uvx airbyte-cdk connector --help
+
+# Run from a specific CDK version:
+pipx run airbyte-cdk==6.5.1 connector --help
+uvx airbyte-cdk==6.5.1 connector --help
+```
+
+**Running within your virtualenv:**
+
+You can also run from your connector's virtualenv:
+
+```bash
+poetry run airbyte-cdk connector --help
+```
+
+"""
+
+import os
+from pathlib import Path
+from types import ModuleType
+
+import rich_click as click
+
+# from airbyte_cdk.test.standard_tests import pytest_hooks
+from airbyte_cdk.cli.airbyte_cdk._util import resolve_connector_name_and_directory
+from airbyte_cdk.test.standard_tests.test_resources import find_connector_root_from_name
+from airbyte_cdk.test.standard_tests.util import create_connector_test_suite
+
+click.rich_click.TEXT_MARKUP = "markdown"
+
+pytest: ModuleType | None
+try:
+    import pytest
+except ImportError:
+    pytest = None
+    # Handle the case where pytest is not installed.
+    # This prevents import errors when running the script without pytest installed.
+    # We will raise an error later if pytest is required for a given command.
+
+
+TEST_FILE_TEMPLATE = '''
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""FAST Airbyte Standard Tests for the source_pokeapi_w_components source."""
+
+#from airbyte_cdk.test.standard_tests import {base_class_name}
+from airbyte_cdk.test.standard_tests.util import create_connector_test_suite
+from pathlib import Path
+
+pytest_plugins = [
+    "airbyte_cdk.test.standard_tests.pytest_hooks",
+]
+
+TestSuite = create_connector_test_suite(
+    connector_directory=Path(),
+)
+
+# class TestSuite({base_class_name}):
+#     """Test suite for the source_pokeapi_w_components source.
+
+#     This class inherits from SourceTestSuiteBase and implements all of the tests in the suite.
+
+#     As long as the class name starts with "Test", pytest will automatically discover and run the
+#     tests in this class.
+#     """
+'''
+
+
+@click.group(
+    name="connector",
+    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown)
+)
+def connector_cli_group() -> None:
+    """Connector related commands."""
+    pass
+
+
+@connector_cli_group.command()
+@click.option(
+    "--connector-name",
+    type=str,
+    help="Name of the connector to test. Ignored if --connector-directory is provided.",
+)
+@click.option(
+    "--connector-directory",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Path to the connector directory.",
+)
+@click.option(
+    "--collect-only",
+    is_flag=True,
+    default=False,
+    help="Only collect tests, do not run them.",
+)
+def test(
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+    *,
+    collect_only: bool = False,
+) -> None:
+    """Run connector tests.
+
+    This command runs the standard connector tests for a specific connector.
+
+    If no connector name or directory is provided, we will look within the current working
+    directory. If the current working directory is not a connector directory (e.g. starting
+    with 'source-') and no connector name or path is provided, the process will fail.
+    """
+    if pytest is None:
+        raise ImportError(
+            "pytest is not installed. Please install pytest to run the connector tests."
+        )
+    click.echo("Connector test command executed.")
+    connector_name, connector_directory = resolve_connector_name_and_directory(
+        connector_name=connector_name,
+        connector_directory=connector_directory,
+    )
+
+    connector_test_suite = create_connector_test_suite(
+        connector_name=connector_name if not connector_directory else None,
+        connector_directory=connector_directory,
+    )
+
+    pytest_args: list[str] = []
+    if connector_directory:
+        pytest_args.append(f"--rootdir={connector_directory}")
+        os.chdir(str(connector_directory))
+    else:
+        print("No connector directory provided. Running tests in the current directory.")
+
+    file_text = TEST_FILE_TEMPLATE.format(
+        base_class_name=connector_test_suite.__bases__[0].__name__,
+        connector_directory=str(connector_directory),
+    )
+    test_file_path = Path() / ".tmp" / "integration_tests/test_airbyte_standards.py"
+    test_file_path = test_file_path.resolve().absolute()
+    test_file_path.parent.mkdir(parents=True, exist_ok=True)
+    test_file_path.write_text(file_text)
+
+    if collect_only:
+        pytest_args.append("--collect-only")
+
+    pytest_args.append(str(test_file_path))
+    click.echo(f"Running tests from connector directory: {connector_directory}...")
+    click.echo(f"Test file: {test_file_path}")
+    click.echo(f"Collect only: {collect_only}")
+    click.echo(f"Pytest args: {pytest_args}")
+    click.echo("Invoking Pytest...")
+    pytest.main(
+        pytest_args,
+        plugins=[],
+    )
+
+
+__all__ = [
+    "connector_cli_group",
+]

airbyte_cdk-6.46.0/airbyte_cdk/cli/airbyte_cdk/_image.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Airbyte CDK 'image' commands.
+
+The `airbyte-cdk image build` command provides a simple way to work with Airbyte
+connector images.
+"""
+
+import sys
+from pathlib import Path
+
+import rich_click as click
+
+from airbyte_cdk.cli.airbyte_cdk._util import resolve_connector_name_and_directory
+from airbyte_cdk.models.connector_metadata import MetadataFile
+from airbyte_cdk.utils.docker import (
+    ConnectorImageBuildError,
+    build_connector_image,
+    verify_docker_installation,
+)
+
+
+@click.group(
+    name="image",
+    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown)
+)
+def image_cli_group() -> None:
+    """Commands for working with connector Docker images."""
+
+
+@image_cli_group.command()
+@click.option(
+    "--connector-name",
+    type=str,
+    help="Name of the connector to test. Ignored if --connector-directory is provided.",
+)
+@click.option(
+    "--connector-directory",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Path to the connector directory.",
+)
+@click.option("--tag", default="dev", help="Tag to apply to the built image (default: dev)")
+@click.option("--no-verify", is_flag=True, help="Skip verification of the built image")
+def build(
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+    *,
+    tag: str = "dev",
+    no_verify: bool = False,
+) -> None:
+    """Build a connector Docker image.
+
+    This command builds a Docker image for a connector, using either
+    the connector's Dockerfile or a base image specified in the metadata.
+    The image is built for both AMD64 and ARM64 architectures.
+    """
+    if not verify_docker_installation():
+        click.echo(
+            "Docker is not installed or not running. Please install Docker and try again.", err=True
+        )
+        sys.exit(1)
+
+    connector_name, connector_directory = resolve_connector_name_and_directory(
+        connector_name=connector_name,
+        connector_directory=connector_directory,
+    )
+
+    metadata_file_path: Path = connector_directory / "metadata.yaml"
+    try:
+        metadata = MetadataFile.from_file(metadata_file_path)
+    except (FileNotFoundError, ValueError) as e:
+        click.echo(
+            f"Error loading metadata file '{metadata_file_path}': {e!s}",
+            err=True,
+        )
+        sys.exit(1)
+    click.echo(f"Building Image for Connector: {metadata.data.dockerRepository}:{tag}")
+    try:
+        build_connector_image(
+            connector_directory=connector_directory,
+            connector_name=connector_name,
+            metadata=metadata,
+            tag=tag,
+            no_verify=no_verify,
+        )
+    except ConnectorImageBuildError as e:
+        click.echo(
+            f"Error building connector image: {e!s}",
+            err=True,
+        )
+        sys.exit(1)
+
+
+__all__ = [
+    "image_cli_group",
+]

airbyte_cdk-6.46.0/airbyte_cdk/cli/airbyte_cdk/_manifest.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Manifest related commands.
+
+Coming soon.
+
+This module is planned to provide a command line interface (CLI) for validating
+Airbyte CDK manifests.
+"""
+
+import rich_click as click
+
+
+@click.group(
+    name="manifest",
+    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown)
+)
+def manifest_cli_group() -> None:
+    """Manifest related commands."""
+    pass
+
+
+__all__ = [
+    "manifest_cli_group",
+]

airbyte_cdk-6.46.0/airbyte_cdk/cli/airbyte_cdk/_secrets.py

@@ -0,0 +1,150 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Secret management commands.
+
+This module provides commands for managing secrets for Airbyte connectors.
+
+Usage:
+    airbyte-cdk secrets fetch --connector-name source-github
+    airbyte-cdk secrets fetch --connector-directory /path/to/connector
+    airbyte-cdk secrets fetch  # Run from within a connector directory
+
+Usage without pre-installing (stateless):
+    pipx run airbyte-cdk secrets fetch ...
+    uvx airbyte-cdk secrets fetch ...
+
+The 'fetch' command retrieves secrets from Google Secret Manager based on connector
+labels and writes them to the connector's `secrets` directory.
+"""
+
+import json
+import os
+from pathlib import Path
+
+import rich_click as click
+
+from airbyte_cdk.cli.airbyte_cdk._util import resolve_connector_name_and_directory
+
+AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing"
+CONNECTOR_LABEL = "connector"
+
+
+@click.group(
+    name="secrets",
+    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown)
+)
+def secrets_cli_group() -> None:
+    """Secret management commands."""
+    pass
+
+
+@secrets_cli_group.command()
+@click.option(
+    "--connector-name",
+    type=str,
+    help="Name of the connector to fetch secrets for. Ignored if --connector-directory is provided.",
+)
+@click.option(
+    "--connector-directory",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Path to the connector directory.",
+)
+@click.option(
+    "--gcp-project-id",
+    type=str,
+    default=AIRBYTE_INTERNAL_GCP_PROJECT,
+    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+)
+def fetch(
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+) -> None:
+    """Fetch secrets for a connector from Google Secret Manager.
+
+    This command fetches secrets for a connector from Google Secret Manager and writes them
+    to the connector's secrets directory.
+
+    If no connector name or directory is provided, we will look within the current working
+    directory. If the current working directory is not a connector directory (e.g. starting
+    with 'source-') and no connector name or path is provided, the process will fail.
+    """
+    try:
+        from google.cloud import secretmanager_v1 as secretmanager
+    except ImportError:
+        raise ImportError(
+            "google-cloud-secret-manager package is required for Secret Manager integration. "
+            "Install it with 'pip install airbyte-cdk[dev]' "
+            "or 'pip install google-cloud-secret-manager'."
+        )
+
+    click.echo("Fetching secrets...")
+
+    # Resolve connector name/directory
+    try:
+        connector_name, connector_directory = resolve_connector_name_and_directory(
+            connector_name=connector_name,
+            connector_directory=connector_directory,
+        )
+    except FileNotFoundError as e:
+        raise FileNotFoundError(
+            f"Could not find connector directory for '{connector_name}'. "
+            "Please provide the --connector-directory option with the path to the connector. "
+            "Note: This command requires either running from within a connector directory, "
+            "being in the airbyte monorepo, or explicitly providing the connector directory path."
+        ) from e
+    except ValueError as e:
+        raise ValueError(str(e))
+
+    # Create secrets directory if it doesn't exist
+    secrets_dir = connector_directory / "secrets"
+    secrets_dir.mkdir(parents=True, exist_ok=True)
+
+    gitignore_path = secrets_dir / ".gitignore"
+    gitignore_path.write_text("*")
+
+    # Get GSM client
+    credentials_json = os.environ.get("GCP_GSM_CREDENTIALS")
+    if not credentials_json:
+        raise ValueError(
+            "No Google Cloud credentials found. Please set the GCP_GSM_CREDENTIALS environment variable."
+        )
+
+    client = secretmanager.SecretManagerServiceClient.from_service_account_info(
+        json.loads(credentials_json)
+    )
+
+    # List all secrets with the connector label
+    parent = f"projects/{gcp_project_id}"
+    filter_string = f"labels.{CONNECTOR_LABEL}={connector_name}"
+    secrets = client.list_secrets(
+        request=secretmanager.ListSecretsRequest(
+            parent=parent,
+            filter=filter_string,
+        )
+    )
+
+    # Fetch and write secrets
+    secret_count = 0
+    for secret in secrets:
+        secret_name = secret.name
+        version_name = f"{secret_name}/versions/latest"
+        response = client.access_secret_version(name=version_name)
+        payload = response.payload.data.decode("UTF-8")
+
+        filename_base = "config"  # Default filename
+        if secret.labels and "filename" in secret.labels:
+            filename_base = secret.labels["filename"]
+
+        secret_file_path = secrets_dir / f"{filename_base}.json"
+        secret_file_path.write_text(payload)
+        secret_file_path.chmod(0o600)  # default to owner read/write only
+        click.echo(f"Secret written to: {secret_file_path.absolute()!s}")
+        secret_count += 1
+
+    if secret_count == 0:
+        click.echo(f"No secrets found for connector: {connector_name}")
+
+
+__all__ = [
+    "secrets_cli_group",
+]

airbyte_cdk-6.46.0/airbyte_cdk/cli/airbyte_cdk/_util.py

@@ -0,0 +1,43 @@
+"""Common utilities for Airbyte CDK CLI."""
+
+from pathlib import Path
+
+from airbyte_cdk.test.standard_tests.test_resources import find_connector_root_from_name
+
+
+def resolve_connector_name_and_directory(
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+) -> tuple[str, Path]:
+    """Resolve the connector name and directory.
+
+    This function will resolve the connector name and directory based on the provided
+    arguments. If no connector name or directory is provided, it will look within the
+    current working directory. If the current working directory is not a connector
+    directory (e.g. starting with 'source-') and no connector name or path is provided,
+    the process will fail.
+    """
+    if not connector_directory:
+        if connector_name:
+            connector_directory = find_connector_root_from_name(connector_name)
+        else:
+            cwd = Path().resolve().absolute()
+            if cwd.name.startswith("source-") or cwd.name.startswith("destination-"):
+                connector_directory = cwd
+            else:
+                raise ValueError(
+                    "Either connector_name or connector_directory must be provided if not "
+                    "running from a connector directory."
+                )
+
+    if not connector_name:
+        connector_name = connector_directory.name
+
+    if connector_directory:
+        connector_directory = connector_directory.resolve().absolute()
+    elif connector_name:
+        connector_directory = find_connector_root_from_name(connector_name)
+    else:
+        raise ValueError("Either connector_name or connector_directory must be provided.")
+
+    return connector_name, connector_directory

airbyte_cdk-6.46.0/airbyte_cdk/cli/airbyte_cdk/_version.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Version information for the Airbyte CDK CLI."""
+
+from airbyte_cdk import __version__
+
+
+def print_version(short: bool = False) -> None:
+    """Print the version of the Airbyte CDK CLI."""
+
+    if short:
+        print(__version__)
+    else:
+        print(f"Airbyte CDK version: {__version__}")

{airbyte_cdk-6.45.10 → airbyte_cdk-6.46.0}/airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -56,12 +56,22 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits:
     return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
 
+def should_normalize_manifest(config: Mapping[str, Any]) -> bool:
+    """
+    Check if the manifest should be normalized.
+    :param config: The configuration to check
+    :return: True if the manifest should be normalized, False otherwise.
+    """
+    return config.get("__should_normalize", False)
+
+
 def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
     return ManifestDeclarativeSource(
         config=config,
         emit_connector_builder_messages=True,
         source_config=manifest,
+        normalize_manifest=should_normalize_manifest(config),
         component_factory=ModelToComponentFactory(
             emit_connector_builder_messages=True,
             limit_pages_fetched_per_slice=limits.max_pages_per_slice,