airbyte-internal-ops 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/poetry/publish/__init__.py

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#

airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/poetry/publish/commands.py

@@ -1,127 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""
-Module exposing the format commands.
-"""
-
-from __future__ import annotations
-
-from typing import Optional
-
-import asyncclick as click
-from consts import (
-    DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL,
-    DEFAULT_PYTHON_PACKAGE_REGISTRY_URL,
-)
-from packaging import version
-from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry
-from pipelines.cli.confirm_prompt import confirm
-from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
-from pipelines.cli.secrets import wrap_in_secret
-from pipelines.models.contexts.click_pipeline_context import (
-    ClickPipelineContext,
-    pass_pipeline_context,
-)
-from pipelines.models.contexts.python_registry_publish import (
-    PythonRegistryPublishContext,
-)
-from pipelines.models.secrets import Secret
-from pipelines.models.steps import StepStatus
-
-
-async def _has_metadata_yaml(context: PythonRegistryPublishContext) -> bool:
-    dir_to_publish = context.get_repo_dir(context.package_path)
-    return "metadata.yaml" in await dir_to_publish.entries()
-
-
-def _validate_python_version(
-    _ctx: dict, _param: dict, value: Optional[str]
-) -> Optional[str]:
-    """
-    Check if an given version is valid.
-    """
-    if value is None:
-        return value
-    try:
-        version.Version(value)
-        return value
-    except version.InvalidVersion:
-        raise click.BadParameter(f"Version {value} is not a valid version.")
-
-
-@click.command(
-    cls=DaggerPipelineCommand,
-    name="publish",
-    help="Publish a Python package to a registry.",
-)
-@click.option(
-    "--python-registry-token",
-    help="Access token",
-    type=click.STRING,
-    required=True,
-    envvar="PYTHON_REGISTRY_TOKEN",
-    callback=wrap_in_secret,
-)
-@click.option(
-    "--python-registry-url",
-    help="Which registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/",
-    type=click.STRING,
-    default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL,
-    envvar="PYTHON_REGISTRY_URL",
-)
-@click.option(
-    "--publish-name",
-    help="The name of the package to publish. If not set, the name will be inferred from the pyproject.toml file of the package.",
-    type=click.STRING,
-)
-@click.option(
-    "--publish-version",
-    help="The version of the package to publish. If not set, the version will be inferred from the pyproject.toml file of the package.",
-    type=click.STRING,
-    callback=_validate_python_version,
-)
-@pass_pipeline_context
-@click.pass_context
-async def publish(
-    ctx: click.Context,
-    click_pipeline_context: ClickPipelineContext,
-    python_registry_token: Secret,
-    python_registry_url: str,
-    publish_name: Optional[str],
-    publish_version: Optional[str],
-) -> bool:
-    context = PythonRegistryPublishContext(
-        is_local=ctx.obj["is_local"],
-        git_branch=ctx.obj["git_branch"],
-        git_revision=ctx.obj["git_revision"],
-        diffed_branch=ctx.obj["diffed_branch"],
-        git_repo_url=ctx.obj["git_repo_url"],
-        ci_report_bucket=ctx.obj["ci_report_bucket_name"],
-        report_output_prefix=ctx.obj["report_output_prefix"],
-        gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
-        dagger_logs_url=ctx.obj.get("dagger_logs_url"),
-        pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
-        ci_context=ctx.obj.get("ci_context"),
-        ci_gcp_credentials=ctx.obj["ci_gcp_credentials"],
-        python_registry_token=python_registry_token,
-        registry=python_registry_url,
-        registry_check_url=DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL,
-        package_path=ctx.obj["package_path"],
-        package_name=publish_name,
-        version=publish_version,
-    )
-
-    dagger_client = await click_pipeline_context.get_dagger_client()
-    context.dagger_client = dagger_client
-
-    if await _has_metadata_yaml(context):
-        confirm(
-            "It looks like you are trying to publish a connector. In most cases, the `connectors` command group should be used instead. Do you want to continue?",
-            abort=True,
-        )
-
-    publish_result = await PublishToPythonRegistry(context).run()
-
-    return publish_result.status is StepStatus.SUCCESS

airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/python_registry.py

@@ -1,238 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import configparser
-import io
-import uuid
-from enum import Enum, auto
-from typing import Dict, Optional
-
-import tomli_w
-import tomllib
-from consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH
-from dagger import Container, Directory
-from pipelines.dagger.actions.python.poetry import with_poetry
-from pipelines.helpers.utils import sh_dash_c
-from pipelines.models.contexts.python_registry_publish import (
-    PythonPackageMetadata,
-    PythonRegistryPublishContext,
-)
-from pipelines.models.steps import Step, StepResult
-
-
-class PackageType(Enum):
-    POETRY = auto()
-    PIP = auto()
-
-
-class PublishToPythonRegistry(Step):
-    context: PythonRegistryPublishContext
-    title = "Publish package to python registry"
-    max_retries = 3
-
-    def _get_base_container(self) -> Container:
-        return with_poetry(self.context)
-
-    async def _get_package_metadata_from_pyproject_toml(
-        self, package_dir_to_publish: Directory
-    ) -> Optional[PythonPackageMetadata]:
-        pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH)
-        pyproject_toml_content = await pyproject_toml.contents()
-        contents = tomllib.loads(pyproject_toml_content)
-        try:
-            return PythonPackageMetadata(
-                contents["tool"]["poetry"]["name"],
-                contents["tool"]["poetry"]["version"],
-            )
-        except KeyError:
-            return None
-
-    async def _get_package_type(
-        self, package_dir_to_publish: Directory
-    ) -> Optional[PackageType]:
-        files = await package_dir_to_publish.entries()
-        has_pyproject_toml = PYPROJECT_TOML_FILE_PATH in files
-        has_setup_py = SETUP_PY_FILE_PATH in files
-        if has_pyproject_toml:
-            return PackageType.POETRY
-        elif has_setup_py:
-            return PackageType.PIP
-        else:
-            return None
-
-    async def _run(self) -> StepResult:
-        package_dir_to_publish = await self.context.get_repo_dir(
-            self.context.package_path
-        )
-        package_type = await self._get_package_type(package_dir_to_publish)
-
-        if not package_type:
-            return self.skip(
-                "Connector does not have a pyproject.toml file or setup.py file, skipping."
-            )
-
-        result = await self._ensure_package_name_and_version(
-            package_dir_to_publish, package_type
-        )
-        if result:
-            return result
-
-        self.logger.info(
-            f"Uploading package {self.context.package_metadata.name} version {self.context.package_metadata.version} to {self.context.registry}..."
-        )
-
-        return await self._publish(package_dir_to_publish, package_type)
-
-    async def _ensure_package_name_and_version(
-        self, package_dir_to_publish: Directory, package_type: PackageType
-    ) -> Optional[StepResult]:
-        """
-        Try to infer package name and version from the pyproject.toml file. If it is not present, we need to have the package name and version set.
-        Setup.py packages need to set package name and version as parameter.
-
-        Returns None if package name and version are set, otherwise a StepResult with a skip message.
-        """
-        if self.context.package_metadata.name and self.context.package_metadata.version:
-            return None
-
-        if package_type is not PackageType.POETRY:
-            return self.skip(
-                "Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping."
-            )
-
-        inferred_package_metadata = (
-            await self._get_package_metadata_from_pyproject_toml(package_dir_to_publish)
-        )
-
-        if not inferred_package_metadata:
-            return self.skip(
-                "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping."
-            )
-
-        if not self.context.package_metadata.name:
-            self.context.package_metadata.name = inferred_package_metadata.name
-        if not self.context.package_metadata.version:
-            self.context.package_metadata.version = inferred_package_metadata.version
-
-        return None
-
-    async def _publish(
-        self, package_dir_to_publish: Directory, package_type: PackageType
-    ) -> StepResult:
-        if package_type is PackageType.PIP:
-            return await self._pip_publish(package_dir_to_publish)
-        else:
-            return await self._poetry_publish(package_dir_to_publish)
-
-    async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult:
-        pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH)
-        pyproject_toml_content = await pyproject_toml.contents()
-        contents = tomllib.loads(pyproject_toml_content)
-        # make sure package name and version are set to the configured one
-        contents["tool"]["poetry"]["name"] = self.context.package_metadata.name
-        contents["tool"]["poetry"]["version"] = self.context.package_metadata.version
-        # enforce consistent author
-        contents["tool"]["poetry"]["authors"] = ["Airbyte <contact@airbyte.io>"]
-        poetry_publish = (
-            self._get_base_container()
-            .with_secret_variable(
-                "PYTHON_REGISTRY_TOKEN",
-                self.context.python_registry_token.as_dagger_secret(self.dagger_client),
-            )
-            .with_directory("package", package_dir_to_publish)
-            .with_workdir("package")
-            .with_new_file(PYPROJECT_TOML_FILE_PATH, contents=tomli_w.dumps(contents))
-            # Make sure these steps are always executed and not cached as they are triggering a side-effect (calling the registry)
-            # Env var setting needs to be in this block as well to make sure a change of the env var will be propagated correctly
-            .with_env_variable("CACHEBUSTER", str(uuid.uuid4()))
-            .with_exec(
-                ["poetry", "config", "repositories.mypypi", self.context.registry],
-                use_entrypoint=True,
-            )
-            .with_exec(
-                sh_dash_c(["poetry config pypi-token.mypypi $PYTHON_REGISTRY_TOKEN"])
-            )
-            # Default timeout is set to 15 seconds
-            # We sometime face 443 HTTP read timeout responses from PyPi
-            # Setting it to 60 seconds to avoid transient publish failures
-            .with_env_variable("POETRY_REQUESTS_TIMEOUT", "60")
-            .with_exec(
-                sh_dash_c(
-                    ["poetry publish --build --repository mypypi -vvv --no-interaction"]
-                )
-            )
-        )
-
-        return await self.get_step_result(poetry_publish)
-
-    async def _pip_publish(self, package_dir_to_publish: Directory) -> StepResult:
-        files = await package_dir_to_publish.entries()
-        metadata: Dict[str, str] = {
-            "name": str(self.context.package_metadata.name),
-            "version": str(self.context.package_metadata.version),
-            # Enforce consistent author
-            "author": "Airbyte",
-            "author_email": "contact@airbyte.io",
-        }
-        if "README.md" in files:
-            metadata["long_description"] = await package_dir_to_publish.file(
-                "README.md"
-            ).contents()
-            metadata["long_description_content_type"] = "text/markdown"
-
-        config = configparser.ConfigParser()
-        config["metadata"] = metadata
-
-        setup_cfg_io = io.StringIO()
-        config.write(setup_cfg_io)
-        setup_cfg = setup_cfg_io.getvalue()
-
-        twine_upload = (
-            self._get_base_container()
-            .with_exec(sh_dash_c(["apt-get update", "apt-get install -y twine"]))
-            .with_directory("package", package_dir_to_publish)
-            .with_workdir("package")
-            .with_exec(
-                [
-                    "sed",
-                    "-i",
-                    "/name=/d; /author=/d; /author_email=/d; /version=/d",
-                    SETUP_PY_FILE_PATH,
-                ],
-                use_entrypoint=True,
-            )
-            .with_new_file("setup.cfg", contents=setup_cfg)
-            .with_exec(
-                ["pip", "install", "--upgrade", "setuptools", "wheel"],
-                use_entrypoint=True,
-            )
-            .with_exec(
-                ["python", SETUP_PY_FILE_PATH, "sdist", "bdist_wheel"],
-                use_entrypoint=True,
-            )
-            # Make sure these steps are always executed and not cached as they are triggering a side-effect (calling the registry)
-            # Env var setting needs to be in this block as well to make sure a change of the env var will be propagated correctly
-            .with_env_variable("CACHEBUSTER", str(uuid.uuid4()))
-            .with_secret_variable(
-                "TWINE_USERNAME",
-                self.context.dagger_client.set_secret("pypi_username", "__token__"),
-            )
-            .with_secret_variable(
-                "TWINE_PASSWORD",
-                self.context.python_registry_token.as_dagger_secret(self.dagger_client),
-            )
-            .with_exec(
-                [
-                    "twine",
-                    "upload",
-                    "--verbose",
-                    "--repository-url",
-                    self.context.registry,
-                    "dist/*",
-                ],
-                use_entrypoint=True,
-            )
-        )
-
-        return await self.get_step_result(twine_upload)

airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/models/contexts/python_registry_publish.py

@@ -1,119 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Optional, Type
-
-from consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL
-from pipelines.airbyte_ci.connectors.context import PipelineContext
-from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext
-from pipelines.models.secrets import Secret
-
-
-@dataclass
-class PythonPackageMetadata:
-    name: Optional[str]
-    version: Optional[str]
-
-
-class PythonRegistryPublishContext(PipelineContext):
-    def __init__(
-        self,
-        python_registry_token: Secret,
-        registry_check_url: str,
-        package_path: str,
-        report_output_prefix: str,
-        is_local: bool,
-        git_branch: str,
-        git_revision: str,
-        diffed_branch: str,
-        git_repo_url: str,
-        ci_report_bucket: Optional[str] = None,
-        registry: str = DEFAULT_PYTHON_PACKAGE_REGISTRY_URL,
-        gha_workflow_run_url: Optional[str] = None,
-        dagger_logs_url: Optional[str] = None,
-        pipeline_start_timestamp: Optional[int] = None,
-        ci_context: Optional[str] = None,
-        ci_gcp_credentials: Optional[Secret] = None,
-        package_name: Optional[str] = None,
-        version: Optional[str] = None,
-    ) -> None:
-        self.python_registry_token = python_registry_token
-        self.registry = registry
-        self.registry_check_url = registry_check_url
-        self.package_path = package_path
-        self.package_metadata = PythonPackageMetadata(package_name, version)
-
-        pipeline_name = f"Publish PyPI {package_path}"
-
-        super().__init__(
-            pipeline_name=pipeline_name,
-            report_output_prefix=report_output_prefix,
-            ci_report_bucket=ci_report_bucket,
-            is_local=is_local,
-            git_branch=git_branch,
-            git_revision=git_revision,
-            diffed_branch=diffed_branch,
-            git_repo_url=git_repo_url,
-            gha_workflow_run_url=gha_workflow_run_url,
-            dagger_logs_url=dagger_logs_url,
-            pipeline_start_timestamp=pipeline_start_timestamp,
-            ci_context=ci_context,
-            ci_gcp_credentials=ci_gcp_credentials,
-        )
-
-    @classmethod
-    async def from_publish_connector_context(
-        cls: Type["PythonRegistryPublishContext"],
-        connector_context: PublishConnectorContext,
-    ) -> Optional["PythonRegistryPublishContext"]:
-        """
-        Create a PythonRegistryPublishContext from a ConnectorContext.
-
-        The metadata of the connector is read from the current workdir to capture changes that are not yet published.
-        If pypi is not enabled, this will return None.
-        """
-
-        current_metadata = connector_context.connector.metadata
-        connector_context.logger.info(f"Current metadata: {current_metadata!s}")
-        if (
-            "remoteRegistries" not in current_metadata
-            or "pypi" not in current_metadata["remoteRegistries"]
-            or not current_metadata["remoteRegistries"]["pypi"]["enabled"]
-        ):
-            return None
-
-        version = current_metadata["dockerImageTag"]
-        if connector_context.pre_release:
-            # use current date as pre-release version
-            # we can't use the git revision because not all python registries allow local version identifiers. Public version identifiers must conform to PEP 440 and only allow digits.
-            release_candidate_tag = datetime.now().strftime("%Y%m%d%H%M")
-            version = f"{version}.dev{release_candidate_tag}"
-
-        assert connector_context.python_registry_token is not None, (
-            "The connector context must have python_registry_token Secret attribute"
-        )
-        pypi_context = cls(
-            python_registry_token=connector_context.python_registry_token,
-            registry=str(connector_context.python_registry_url),
-            registry_check_url=str(connector_context.python_registry_check_url),
-            package_path=str(connector_context.connector.code_directory),
-            package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"],
-            version=version,
-            ci_report_bucket=connector_context.ci_report_bucket,
-            report_output_prefix=connector_context.report_output_prefix,
-            is_local=connector_context.is_local,
-            git_branch=connector_context.git_branch,
-            git_revision=connector_context.git_revision,
-            diffed_branch=connector_context.diffed_branch,
-            git_repo_url=connector_context.git_repo_url,
-            gha_workflow_run_url=connector_context.gha_workflow_run_url,
-            dagger_logs_url=connector_context.dagger_logs_url,
-            pipeline_start_timestamp=connector_context.pipeline_start_timestamp,
-            ci_context=connector_context.ci_context,
-            ci_gcp_credentials=connector_context.ci_gcp_credentials,
-        )
-        pypi_context.dagger_client = connector_context.dagger_client
-        return pypi_context

airbyte_ops_mcp/live_tests/_connection_retriever/__init__.py

@@ -1,35 +0,0 @@
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-"""Vendored subset of connection-retriever from airbyte-platform-internal.
-
-This module contains a minimal subset of the connection-retriever tool,
-vendored to avoid depending on an unpublished internal package. It provides
-functionality to retrieve unmasked source configuration from Airbyte Cloud's
-internal database.
-
-Original source: airbyte-platform-internal/tools/connection-retriever
-Vendored: 2025-01-XX
-
-Only the following functionality is included:
-- Retrieve unmasked source config for a given connection ID
-- Secret resolution from GCP Secret Manager
-- Audit logging to GCP Cloud Logging
-
-NOT included (see issue #91 for future work):
-- retrieve_testing_candidates() - BigQuery-based candidate discovery
-- Destination config retrieval
-- CLI interface
-"""
-
-from airbyte_ops_mcp.live_tests._connection_retriever.consts import (
-    ConnectionObject,
-)
-from airbyte_ops_mcp.live_tests._connection_retriever.retrieval import (
-    TestingCandidate,
-    retrieve_objects,
-)
-
-__all__ = [
-    "ConnectionObject",
-    "TestingCandidate",
-    "retrieve_objects",
-]

airbyte_ops_mcp/live_tests/_connection_retriever/consts.py

@@ -1,33 +0,0 @@
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-"""Constants for vendored connection-retriever.
-
-Vendored from: airbyte-platform-internal/tools/connection-retriever/src/connection_retriever/consts.py
-"""
-
-from enum import Enum
-
-GCP_PROJECT_NAME = "prod-ab-cloud-proj"
-
-CLOUD_REGISTRY_URL = (
-    "https://connectors.airbyte.com/files/registries/v0/cloud_registry.json"
-)
-
-CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS_SECRET_ID = (
-    "projects/587336813068/secrets/CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS"
-)
-
-
-class ConnectionObject(Enum):
-    """Types of connection objects that can be retrieved."""
-
-    CONNECTION = "connection"
-    SOURCE_ID = "source-id"
-    DESTINATION_ID = "destination-id"
-    DESTINATION_CONFIG = "destination-config"
-    SOURCE_CONFIG = "source-config"
-    CATALOG = "catalog"
-    CONFIGURED_CATALOG = "configured-catalog"
-    STATE = "state"
-    WORKSPACE_ID = "workspace-id"
-    DESTINATION_DOCKER_IMAGE = "destination-docker-image"
-    SOURCE_DOCKER_IMAGE = "source-docker-image"

airbyte_ops_mcp/live_tests/_connection_retriever/db_access.py

@@ -1,82 +0,0 @@
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-"""Database access for vendored connection-retriever.
-
-Vendored from: airbyte-platform-internal/tools/connection-retriever/src/connection_retriever/db_access.py
-"""
-
-from __future__ import annotations
-
-import json
-import os
-import traceback
-from typing import Any, Callable
-
-import sqlalchemy
-from google.cloud import secretmanager
-from google.cloud.sql.connector import Connector
-from google.cloud.sql.connector.enums import IPTypes
-
-from airbyte_ops_mcp.live_tests._connection_retriever.consts import (
-    CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS_SECRET_ID,
-)
-from airbyte_ops_mcp.live_tests._connection_retriever.secrets_resolution import (
-    get_secret_value,
-)
-
-PG_DRIVER = "pg8000"
-
-# Lazy-initialized to avoid import-time GCP auth
-_connector: Connector | None = None
-
-
-def _get_connector() -> Connector:
-    """Get the Cloud SQL connector, initializing lazily on first use."""
-    global _connector
-    if _connector is None:
-        _connector = Connector()
-    return _connector
-
-
-def get_database_creator(pg_connection_details: dict) -> Callable:
-    """Create a database connection creator function."""
-
-    def creator() -> Any:
-        return _get_connector().connect(
-            pg_connection_details["database_address"],
-            PG_DRIVER,
-            user=pg_connection_details["pg_user"],
-            password=pg_connection_details["pg_password"],
-            db=pg_connection_details["database_name"],
-            ip_type=IPTypes.PRIVATE,
-        )
-
-    return creator
-
-
-def get_pool(
-    secret_manager_client: secretmanager.SecretManagerServiceClient,
-) -> sqlalchemy.Engine:
-    """Get a SQLAlchemy connection pool for the Airbyte Cloud database."""
-    pg_connection_details = json.loads(
-        get_secret_value(
-            secret_manager_client, CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS_SECRET_ID
-        )
-    )
-
-    if os.getenv("CI"):
-        # In CI we connect via Cloud SQL Auth Proxy, running on localhost
-        host = "127.0.0.1"
-        try:
-            return sqlalchemy.create_engine(
-                f"postgresql+{PG_DRIVER}://{pg_connection_details['pg_user']}:{pg_connection_details['pg_password']}@127.0.0.1/{pg_connection_details['database_name']}",
-            )
-        except Exception as e:
-            raise AssertionError(
-                f"sqlalchemy.create_engine exception; could not connect to the proxy at {host}. "
-                f"Error: {traceback.format_exception(e)}"
-            ) from e
-    else:
-        return sqlalchemy.create_engine(
-            f"postgresql+{PG_DRIVER}://",
-            creator=get_database_creator(pg_connection_details),
-        )