airbyte-internal-ops 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

airbyte_ops_mcp/_legacy/airbyte_ci/connector_live_tests/commons/proxy.py

@@ -1,207 +0,0 @@
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-from __future__ import annotations
-
-import logging
-import uuid
-
-import dagger
-
-from . import mitm_addons
-
-
-class Proxy:
-    """
-    This class is a wrapper around a mitmproxy container. It allows to declare a mitmproxy container,
-    bind it as a service to a different container and retrieve the mitmproxy stream file.
-    """
-
-    MITMPROXY_IMAGE = "mitmproxy/mitmproxy:10.2.4"
-    MITM_STREAM_FILE = "stream.mitm"
-    PROXY_PORT = 8080
-    MITM_ADDONS_PATH = mitm_addons.__file__
-
-    def __init__(
-        self,
-        dagger_client: dagger.Client,
-        hostname: str,
-        session_id: str,
-        stream_for_server_replay: dagger.File | None = None,
-    ) -> None:
-        self.dagger_client = dagger_client
-        self.hostname = hostname
-        self.session_id = session_id
-        self.stream_for_server_replay = stream_for_server_replay
-
-    @property
-    def dump_cache_volume(self) -> dagger.CacheVolume:
-        # We namespace the cache by:
-        # - Mitmproxy image name to make sure we're not re-using a cached artifact on a different and potentially incompatible mitmproxy version
-        # - Hostname to avoid sharing the same https dump between different tests
-        # - Session id to avoid sharing the same https dump between different runs of the same tests
-        # The session id is set to the Airbyte Connection ID to ensure that no cache is shared between connections
-        return self.dagger_client.cache_volume(
-            f"{self.MITMPROXY_IMAGE}{self.hostname}{self.session_id}"
-        )
-
-    @property
-    def mitmproxy_dir_cache(self) -> dagger.CacheVolume:
-        return self.dagger_client.cache_volume(self.MITMPROXY_IMAGE)
-
-    def generate_mitmconfig(self):
-        return (
-            self.dagger_client.container()
-            .from_(self.MITMPROXY_IMAGE)
-            # Mitmproxy generates its self signed certs at first run, we need to run it once to generate the certs
-            # They are stored in /root/.mitmproxy
-            .with_exec(["timeout", "--preserve-status", "1", "mitmdump"])
-            .directory("/root/.mitmproxy")
-        )
-
-    async def get_container(self, mitm_config: dagger.Directory) -> dagger.Container:
-        """Get a container for the mitmproxy service.
-        If a stream for server replay is provided, it will be used to replay requests to the same URL.
-
-        Returns:
-            dagger.Container: The container for the mitmproxy service.
-            mitm_config (dagger.Directory): The directory containing the mitmproxy configuration.
-        """
-        container_addons_path = "/addons.py"
-        proxy_container = (
-            self.dagger_client.container()
-            .from_(self.MITMPROXY_IMAGE)
-            .with_exec(["mkdir", "/dumps"])
-            # This is caching the mitmproxy stream files, which can contain sensitive information
-            # We want to nuke this cache after test suite execution.
-            .with_mounted_cache("/dumps", self.dump_cache_volume)
-            # This is caching the mitmproxy self-signed certificate, no sensitive information is stored in it
-            .with_file(
-                container_addons_path,
-                self.dagger_client.host().file(self.MITM_ADDONS_PATH),
-            )
-            .with_directory("/root/.mitmproxy", mitm_config)
-        )
-
-        # If the proxy was instantiated with a stream for server replay from a previous run, we want to use it.
-        # Requests to the same URL will be replayed from the stream instead of being sent to the server.
-        # This is useful to avoid rate limiting issues and limits responses drifts due to time based logics.
-        if (
-            self.stream_for_server_replay is not None
-            and await self.stream_for_server_replay.size() > 0
-        ):
-            proxy_container = proxy_container.with_file(
-                "/cache.mitm", self.stream_for_server_replay
-            )
-            command = [
-                "mitmdump",
-                "-s",
-                container_addons_path,
-                "--flow-detail",
-                "2",
-                "--server-replay",
-                "/cache.mitm",
-                "--save-stream-file",
-                f"/dumps/{self.MITM_STREAM_FILE}",
-            ]
-        else:
-            command = [
-                "mitmdump",
-                "-s",
-                container_addons_path,
-                "--flow-detail",
-                "2",
-                "--save-stream-file",
-                f"/dumps/{self.MITM_STREAM_FILE}",
-            ]
-
-        return proxy_container.with_exec(command)
-
-    async def get_service(self, mitm_config: dagger.Directory) -> dagger.Service:
-        return (
-            (await self.get_container(mitm_config))
-            .with_exposed_port(self.PROXY_PORT)
-            .as_service()
-        )
-
-    async def bind_container(self, container: dagger.Container) -> dagger.Container:
-        """Bind a container to the proxy service and set environment variables to use the proxy for HTTP(S) traffic.
-
-        Args:
-            container (dagger.Container): The container to bind to the proxy service.
-
-        Returns:
-            dagger.Container: The container with the proxy service bound and environment variables set.
-        """
-        mitmconfig_dir = self.generate_mitmconfig()
-        pem = mitmconfig_dir.file("mitmproxy-ca.pem")
-
-        # Find the python version in the container to get the correct path for the requests cert file
-        # We will overwrite this file with the mitmproxy self-signed certificate
-        # I could not find a less brutal way to make Requests trust the mitmproxy self-signed certificate
-        # I tried running update-ca-certificates + setting REQUESTS_CA_BUNDLE in the container but it did not work
-        python_version_output = (
-            await container.with_exec(["python", "--version"]).stdout()
-        ).strip()
-        python_version = python_version_output.split(" ")[-1]
-        python_version_minor_only = ".".join(python_version.split(".")[:-1])
-        requests_cert_path = f"/usr/local/lib/python{python_version_minor_only}/site-packages/certifi/cacert.pem"
-        current_user = (await container.with_exec(["whoami"]).stdout()).strip()
-        try:
-            return await (
-                container.with_user("root")
-                # Overwrite the requests cert file with the mitmproxy self-signed certificate
-                .with_file(requests_cert_path, pem, owner=current_user)
-                .with_env_variable("http_proxy", f"{self.hostname}:{self.PROXY_PORT}")
-                .with_env_variable("https_proxy", f"{self.hostname}:{self.PROXY_PORT}")
-                .with_user(current_user)
-                .with_service_binding(
-                    self.hostname, await self.get_service(mitmconfig_dir)
-                )
-            )
-        except dagger.DaggerError as e:
-            # This is likely hapenning on Java connector images whose certificates location is different
-            # TODO handle this case
-            logging.warn(f"Failed to bind container to proxy: {e}")
-            return container
-
-    async def retrieve_http_dump(self) -> dagger.File | None:
-        """We mount the cache volume, where the mitmproxy container saves the stream file, to a fresh container.
-        We then copy the stream file to a new directory and return it as a dagger.File.
-        The copy operation to /to_export is required as Dagger does not support direct access to files in cache volumes.
-
-
-        Returns:
-            dagger.File | None: The mitmproxy stream file if it exists, None otherwise.
-        """
-        container = (
-            self.dagger_client.container()
-            .from_("alpine:latest")
-            .with_env_variable("CACHEBUSTER", str(uuid.uuid4()))
-            .with_mounted_cache("/dumps", self.dump_cache_volume)
-        )
-        dump_files = (await container.with_exec(["ls", "/dumps"]).stdout()).splitlines()
-        if self.MITM_STREAM_FILE not in dump_files:
-            return None
-        return await (
-            container.with_exec(["mkdir", "/to_export"], use_entrypoint=True)
-            .with_exec(
-                [
-                    "cp",
-                    "-r",
-                    f"/dumps/{self.MITM_STREAM_FILE}",
-                    f"/to_export/{self.MITM_STREAM_FILE}",
-                ],
-                use_entrypoint=True,
-            )
-            .file(f"/to_export/{self.MITM_STREAM_FILE}")
-        )
-
-    async def clear_cache_volume(self) -> None:
-        """Delete all files in the cache volume. This is useful to avoid caching sensitive information between tests."""
-        await (
-            self.dagger_client.container()
-            .from_("alpine:latest")
-            .with_mounted_cache("/to_clear", self.dump_cache_volume)
-            .with_exec(["rm", "-rf", "/to_clear/*"], use_entrypoint=True)
-            .sync()
-        )
-        logging.info(f"Cache volume {self.dump_cache_volume} cleared")

airbyte_ops_mcp/_legacy/airbyte_ci/connector_live_tests/commons/secret_access.py

@@ -1,47 +0,0 @@
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-from __future__ import annotations
-
-import logging
-
-from google.api_core.exceptions import PermissionDenied
-from google.cloud import secretmanager
-
-LIVE_TESTS_AIRBYTE_API_KEY_SECRET_ID = (
-    "projects/587336813068/secrets/live_tests_airbyte_api_key"
-)
-
-
-def get_secret_value(
-    secret_manager_client: secretmanager.SecretManagerServiceClient, secret_id: str
-) -> str:
-    """Get the value of the enabled version of a secret
-
-    Args:
-        secret_manager_client (secretmanager.SecretManagerServiceClient): The secret manager client
-        secret_id (str): The id of the secret
-
-    Returns:
-        str: The value of the enabled version of the secret
-    """
-    try:
-        response = secret_manager_client.list_secret_versions(
-            request={"parent": secret_id, "filter": "state:ENABLED"}
-        )
-        if len(response.versions) == 0:
-            raise ValueError(f"No enabled version of secret {secret_id} found")
-        enabled_version = response.versions[0]
-        response = secret_manager_client.access_secret_version(
-            name=enabled_version.name
-        )
-        return response.payload.data.decode("UTF-8")
-    except PermissionDenied as e:
-        logging.exception(
-            f"Permission denied while trying to access secret {secret_id}. Please write to #dev-tooling in Airbyte Slack for help.",
-            exc_info=e,
-        )
-        raise e
-
-
-def get_airbyte_api_key() -> str:
-    secret_manager_client = secretmanager.SecretManagerServiceClient()
-    return get_secret_value(secret_manager_client, LIVE_TESTS_AIRBYTE_API_KEY_SECRET_ID)

airbyte_ops_mcp/_legacy/airbyte_ci/connector_live_tests/commons/segment_tracking.py

@@ -1,45 +0,0 @@
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-from __future__ import annotations
-
-import logging
-import os
-from importlib.metadata import version
-from typing import Any, Optional
-
-from segment import analytics  # type: ignore
-
-ENABLE_TRACKING = os.getenv("REGRESSION_TEST_DISABLE_TRACKING") is None
-DEBUG_SEGMENT = os.getenv("DEBUG_SEGMENT") is not None
-EVENT_NAME = "regression_test_start"
-CURRENT_VERSION = version(__name__.split(".")[0])
-
-
-def on_error(error: Exception, items: Any) -> None:
-    logging.warning("An error occurred in Segment Tracking", exc_info=error)
-
-
-# This is not a secret key, it is a public key that is used to identify the Segment project
-analytics.write_key = "hnWfMdEtXNKBjvmJ258F72wShsLmcsZ8"
-analytics.send = ENABLE_TRACKING
-analytics.debug = DEBUG_SEGMENT
-analytics.on_error = on_error
-
-
-def track_usage(
-    user_id: Optional[str],
-    pytest_options: dict[str, Any],
-) -> None:
-    if user_id:
-        analytics.identify(user_id)
-    else:
-        user_id = "airbyte-ci"
-
-    # It contains default pytest option and the custom one passed by the user
-    analytics.track(
-        user_id,
-        EVENT_NAME,
-        {
-            "pytest_options": pytest_options,
-            "package_version": CURRENT_VERSION,
-        },
-    )

airbyte_ops_mcp/_legacy/airbyte_ci/connector_live_tests/commons/utils.py

@@ -1,214 +0,0 @@
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-from __future__ import annotations
-
-import logging
-import os
-import re
-import shutil
-from pathlib import Path
-from typing import Optional
-
-import dagger
-import docker  # type: ignore
-import pytest
-from mitmproxy import http, io  # type: ignore
-from mitmproxy.addons.savehar import SaveHar  # type: ignore
-from slugify import slugify
-
-
-async def get_container_from_id(
-    dagger_client: dagger.Client, container_id: str
-) -> dagger.Container:
-    """Get a dagger container from its id.
-    Please remind that container id are not persistent and can change between Dagger sessions.
-
-    Args:
-        dagger_client (dagger.Client): The dagger client to use to import the connector image
-    """
-    try:
-        return await dagger_client.load_container_from_id(
-            dagger.ContainerID(container_id)
-        )
-    except dagger.DaggerError as e:
-        pytest.exit(f"Failed to load connector container: {e}")
-
-
-async def get_container_from_tarball_path(
-    dagger_client: dagger.Client, tarball_path: Path
-) -> dagger.Container:
-    if not tarball_path.exists():
-        pytest.exit(f"Connector image tarball {tarball_path} does not exist")
-    container_under_test_tar_file = (
-        dagger_client.host()
-        .directory(str(tarball_path.parent), include=[tarball_path.name])
-        .file(tarball_path.name)
-    )
-    try:
-        return await dagger_client.container().import_(container_under_test_tar_file)
-    except dagger.DaggerError as e:
-        pytest.exit(f"Failed to import connector image from tarball: {e}")
-
-
-async def get_container_from_local_image(
-    dagger_client: dagger.Client, local_image_name: str
-) -> Optional[dagger.Container]:
-    """Get a dagger container from a local image.
-    It will use Docker python client to export the image to a tarball and then import it into dagger.
-
-    Args:
-        dagger_client (dagger.Client): The dagger client to use to import the connector image
-        local_image_name (str): The name of the local image to import
-
-    Returns:
-        Optional[dagger.Container]: The dagger container for the local image or None if the image does not exist
-    """
-    docker_client = docker.from_env()
-
-    try:
-        image = docker_client.images.get(local_image_name)
-    except docker.errors.ImageNotFound:
-        return None
-
-    image_digest = image.id.replace("sha256:", "")
-    tarball_path = Path(f"/tmp/{image_digest}.tar")
-    if not tarball_path.exists():
-        logging.info(
-            f"Exporting local connector image {local_image_name} to tarball {tarball_path}"
-        )
-        with open(tarball_path, "wb") as f:
-            for chunk in image.save(named=True):
-                f.write(chunk)
-    return await get_container_from_tarball_path(dagger_client, tarball_path)
-
-
-async def get_container_from_dockerhub_image(
-    dagger_client: dagger.Client, dockerhub_image_name: str
-) -> dagger.Container:
-    """Get a dagger container from a dockerhub image.
-
-    Args:
-        dagger_client (dagger.Client): The dagger client to use to import the connector image
-        dockerhub_image_name (str): The name of the dockerhub image to import
-
-    Returns:
-        dagger.Container: The dagger container for the dockerhub image
-    """
-    try:
-        return await dagger_client.container().from_(dockerhub_image_name)
-    except dagger.DaggerError as e:
-        pytest.exit(f"Failed to import connector image from DockerHub: {e}")
-
-
-async def get_connector_container(
-    dagger_client: dagger.Client, image_name_with_tag: str
-) -> dagger.Container:
-    """Get a dagger container for the connector image to test.
-
-    Args:
-        dagger_client (dagger.Client): The dagger client to use to import the connector image
-        image_name_with_tag (str): The docker image name and tag of the connector image to test
-
-    Returns:
-        dagger.Container: The dagger container for the connector image to test
-    """
-    # If a container_id.txt file is available, we'll use it to load the connector container
-    # We use a txt file as container ids can be too long to be passed as env vars
-    # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation
-    container_id_path = Path(f"/tmp/{slugify(image_name_with_tag)}_container_id.txt")
-    if container_id_path.exists():
-        return await get_container_from_id(dagger_client, container_id_path.read_text())
-
-    # If the CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH env var is set, we'll use it to import the connector image from the tarball
-    if connector_image_tarball_path := os.environ.get(
-        "CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"
-    ):
-        tarball_path = Path(connector_image_tarball_path)
-        return await get_container_from_tarball_path(dagger_client, tarball_path)
-
-    # Let's try to load the connector container from a local image
-    if connector_container := await get_container_from_local_image(
-        dagger_client, image_name_with_tag
-    ):
-        return connector_container
-
-    # If we get here, we'll try to pull the connector image from DockerHub
-    return await get_container_from_dockerhub_image(dagger_client, image_name_with_tag)
-
-
-def sh_dash_c(lines: list[str]) -> list[str]:
-    """Wrap sequence of commands in shell for safe usage of dagger Container's with_exec method."""
-    return ["sh", "-c", " && ".join(["set -o xtrace"] + lines)]
-
-
-def clean_up_artifacts(directory: Path, logger: logging.Logger) -> None:
-    if directory.exists():
-        shutil.rmtree(directory)
-        logger.info(f"🧹 Test artifacts cleaned up from {directory}")
-
-
-def get_http_flows_from_mitm_dump(mitm_dump_path: Path) -> list[http.HTTPFlow]:
-    """Get http flows from a mitmproxy dump file.
-
-    Args:
-        mitm_dump_path (Path): Path to the mitmproxy dump file.
-
-    Returns:
-        List[http.HTTPFlow]: List of http flows.
-    """
-    with open(mitm_dump_path, "rb") as dump_file:
-        return [
-            f for f in io.FlowReader(dump_file).stream() if isinstance(f, http.HTTPFlow)
-        ]
-
-
-def mitm_http_stream_to_har(mitm_http_stream_path: Path, har_file_path: Path) -> Path:
-    """Converts a mitmproxy http stream file to a har file.
-
-    Args:
-        mitm_http_stream_path (Path): Path to the mitmproxy http stream file.
-        har_file_path (Path): Path where the har file will be saved.
-
-    Returns:
-        Path: Path to the har file.
-    """
-    flows = get_http_flows_from_mitm_dump(mitm_http_stream_path)
-    SaveHar().export_har(flows, str(har_file_path))
-    return har_file_path
-
-
-def extract_connection_id_from_url(url: str) -> str:
-    pattern = r"/connections/([a-f0-9\-]+)"
-    match = re.search(pattern, url)
-    if match:
-        return match.group(1)
-    else:
-        raise ValueError(f"Could not extract connection id from url {url}")
-
-
-def extract_workspace_id_from_url(url: str) -> str:
-    pattern = r"/workspaces/([a-f0-9\-]+)"
-    match = re.search(pattern, url)
-    if match:
-        return match.group(1)
-    else:
-        raise ValueError(f"Could not extract workspace id from url {url}")
-
-
-def build_connection_url(workspace_id: str | None, connection_id: str | None) -> str:
-    if not workspace_id or not connection_id:
-        raise ValueError("Both workspace_id and connection_id must be provided")
-    return f"https://cloud.airbyte.com/workspaces/{workspace_id}/connections/{connection_id}"
-
-
-def sort_dict_keys(d: dict) -> dict:
-    if isinstance(d, dict):
-        sorted_dict = {}
-        for key in sorted(d.keys()):
-            sorted_dict[key] = sort_dict_keys(d[key])
-        return sorted_dict
-    else:
-        return d
-
-
-def sanitize_stream_name(stream_name: str) -> str:
-    return stream_name.replace("/", "_").replace(" ", "_").lower()