airbyte-internal-ops 0.1.9__tar.gz → 0.1.11__tar.gz

{airbyte_internal_ops-0.1.9 → airbyte_internal_ops-0.1.11}/.github/workflows/connector-live-test.yml

@@ -13,6 +13,11 @@ on:
         description: "Connector name to build from source (e.g., 'source-pokeapi'). If provided, builds the image locally."
         required: false
         type: string
+      airbyte_ref:
+        description: "Git ref or PR number to checkout from airbyte repo (e.g., 'master', '70847', 'refs/pull/70847/head'). Only used when connector_name is provided."
+        required: false
+        type: string
+        default: "master"
       command:
         description: "Airbyte command to run"
         required: false
@@ -53,6 +58,29 @@ jobs:
       - name: Install dependencies
         run: uv sync --group dev
 
+      - name: Resolve Airbyte ref
+        if: ${{ inputs.connector_name != '' }}
+        id: resolve-airbyte-ref
+        run: |
+          REF="${{ inputs.airbyte_ref }}"
+          # If the input is all digits, treat it as a PR number
+          if [[ "$REF" =~ ^[0-9]+$ ]]; then
+            echo "resolved_ref=refs/pull/${REF}/head" >> "$GITHUB_OUTPUT"
+            echo "Resolved PR number $REF to refs/pull/${REF}/head"
+          else
+            echo "resolved_ref=$REF" >> "$GITHUB_OUTPUT"
+            echo "Using ref as-is: $REF"
+          fi
+
+      - name: Checkout Airbyte repo
+        if: ${{ inputs.connector_name != '' }}
+        uses: actions/checkout@v6
+        with:
+          repository: airbytehq/airbyte
+          ref: ${{ steps.resolve-airbyte-ref.outputs.resolved_ref }}
+          path: airbyte
+          fetch-depth: 1
+
       - name: Run Live Test
         id: live-test
         env:
@@ -67,6 +95,7 @@ jobs:
 
           if [ -n "${{ inputs.connector_name }}" ]; then
             ARGS+=(--connector-name "${{ inputs.connector_name }}")
+            ARGS+=(--repo-root "${{ github.workspace }}/airbyte")
           fi
 
           if [ -n "${{ inputs.connection_id }}" ]; then

{airbyte_internal_ops-0.1.9 → airbyte_internal_ops-0.1.11}/.github/workflows/connector-regression-test.yml

@@ -18,6 +18,11 @@ on:
         description: "Connector name to build target image from source (e.g., 'source-pokeapi'). If provided, builds the target image locally."
         required: false
         type: string
+      airbyte_ref:
+        description: "Git ref or PR number to checkout from airbyte repo (e.g., 'master', '70847', 'refs/pull/70847/head'). Only used when connector_name is provided."
+        required: false
+        type: string
+        default: "master"
       command:
         description: "Airbyte command to run"
         required: false
@@ -51,6 +56,11 @@ on:
         description: "Connector name to build target image from source (e.g., 'source-pokeapi')."
         required: false
         type: string
+      airbyte_ref:
+        description: "Git ref or PR number to checkout from airbyte repo. Only used when connector_name is provided."
+        required: false
+        type: string
+        default: "master"
       command:
         description: "Airbyte command to run (spec, check, discover, read)"
         required: false
@@ -79,6 +89,8 @@ on:
         required: false
       AIRBYTE_CLOUD_CLIENT_SECRET:
         required: false
+      GCP_GSM_CREDENTIALS:
+        required: false
 
 jobs:
   regression-test:
@@ -105,11 +117,35 @@ jobs:
       - name: Install dependencies
         run: uv sync --group dev
 
+      - name: Resolve Airbyte ref
+        if: ${{ inputs.connector_name != '' }}
+        id: resolve-airbyte-ref
+        run: |
+          REF="${{ inputs.airbyte_ref }}"
+          # If the input is all digits, treat it as a PR number
+          if [[ "$REF" =~ ^[0-9]+$ ]]; then
+            echo "resolved_ref=refs/pull/${REF}/head" >> "$GITHUB_OUTPUT"
+            echo "Resolved PR number $REF to refs/pull/${REF}/head"
+          else
+            echo "resolved_ref=$REF" >> "$GITHUB_OUTPUT"
+            echo "Using ref as-is: $REF"
+          fi
+
+      - name: Checkout Airbyte repo
+        if: ${{ inputs.connector_name != '' }}
+        uses: actions/checkout@v6
+        with:
+          repository: airbytehq/airbyte
+          ref: ${{ steps.resolve-airbyte-ref.outputs.resolved_ref }}
+          path: airbyte
+          fetch-depth: 1
+
       - name: Run Regression Test
         id: regression-test
         env:
           AIRBYTE_CLOUD_CLIENT_ID: ${{ secrets.AIRBYTE_CLOUD_CLIENT_ID }}
           AIRBYTE_CLOUD_CLIENT_SECRET: ${{ secrets.AIRBYTE_CLOUD_CLIENT_SECRET }}
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
         run: |
           ARGS=(--command "${{ inputs.command }}")
 
@@ -123,6 +159,7 @@ jobs:
 
           if [ -n "${{ inputs.connector_name }}" ]; then
             ARGS+=(--connector-name "${{ inputs.connector_name }}")
+            ARGS+=(--repo-root "${{ github.workspace }}/airbyte")
           fi
 
           if [ -n "${{ inputs.connection_id }}" ]; then

airbyte_internal_ops-0.1.11/.github/workflows/live-tests-ci.yml

@@ -0,0 +1,33 @@
+# CI smoke test for live_tests code changes
+# Runs a lightweight regression test to validate live_tests code works correctly
+name: Live Tests CI
+
+permissions:
+  contents: read
+  checks: write
+
+on:
+  pull_request:
+    paths:
+      - 'src/airbyte_ops_mcp/live_tests/**'
+      - '.github/workflows/live-tests-ci.yml'
+      - '.github/workflows/connector-regression-test.yml'
+
+  # Allow manual triggering for testing
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke-test:
+    name: Live Tests Smoke Test
+    uses: ./.github/workflows/connector-regression-test.yml
+    with:
+      # Use source-pokeapi as a stable, public connector for smoke testing
+      target_image: "airbyte/source-pokeapi:0.3.0"
+      control_image: "airbyte/source-pokeapi:0.2.0"
+      # spec is the fastest command and doesn't require credentials
+      command: "spec"
+    # No secrets needed for spec command with public connector

{airbyte_internal_ops-0.1.9 → airbyte_internal_ops-0.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airbyte-internal-ops
-Version: 0.1.9
+Version: 0.1.11
 Summary: MCP and API interfaces that let the agents do the admin work
 Author-email: Aaron Steers <aj@airbyte.io>
 Keywords: admin,airbyte,api,mcp

{airbyte_internal_ops-0.1.9 → airbyte_internal_ops-0.1.11}/src/airbyte_ops_mcp/cli/cloud.py

@@ -13,9 +13,18 @@ Commands:
 from __future__ import annotations
 
 import json
+import os
+import shutil
+import signal
+import socket
+import subprocess
+import tempfile
+import time
 from pathlib import Path
 from typing import Annotated, Literal
 
+import requests
+import yaml
 from airbyte_cdk.models.connector_metadata import MetadataFile
 from airbyte_cdk.utils.connector_paths import find_connector_root_from_name
 from airbyte_cdk.utils.docker import build_connector_image, verify_docker_installation
@@ -30,6 +39,13 @@ from airbyte_ops_mcp.cli._shared import (
     print_success,
 )
 from airbyte_ops_mcp.cloud_admin.connection_config import fetch_connection_config
+from airbyte_ops_mcp.constants import (
+    CLOUD_SQL_INSTANCE,
+    CLOUD_SQL_PROXY_PID_FILE,
+    DEFAULT_CLOUD_SQL_PROXY_PORT,
+    ENV_GCP_PROD_DB_ACCESS_CREDENTIALS,
+)
+from airbyte_ops_mcp.live_tests.cdk_secrets import get_first_config_from_secrets
 from airbyte_ops_mcp.live_tests.ci_output import (
     generate_regression_report,
     get_report_summary,
@@ -75,6 +91,184 @@ connector_app = App(
 )
 cloud_app.command(connector_app)
 
+# Create the db sub-app under cloud
+db_app = App(name="db", help="Database operations for Airbyte Cloud Prod DB Replica.")
+cloud_app.command(db_app)
+
+
+@db_app.command(name="start-proxy")
+def start_proxy(
+    port: Annotated[
+        int,
+        Parameter(help="Port for the Cloud SQL Proxy to listen on."),
+    ] = DEFAULT_CLOUD_SQL_PROXY_PORT,
+    daemon: Annotated[
+        bool,
+        Parameter(
+            help="Run as daemon in background (default). Use --no-daemon for foreground."
+        ),
+    ] = True,
+) -> None:
+    """Start the Cloud SQL Proxy for database access.
+
+    This command starts the Cloud SQL Auth Proxy to enable connections to the
+    Airbyte Cloud Prod DB Replica. The proxy is required for database query tools.
+
+    By default, runs as a daemon (background process). Use --no-daemon to run in
+    foreground mode where you can see logs and stop with Ctrl+C.
+
+    Credentials are read from the GCP_PROD_DB_ACCESS_CREDENTIALS environment variable,
+    which should contain the service account JSON credentials.
+
+    After starting the proxy, set these environment variables to use database tools:
+        export USE_CLOUD_SQL_PROXY=1
+        export DB_PORT={port}
+
+    Example:
+        airbyte-ops cloud db start-proxy
+        airbyte-ops cloud db start-proxy --port 15432
+        airbyte-ops cloud db start-proxy --no-daemon
+    """
+    # Check if proxy is already running on the requested port (idempotency)
+    try:
+        with socket.create_connection(("127.0.0.1", port), timeout=0.5):
+            # Something is already listening on this port
+            pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
+            pid_info = ""
+            if pid_file.exists():
+                pid_info = f" (PID: {pid_file.read_text().strip()})"
+            print_success(
+                f"Cloud SQL Proxy is already running on port {port}{pid_info}"
+            )
+            print_success("")
+            print_success("To use database tools, set these environment variables:")
+            print_success("  export USE_CLOUD_SQL_PROXY=1")
+            print_success(f"  export DB_PORT={port}")
+            return
+    except (OSError, TimeoutError, ConnectionRefusedError):
+        pass  # Port not in use, proceed with starting proxy
+
+    # Check if cloud-sql-proxy is installed
+    proxy_path = shutil.which("cloud-sql-proxy")
+    if not proxy_path:
+        exit_with_error(
+            "cloud-sql-proxy not found in PATH. "
+            "Install it from: https://cloud.google.com/sql/docs/mysql/sql-proxy"
+        )
+
+    # Get credentials from environment
+    creds_json = os.getenv(ENV_GCP_PROD_DB_ACCESS_CREDENTIALS)
+    if not creds_json:
+        exit_with_error(
+            f"{ENV_GCP_PROD_DB_ACCESS_CREDENTIALS} environment variable is not set. "
+            "This should contain the GCP service account JSON credentials."
+        )
+
+    # Build the command using --json-credentials to avoid writing to disk
+    cmd = [
+        proxy_path,
+        CLOUD_SQL_INSTANCE,
+        f"--port={port}",
+        f"--json-credentials={creds_json}",
+    ]
+
+    print_success(f"Starting Cloud SQL Proxy on port {port}...")
+    print_success(f"Instance: {CLOUD_SQL_INSTANCE}")
+    print_success("")
+    print_success("To use database tools, set these environment variables:")
+    print_success("  export USE_CLOUD_SQL_PROXY=1")
+    print_success(f"  export DB_PORT={port}")
+    print_success("")
+
+    if daemon:
+        # Run in background (daemon mode) with log file for diagnostics
+        log_file_path = Path("/tmp/airbyte-cloud-sql-proxy.log")
+        log_file = log_file_path.open("ab")
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.DEVNULL,
+            stderr=log_file,
+            start_new_session=True,
+        )
+
+        # Brief wait to verify the process started successfully
+        time.sleep(0.5)
+        if process.poll() is not None:
+            # Process exited immediately - read any error output
+            log_file.close()
+            error_output = ""
+            if log_file_path.exists():
+                error_output = log_file_path.read_text()[-1000:]  # Last 1000 chars
+            exit_with_error(
+                f"Cloud SQL Proxy failed to start (exit code: {process.returncode}).\n"
+                f"Check logs at {log_file_path}\n"
+                f"Recent output: {error_output}"
+            )
+
+        # Write PID to file for stop-proxy command
+        pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
+        pid_file.write_text(str(process.pid))
+        print_success(f"Cloud SQL Proxy started as daemon (PID: {process.pid})")
+        print_success(f"Logs: {log_file_path}")
+        print_success("To stop: airbyte-ops cloud db stop-proxy")
+    else:
+        # Run in foreground - replace current process
+        # Signals (Ctrl+C) will be handled directly by the cloud-sql-proxy process
+        print_success("Running in foreground. Press Ctrl+C to stop the proxy.")
+        print_success("")
+        os.execv(proxy_path, cmd)
+
+
+@db_app.command(name="stop-proxy")
+def stop_proxy() -> None:
+    """Stop the Cloud SQL Proxy daemon.
+
+    This command stops a Cloud SQL Proxy that was started with 'start-proxy'.
+    It reads the PID from the PID file and sends a SIGTERM signal to stop the process.
+
+    Example:
+        airbyte-ops cloud db stop-proxy
+    """
+    pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
+
+    if not pid_file.exists():
+        exit_with_error(
+            f"PID file not found at {CLOUD_SQL_PROXY_PID_FILE}. "
+            "No Cloud SQL Proxy daemon appears to be running."
+        )
+
+    pid_str = pid_file.read_text().strip()
+    if not pid_str.isdigit():
+        pid_file.unlink()
+        exit_with_error(f"Invalid PID in {CLOUD_SQL_PROXY_PID_FILE}: {pid_str}")
+
+    pid = int(pid_str)
+
+    # Check if process is still running
+    try:
+        os.kill(pid, 0)  # Signal 0 just checks if process exists
+    except ProcessLookupError:
+        pid_file.unlink()
+        print_success(
+            f"Cloud SQL Proxy (PID: {pid}) is not running. Cleaned up PID file."
+        )
+        return
+    except PermissionError:
+        exit_with_error(f"Permission denied to check process {pid}.")
+
+    # Send SIGTERM to stop the process
+    try:
+        os.kill(pid, signal.SIGTERM)
+        print_success(f"Sent SIGTERM to Cloud SQL Proxy (PID: {pid}).")
+    except ProcessLookupError:
+        print_success(f"Cloud SQL Proxy (PID: {pid}) already stopped.")
+    except PermissionError:
+        exit_with_error(f"Permission denied to stop process {pid}.")
+
+    # Clean up PID file
+    pid_file.unlink(missing_ok=True)
+    print_success("Cloud SQL Proxy stopped.")
+
 
 @connector_app.command(name="get-version-info")
 def get_version_info(
@@ -317,6 +511,49 @@ def _build_connector_image_from_source(
     return built_image
 
 
+def _fetch_control_image_from_metadata(connector_name: str) -> str | None:
+    """Fetch the current released connector image from metadata.yaml on main branch.
+
+    This fetches the connector's metadata.yaml from the airbyte monorepo's master branch
+    and extracts the dockerRepository and dockerImageTag to construct the control image.
+
+    Args:
+        connector_name: The connector name (e.g., 'source-github').
+
+    Returns:
+        The full connector image with tag (e.g., 'airbyte/source-github:1.0.0'),
+        or None if the metadata could not be fetched or parsed.
+    """
+    metadata_url = (
+        f"https://raw.githubusercontent.com/airbytehq/airbyte/master/"
+        f"airbyte-integrations/connectors/{connector_name}/metadata.yaml"
+    )
+    response = requests.get(metadata_url, timeout=30)
+    if not response.ok:
+        print_error(
+            f"Failed to fetch metadata for {connector_name}: "
+            f"HTTP {response.status_code} from {metadata_url}"
+        )
+        return None
+
+    metadata = yaml.safe_load(response.text)
+    if not isinstance(metadata, dict):
+        print_error(f"Invalid metadata format for {connector_name}: expected dict")
+        return None
+
+    data = metadata.get("data", {})
+    docker_repository = data.get("dockerRepository")
+    docker_image_tag = data.get("dockerImageTag")
+
+    if not docker_repository or not docker_image_tag:
+        print_error(
+            f"Could not find dockerRepository/dockerImageTag in metadata for {connector_name}"
+        )
+        return None
+
+    return f"{docker_repository}:{docker_image_tag}"
+
+
 @connector_app.command(name="live-test")
 def live_test(
     connector_image: Annotated[
@@ -716,10 +953,48 @@ def regression_test(
         if not resolved_control_image and connection_data.connector_image:
             resolved_control_image = connection_data.connector_image
             print_success(f"Auto-detected control image: {resolved_control_image}")
+    elif config_path:
+        config_file = Path(config_path)
+        catalog_file = Path(catalog_path) if catalog_path else None
+    elif connector_name:
+        # Fallback: fetch integration test secrets from GSM using PyAirbyte API
+        print_success(
+            f"No connection_id or config_path provided. "
+            f"Attempting to fetch integration test config from GSM for {connector_name}..."
+        )
+        gsm_config = get_first_config_from_secrets(connector_name)
+        if gsm_config:
+            # Write config to a temp file (not in output_path to avoid artifact upload)
+            gsm_config_dir = Path(
+                tempfile.mkdtemp(prefix=f"gsm-config-{connector_name}-")
+            )
+            gsm_config_dir.chmod(0o700)
+            gsm_config_file = gsm_config_dir / "config.json"
+            gsm_config_file.write_text(json.dumps(gsm_config, indent=2))
+            gsm_config_file.chmod(0o600)
+            config_file = gsm_config_file
+            catalog_file = None
+            print_success(
+                f"Fetched integration test config from GSM for {connector_name}"
+            )
+        else:
+            print_error(
+                f"Failed to fetch integration test config from GSM for {connector_name}."
+            )
+            config_file = None
+            catalog_file = None
     else:
-        config_file = Path(config_path) if config_path else None
+        config_file = None
         catalog_file = Path(catalog_path) if catalog_path else None
 
+    # Auto-detect control_image from metadata.yaml if connector_name is provided
+    if not resolved_control_image and connector_name:
+        resolved_control_image = _fetch_control_image_from_metadata(connector_name)
+        if resolved_control_image:
+            print_success(
+                f"Auto-detected control image from metadata.yaml: {resolved_control_image}"
+            )
+
     # Validate that we have both images
     if not resolved_target_image:
         write_github_output("success", False)
@@ -733,8 +1008,9 @@ def regression_test(
         write_github_output("success", False)
         write_github_output("error", "No control image specified")
         exit_with_error(
-            "You must provide one of the following: a control_image or a connection_id "
-            "for a connection that has an associated connector image."
+            "You must provide one of the following: a control_image, a connection_id "
+            "for a connection that has an associated connector image, or a connector_name "
+            "to auto-detect the control image from the airbyte repo's metadata.yaml."
         )
 
     # Pull images if they weren't just built locally

{airbyte_internal_ops-0.1.9 → airbyte_internal_ops-0.1.11}/src/airbyte_ops_mcp/constants.py

@@ -27,6 +27,15 @@ EXPECTED_ADMIN_EMAIL_DOMAIN = "@airbyte.io"
 GCP_PROJECT_NAME = "prod-ab-cloud-proj"
 """The GCP project name for Airbyte Cloud production."""
 
+CLOUD_SQL_INSTANCE = "prod-ab-cloud-proj:us-west3:prod-pgsql-replica"
+"""The Cloud SQL instance connection name for the Prod DB Replica."""
+
+DEFAULT_CLOUD_SQL_PROXY_PORT = 15432
+"""Default port for Cloud SQL Proxy connections."""
+
+CLOUD_SQL_PROXY_PID_FILE = "/tmp/airbyte-cloud-sql-proxy.pid"
+"""PID file for tracking the Cloud SQL Proxy process."""
+
 CLOUD_REGISTRY_URL = (
     "https://connectors.airbyte.com/files/registries/v0/cloud_registry.json"
 )