airbyte-internal-ops 0.1.9__py3-none-any.whl → 0.1.11__py3-none-any.whl

airbyte_ops_mcp/live_tests/cdk_secrets.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Fetch connector test secrets using the PyAirbyte secrets API.
+
+This module uses the PyAirbyte GoogleGSMSecretManager to retrieve
+integration test secrets from Google Secret Manager for connectors.
+
+Usage:
+    from airbyte_ops_mcp.live_tests.cdk_secrets import get_first_config_from_secrets
+
+    # Fetch the first config for a connector
+    config = get_first_config_from_secrets("source-github")
+    if config:
+        # Use the config dict
+        ...
+
+Note: Requires GCP credentials with access to the integration testing project.
+The credentials can be provided via:
+- GOOGLE_APPLICATION_CREDENTIALS environment variable
+- GCP_GSM_CREDENTIALS environment variable (JSON string)
+- Application Default Credentials
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+from airbyte.secrets import GoogleGSMSecretManager
+
+logger = logging.getLogger(__name__)
+
+# Default GCP project for integration test secrets
+DEFAULT_GSM_PROJECT = "dataline-integration-testing"
+
+
+def get_first_config_from_secrets(
+    connector_name: str,
+    project: str = DEFAULT_GSM_PROJECT,
+) -> dict | None:
+    """Fetch the first integration test config for a connector from GSM.
+
+    This function uses the PyAirbyte GoogleGSMSecretManager to fetch secrets
+    labeled with the connector name and returns the first one as a parsed dict.
+
+    Args:
+        connector_name: The connector name (e.g., 'source-github').
+        project: The GCP project ID containing the secrets.
+
+    Returns:
+        The parsed config dict, or None if no secrets are found or fetching fails.
+    """
+    # Get credentials from environment
+    credentials_json: str | None = None
+    credentials_path: str | None = None
+
+    if "GCP_GSM_CREDENTIALS" in os.environ:
+        credentials_json = os.environ["GCP_GSM_CREDENTIALS"]
+    elif "GOOGLE_APPLICATION_CREDENTIALS" in os.environ:
+        credentials_path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
+
+    # If no explicit credentials, GoogleGSMSecretManager will try ADC
+    try:
+        gsm = GoogleGSMSecretManager(
+            project=project,
+            credentials_json=credentials_json,
+            credentials_path=credentials_path,
+        )
+    except Exception as e:
+        logger.warning(f"Failed to initialize GSM client: {e}")
+        return None
+
+    logger.info(f"Fetching integration test config for {connector_name} from GSM")
+
+    try:
+        # fetch_connector_secret returns the first secret matching the connector label
+        secret_handle = gsm.fetch_connector_secret(connector_name)
+        # parse_json() calls get_value() internally and parses the result
+        config = secret_handle.parse_json()
+        logger.info(f"Successfully fetched config for {connector_name}")
+        return dict(config) if config else None
+
+    except StopIteration:
+        logger.warning(f"No secrets found for connector {connector_name}")
+        return None
+    except Exception as e:
+        # Log the exception type but not the message (may contain sensitive info)
+        logger.warning(
+            f"Failed to fetch config for {connector_name}: {type(e).__name__}"
+        )
+        return None

airbyte_ops_mcp/live_tests/ci_output.py

@@ -142,6 +142,35 @@ def _format_delta(delta: int) -> str:
     return "0"
 
 
+def _get_github_run_url() -> str | None:
+    """Get the URL to the current GitHub Actions workflow run.
+
+    Returns:
+        The workflow run URL, or None if not running in GitHub Actions.
+    """
+    server_url = os.getenv("GITHUB_SERVER_URL")
+    repository = os.getenv("GITHUB_REPOSITORY")
+    run_id = os.getenv("GITHUB_RUN_ID")
+
+    if not all([server_url, repository, run_id]):
+        return None
+
+    return f"{server_url}/{repository}/actions/runs/{run_id}"
+
+
+def _get_github_artifacts_url() -> str | None:
+    """Get the URL to the artifacts section of the current workflow run.
+
+    Returns:
+        The artifacts section URL, or None if not running in GitHub Actions.
+    """
+    run_url = _get_github_run_url()
+    if not run_url:
+        return None
+
+    return f"{run_url}#artifacts"
+
+
 def generate_regression_report(
     target_image: str,
     control_image: str,
@@ -190,6 +219,9 @@ def generate_regression_report(
         target_image.rsplit(":", 1)[0] if ":" in target_image else target_image
     )
 
+    run_url = _get_github_run_url()
+    artifacts_url = _get_github_artifacts_url()
+
     lines: list[str] = [
         "# Regression Test Report",
         "",
@@ -200,12 +232,23 @@ def generate_regression_report(
         f"- **Control Version:** `{control_version}`",
         f"- **Target Version:** `{target_version}`",
         f"- **Command:** `{command.upper()}`",
-        f"- **Artifacts:** `{artifact_name}`",
-        "",
-        "## Summary",
-        "",
     ]
 
+    if run_url:
+        lines.append(f"- **Workflow Run:** [View Execution]({run_url})")
+    if artifacts_url:
+        lines.append(f"- **Artifacts:** [Download `{artifact_name}`]({artifacts_url})")
+    else:
+        lines.append(f"- **Artifacts:** `{artifact_name}`")
+
+    lines.extend(
+        [
+            "",
+            "## Summary",
+            "",
+        ]
+    )
+
     if regression_detected:
         if target_result["success"] and not control_result["success"]:
             lines.append("**Result:** Target succeeded, control failed (improvement)")
@@ -341,9 +384,16 @@ def get_report_summary(report_path: Path) -> str:
         f"regression-test-artifacts-{run_id}" if run_id else "regression-test-artifacts"
     )
 
+    artifacts_url = _get_github_artifacts_url()
+    artifact_link = (
+        f"[`{artifact_name}`]({artifacts_url})"
+        if artifacts_url
+        else f"`{artifact_name}`"
+    )
+
     return f"""## Regression Test Report
 
-Full report available in the **Regression Test Report** check or in artifact `{artifact_name}`.
+Full report available in the **Regression Test Report** check or in artifact {artifact_link}.
 
 See the Checks tab for the complete report with message counts and execution details.
 """

airbyte_ops_mcp/live_tests/connector_runner.py

@@ -168,6 +168,9 @@ class ConnectorRunner:
 
         with tempfile.TemporaryDirectory() as temp_dir:
             temp_path = Path(temp_dir)
+            # Make temp directory world-readable so non-root container users can access it
+            # Many connector images run as non-root users (e.g., 'airbyte' user)
+            temp_path.chmod(0o755)
             self._prepare_data_directory(temp_path)
 
             docker_cmd = self._build_docker_command(temp_path)

airbyte_ops_mcp/mcp/github.py

@@ -7,7 +7,6 @@ Docker image availability, and other related operations.
 
 from __future__ import annotations
 
-import os
 import re
 from typing import Annotated
 
@@ -15,9 +14,9 @@ import requests
 from fastmcp import FastMCP
 from pydantic import BaseModel, Field
 
+from airbyte_ops_mcp.github_actions import GITHUB_API_BASE, resolve_github_token
 from airbyte_ops_mcp.mcp._mcp_utils import mcp_tool, register_mcp_tools
 
-GITHUB_API_BASE = "https://api.github.com"
 DOCKERHUB_API_BASE = "https://hub.docker.com/v2"
 
 
@@ -37,24 +36,6 @@ class WorkflowRunStatus(BaseModel):
     jobs_url: str
 
 
-def _get_github_token() -> str:
-    """Get GitHub token from environment.
-
-    Returns:
-        GitHub token string.
-
-    Raises:
-        ValueError: If GITHUB_TOKEN environment variable is not set.
-    """
-    token = os.getenv("GITHUB_TOKEN")
-    if not token:
-        raise ValueError(
-            "GITHUB_TOKEN environment variable is required. "
-            "Please set it to a GitHub personal access token."
-        )
-    return token
-
-
 def _parse_workflow_url(url: str) -> tuple[str, str, int]:
     """Parse a GitHub Actions workflow run URL into components.
 
@@ -162,7 +143,7 @@ def check_workflow_status(
         )
 
     # Guard: Check for required token
-    token = _get_github_token()
+    token = resolve_github_token()
 
     # Get workflow run details
     run_data = _get_workflow_run(owner, repo, run_id, token)

airbyte_ops_mcp/mcp/live_tests.py

@@ -9,7 +9,6 @@ in GitHub Actions and results can be polled via workflow status.
 from __future__ import annotations
 
 import logging
-import os
 import uuid
 from datetime import datetime
 from enum import Enum
@@ -21,6 +20,11 @@ from airbyte.cloud.auth import resolve_cloud_client_id, resolve_cloud_client_sec
 from fastmcp import FastMCP
 from pydantic import BaseModel, Field
 
+from airbyte_ops_mcp.github_actions import (
+    GITHUB_API_BASE,
+    resolve_github_token,
+    trigger_workflow_dispatch,
+)
 from airbyte_ops_mcp.mcp._mcp_utils import mcp_tool, register_mcp_tools
 
 logger = logging.getLogger(__name__)
@@ -29,7 +33,6 @@ logger = logging.getLogger(__name__)
 # GitHub Workflow Configuration
 # =============================================================================
 
-GITHUB_API_BASE = "https://api.github.com"
 LIVE_TEST_REPO_OWNER = "airbytehq"
 LIVE_TEST_REPO_NAME = "airbyte-ops-mcp"
 LIVE_TEST_DEFAULT_BRANCH = "main"
@@ -37,76 +40,6 @@ LIVE_TEST_WORKFLOW_FILE = "connector-live-test.yml"
 REGRESSION_TEST_WORKFLOW_FILE = "connector-regression-test.yml"
 
 
-# =============================================================================
-# GitHub API Helper Functions
-# =============================================================================
-
-
-def _get_github_token() -> str:
-    """Get GitHub token from environment.
-
-    Checks for tokens in order of specificity:
-    1. GITHUB_CI_WORKFLOW_TRIGGER_PAT (general workflow triggering)
-    2. GITHUB_TOKEN (fallback)
-
-    Returns:
-        GitHub token string.
-
-    Raises:
-        ValueError: If no GitHub token environment variable is set.
-    """
-    token = os.getenv("GITHUB_CI_WORKFLOW_TRIGGER_PAT") or os.getenv("GITHUB_TOKEN")
-    if not token:
-        raise ValueError(
-            "No GitHub token found. Set GITHUB_CI_WORKFLOW_TRIGGER_PAT or GITHUB_TOKEN "
-            "environment variable with 'actions:write' permission."
-        )
-    return token
-
-
-def _trigger_workflow_dispatch(
-    owner: str,
-    repo: str,
-    workflow_file: str,
-    ref: str,
-    inputs: dict[str, Any],
-    token: str,
-) -> str:
-    """Trigger a GitHub Actions workflow via workflow_dispatch.
-
-    Args:
-        owner: Repository owner (e.g., "airbytehq")
-        repo: Repository name (e.g., "airbyte-ops-mcp")
-        workflow_file: Workflow file name (e.g., "connector-live-test.yml")
-        ref: Git ref to run the workflow on (branch name)
-        inputs: Workflow inputs dictionary
-        token: GitHub API token
-
-    Returns:
-        URL to view workflow runs.
-
-    Raises:
-        requests.HTTPError: If API request fails.
-    """
-    url = f"{GITHUB_API_BASE}/repos/{owner}/{repo}/actions/workflows/{workflow_file}/dispatches"
-    headers = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github+json",
-        "X-GitHub-Api-Version": "2022-11-28",
-    }
-    payload = {
-        "ref": ref,
-        "inputs": inputs,
-    }
-
-    response = requests.post(url, headers=headers, json=payload, timeout=30)
-    response.raise_for_status()
-
-    # workflow_dispatch returns 204 No Content on success
-    # Return URL to view workflow runs
-    return f"https://github.com/{owner}/{repo}/actions/workflows/{workflow_file}"
-
-
 def _get_workflow_run_status(
     owner: str,
     repo: str,
@@ -293,12 +226,22 @@ class LiveConnectionTestResult(BaseModel):
 class RunLiveConnectionTestsResponse(BaseModel):
     """Response from starting a live connection test via GitHub Actions workflow."""
 
-    run_id: str = Field(description="Unique identifier for the test run")
+    run_id: str = Field(
+        description="Unique identifier for the test run (internal tracking ID)"
+    )
     status: TestRunStatus = Field(description="Initial status of the test run")
     message: str = Field(description="Human-readable status message")
     workflow_url: str | None = Field(
         default=None,
-        description="URL to view the GitHub Actions workflow runs",
+        description="URL to view the GitHub Actions workflow file",
+    )
+    github_run_id: int | None = Field(
+        default=None,
+        description="GitHub Actions workflow run ID (use with check_workflow_status)",
+    )
+    github_run_url: str | None = Field(
+        default=None,
+        description="Direct URL to the GitHub Actions workflow run",
     )
 
 
@@ -348,9 +291,16 @@ def run_live_connection_tests(
     ] = None,
     connector_name: Annotated[
         str | None,
-        "Connector name to build target image from source for regression tests "
-        "(e.g., 'source-pokeapi'). If provided, builds the target image locally. "
-        "Only used when skip_regression_tests=False.",
+        "Connector name to build the connector image from source "
+        "(e.g., 'source-pokeapi'). If provided, builds the image locally with tag 'dev'. "
+        "For live tests, this builds the test image. For regression tests, this builds "
+        "the target image while control is auto-detected from the connection.",
+    ] = None,
+    airbyte_ref: Annotated[
+        str | None,
+        "Git ref or PR number to checkout from the airbyte monorepo "
+        "(e.g., 'master', '70847', 'refs/pull/70847/head'). "
+        "Only used when connector_name is provided. Defaults to 'master' if not specified.",
     ] = None,
 ) -> RunLiveConnectionTestsResponse:
     """Start a live connection test run via GitHub Actions workflow.
@@ -377,7 +327,7 @@ def run_live_connection_tests(
 
     # Get GitHub token
     try:
-        token = _get_github_token()
+        token = resolve_github_token()
     except ValueError as e:
         return RunLiveConnectionTestsResponse(
             run_id=run_id,
@@ -422,9 +372,11 @@ def run_live_connection_tests(
         }
         if connector_image:
             workflow_inputs["connector_image"] = connector_image
+        if connector_name:
+            workflow_inputs["connector_name"] = connector_name
 
         try:
-            workflow_url = _trigger_workflow_dispatch(
+            dispatch_result = trigger_workflow_dispatch(
                 owner=LIVE_TEST_REPO_OWNER,
                 repo=LIVE_TEST_REPO_NAME,
                 workflow_file=LIVE_TEST_WORKFLOW_FILE,
@@ -441,12 +393,15 @@ def run_live_connection_tests(
                 workflow_url=None,
             )
 
+        view_url = dispatch_result.run_url or dispatch_result.workflow_url
         return RunLiveConnectionTestsResponse(
             run_id=run_id,
             status=TestRunStatus.QUEUED,
             message=f"Live-test workflow triggered for connection {connection_id}. "
-            f"View progress at: {workflow_url}",
-            workflow_url=workflow_url,
+            f"View progress at: {view_url}",
+            workflow_url=dispatch_result.workflow_url,
+            github_run_id=dispatch_result.run_id,
+            github_run_url=dispatch_result.run_url,
         )
 
     # Regression test workflow (skip_regression_tests=False)
@@ -472,9 +427,11 @@ def run_live_connection_tests(
         workflow_inputs["control_image"] = control_image
     if connector_name:
         workflow_inputs["connector_name"] = connector_name
+        if airbyte_ref:
+            workflow_inputs["airbyte_ref"] = airbyte_ref
 
     try:
-        workflow_url = _trigger_workflow_dispatch(
+        dispatch_result = trigger_workflow_dispatch(
             owner=LIVE_TEST_REPO_OWNER,
             repo=LIVE_TEST_REPO_NAME,
             workflow_file=REGRESSION_TEST_WORKFLOW_FILE,
@@ -491,12 +448,15 @@ def run_live_connection_tests(
             workflow_url=None,
         )
 
+    view_url = dispatch_result.run_url or dispatch_result.workflow_url
     return RunLiveConnectionTestsResponse(
         run_id=run_id,
         status=TestRunStatus.QUEUED,
         message=f"Regression-test workflow triggered for connection {connection_id}. "
-        f"View progress at: {workflow_url}",
-        workflow_url=workflow_url,
+        f"View progress at: {view_url}",
+        workflow_url=dispatch_result.workflow_url,
+        github_run_id=dispatch_result.run_id,
+        github_run_url=dispatch_result.run_url,
     )
 
 

airbyte_ops_mcp/mcp/prerelease.py

@@ -8,7 +8,6 @@ workflow in the airbytehq/airbyte repository via GitHub's workflow dispatch API.
 from __future__ import annotations
 
 import base64
-import os
 from typing import Annotated, Literal
 
 import requests
@@ -16,15 +15,22 @@ import yaml
 from fastmcp import FastMCP
 from pydantic import BaseModel, Field
 
+from airbyte_ops_mcp.github_actions import GITHUB_API_BASE, resolve_github_token
 from airbyte_ops_mcp.mcp._mcp_utils import mcp_tool, register_mcp_tools
 
-GITHUB_API_BASE = "https://api.github.com"
 DEFAULT_REPO_OWNER = "airbytehq"
 DEFAULT_REPO_NAME = "airbyte"
 DEFAULT_BRANCH = "master"
 PRERELEASE_WORKFLOW_FILE = "publish-connectors-prerelease-command.yml"
 CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors"
 
+# Token env vars for prerelease publishing (in order of preference)
+PRERELEASE_TOKEN_ENV_VARS = [
+    "GITHUB_CONNECTOR_PUBLISHING_PAT",
+    "GITHUB_CI_WORKFLOW_TRIGGER_PAT",
+    "GITHUB_TOKEN",
+]
+
 
 class PRHeadInfo(BaseModel):
     """Information about a PR's head commit."""
@@ -46,34 +52,6 @@ class PrereleaseWorkflowResult(BaseModel):
     docker_image_tag: str | None = None
 
 
-def _get_github_token() -> str:
-    """Get GitHub token from environment.
-
-    Checks for tokens in order of specificity:
-    1. GITHUB_CONNECTOR_PUBLISHING_PAT (most specific)
-    2. GITHUB_CI_WORKFLOW_TRIGGER_PAT (general workflow triggering)
-    3. GITHUB_TOKEN (fallback)
-
-    Returns:
-        GitHub token string.
-
-    Raises:
-        ValueError: If no GitHub token environment variable is set.
-    """
-    token = (
-        os.getenv("GITHUB_CONNECTOR_PUBLISHING_PAT")
-        or os.getenv("GITHUB_CI_WORKFLOW_TRIGGER_PAT")
-        or os.getenv("GITHUB_TOKEN")
-    )
-    if not token:
-        raise ValueError(
-            "No GitHub token found. Set GITHUB_CONNECTOR_PUBLISHING_PAT, "
-            "GITHUB_CI_WORKFLOW_TRIGGER_PAT, or GITHUB_TOKEN environment variable "
-            "with 'actions:write' permission."
-        )
-    return token
-
-
 def _get_pr_head_info(
     owner: str,
     repo: str,
@@ -248,7 +226,7 @@ def publish_connector_to_airbyte_registry(
         )
 
     # Guard: Check for required token
-    token = _get_github_token()
+    token = resolve_github_token(PRERELEASE_TOKEN_ENV_VARS)
 
     # Get the PR's head ref and SHA
     head_info = _get_pr_head_info(