airbyte-internal-ops 0.1.11__py3-none-any.whl → 0.2.1__py3-none-any.whl

airbyte_ops_mcp/mcp/{live_tests.py → regression_tests.py}

@@ -1,9 +1,16 @@
 # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-"""MCP tools for live connection tests.
+"""MCP tools for connector regression tests.
 
-This module provides MCP tools for triggering live validation and regression tests
-on Airbyte Cloud connections via GitHub Actions workflows. Tests run asynchronously
-in GitHub Actions and results can be polled via workflow status.
+This module provides MCP tools for triggering regression tests on Airbyte Cloud
+connections via GitHub Actions workflows. Regression tests can run in two modes:
+- Single version mode: Tests a connector version against a connection config
+- Comparison mode: Compares a target version against a control (baseline) version
+
+Tests run asynchronously in GitHub Actions and results can be polled via workflow status.
+
+Note: The term "regression tests" encompasses all connector validation testing.
+The term "live tests" is reserved for scenarios where actual Cloud connections
+are pinned to pre-release versions for real-world validation.
 """
 
 from __future__ import annotations
@@ -17,6 +24,10 @@ from typing import Annotated, Any
 import requests
 from airbyte.cloud import CloudWorkspace
 from airbyte.cloud.auth import resolve_cloud_client_id, resolve_cloud_client_secret
+from airbyte.exceptions import (
+    AirbyteMissingResourceError,
+    AirbyteWorkspaceMismatchError,
+)
 from fastmcp import FastMCP
 from pydantic import BaseModel, Field
 
@@ -33,13 +44,49 @@ logger = logging.getLogger(__name__)
 # GitHub Workflow Configuration
 # =============================================================================
 
-LIVE_TEST_REPO_OWNER = "airbytehq"
-LIVE_TEST_REPO_NAME = "airbyte-ops-mcp"
-LIVE_TEST_DEFAULT_BRANCH = "main"
-LIVE_TEST_WORKFLOW_FILE = "connector-live-test.yml"
+REGRESSION_TEST_REPO_OWNER = "airbytehq"
+REGRESSION_TEST_REPO_NAME = "airbyte-ops-mcp"
+REGRESSION_TEST_DEFAULT_BRANCH = "main"
+# Unified regression test workflow (handles both single-version and comparison modes)
 REGRESSION_TEST_WORKFLOW_FILE = "connector-regression-test.yml"
 
 
+# =============================================================================
+# Workspace Validation Helpers
+# =============================================================================
+
+
+def validate_connection_workspace(
+    connection_id: str,
+    workspace_id: str,
+) -> None:
+    """Validate that a connection belongs to the expected workspace.
+
+    Uses PyAirbyte's CloudConnection.check_is_valid() method to verify that
+    the connection exists and belongs to the specified workspace.
+
+    Raises:
+        ValueError: If Airbyte Cloud credentials are missing.
+        AirbyteWorkspaceMismatchError: If connection belongs to a different workspace.
+        AirbyteMissingResourceError: If connection is not found.
+    """
+    client_id = resolve_cloud_client_id()
+    client_secret = resolve_cloud_client_secret()
+    if not client_id or not client_secret:
+        raise ValueError(
+            "Missing Airbyte Cloud credentials. "
+            "Set AIRBYTE_CLOUD_CLIENT_ID and AIRBYTE_CLOUD_CLIENT_SECRET env vars."
+        )
+
+    workspace = CloudWorkspace(
+        workspace_id=workspace_id,
+        client_id=client_id,
+        client_secret=client_secret,
+    )
+    connection = workspace.get_connection(connection_id)
+    connection.check_is_valid()
+
+
 def _get_workflow_run_status(
     owner: str,
     repo: str,
@@ -90,8 +137,8 @@ class TestRunStatus(str, Enum):
     FAILED = "failed"
 
 
-class TestPhaseStatus(str, Enum):
-    """Status of a test phase (live or regression)."""
+class TestOutcome(str, Enum):
+    """Outcome of a test (execution or comparison)."""
 
     PENDING = "pending"
     RUNNING = "running"
@@ -130,10 +177,10 @@ class StreamComparisonResultModel(BaseModel):
     message: str = Field(description="Human-readable comparison summary")
 
 
-class LivePhaseResult(BaseModel):
-    """Results from the live test phase."""
+class RegressionTestExecutionResult(BaseModel):
+    """Results from executing the connector (validations and record counts)."""
 
-    status: TestPhaseStatus = Field(description="Status of the live phase")
+    outcome: TestOutcome = Field(description="Outcome of the execution")
     catalog_validations: list[ValidationResultModel] = Field(
         default_factory=list,
         description="Results of catalog validation checks",
@@ -148,14 +195,14 @@ class LivePhaseResult(BaseModel):
     )
     error_message: str | None = Field(
         default=None,
-        description="Error message if the phase failed",
+        description="Error message if the execution failed",
     )
 
 
-class RegressionPhaseResult(BaseModel):
-    """Results from the regression test phase."""
+class RegressionTestComparisonResult(BaseModel):
+    """Results from comparing target vs control connector versions."""
 
-    status: TestPhaseStatus = Field(description="Status of the regression phase")
+    outcome: TestOutcome = Field(description="Outcome of the comparison")
     baseline_version: str | None = Field(
         default=None,
         description="Version of the baseline (control) connector",
@@ -166,12 +213,12 @@ class RegressionPhaseResult(BaseModel):
     )
     error_message: str | None = Field(
         default=None,
-        description="Error message if the phase failed",
+        description="Error message if the comparison failed",
     )
 
 
-class LiveConnectionTestResult(BaseModel):
-    """Complete result of a live connection test run."""
+class RegressionTestResult(BaseModel):
+    """Complete result of a regression test run."""
 
     run_id: str = Field(description="Unique identifier for this test run")
     connection_id: str = Field(description="The connection being tested")
@@ -183,23 +230,23 @@ class LiveConnectionTestResult(BaseModel):
     )
     baseline_version: str | None = Field(
         default=None,
-        description="Version of the baseline connector (if regression ran)",
+        description="Version of the baseline connector (if comparison mode)",
     )
     evaluation_mode: str = Field(
         default="diagnostic",
         description="Evaluation mode used (diagnostic or strict)",
     )
-    skip_regression_tests: bool = Field(
+    compare_versions: bool = Field(
         default=False,
-        description="Whether regression tests were skipped by request",
+        description="Whether comparison mode was used (target vs control)",
     )
-    live_phase: LivePhaseResult | None = Field(
+    execution_result: RegressionTestExecutionResult | None = Field(
         default=None,
-        description="Results from the live test phase",
+        description="Results from executing the connector (validations and record counts)",
     )
-    regression_phase: RegressionPhaseResult | None = Field(
+    comparison_result: RegressionTestComparisonResult | None = Field(
         default=None,
-        description="Results from the regression test phase",
+        description="Results from comparing target vs control connector versions",
     )
     artifacts: dict[str, str] = Field(
         default_factory=dict,
@@ -223,8 +270,8 @@ class LiveConnectionTestResult(BaseModel):
     )
 
 
-class RunLiveConnectionTestsResponse(BaseModel):
-    """Response from starting a live connection test via GitHub Actions workflow."""
+class RunRegressionTestsResponse(BaseModel):
+    """Response from starting a regression test via GitHub Actions workflow."""
 
     run_id: str = Field(
         description="Unique identifier for the test run (internal tracking ID)"
@@ -255,66 +302,68 @@ class RunLiveConnectionTestsResponse(BaseModel):
     idempotent=False,
     open_world=True,
 )
-def run_live_connection_tests(
+def run_regression_tests(
     connection_id: Annotated[str, "The Airbyte Cloud connection ID to test"],
-    command: Annotated[
-        str,
-        "Airbyte command to run: 'spec', 'check', 'discover', or 'read'",
-    ] = "check",
+    skip_read_action: Annotated[
+        bool,
+        "If True, skip the read action (run only spec, check, discover). "
+        "If False (default), run all verbs including read.",
+    ] = False,
     workspace_id: Annotated[
         str | None,
         "Optional Airbyte Cloud workspace ID. If provided, validates that the connection "
         "belongs to this workspace before triggering tests. If omitted, no validation is done.",
     ] = None,
-    skip_regression_tests: Annotated[
+    skip_compare: Annotated[
         bool,
-        "If True, run only live tests (connector-live-test workflow). "
-        "If False, run regression tests comparing target vs control versions "
-        "(connector-regression-test workflow).",
-    ] = True,
+        "If True, skip comparison and run single-version tests only. "
+        "If False (default), run comparison tests (target vs control versions).",
+    ] = False,
     connector_image: Annotated[
         str | None,
-        "Optional connector image with tag for live tests (e.g., 'airbyte/source-github:1.0.0'). "
-        "If not provided, auto-detected from connection. Only used when skip_regression_tests=True.",
+        "Optional connector image with tag for single-version tests "
+        "(e.g., 'airbyte/source-github:1.0.0'). "
+        "If not provided, auto-detected from connection. Only used when skip_compare=True.",
     ] = None,
     target_image: Annotated[
         str | None,
-        "Target connector image (new version) with tag for regression tests "
+        "Target connector image (new version) with tag for comparison tests "
         "(e.g., 'airbyte/source-github:2.0.0'). Optional if connector_name is provided. "
-        "Only used when skip_regression_tests=False.",
+        "Only used when skip_compare=False (default).",
     ] = None,
     control_image: Annotated[
         str | None,
-        "Control connector image (baseline version) with tag for regression tests "
+        "Control connector image (baseline version) with tag for comparison tests "
         "(e.g., 'airbyte/source-github:1.0.0'). Optional if connection_id is provided "
-        "(auto-detected from connection). Only used when skip_regression_tests=False.",
+        "(auto-detected from connection). Only used when skip_compare=False (default).",
     ] = None,
     connector_name: Annotated[
         str | None,
         "Connector name to build the connector image from source "
         "(e.g., 'source-pokeapi'). If provided, builds the image locally with tag 'dev'. "
-        "For live tests, this builds the test image. For regression tests, this builds "
-        "the target image while control is auto-detected from the connection.",
+        "For comparison tests (default), this builds the target image while control is "
+        "auto-detected from the connection. For single-version tests (skip_compare=True), "
+        "this builds the test image.",
     ] = None,
-    airbyte_ref: Annotated[
-        str | None,
-        "Git ref or PR number to checkout from the airbyte monorepo "
-        "(e.g., 'master', '70847', 'refs/pull/70847/head'). "
-        "Only used when connector_name is provided. Defaults to 'master' if not specified.",
+    pr: Annotated[
+        int | None,
+        "PR number from the airbyte monorepo to checkout and build from "
+        "(e.g., 70847). Only used when connector_name is provided. "
+        "If not specified, builds from the default branch (master).",
     ] = None,
-) -> RunLiveConnectionTestsResponse:
-    """Start a live connection test run via GitHub Actions workflow.
-
-    This tool triggers either the live-test or regression-test workflow depending
-    on the skip_regression_tests parameter:
+) -> RunRegressionTestsResponse:
+    """Start a regression test run via GitHub Actions workflow.
 
-    - skip_regression_tests=True (default): Triggers connector-live-test workflow.
-      Runs the specified command against the connection and validates the output.
+    This tool triggers either the comparison or single-version regression test
+    workflow depending on the skip_compare parameter:
 
-    - skip_regression_tests=False: Triggers connector-regression-test workflow.
+    - skip_compare=False (default): Triggers comparison regression test workflow.
       Compares the target connector version against a control (baseline) version.
-      For regression tests, provide either target_image or connector_name to specify
-      the target version.
+      Provide either target_image or connector_name to specify the target version.
+
+    - skip_compare=True: Triggers single-version regression test workflow.
+      Runs the specified command against the connection and validates the output.
+      No comparison is performed.
 
     Returns immediately with a run_id and workflow URL. Check the workflow URL
     to monitor progress and view results.
@@ -329,7 +378,7 @@ def run_live_connection_tests(
     try:
         token = resolve_github_token()
     except ValueError as e:
-        return RunLiveConnectionTestsResponse(
+        return RunRegressionTestsResponse(
             run_id=run_id,
             status=TestRunStatus.FAILED,
             message=str(e),
@@ -338,122 +387,86 @@ def run_live_connection_tests(
 
     # Validate workspace membership if workspace_id is provided
     if workspace_id:
-        client_id = resolve_cloud_client_id()
-        client_secret = resolve_cloud_client_secret()
-        if not client_id or not client_secret:
-            return RunLiveConnectionTestsResponse(
-                run_id=run_id,
-                status=TestRunStatus.FAILED,
-                message=(
-                    "Missing Airbyte Cloud credentials. "
-                    "Set AIRBYTE_CLOUD_CLIENT_ID and AIRBYTE_CLOUD_CLIENT_SECRET env vars."
-                ),
-                workflow_url=None,
-            )
-        workspace = CloudWorkspace(
-            workspace_id=workspace_id,
-            client_id=client_id,
-            client_secret=client_secret,
-        )
-        connection = workspace.get_connection(connection_id)
-        if connection is None:
-            return RunLiveConnectionTestsResponse(
+        try:
+            validate_connection_workspace(connection_id, workspace_id)
+        except (
+            ValueError,
+            AirbyteWorkspaceMismatchError,
+            AirbyteMissingResourceError,
+        ) as e:
+            return RunRegressionTestsResponse(
                 run_id=run_id,
                 status=TestRunStatus.FAILED,
-                message=f"Connection {connection_id} not found in workspace {workspace_id}",
+                message=str(e),
                 workflow_url=None,
             )
 
-    if skip_regression_tests:
-        # Live test workflow
-        workflow_inputs: dict[str, str] = {
-            "connection_id": connection_id,
-            "command": command,
-        }
+    # Build workflow inputs for the unified regression test workflow
+    workflow_inputs: dict[str, str] = {
+        "connection_id": connection_id,
+    }
+
+    if skip_compare:
+        # Single-version mode
+        workflow_inputs["skip_compare"] = "true"
         if connector_image:
             workflow_inputs["connector_image"] = connector_image
-        if connector_name:
-            workflow_inputs["connector_name"] = connector_name
-
-        try:
-            dispatch_result = trigger_workflow_dispatch(
-                owner=LIVE_TEST_REPO_OWNER,
-                repo=LIVE_TEST_REPO_NAME,
-                workflow_file=LIVE_TEST_WORKFLOW_FILE,
-                ref=LIVE_TEST_DEFAULT_BRANCH,
-                inputs=workflow_inputs,
-                token=token,
-            )
-        except Exception as e:
-            logger.exception("Failed to trigger live test workflow")
-            return RunLiveConnectionTestsResponse(
+    else:
+        # Comparison mode (default): validate that we have enough info
+        if not target_image and not connector_name:
+            return RunRegressionTestsResponse(
                 run_id=run_id,
                 status=TestRunStatus.FAILED,
-                message=f"Failed to trigger live-test workflow: {e}",
+                message=(
+                    "For comparison regression tests (skip_compare=False, the default), "
+                    "provide either target_image or connector_name so the workflow can "
+                    "determine the target image."
+                ),
                 workflow_url=None,
             )
-
-        view_url = dispatch_result.run_url or dispatch_result.workflow_url
-        return RunLiveConnectionTestsResponse(
-            run_id=run_id,
-            status=TestRunStatus.QUEUED,
-            message=f"Live-test workflow triggered for connection {connection_id}. "
-            f"View progress at: {view_url}",
-            workflow_url=dispatch_result.workflow_url,
-            github_run_id=dispatch_result.run_id,
-            github_run_url=dispatch_result.run_url,
-        )
-
-    # Regression test workflow (skip_regression_tests=False)
-    # Validate that we have enough info to run regression tests
-    if not target_image and not connector_name:
-        return RunLiveConnectionTestsResponse(
-            run_id=run_id,
-            status=TestRunStatus.FAILED,
-            message=(
-                "For regression tests (skip_regression_tests=False), provide either "
-                "target_image or connector_name so the workflow can determine the target image."
-            ),
-            workflow_url=None,
-        )
-
-    workflow_inputs = {
-        "connection_id": connection_id,
-        "command": command,
-    }
-    if target_image:
-        workflow_inputs["target_image"] = target_image
-    if control_image:
-        workflow_inputs["control_image"] = control_image
+        workflow_inputs["skip_compare"] = "false"
+        if target_image:
+            workflow_inputs["target_image"] = target_image
+        if control_image:
+            workflow_inputs["control_image"] = control_image
+
+    # Common inputs for both modes
+    if skip_read_action:
+        workflow_inputs["skip_read_action"] = "true"
     if connector_name:
         workflow_inputs["connector_name"] = connector_name
-        if airbyte_ref:
-            workflow_inputs["airbyte_ref"] = airbyte_ref
+        if pr:
+            workflow_inputs["pr"] = str(pr)
 
+    mode_description = "single-version" if skip_compare else "comparison"
     try:
         dispatch_result = trigger_workflow_dispatch(
-            owner=LIVE_TEST_REPO_OWNER,
-            repo=LIVE_TEST_REPO_NAME,
+            owner=REGRESSION_TEST_REPO_OWNER,
+            repo=REGRESSION_TEST_REPO_NAME,
             workflow_file=REGRESSION_TEST_WORKFLOW_FILE,
-            ref=LIVE_TEST_DEFAULT_BRANCH,
+            ref=REGRESSION_TEST_DEFAULT_BRANCH,
             inputs=workflow_inputs,
             token=token,
         )
     except Exception as e:
-        logger.exception("Failed to trigger regression test workflow")
-        return RunLiveConnectionTestsResponse(
+        logger.exception(
+            f"Failed to trigger {mode_description} regression test workflow"
+        )
+        return RunRegressionTestsResponse(
             run_id=run_id,
             status=TestRunStatus.FAILED,
-            message=f"Failed to trigger regression-test workflow: {e}",
+            message=f"Failed to trigger {mode_description} regression test workflow: {e}",
             workflow_url=None,
         )
 
     view_url = dispatch_result.run_url or dispatch_result.workflow_url
-    return RunLiveConnectionTestsResponse(
+    return RunRegressionTestsResponse(
         run_id=run_id,
         status=TestRunStatus.QUEUED,
-        message=f"Regression-test workflow triggered for connection {connection_id}. "
-        f"View progress at: {view_url}",
+        message=(
+            f"{mode_description.capitalize()} regression test workflow triggered for "
+            f"connection {connection_id}. View progress at: {view_url}"
+        ),
         workflow_url=dispatch_result.workflow_url,
         github_run_id=dispatch_result.run_id,
         github_run_url=dispatch_result.run_url,
@@ -465,8 +478,8 @@ def run_live_connection_tests(
 # =============================================================================
 
 
-def register_live_tests_tools(app: FastMCP) -> None:
-    """Register live tests tools with the FastMCP app.
+def register_regression_tests_tools(app: FastMCP) -> None:
+    """Register regression tests tools with the FastMCP app.
 
     Args:
         app: FastMCP application instance

airbyte_ops_mcp/mcp/server.py

@@ -2,9 +2,18 @@
 """Airbyte Admin MCP server implementation.
 
 This module provides the main MCP server for Airbyte admin operations.
+
+The server can run in two modes:
+- **stdio mode** (default): For direct MCP client connections via stdin/stdout
+- **HTTP mode**: For HTTP-based MCP connections, useful for containerized deployments
+
+Environment Variables:
+    MCP_HTTP_HOST: Host to bind HTTP server to (default: 127.0.0.1)
+    MCP_HTTP_PORT: Port for HTTP server (default: 8082)
 """
 
 import asyncio
+import os
 import sys
 from pathlib import Path
 
@@ -17,12 +26,16 @@ from airbyte_ops_mcp.mcp.cloud_connector_versions import (
 )
 from airbyte_ops_mcp.mcp.github import register_github_tools
 from airbyte_ops_mcp.mcp.github_repo_ops import register_github_repo_ops_tools
-from airbyte_ops_mcp.mcp.live_tests import register_live_tests_tools
 from airbyte_ops_mcp.mcp.prerelease import register_prerelease_tools
 from airbyte_ops_mcp.mcp.prod_db_queries import register_prod_db_query_tools
 from airbyte_ops_mcp.mcp.prompts import register_prompts
+from airbyte_ops_mcp.mcp.regression_tests import register_regression_tests_tools
 from airbyte_ops_mcp.mcp.server_info import register_server_info_resources
 
+# Default HTTP server configuration
+DEFAULT_HTTP_HOST = "127.0.0.1"
+DEFAULT_HTTP_PORT = 8082
+
 app: FastMCP = FastMCP(MCP_SERVER_NAME)
 
 
@@ -34,7 +47,7 @@ def register_server_assets(app: FastMCP) -> None:
     - REPO: GitHub repository operations
     - CLOUD: Cloud connector version management
     - PROMPTS: Prompt templates for common workflows
-    - LIVE_TESTS: Live connection validation and regression tests
+    - REGRESSION_TESTS: Connector regression tests (single-version and comparison)
     - REGISTRY: Connector registry operations (future)
     - METADATA: Connector metadata operations (future)
     - QA: Connector quality assurance (future)
@@ -50,33 +63,93 @@ def register_server_assets(app: FastMCP) -> None:
     register_cloud_connector_version_tools(app)
     register_prod_db_query_tools(app)
     register_prompts(app)
-    register_live_tests_tools(app)
+    register_regression_tests_tools(app)
 
 
 register_server_assets(app)
 
 
-def main() -> None:
-    """Main entry point for the Airbyte Admin MCP server."""
-    # Load environment variables from .env file in current working directory
+def _load_env() -> None:
+    """Load environment variables from .env file if present."""
     env_file = Path.cwd() / ".env"
     if env_file.exists():
         load_dotenv(env_file)
         print(f"Loaded environment from: {env_file}", flush=True, file=sys.stderr)
 
+
+def main() -> None:
+    """Main entry point for the Airbyte Admin MCP server (stdio mode).
+
+    This is the default entry point that runs the server in stdio mode,
+    suitable for direct MCP client connections.
+    """
+    _load_env()
+
     print("=" * 60, flush=True, file=sys.stderr)
-    print("Starting Airbyte Admin MCP server.", file=sys.stderr)
+    print("Starting Airbyte Admin MCP server (stdio mode).", file=sys.stderr)
     try:
         asyncio.run(app.run_stdio_async(show_banner=False))
     except KeyboardInterrupt:
         print("Airbyte Admin MCP server interrupted by user.", file=sys.stderr)
-    except Exception as ex:
-        print(f"Error running Airbyte Admin MCP server: {ex}", file=sys.stderr)
-        sys.exit(1)
 
     print("Airbyte Admin MCP server stopped.", file=sys.stderr)
     print("=" * 60, flush=True, file=sys.stderr)
-    sys.exit(0)
+
+
+def _parse_port(port_str: str | None, default: int) -> int:
+    """Parse and validate a port number from string.
+
+    Args:
+        port_str: Port string from environment variable, or None if not set
+        default: Default port to use if port_str is None
+
+    Returns:
+        Validated port number
+
+    Raises:
+        ValueError: If port_str is not a valid integer or out of range
+    """
+    if port_str is None:
+        return default
+
+    port_str = port_str.strip()
+    if not port_str.isdecimal():
+        raise ValueError(f"MCP_HTTP_PORT must be a valid integer, got: {port_str!r}")
+
+    port = int(port_str)
+    if not 1 <= port <= 65535:
+        raise ValueError(f"MCP_HTTP_PORT must be between 1 and 65535, got: {port}")
+
+    return port
+
+
+def main_http() -> None:
+    """HTTP entry point for the Airbyte Admin MCP server.
+
+    This entry point runs the server in HTTP mode, suitable for containerized
+    deployments where the server needs to be accessible over HTTP.
+
+    Environment Variables:
+        MCP_HTTP_HOST: Host to bind to (default: 127.0.0.1)
+        MCP_HTTP_PORT: Port to listen on (default: 8082)
+    """
+    _load_env()
+
+    host = os.getenv("MCP_HTTP_HOST", DEFAULT_HTTP_HOST)
+    port = _parse_port(os.getenv("MCP_HTTP_PORT"), DEFAULT_HTTP_PORT)
+
+    print("=" * 60, flush=True, file=sys.stderr)
+    print(
+        f"Starting Airbyte Admin MCP server (HTTP mode) on {host}:{port}",
+        file=sys.stderr,
+    )
+    try:
+        app.run(transport="http", host=host, port=port)
+    except KeyboardInterrupt:
+        print("Airbyte Admin MCP server interrupted by user.", file=sys.stderr)
+
+    print("Airbyte Admin MCP server stopped.", file=sys.stderr)
+    print("=" * 60, flush=True, file=sys.stderr)
 
 
 if __name__ == "__main__":