airbyte-internal-ops 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

airbyte_ops_mcp/cli/cloud.py

@@ -5,8 +5,7 @@ Commands:
     airbyte-ops cloud connector get-version-info - Get connector version info
     airbyte-ops cloud connector set-version-override - Set connector version override
     airbyte-ops cloud connector clear-version-override - Clear connector version override
-    airbyte-ops cloud connector live-test - Run live validation tests on a connector
-    airbyte-ops cloud connector regression-test - Run regression tests comparing connector versions
+    airbyte-ops cloud connector regression-test - Run regression tests (single-version or comparison)
     airbyte-ops cloud connector fetch-connection-config - Fetch connection config to local file
 """
 
@@ -45,8 +44,12 @@ from airbyte_ops_mcp.constants import (
     DEFAULT_CLOUD_SQL_PROXY_PORT,
     ENV_GCP_PROD_DB_ACCESS_CREDENTIALS,
 )
-from airbyte_ops_mcp.live_tests.cdk_secrets import get_first_config_from_secrets
-from airbyte_ops_mcp.live_tests.ci_output import (
+from airbyte_ops_mcp.mcp.cloud_connector_versions import (
+    get_cloud_connector_version,
+    set_cloud_connector_version_override,
+)
+from airbyte_ops_mcp.regression_tests.cdk_secrets import get_first_config_from_secrets
+from airbyte_ops_mcp.regression_tests.ci_output import (
     generate_regression_report,
     get_report_summary,
     write_github_output,
@@ -55,32 +58,28 @@ from airbyte_ops_mcp.live_tests.ci_output import (
     write_json_output,
     write_test_summary,
 )
-from airbyte_ops_mcp.live_tests.connection_fetcher import (
+from airbyte_ops_mcp.regression_tests.connection_fetcher import (
     fetch_connection_data,
     save_connection_data_to_files,
 )
-from airbyte_ops_mcp.live_tests.connection_secret_retriever import (
+from airbyte_ops_mcp.regression_tests.connection_secret_retriever import (
     enrich_config_with_secrets,
     should_use_secret_retriever,
 )
-from airbyte_ops_mcp.live_tests.connector_runner import (
+from airbyte_ops_mcp.regression_tests.connector_runner import (
     ConnectorRunner,
     ensure_image_available,
 )
-from airbyte_ops_mcp.live_tests.http_metrics import (
+from airbyte_ops_mcp.regression_tests.http_metrics import (
     MitmproxyManager,
     parse_http_dump,
 )
-from airbyte_ops_mcp.live_tests.models import (
+from airbyte_ops_mcp.regression_tests.models import (
     Command,
     ConnectorUnderTest,
     ExecutionInputs,
     TargetOrControl,
 )
-from airbyte_ops_mcp.mcp.cloud_connector_versions import (
-    get_cloud_connector_version,
-    set_cloud_connector_version_override,
-)
 
 # Path to connectors directory within the airbyte repo
 CONNECTORS_SUBDIR = Path("airbyte-integrations") / "connectors"
@@ -596,187 +595,6 @@ def _fetch_control_image_from_metadata(connector_name: str) -> str | None:
     return f"{docker_repository}:{docker_image_tag}"
 
 
-@connector_app.command(name="live-test")
-def live_test(
-    connector_image: Annotated[
-        str | None,
-        Parameter(
-            help="Full connector image name with tag (e.g., airbyte/source-github:1.0.0). "
-            "Optional if connector_name or connection_id is provided."
-        ),
-    ] = None,
-    connector_name: Annotated[
-        str | None,
-        Parameter(
-            help="Connector name to build from source (e.g., 'source-pokeapi'). "
-            "If provided, builds the image locally with tag 'dev'."
-        ),
-    ] = None,
-    repo_root: Annotated[
-        str | None,
-        Parameter(
-            help="Path to the airbyte repo root. Required if connector_name is provided "
-            "and the repo cannot be auto-detected."
-        ),
-    ] = None,
-    command: Annotated[
-        Literal["spec", "check", "discover", "read"],
-        Parameter(help="The Airbyte command to run."),
-    ] = "check",
-    connection_id: Annotated[
-        str | None,
-        Parameter(
-            help="Airbyte Cloud connection ID to fetch config/catalog from. "
-            "Mutually exclusive with config-path/catalog-path. "
-            "If provided, connector_image can be auto-detected."
-        ),
-    ] = None,
-    config_path: Annotated[
-        str | None,
-        Parameter(help="Path to the connector config JSON file."),
-    ] = None,
-    catalog_path: Annotated[
-        str | None,
-        Parameter(help="Path to the configured catalog JSON file (required for read)."),
-    ] = None,
-    state_path: Annotated[
-        str | None,
-        Parameter(help="Path to the state JSON file (optional for read)."),
-    ] = None,
-    output_dir: Annotated[
-        str,
-        Parameter(help="Directory to store test artifacts."),
-    ] = "/tmp/live_test_artifacts",
-) -> None:
-    """Run live validation tests on a connector.
-
-    This command runs the specified Airbyte protocol command against a connector
-    and validates the output. Results are written to the output directory and
-    to GitHub Actions outputs if running in CI.
-
-    You can provide the connector image in three ways:
-    1. --connector-image: Use a pre-built image from Docker registry
-    2. --connector-name: Build the image locally from source code
-    3. --connection-id: Auto-detect from an Airbyte Cloud connection
-
-    You can provide config/catalog either via file paths OR via a connection_id
-    that fetches them from Airbyte Cloud.
-    """
-    output_path = Path(output_dir)
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    cmd = Command(command)
-
-    config_file: Path | None = None
-    catalog_file: Path | None = None
-    state_file = Path(state_path) if state_path else None
-    resolved_connector_image: str | None = connector_image
-
-    # If connector_name is provided, build the image from source
-    if connector_name:
-        if connector_image:
-            write_github_output("success", False)
-            write_github_output(
-                "error", "Cannot specify both connector_image and connector_name"
-            )
-            exit_with_error("Cannot specify both connector_image and connector_name")
-
-        repo_root_path = Path(repo_root) if repo_root else None
-        built_image = _build_connector_image_from_source(
-            connector_name=connector_name,
-            repo_root=repo_root_path,
-            tag="dev",
-        )
-        if not built_image:
-            write_github_output("success", False)
-            write_github_output("error", f"Failed to build image for {connector_name}")
-            exit_with_error(f"Failed to build image for {connector_name}")
-        resolved_connector_image = built_image
-
-    if connection_id:
-        if config_path or catalog_path:
-            write_github_output("success", False)
-            write_github_output(
-                "error", "Cannot specify both connection_id and file paths"
-            )
-            exit_with_error(
-                "Cannot specify both connection_id and config_path/catalog_path"
-            )
-
-        print_success(f"Fetching config/catalog from connection: {connection_id}")
-        connection_data = fetch_connection_data(connection_id)
-        config_file, catalog_file = save_connection_data_to_files(
-            connection_data, output_path / "connection_data"
-        )
-        print_success(
-            f"Fetched config for source: {connection_data.source_name} "
-            f"with {len(connection_data.stream_names)} streams"
-        )
-
-        if not resolved_connector_image and connection_data.connector_image:
-            resolved_connector_image = connection_data.connector_image
-            print_success(f"Auto-detected connector image: {resolved_connector_image}")
-    else:
-        config_file = Path(config_path) if config_path else None
-        catalog_file = Path(catalog_path) if catalog_path else None
-
-    if not resolved_connector_image:
-        write_github_output("success", False)
-        write_github_output("error", "Missing connector image")
-        exit_with_error(
-            "You must provide one of the following: a connector_image, a connector_name, "
-            "or a connection_id for a connection that has an associated connector image. "
-            "If using connection_id, ensure the connection has a connector image configured."
-        )
-
-    # If connector_name was provided, we just built the image locally and it is already
-    # available in Docker, so we skip the image availability check/pull. Only try to pull
-    # if we didn't just build it (i.e., using a pre-built image from registry).
-    if not connector_name and not ensure_image_available(resolved_connector_image):
-        write_github_output("success", False)
-        write_github_output(
-            "error", f"Failed to pull image: {resolved_connector_image}"
-        )
-        exit_with_error(f"Failed to pull connector image: {resolved_connector_image}")
-
-    result = _run_connector_command(
-        connector_image=resolved_connector_image,
-        command=cmd,
-        output_dir=output_path,
-        target_or_control=TargetOrControl.TARGET,
-        config_path=config_file,
-        catalog_path=catalog_file,
-        state_path=state_file,
-    )
-
-    print_json(result)
-
-    write_github_outputs(
-        {
-            "success": result["success"],
-            "connector": resolved_connector_image,
-            "command": command,
-            "exit_code": result["exit_code"],
-        }
-    )
-
-    write_test_summary(
-        connector_image=resolved_connector_image,
-        test_type="live-test",
-        success=result["success"],
-        results={
-            "command": command,
-            "exit_code": result["exit_code"],
-            "output_dir": output_dir,
-        },
-    )
-
-    if result["success"]:
-        print_success(f"Live test passed for {resolved_connector_image}")
-    else:
-        exit_with_error(f"Live test failed for {resolved_connector_image}")
-
-
 def _run_with_optional_http_metrics(
     connector_image: str,
     command: Command,
@@ -855,25 +673,34 @@ def _run_with_optional_http_metrics(
 
 @connector_app.command(name="regression-test")
 def regression_test(
-    target_image: Annotated[
+    skip_compare: Annotated[
+        bool,
+        Parameter(
+            help="If True, skip comparison and run single-version tests only. "
+            "If False (default), run comparison tests (target vs control)."
+        ),
+    ] = False,
+    test_image: Annotated[
         str | None,
         Parameter(
-            help="Target connector image (new version) with tag (e.g., airbyte/source-github:2.0.0). "
-            "Optional if connector_name is provided."
+            help="Test connector image with tag (e.g., airbyte/source-github:1.0.0). "
+            "This is the image under test - in comparison mode, it's compared against control_image."
         ),
     ] = None,
     control_image: Annotated[
         str | None,
         Parameter(
             help="Control connector image (baseline version) with tag (e.g., airbyte/source-github:1.0.0). "
-            "Optional if connection_id is provided (auto-detected from connection)."
+            "Ignored if `skip_compare=True`."
         ),
     ] = None,
     connector_name: Annotated[
         str | None,
         Parameter(
-            help="Connector name to build target image from source (e.g., 'source-pokeapi'). "
-            "If provided, builds the target image locally with tag 'dev'."
+            help="Connector name to build image from source (e.g., 'source-pokeapi'). "
+            "If provided, builds the image locally with tag 'dev'. "
+            "For comparison tests (default), this builds the target image. "
+            "For single-version tests (skip_compare=True), this builds the test image."
         ),
     ] = None,
     repo_root: Annotated[
@@ -892,7 +719,7 @@ def regression_test(
         Parameter(
             help="Airbyte Cloud connection ID to fetch config/catalog from. "
             "Mutually exclusive with config-path/catalog-path. "
-            "If provided, control_image can be auto-detected."
+            "If provided, test_image/control_image can be auto-detected."
         ),
     ] = None,
     config_path: Annotated[
@@ -915,26 +742,30 @@ def regression_test(
         bool,
         Parameter(
             help="Capture HTTP traffic metrics via mitmproxy (experimental). "
-            "Requires mitmdump to be installed."
+            "Requires mitmdump to be installed. Only used in comparison mode."
         ),
     ] = False,
 ) -> None:
-    """Run regression tests comparing two connector versions.
+    """Run regression tests on connectors.
+
+    This command supports two modes:
 
-    This command runs the specified Airbyte protocol command against both the
-    target (new) and control (baseline) connector versions, then compares the
-    results. This helps identify regressions between versions.
+    Comparison mode (skip_compare=False, default):
+        Runs the specified Airbyte protocol command against both the target (new)
+        and control (baseline) connector versions, then compares the results.
+        This helps identify regressions between versions.
+
+    Single-version mode (skip_compare=True):
+        Runs the specified Airbyte protocol command against a single connector
+        and validates the output. No comparison is performed.
 
     Results are written to the output directory and to GitHub Actions outputs
     if running in CI.
 
-    You can provide the target image in two ways:
-    1. --target-image: Use a pre-built image from Docker registry
-    2. --connector-name: Build the target image locally from source code
-
-    You can provide the control image in two ways:
-    1. --control-image: Use a pre-built image from Docker registry
-    2. --connection-id: Auto-detect from an Airbyte Cloud connection
+    You can provide the test image in three ways:
+    1. --test-image: Use a pre-built image from Docker registry
+    2. --connector-name: Build the image locally from source code
+    3. --connection-id: Auto-detect from an Airbyte Cloud connection
 
     You can provide config/catalog either via file paths OR via a connection_id
     that fetches them from Airbyte Cloud.
@@ -947,17 +778,31 @@ def regression_test(
     config_file: Path | None = None
     catalog_file: Path | None = None
     state_file = Path(state_path) if state_path else None
-    resolved_target_image: str | None = target_image
+
+    # Resolve the test image (used in both single-version and comparison modes)
+    resolved_test_image: str | None = test_image
     resolved_control_image: str | None = control_image
 
-    # If connector_name is provided, build the target image from source
+    # Validate conflicting parameters
+    # Single-version mode: reject comparison-specific parameters
+    if skip_compare and control_image:
+        write_github_output("success", False)
+        write_github_output(
+            "error", "Cannot specify control_image with skip_compare=True"
+        )
+        exit_with_error(
+            "Cannot specify --control-image with --skip-compare. "
+            "Control image is only used in comparison mode."
+        )
+
+    # If connector_name is provided, build the image from source
     if connector_name:
-        if target_image:
+        if resolved_test_image:
             write_github_output("success", False)
             write_github_output(
-                "error", "Cannot specify both target_image and connector_name"
+                "error", "Cannot specify both test_image and connector_name"
             )
-            exit_with_error("Cannot specify both target_image and connector_name")
+            exit_with_error("Cannot specify both --test-image and --connector-name")
 
         repo_root_path = Path(repo_root) if repo_root else None
         built_image = _build_connector_image_from_source(
@@ -969,7 +814,7 @@ def regression_test(
             write_github_output("success", False)
             write_github_output("error", f"Failed to build image for {connector_name}")
             exit_with_error(f"Failed to build image for {connector_name}")
-        resolved_target_image = built_image
+        resolved_test_image = built_image
 
     if connection_id:
         if config_path or catalog_path:
@@ -997,10 +842,11 @@ def regression_test(
                 print_success("Successfully retrieved unmasked secrets from database")
             except Exception as e:
                 print_error(f"Failed to retrieve unmasked secrets: {e}")
-                print_error(
-                    "Proceeding with masked config from public API - tests may fail due to masked credentials. "
-                    "If you expected unmasked secrets, verify that the USE_CONNECTION_SECRET_RETRIEVER flag is enabled "
-                    f"and that the {ENV_GCP_PROD_DB_ACCESS_CREDENTIALS} environment variable is set with valid database credentials."
+                exit_with_error(
+                    f"Failed to retrieve unmasked secrets: {e}\n"
+                    f"Unset USE_CONNECTION_SECRET_RETRIEVER or verify that the "
+                    f"{ENV_GCP_PROD_DB_ACCESS_CREDENTIALS} environment variable is set "
+                    f"with valid database credentials and that the Cloud SQL Proxy is running."
                 )
 
         config_file, catalog_file = save_connection_data_to_files(
@@ -1011,8 +857,16 @@ def regression_test(
             f"with {len(connection_data.stream_names)} streams"
         )
 
-        # Auto-detect control_image from connection if not provided
-        if not resolved_control_image and connection_data.connector_image:
+        # Auto-detect test/control image from connection if not provided
+        if not resolved_test_image and connection_data.connector_image:
+            resolved_test_image = connection_data.connector_image
+            print_success(f"Auto-detected test image: {resolved_test_image}")
+
+        if (
+            not skip_compare
+            and not resolved_control_image
+            and connection_data.connector_image
+        ):
             resolved_control_image = connection_data.connector_image
             print_success(f"Auto-detected control image: {resolved_control_image}")
     elif config_path:
@@ -1049,24 +903,24 @@ def regression_test(
         config_file = None
         catalog_file = Path(catalog_path) if catalog_path else None
 
-    # Auto-detect control_image from metadata.yaml if connector_name is provided
-    if not resolved_control_image and connector_name:
+    # Auto-detect control_image from metadata.yaml if connector_name is provided (comparison mode only)
+    if not skip_compare and not resolved_control_image and connector_name:
         resolved_control_image = _fetch_control_image_from_metadata(connector_name)
         if resolved_control_image:
             print_success(
                 f"Auto-detected control image from metadata.yaml: {resolved_control_image}"
             )
 
-    # Validate that we have both images
-    if not resolved_target_image:
+    # Validate that we have the required images
+    if not resolved_test_image:
         write_github_output("success", False)
-        write_github_output("error", "No target image specified")
+        write_github_output("error", "No test image specified")
         exit_with_error(
-            "You must provide one of the following: a target_image or a connector_name "
-            "to build the target image from source."
+            "You must provide one of the following: a test_image, a connector_name "
+            "to build the image from source, or a connection_id to auto-detect the image."
         )
 
-    if not resolved_control_image:
+    if not skip_compare and not resolved_control_image:
         write_github_output("success", False)
         write_github_output("error", "No control image specified")
         exit_with_error(
@@ -1076,97 +930,144 @@ def regression_test(
         )
 
     # Pull images if they weren't just built locally
-    # If connector_name was provided, we just built the target image locally
-    if not connector_name and not ensure_image_available(resolved_target_image):
+    # If connector_name was provided, we just built the test image locally
+    if not connector_name and not ensure_image_available(resolved_test_image):
         write_github_output("success", False)
-        write_github_output("error", f"Failed to pull image: {resolved_target_image}")
-        exit_with_error(
-            f"Failed to pull target connector image: {resolved_target_image}"
-        )
-
-    if not ensure_image_available(resolved_control_image):
+        write_github_output("error", f"Failed to pull image: {resolved_test_image}")
+        exit_with_error(f"Failed to pull test image: {resolved_test_image}")
+
+    if (
+        not skip_compare
+        and resolved_control_image
+        and not ensure_image_available(resolved_control_image)
+    ):
         write_github_output("success", False)
         write_github_output("error", f"Failed to pull image: {resolved_control_image}")
         exit_with_error(
             f"Failed to pull control connector image: {resolved_control_image}"
         )
 
-    target_output = output_path / "target"
-    control_output = output_path / "control"
-
-    target_result = _run_with_optional_http_metrics(
-        connector_image=resolved_target_image,
-        command=cmd,
-        output_dir=target_output,
-        target_or_control=TargetOrControl.TARGET,
-        enable_http_metrics=enable_http_metrics,
-        config_path=config_file,
-        catalog_path=catalog_file,
-        state_path=state_file,
-    )
+    # Execute the appropriate mode
+    if skip_compare:
+        # Single-version mode: run only the connector image
+        result = _run_connector_command(
+            connector_image=resolved_test_image,
+            command=cmd,
+            output_dir=output_path,
+            target_or_control=TargetOrControl.TARGET,
+            config_path=config_file,
+            catalog_path=catalog_file,
+            state_path=state_file,
+        )
 
-    control_result = _run_with_optional_http_metrics(
-        connector_image=resolved_control_image,
-        command=cmd,
-        output_dir=control_output,
-        target_or_control=TargetOrControl.CONTROL,
-        enable_http_metrics=enable_http_metrics,
-        config_path=config_file,
-        catalog_path=catalog_file,
-        state_path=state_file,
-    )
+        print_json(result)
 
-    both_succeeded = target_result["success"] and control_result["success"]
-    regression_detected = target_result["success"] != control_result["success"]
+        write_github_outputs(
+            {
+                "success": result["success"],
+                "connector": resolved_test_image,
+                "command": command,
+                "exit_code": result["exit_code"],
+            }
+        )
 
-    combined_result = {
-        "target": target_result,
-        "control": control_result,
-        "both_succeeded": both_succeeded,
-        "regression_detected": regression_detected,
-    }
+        write_test_summary(
+            connector_image=resolved_test_image,
+            test_type="regression-test",
+            success=result["success"],
+            results={
+                "command": command,
+                "exit_code": result["exit_code"],
+                "output_dir": output_dir,
+            },
+        )
+
+        if result["success"]:
+            print_success(
+                f"Single-version regression test passed for {resolved_test_image}"
+            )
+        else:
+            exit_with_error(
+                f"Single-version regression test failed for {resolved_test_image}"
+            )
+    else:
+        # Comparison mode: run both target and control images
+        target_output = output_path / "target"
+        control_output = output_path / "control"
+
+        target_result = _run_with_optional_http_metrics(
+            connector_image=resolved_test_image,
+            command=cmd,
+            output_dir=target_output,
+            target_or_control=TargetOrControl.TARGET,
+            enable_http_metrics=enable_http_metrics,
+            config_path=config_file,
+            catalog_path=catalog_file,
+            state_path=state_file,
+        )
 
-    print_json(combined_result)
+        control_result = _run_with_optional_http_metrics(
+            connector_image=resolved_control_image,  # type: ignore[arg-type]
+            command=cmd,
+            output_dir=control_output,
+            target_or_control=TargetOrControl.CONTROL,
+            enable_http_metrics=enable_http_metrics,
+            config_path=config_file,
+            catalog_path=catalog_file,
+            state_path=state_file,
+        )
+
+        both_succeeded = target_result["success"] and control_result["success"]
+        regression_detected = target_result["success"] != control_result["success"]
 
-    write_github_outputs(
-        {
-            "success": both_succeeded and not regression_detected,
-            "target_image": resolved_target_image,
-            "control_image": resolved_control_image,
-            "command": command,
-            "target_exit_code": target_result["exit_code"],
-            "control_exit_code": control_result["exit_code"],
+        combined_result = {
+            "target": target_result,
+            "control": control_result,
+            "both_succeeded": both_succeeded,
             "regression_detected": regression_detected,
         }
-    )
 
-    write_json_output("regression_report", combined_result)
-
-    report_path = generate_regression_report(
-        target_image=resolved_target_image,
-        control_image=resolved_control_image,
-        command=command,
-        target_result=target_result,
-        control_result=control_result,
-        output_dir=output_path,
-    )
-    print_success(f"Generated regression report: {report_path}")
+        print_json(combined_result)
+
+        write_github_outputs(
+            {
+                "success": both_succeeded and not regression_detected,
+                "target_image": resolved_test_image,
+                "control_image": resolved_control_image,
+                "command": command,
+                "target_exit_code": target_result["exit_code"],
+                "control_exit_code": control_result["exit_code"],
+                "regression_detected": regression_detected,
+            }
+        )
 
-    summary = get_report_summary(report_path)
-    write_github_summary(summary)
+        write_json_output("regression_report", combined_result)
 
-    if regression_detected:
-        exit_with_error(
-            f"Regression detected between {resolved_target_image} and {resolved_control_image}"
-        )
-    elif both_succeeded:
-        print_success(
-            f"Regression test passed for {resolved_target_image} vs {resolved_control_image}"
-        )
-    else:
-        exit_with_error(
-            f"Both versions failed for {resolved_target_image} vs {resolved_control_image}"
+        report_path = generate_regression_report(
+            target_image=resolved_test_image,
+            control_image=resolved_control_image,  # type: ignore[arg-type]
+            command=command,
+            target_result=target_result,
+            control_result=control_result,
+            output_dir=output_path,
         )
+        print_success(f"Generated regression report: {report_path}")
+
+        summary = get_report_summary(report_path)
+        write_github_summary(summary)
+
+        if regression_detected:
+            exit_with_error(
+                f"Regression detected between {resolved_test_image} and {resolved_control_image}"
+            )
+        elif both_succeeded:
+            print_success(
+                f"Regression test passed for {resolved_test_image} vs {resolved_control_image}"
+            )
+        else:
+            exit_with_error(
+                f"Both versions failed for {resolved_test_image} vs {resolved_control_image}"
+            )
 
 
 @connector_app.command(name="fetch-connection-config")