airbyte-internal-ops 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

airbyte_ops_mcp/cli/cloud.py

@@ -7,6 +7,7 @@ Commands:
     airbyte-ops cloud connector clear-version-override - Clear connector version override
     airbyte-ops cloud connector live-test - Run live validation tests on a connector
     airbyte-ops cloud connector regression-test - Run regression tests comparing connector versions
+    airbyte-ops cloud connector fetch-connection-config - Fetch connection config to local file
 """
 
 from __future__ import annotations
@@ -15,11 +16,20 @@ import json
 from pathlib import Path
 from typing import Annotated, Literal
 
+from airbyte_cdk.models.connector_metadata import MetadataFile
+from airbyte_cdk.utils.connector_paths import find_connector_root_from_name
+from airbyte_cdk.utils.docker import build_connector_image, verify_docker_installation
 from airbyte_protocol.models import ConfiguredAirbyteCatalog
 from cyclopts import App, Parameter
 
 from airbyte_ops_mcp.cli._base import app
-from airbyte_ops_mcp.cli._shared import print_error, print_json, print_success
+from airbyte_ops_mcp.cli._shared import (
+    exit_with_error,
+    print_error,
+    print_json,
+    print_success,
+)
+from airbyte_ops_mcp.cloud_admin.connection_config import fetch_connection_config
 from airbyte_ops_mcp.live_tests.ci_output import (
     generate_regression_report,
     get_report_summary,
@@ -52,6 +62,9 @@ from airbyte_ops_mcp.mcp.cloud_connector_versions import (
     set_cloud_connector_version_override,
 )
 
+# Path to connectors directory within the airbyte repo
+CONNECTORS_SUBDIR = Path("airbyte-integrations") / "connectors"
+
 # Create the cloud sub-app
 cloud_app = App(name="cloud", help="Airbyte Cloud operations.")
 app.command(cloud_app)
@@ -250,14 +263,83 @@ def _run_connector_command(
     }
 
 
+def _build_connector_image_from_source(
+    connector_name: str,
+    repo_root: Path | None = None,
+    tag: str = "dev",
+) -> str | None:
+    """Build a connector image from source code.
+
+    Args:
+        connector_name: Name of the connector (e.g., 'source-pokeapi').
+        repo_root: Optional path to the airbyte repo root. If not provided,
+            will attempt to auto-detect from current directory.
+        tag: Tag to apply to the built image (default: 'dev').
+
+    Returns:
+        The full image name with tag if successful, None if build fails.
+    """
+    if not verify_docker_installation():
+        print_error("Docker is not installed or not running")
+        return None
+
+    try:
+        connector_directory = find_connector_root_from_name(connector_name)
+    except FileNotFoundError:
+        if repo_root:
+            connector_directory = repo_root / CONNECTORS_SUBDIR / connector_name
+            if not connector_directory.exists():
+                print_error(f"Connector directory not found: {connector_directory}")
+                return None
+        else:
+            print_error(
+                f"Could not find connector '{connector_name}'. "
+                "Try providing --repo-root to specify the airbyte repo location."
+            )
+            return None
+
+    metadata_file_path = connector_directory / "metadata.yaml"
+    if not metadata_file_path.exists():
+        print_error(f"metadata.yaml not found at {metadata_file_path}")
+        return None
+
+    metadata = MetadataFile.from_file(metadata_file_path)
+    print_success(f"Building image for connector: {connector_name}")
+
+    built_image = build_connector_image(
+        connector_name=connector_name,
+        connector_directory=connector_directory,
+        metadata=metadata,
+        tag=tag,
+        no_verify=False,
+    )
+    print_success(f"Successfully built image: {built_image}")
+    return built_image
+
+
 @connector_app.command(name="live-test")
 def live_test(
     connector_image: Annotated[
-        str,
+        str | None,
         Parameter(
-            help="Full connector image name with tag (e.g., airbyte/source-github:1.0.0)."
+            help="Full connector image name with tag (e.g., airbyte/source-github:1.0.0). "
+            "Optional if connector_name or connection_id is provided."
         ),
-    ],
+    ] = None,
+    connector_name: Annotated[
+        str | None,
+        Parameter(
+            help="Connector name to build from source (e.g., 'source-pokeapi'). "
+            "If provided, builds the image locally with tag 'dev'."
+        ),
+    ] = None,
+    repo_root: Annotated[
+        str | None,
+        Parameter(
+            help="Path to the airbyte repo root. Required if connector_name is provided "
+            "and the repo cannot be auto-detected."
+        ),
+    ] = None,
     command: Annotated[
         Literal["spec", "check", "discover", "read"],
         Parameter(help="The Airbyte command to run."),
@@ -266,7 +348,8 @@ def live_test(
         str | None,
         Parameter(
             help="Airbyte Cloud connection ID to fetch config/catalog from. "
-            "Mutually exclusive with config-path/catalog-path."
+            "Mutually exclusive with config-path/catalog-path. "
+            "If provided, connector_image can be auto-detected."
         ),
     ] = None,
     config_path: Annotated[
@@ -292,6 +375,11 @@ def live_test(
     and validates the output. Results are written to the output directory and
     to GitHub Actions outputs if running in CI.
 
+    You can provide the connector image in three ways:
+    1. --connector-image: Use a pre-built image from Docker registry
+    2. --connector-name: Build the image locally from source code
+    3. --connection-id: Auto-detect from an Airbyte Cloud connection
+
     You can provide config/catalog either via file paths OR via a connection_id
     that fetches them from Airbyte Cloud.
     """
@@ -300,26 +388,41 @@ def live_test(
 
     cmd = Command(command)
 
-    if not ensure_image_available(connector_image):
-        print_error(f"Failed to pull connector image: {connector_image}")
-        write_github_output("success", False)
-        write_github_output("error", f"Failed to pull image: {connector_image}")
-        return
-
     config_file: Path | None = None
     catalog_file: Path | None = None
     state_file = Path(state_path) if state_path else None
+    resolved_connector_image: str | None = connector_image
+
+    # If connector_name is provided, build the image from source
+    if connector_name:
+        if connector_image:
+            write_github_output("success", False)
+            write_github_output(
+                "error", "Cannot specify both connector_image and connector_name"
+            )
+            exit_with_error("Cannot specify both connector_image and connector_name")
+
+        repo_root_path = Path(repo_root) if repo_root else None
+        built_image = _build_connector_image_from_source(
+            connector_name=connector_name,
+            repo_root=repo_root_path,
+            tag="dev",
+        )
+        if not built_image:
+            write_github_output("success", False)
+            write_github_output("error", f"Failed to build image for {connector_name}")
+            exit_with_error(f"Failed to build image for {connector_name}")
+        resolved_connector_image = built_image
 
     if connection_id:
         if config_path or catalog_path:
-            print_error(
-                "Cannot specify both connection_id and config_path/catalog_path"
-            )
             write_github_output("success", False)
             write_github_output(
                 "error", "Cannot specify both connection_id and file paths"
             )
-            return
+            exit_with_error(
+                "Cannot specify both connection_id and config_path/catalog_path"
+            )
 
         print_success(f"Fetching config/catalog from connection: {connection_id}")
         connection_data = fetch_connection_data(connection_id)
@@ -330,12 +433,35 @@ def live_test(
             f"Fetched config for source: {connection_data.source_name} "
             f"with {len(connection_data.stream_names)} streams"
         )
+
+        if not resolved_connector_image and connection_data.connector_image:
+            resolved_connector_image = connection_data.connector_image
+            print_success(f"Auto-detected connector image: {resolved_connector_image}")
     else:
         config_file = Path(config_path) if config_path else None
         catalog_file = Path(catalog_path) if catalog_path else None
 
+    if not resolved_connector_image:
+        write_github_output("success", False)
+        write_github_output("error", "Missing connector image")
+        exit_with_error(
+            "You must provide one of the following: a connector_image, a connector_name, "
+            "or a connection_id for a connection that has an associated connector image. "
+            "If using connection_id, ensure the connection has a connector image configured."
+        )
+
+    # If connector_name was provided, we just built the image locally and it is already
+    # available in Docker, so we skip the image availability check/pull. Only try to pull
+    # if we didn't just build it (i.e., using a pre-built image from registry).
+    if not connector_name and not ensure_image_available(resolved_connector_image):
+        write_github_output("success", False)
+        write_github_output(
+            "error", f"Failed to pull image: {resolved_connector_image}"
+        )
+        exit_with_error(f"Failed to pull connector image: {resolved_connector_image}")
+
     result = _run_connector_command(
-        connector_image=connector_image,
+        connector_image=resolved_connector_image,
         command=cmd,
         output_dir=output_path,
         target_or_control=TargetOrControl.TARGET,
@@ -349,14 +475,14 @@ def live_test(
     write_github_outputs(
         {
             "success": result["success"],
-            "connector": connector_image,
+            "connector": resolved_connector_image,
             "command": command,
             "exit_code": result["exit_code"],
         }
     )
 
     write_test_summary(
-        connector_image=connector_image,
+        connector_image=resolved_connector_image,
         test_type="live-test",
         success=result["success"],
         results={
@@ -367,9 +493,9 @@ def live_test(
     )
 
     if result["success"]:
-        print_success(f"Live test passed for {connector_image}")
+        print_success(f"Live test passed for {resolved_connector_image}")
     else:
-        print_error(f"Live test failed for {connector_image}")
+        exit_with_error(f"Live test failed for {resolved_connector_image}")
 
 
 def _run_with_optional_http_metrics(
@@ -451,17 +577,33 @@ def _run_with_optional_http_metrics(
 @connector_app.command(name="regression-test")
 def regression_test(
     target_image: Annotated[
-        str,
+        str | None,
         Parameter(
-            help="Target connector image (new version) with tag (e.g., airbyte/source-github:2.0.0)."
+            help="Target connector image (new version) with tag (e.g., airbyte/source-github:2.0.0). "
+            "Optional if connector_name is provided."
         ),
-    ],
+    ] = None,
     control_image: Annotated[
-        str,
+        str | None,
         Parameter(
-            help="Control connector image (baseline version) with tag (e.g., airbyte/source-github:1.0.0)."
+            help="Control connector image (baseline version) with tag (e.g., airbyte/source-github:1.0.0). "
+            "Optional if connection_id is provided (auto-detected from connection)."
         ),
-    ],
+    ] = None,
+    connector_name: Annotated[
+        str | None,
+        Parameter(
+            help="Connector name to build target image from source (e.g., 'source-pokeapi'). "
+            "If provided, builds the target image locally with tag 'dev'."
+        ),
+    ] = None,
+    repo_root: Annotated[
+        str | None,
+        Parameter(
+            help="Path to the airbyte repo root. Required if connector_name is provided "
+            "and the repo cannot be auto-detected."
+        ),
+    ] = None,
     command: Annotated[
         Literal["spec", "check", "discover", "read"],
         Parameter(help="The Airbyte command to run."),
@@ -470,7 +612,8 @@ def regression_test(
         str | None,
         Parameter(
             help="Airbyte Cloud connection ID to fetch config/catalog from. "
-            "Mutually exclusive with config-path/catalog-path."
+            "Mutually exclusive with config-path/catalog-path. "
+            "If provided, control_image can be auto-detected."
         ),
     ] = None,
     config_path: Annotated[
@@ -506,6 +649,14 @@ def regression_test(
     Results are written to the output directory and to GitHub Actions outputs
     if running in CI.
 
+    You can provide the target image in two ways:
+    1. --target-image: Use a pre-built image from Docker registry
+    2. --connector-name: Build the target image locally from source code
+
+    You can provide the control image in two ways:
+    1. --control-image: Use a pre-built image from Docker registry
+    2. --connection-id: Auto-detect from an Airbyte Cloud connection
+
     You can provide config/catalog either via file paths OR via a connection_id
     that fetches them from Airbyte Cloud.
     """
@@ -514,27 +665,42 @@ def regression_test(
 
     cmd = Command(command)
 
-    for image in [target_image, control_image]:
-        if not ensure_image_available(image):
-            print_error(f"Failed to pull connector image: {image}")
-            write_github_output("success", False)
-            write_github_output("error", f"Failed to pull image: {image}")
-            return
-
     config_file: Path | None = None
     catalog_file: Path | None = None
     state_file = Path(state_path) if state_path else None
+    resolved_target_image: str | None = target_image
+    resolved_control_image: str | None = control_image
+
+    # If connector_name is provided, build the target image from source
+    if connector_name:
+        if target_image:
+            write_github_output("success", False)
+            write_github_output(
+                "error", "Cannot specify both target_image and connector_name"
+            )
+            exit_with_error("Cannot specify both target_image and connector_name")
+
+        repo_root_path = Path(repo_root) if repo_root else None
+        built_image = _build_connector_image_from_source(
+            connector_name=connector_name,
+            repo_root=repo_root_path,
+            tag="dev",
+        )
+        if not built_image:
+            write_github_output("success", False)
+            write_github_output("error", f"Failed to build image for {connector_name}")
+            exit_with_error(f"Failed to build image for {connector_name}")
+        resolved_target_image = built_image
 
     if connection_id:
         if config_path or catalog_path:
-            print_error(
-                "Cannot specify both connection_id and config_path/catalog_path"
-            )
             write_github_output("success", False)
             write_github_output(
                 "error", "Cannot specify both connection_id and file paths"
             )
-            return
+            exit_with_error(
+                "Cannot specify both connection_id and config_path/catalog_path"
+            )
 
         print_success(f"Fetching config/catalog from connection: {connection_id}")
         connection_data = fetch_connection_data(connection_id)
@@ -545,15 +711,53 @@ def regression_test(
             f"Fetched config for source: {connection_data.source_name} "
             f"with {len(connection_data.stream_names)} streams"
         )
+
+        # Auto-detect control_image from connection if not provided
+        if not resolved_control_image and connection_data.connector_image:
+            resolved_control_image = connection_data.connector_image
+            print_success(f"Auto-detected control image: {resolved_control_image}")
     else:
         config_file = Path(config_path) if config_path else None
         catalog_file = Path(catalog_path) if catalog_path else None
 
+    # Validate that we have both images
+    if not resolved_target_image:
+        write_github_output("success", False)
+        write_github_output("error", "No target image specified")
+        exit_with_error(
+            "You must provide one of the following: a target_image or a connector_name "
+            "to build the target image from source."
+        )
+
+    if not resolved_control_image:
+        write_github_output("success", False)
+        write_github_output("error", "No control image specified")
+        exit_with_error(
+            "You must provide one of the following: a control_image or a connection_id "
+            "for a connection that has an associated connector image."
+        )
+
+    # Pull images if they weren't just built locally
+    # If connector_name was provided, we just built the target image locally
+    if not connector_name and not ensure_image_available(resolved_target_image):
+        write_github_output("success", False)
+        write_github_output("error", f"Failed to pull image: {resolved_target_image}")
+        exit_with_error(
+            f"Failed to pull target connector image: {resolved_target_image}"
+        )
+
+    if not ensure_image_available(resolved_control_image):
+        write_github_output("success", False)
+        write_github_output("error", f"Failed to pull image: {resolved_control_image}")
+        exit_with_error(
+            f"Failed to pull control connector image: {resolved_control_image}"
+        )
+
     target_output = output_path / "target"
     control_output = output_path / "control"
 
     target_result = _run_with_optional_http_metrics(
-        connector_image=target_image,
+        connector_image=resolved_target_image,
         command=cmd,
         output_dir=target_output,
         target_or_control=TargetOrControl.TARGET,
@@ -564,7 +768,7 @@ def regression_test(
     )
 
     control_result = _run_with_optional_http_metrics(
-        connector_image=control_image,
+        connector_image=resolved_control_image,
         command=cmd,
         output_dir=control_output,
         target_or_control=TargetOrControl.CONTROL,
@@ -589,8 +793,8 @@ def regression_test(
     write_github_outputs(
         {
             "success": both_succeeded and not regression_detected,
-            "target_image": target_image,
-            "control_image": control_image,
+            "target_image": resolved_target_image,
+            "control_image": resolved_control_image,
             "command": command,
             "target_exit_code": target_result["exit_code"],
             "control_exit_code": control_result["exit_code"],
@@ -601,8 +805,8 @@ def regression_test(
     write_json_output("regression_report", combined_result)
 
     report_path = generate_regression_report(
-        target_image=target_image,
-        control_image=control_image,
+        target_image=resolved_target_image,
+        control_image=resolved_control_image,
         command=command,
         target_result=target_result,
         control_result=control_result,
@@ -614,8 +818,74 @@ def regression_test(
     write_github_summary(summary)
 
     if regression_detected:
-        print_error(f"Regression detected between {target_image} and {control_image}")
+        exit_with_error(
+            f"Regression detected between {resolved_target_image} and {resolved_control_image}"
+        )
     elif both_succeeded:
-        print_success(f"Regression test passed for {target_image} vs {control_image}")
+        print_success(
+            f"Regression test passed for {resolved_target_image} vs {resolved_control_image}"
+        )
+    else:
+        exit_with_error(
+            f"Both versions failed for {resolved_target_image} vs {resolved_control_image}"
+        )
+
+
+@connector_app.command(name="fetch-connection-config")
+def fetch_connection_config_cmd(
+    connection_id: Annotated[
+        str,
+        Parameter(help="The UUID of the Airbyte Cloud connection."),
+    ],
+    output_path: Annotated[
+        str | None,
+        Parameter(
+            help="Path to output file or directory. "
+            "If directory, writes connection-<id>-config.json. "
+            "Default: ./connection-<id>-config.json"
+        ),
+    ] = None,
+    with_secrets: Annotated[
+        bool,
+        Parameter(
+            name="--with-secrets",
+            negative="--no-secrets",
+            help="If set, fetches unmasked secrets from the internal database. "
+            "Requires GCP_PROD_DB_ACCESS_CREDENTIALS env var or `gcloud auth application-default login`. "
+            "Must be used with --oc-issue-url.",
+        ),
+    ] = False,
+    oc_issue_url: Annotated[
+        str | None,
+        Parameter(
+            help="OC issue URL for audit logging. Required when using --with-secrets."
+        ),
+    ] = None,
+) -> None:
+    """Fetch connection configuration from Airbyte Cloud to a local file.
+
+    This command retrieves the source configuration for a given connection ID
+    and writes it to a local JSON file.
+
+    Requires authentication via AIRBYTE_CLOUD_CLIENT_ID and
+    AIRBYTE_CLOUD_CLIENT_SECRET environment variables.
+
+    When --with-secrets is specified, the command fetches unmasked secrets from
+    the internal database using the connection-retriever. This additionally requires:
+    - An OC issue URL for audit logging (--oc-issue-url)
+    - GCP credentials via `GCP_PROD_DB_ACCESS_CREDENTIALS` env var or `gcloud auth application-default login`
+    - If `CI=true`: expects `cloud-sql-proxy` running on localhost, or
+      direct network access to the Cloud SQL instance.
+    """
+    path = Path(output_path) if output_path else None
+    result = fetch_connection_config(
+        connection_id=connection_id,
+        output_path=path,
+        with_secrets=with_secrets,
+        oc_issue_url=oc_issue_url,
+    )
+    if result.success:
+        print_success(result.message)
     else:
-        print_error(f"Both versions failed for {target_image} vs {control_image}")
+        print_error(result.message)
+    print_json(result.model_dump())

airbyte_ops_mcp/cli/repo.py

@@ -27,6 +27,7 @@ from airbyte_ops_mcp.airbyte_repo.list_connectors import (
     CONNECTOR_PATH_PREFIX,
     METADATA_FILE_NAME,
     _detect_connector_language,
+    get_connectors_with_local_cdk,
 )
 from airbyte_ops_mcp.cli._base import app
 from airbyte_ops_mcp.cli._shared import exit_with_error, print_json
@@ -160,6 +161,15 @@ def list_connectors(
         bool,
         Parameter(help="Include only modified connectors (requires PR context)."),
     ] = False,
+    local_cdk: Annotated[
+        bool,
+        Parameter(
+            help=(
+                "Include connectors using local CDK reference. "
+                "When combined with --modified-only, adds local-CDK connectors to the modified set."
+            )
+        ),
+    ] = False,
     language: Annotated[
         list[str] | None,
         Parameter(help="Languages to include (python, java, low-code, manifest-only)."),
@@ -286,6 +296,11 @@ def list_connectors(
     connectors = list(result.connectors)
     repo_path_obj = Path(repo_path)
 
+    # Add connectors with local CDK reference if --local-cdk flag is set
+    if local_cdk:
+        local_cdk_connectors = get_connectors_with_local_cdk(repo_path)
+        connectors = sorted(set(connectors) | local_cdk_connectors)
+
     # Apply connector type filter
     if connector_type_filter:
         connectors = [