airbyte-internal-ops 0.3.0__tar.gz → 0.4.0__tar.gz

airbyte_internal_ops-0.4.0/.github/workflows/connector-regression-test.yml

@@ -0,0 +1,407 @@
+name: Connector Regression Test
+permissions:
+  contents: read
+  checks: write
+
+on:
+  workflow_dispatch:
+    inputs:
+      connector_name:
+        description: "Connector name to build from source (e.g., 'source-pokeapi')."
+        required: true
+        type: string
+      pr:
+        description: "PR number to checkout and build from (e.g., 70847). Must be from the repository specified by 'repo'."
+        required: true
+        type: string
+      repo:
+        description: "Repository where the connector PR is located."
+        required: false
+        type: choice
+        options:
+          - airbyte
+          - airbyte-enterprise
+        default: "airbyte"
+      connection_id:
+        description: "Airbyte Cloud connection ID to fetch config/catalog from."
+        required: false
+        type: string
+      skip_compare:
+        description: "If true, skip comparison and run single-version tests only. If false (default), run comparison tests (target vs control)."
+        required: false
+        type: boolean
+        default: false
+      skip_read_action:
+        description: "If true, skip the read action (run only spec, check, discover)."
+        required: false
+        type: boolean
+        default: false
+      override_test_image:
+        description: "Override test connector image with tag (e.g., airbyte/source-github:1.0.0). Ignored if skip_compare=false."
+        required: false
+        type: string
+      override_control_image:
+        description: "Override control connector image (baseline version) with tag. Ignored if skip_compare=true."
+        required: false
+        type: string
+
+  workflow_call:
+    inputs:
+      connector_name:
+        description: "Connector name to build from source (e.g., 'source-pokeapi')."
+        required: true
+        type: string
+      pr:
+        description: "PR number to checkout and build from. Must be from the repository specified by 'repo'."
+        required: true
+        type: string
+      repo:
+        description: "Repository where the connector PR is located. Use 'airbyte' or 'airbyte-enterprise'."
+        required: false
+        type: string
+        default: "airbyte"
+      connection_id:
+        description: "Airbyte Cloud connection ID to fetch config/catalog from."
+        required: false
+        type: string
+      skip_compare:
+        description: "If true, skip comparison and run single-version tests only. If false (default), run comparison tests (target vs control)."
+        required: false
+        type: boolean
+        default: false
+      skip_read_action:
+        description: "If true, skip the read action (run only spec, check, discover)."
+        required: false
+        type: boolean
+        default: false
+      override_test_image:
+        description: "Override test connector image with tag. Ignored if skip_compare=false."
+        required: false
+        type: string
+      override_control_image:
+        description: "Override control connector image (baseline version) with tag. Ignored if skip_compare=true."
+        required: false
+        type: string
+    outputs:
+      success:
+        description: "True if regression test passed (no regression detected)"
+        value: ${{ jobs.regression-test.outputs.success }}
+      summary_url:
+        description: "GitHub Actions run summary URL"
+        value: ${{ jobs.regression-test.outputs.summary_url }}
+      report_url:
+        description: "Detailed regression report (GitHub Check URL)"
+        value: ${{ jobs.regression-test.outputs.report_url }}
+    secrets:
+      AIRBYTE_CLOUD_CLIENT_ID:
+        required: false
+      AIRBYTE_CLOUD_CLIENT_SECRET:
+        required: false
+      GCP_GSM_CREDENTIALS:
+        required: false
+      GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL:
+        required: false
+
+jobs:
+  regression-test:
+    name: Regression Test: '${{ inputs.connector_name }}'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Resolve connector repo ref
+        id: resolve-connector-ref
+        run: |
+          REPO="${{ inputs.repo }}"
+          # Default to 'airbyte' if not specified
+          if [ -z "$REPO" ]; then
+            REPO="airbyte"
+          fi
+          echo "repo=$REPO" >> "$GITHUB_OUTPUT"
+          echo "resolved_ref=refs/pull/${{ inputs.pr }}/head" >> "$GITHUB_OUTPUT"
+          echo "Using PR ${{ inputs.pr }} from airbytehq/$REPO -> refs/pull/${{ inputs.pr }}/head"
+
+      - name: Checkout connector repo
+        uses: actions/checkout@v6
+        with:
+          repository: airbytehq/${{ steps.resolve-connector-ref.outputs.repo }}
+          ref: ${{ steps.resolve-connector-ref.outputs.resolved_ref }}
+          path: connector-repo
+          fetch-depth: 1
+
+      - name: Install Cloud SQL Proxy
+        if: ${{ inputs.connection_id != '' }}
+        run: |
+          set -euo pipefail
+          echo "Installing Cloud SQL Proxy..."
+          VERSION="v2.15.0"
+          BASE_URL="https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/${VERSION}"
+          BINARY_NAME="cloud-sql-proxy.linux.amd64"
+          # Checksum from GitHub release: https://github.com/GoogleCloudPlatform/cloud-sql-proxy/releases/tag/v2.15.0
+          EXPECTED_SHA256="cf3e9d069ea0d09cdfddce77daa36811bb0b963b1169e9c3f1586433ca7325fa"
+
+          echo "Downloading Cloud SQL Proxy binary..."
+          curl -fsSLO "${BASE_URL}/${BINARY_NAME}"
+
+          echo "Verifying Cloud SQL Proxy checksum..."
+          echo "${EXPECTED_SHA256}  ${BINARY_NAME}" | sha256sum -c -
+
+          chmod +x "${BINARY_NAME}"
+          sudo mv "${BINARY_NAME}" /usr/local/bin/cloud-sql-proxy
+          cloud-sql-proxy --version
+
+      - name: Start Cloud SQL Proxy
+        if: ${{ inputs.connection_id != '' }}
+        env:
+          GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
+        run: |
+          echo "Starting Cloud SQL Proxy for connection secret retrieval..."
+          # Start proxy in background on port 15432
+          uv run airbyte-ops cloud db start-proxy --port 15432 &
+          # Wait for proxy to be ready
+          for i in {1..30}; do
+            if nc -z localhost 15432 2>/dev/null; then
+              echo "Cloud SQL Proxy is ready on port 15432"
+              break
+            fi
+            echo "Waiting for proxy to start... ($i/30)"
+            sleep 1
+          done
+          if ! nc -z localhost 15432 2>/dev/null; then
+            echo "ERROR: Cloud SQL Proxy failed to start"
+            exit 1
+          fi
+
+      - name: Build common args
+        id: build-args
+        run: |
+          SKIP_COMPARE="${{ inputs.skip_compare }}"
+          ARGS=""
+
+          # Always add connector_name and repo-root (both are required)
+          ARGS+=" --connector-name ${{ inputs.connector_name }}"
+          ARGS+=" --repo-root ${{ github.workspace }}/connector-repo"
+
+          # Add connection_id if provided
+          if [ -n "${{ inputs.connection_id }}" ]; then
+            ARGS+=" --connection-id ${{ inputs.connection_id }}"
+          fi
+
+          if [ "$SKIP_COMPARE" = "true" ]; then
+            # Single-version mode: use --skip-compare flag
+            ARGS+=" --skip-compare"
+
+            if [ -n "${{ inputs.override_test_image }}" ]; then
+              ARGS+=" --test-image ${{ inputs.override_test_image }}"
+            fi
+          else
+            # Comparison mode (default): run target vs control
+            if [ -n "${{ inputs.override_control_image }}" ]; then
+              ARGS+=" --control-image ${{ inputs.override_control_image }}"
+            fi
+          fi
+
+          echo "common_args=$ARGS" >> "$GITHUB_OUTPUT"
+
+      - name: Run SPEC
+        id: run-spec
+        env:
+          AIRBYTE_CLOUD_CLIENT_ID: ${{ secrets.AIRBYTE_CLOUD_CLIENT_ID }}
+          AIRBYTE_CLOUD_CLIENT_SECRET: ${{ secrets.AIRBYTE_CLOUD_CLIENT_SECRET }}
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
+          USE_CONNECTION_SECRET_RETRIEVER: "true"
+        run: |
+          echo "Running: uv run airbyte-ops cloud connector regression-test --command spec ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/spec"
+          uv run airbyte-ops cloud connector regression-test --command spec ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/spec
+
+      - name: Upload SPEC Artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: regression-test-artifacts-spec-${{ github.run_id }}
+          path: /tmp/regression_test_artifacts/spec/
+          retention-days: 7
+
+      - name: Run CHECK
+        id: run-check
+        env:
+          AIRBYTE_CLOUD_CLIENT_ID: ${{ secrets.AIRBYTE_CLOUD_CLIENT_ID }}
+          AIRBYTE_CLOUD_CLIENT_SECRET: ${{ secrets.AIRBYTE_CLOUD_CLIENT_SECRET }}
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
+          USE_CONNECTION_SECRET_RETRIEVER: "true"
+        run: |
+          echo "Running: uv run airbyte-ops cloud connector regression-test --command check ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/check"
+          uv run airbyte-ops cloud connector regression-test --command check ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/check
+
+      - name: Upload CHECK Artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: regression-test-artifacts-check-${{ github.run_id }}
+          path: /tmp/regression_test_artifacts/check/
+          retention-days: 7
+
+      - name: Run DISCOVER
+        id: run-discover
+        env:
+          AIRBYTE_CLOUD_CLIENT_ID: ${{ secrets.AIRBYTE_CLOUD_CLIENT_ID }}
+          AIRBYTE_CLOUD_CLIENT_SECRET: ${{ secrets.AIRBYTE_CLOUD_CLIENT_SECRET }}
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
+          USE_CONNECTION_SECRET_RETRIEVER: "true"
+        run: |
+          echo "Running: uv run airbyte-ops cloud connector regression-test --command discover ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/discover"
+          uv run airbyte-ops cloud connector regression-test --command discover ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/discover
+
+      - name: Upload DISCOVER Artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: regression-test-artifacts-discover-${{ github.run_id }}
+          path: /tmp/regression_test_artifacts/discover/
+          retention-days: 7
+
+      - name: Run READ
+        id: run-read
+        if: inputs.skip_read_action != true
+        env:
+          AIRBYTE_CLOUD_CLIENT_ID: ${{ secrets.AIRBYTE_CLOUD_CLIENT_ID }}
+          AIRBYTE_CLOUD_CLIENT_SECRET: ${{ secrets.AIRBYTE_CLOUD_CLIENT_SECRET }}
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
+          USE_CONNECTION_SECRET_RETRIEVER: "true"
+        run: |
+          echo "Running: uv run airbyte-ops cloud connector regression-test --command read ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/read"
+          uv run airbyte-ops cloud connector regression-test --command read ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/read
+
+      - name: Upload READ Artifacts
+        if: always() && inputs.skip_read_action != true
+        uses: actions/upload-artifact@v5
+        with:
+          name: regression-test-artifacts-read-${{ github.run_id }}
+          path: /tmp/regression_test_artifacts/read/
+          retention-days: 7
+
+      - name: Determine Final Status
+        id: final-status
+        if: always()
+        run: |
+          SPEC_RESULT="${{ steps.run-spec.outcome }}"
+          CHECK_RESULT="${{ steps.run-check.outcome }}"
+          DISCOVER_RESULT="${{ steps.run-discover.outcome }}"
+          READ_RESULT="${{ steps.run-read.outcome }}"
+          SKIP_READ="${{ inputs.skip_read_action }}"
+
+          echo "SPEC: $SPEC_RESULT"
+          echo "CHECK: $CHECK_RESULT"
+          echo "DISCOVER: $DISCOVER_RESULT"
+          echo "READ: $READ_RESULT (skip_read=$SKIP_READ)"
+
+          if [ "$SPEC_RESULT" != "success" ] || [ "$CHECK_RESULT" != "success" ] || [ "$DISCOVER_RESULT" != "success" ]; then
+            echo "success=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if [ "$SKIP_READ" != "true" ] && [ "$READ_RESULT" != "success" ]; then
+            echo "success=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          echo "success=true" >> "$GITHUB_OUTPUT"
+
+      - name: Create Regression Test Report Check
+        id: create-report-check
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+
+            // Collect results from each command
+            const specResult = '${{ steps.run-spec.outcome }}';
+            const checkResult = '${{ steps.run-check.outcome }}';
+            const discoverResult = '${{ steps.run-discover.outcome }}';
+            const readResult = '${{ steps.run-read.outcome }}';
+            const skipRead = '${{ inputs.skip_read_action }}' === 'true';
+
+            const results = [
+              { command: 'spec', result: specResult },
+              { command: 'check', result: checkResult },
+              { command: 'discover', result: discoverResult },
+            ];
+            if (!skipRead) {
+              results.push({ command: 'read', result: readResult });
+            }
+
+            // Build report content
+            let reportContent = '# Sequential Regression Test Report\n\n';
+            reportContent += '| Command | Result |\n';
+            reportContent += '|---------|--------|\n';
+            for (const r of results) {
+              const status = r.result === 'success' ? 'PASS' : 'FAIL';
+              reportContent += `| ${r.command.toUpperCase()} | ${status} |\n`;
+            }
+
+            const allPassed = results.every(r => r.result === 'success');
+            const conclusion = allPassed ? 'success' : 'failure';
+
+            const skipCompare = '${{ inputs.skip_compare }}' === 'true';
+            const connectorName = '${{ inputs.connector_name }}';
+            const prNumber = '${{ inputs.pr }}';
+            const commands = skipRead ? 'spec, check, discover' : 'spec, check, discover, read';
+            let summary;
+            if (skipCompare) {
+              const image = '${{ inputs.override_test_image }}' || `(built from ${connectorName} PR #${prNumber})`;
+              summary = `Single-version sequential regression test: \`${image}\` running \`${commands}\` commands.`;
+            } else {
+              const controlImage = '${{ inputs.override_control_image }}' || '(auto-detected)';
+              summary = `Comparison sequential regression test: \`${connectorName}\` (PR #${prNumber}) vs \`${controlImage}\` running \`${commands}\` commands.`;
+            }
+
+            const check = await github.rest.checks.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'Regression Test Report',
+              head_sha: context.sha,
+              status: 'completed',
+              conclusion: conclusion,
+              output: {
+                title: 'Regression Test Report',
+                summary: summary,
+                text: reportContent
+              }
+            });
+
+            core.setOutput('report_url', check.data.html_url);
+
+      - name: Add Report Link to Summary
+        if: always()
+        run: |
+          {
+            echo ""
+            echo "---"
+            echo ""
+            echo "### View Full Report"
+            echo ""
+            echo "[Open Regression Test Report](${{ steps.create-report-check.outputs.report_url }})"
+          } >> "$GITHUB_STEP_SUMMARY"
+    outputs:
+      success: ${{ steps.final-status.outputs.success }}
+      summary_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      report_url: ${{ steps.create-report-check.outputs.report_url }}

airbyte_internal_ops-0.4.0/CONTRIBUTING.md

@@ -0,0 +1,219 @@
+# Contributing
+
+First clone the repo, then use something like the following MCP config.
+
+```json
+{
+  "mcpServers": {
+    "airbyte-ops-mcp": {
+      "type": "stdio",
+      "command": "uv",
+      "args": [
+        "run",
+        "--project=/Users/{my-user-id}/repos/airbyte-ops-mcp/",
+        "airbyte-ops-mcp"
+      ],
+      "env": {
+        "AIRBYTE_MCP_ENV_FILE": "/path/to/airbyte-ops-mcp/.env"
+      }
+    },
+    "airbyte-coral-mcp": {
+      "type": "stdio",
+      "command": "uvx",
+      "args": [
+        "--python=3.11",
+        "--from=airbyte@latest",
+        "airbyte-mcp"
+      ],
+      "env": {
+        "AIRBYTE_MCP_ENV_FILE": "/Users/{user-id}/.mcp/airbyte_mcp.env"
+      }
+    }
+  }
+}
+```
+
+## Internal Secrets for Live Tests
+
+The live tests feature can retrieve unmasked connection secrets from Airbyte Cloud's internal database. This requires:
+
+- **GCP_PROD_DB_ACCESS_CREDENTIALS** - Access to prod Cloud SQL and Google Secret Manager for DB connection details
+
+To test locally:
+
+1. Set up GCP Application Default Credentials: `gcloud auth application-default login`
+2. Ensure you have access to the `prod-ab-cloud-proj` project
+3. Connect to Tailscale (required for private network access)
+
+In CI, these secrets are available at the org level and a Cloud SQL Auth Proxy handles connectivity.
+
+## Authorizing Service Accounts
+
+To grant a service account access to the prod database query tools, you need to grant permissions on the `CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS` secret in Google Secret Manager (see `CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS_SECRET_ID` in `src/airbyte_ops_mcp/constants.py`). The service account needs both `secretAccessor` (to read the secret value) and `viewer` (to list secret versions) roles.
+
+### Known Service Accounts
+
+The following service accounts are currently used for specific purposes:
+
+- **`devin-ai-support-service@prod-ab-cloud-proj.iam.gserviceaccount.com`** - Used by Devin AI for interactive development and testing
+- **`e2e-testing-gsm-reader@dataline-integration-testing.iam.gserviceaccount.com`** - Used by CI workflows for regression testing (note: this is a cross-project service account from `dataline-integration-testing`)
+
+### Required Permissions
+
+#### Secret Manager Permissions
+
+Grant both roles scoped to the specific secret:
+
+```bash
+# Grant secretAccessor role (for reading secret values)
+gcloud secrets add-iam-policy-binding projects/587336813068/secrets/CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS \
+  --member="serviceAccount:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --role="roles/secretmanager.secretAccessor"
+
+# Grant viewer role (for listing secret versions)
+gcloud secrets add-iam-policy-binding projects/587336813068/secrets/CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS \
+  --member="serviceAccount:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --role="roles/secretmanager.viewer"
+```
+
+#### Cloud SQL Permissions
+
+Grant Cloud SQL access at the project level:
+
+```bash
+# Grant Cloud SQL Client role (required for connecting via Cloud SQL Proxy or Python Connector)
+gcloud projects add-iam-policy-binding prod-ab-cloud-proj \
+  --member="serviceAccount:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --role="roles/cloudsql.client"
+
+# Grant Cloud SQL Instance User role (may be required depending on org policies)
+gcloud projects add-iam-policy-binding prod-ab-cloud-proj \
+  --member="serviceAccount:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --role="roles/cloudsql.instanceUser"
+```
+
+Note: `roles/cloudsql.client` is required for the Cloud SQL Python Connector and Cloud SQL Auth Proxy to establish connections. `roles/cloudsql.instanceUser` is typically needed for IAM database authentication; it may be redundant when using username/password authentication (as this codebase does), but some environments may still require it.
+
+#### Cloud Logging Permissions
+
+For tools that query GCP Cloud Logging (e.g., `lookup_cloud_backend_error`), grant the Logs Viewer role at the project level:
+
+```bash
+# Grant Logs Viewer role (for reading log entries)
+# Note: --condition=None is required to avoid interactive prompts when the project has conditional bindings
+gcloud projects add-iam-policy-binding prod-ab-cloud-proj \
+  --member="serviceAccount:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --role="roles/logging.viewer" \
+  --condition=None
+```
+
+For tools that retrieve connection secrets with audit logging (e.g., `fetch-connection-config --with-secrets`), also grant the Logs Writer role:
+
+```bash
+# Grant Logs Writer role (for writing audit log entries when retrieving secrets)
+gcloud projects add-iam-policy-binding prod-ab-cloud-proj \
+  --member="serviceAccount:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --role="roles/logging.logWriter" \
+  --condition=None
+```
+
+#### Workspace-Specific Secrets (for `--with-secrets`)
+
+The `fetch-connection-config --with-secrets` feature retrieves unmasked connector credentials. This requires access to **two types of secrets**:
+
+1. **`CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS`** - Database connection details (documented above)
+2. **Workspace-specific secrets** - The actual connector credentials stored in Secret Manager with names like `airbyte_workspace_{workspace_id}_secret_{secret_id}_v1`
+
+The permissions documented above only grant access to the database connection secret. To retrieve actual connector credentials, the service account also needs Secret Manager access to the workspace-specific secrets. This can be granted in two ways:
+
+**Option A: Project-level access (broad)**
+```bash
+# Grant project-level Secret Manager access (allows reading ALL secrets in the project)
+gcloud projects add-iam-policy-binding prod-ab-cloud-proj \
+  --member="serviceAccount:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --role="roles/secretmanager.secretAccessor" \
+  --condition=None
+
+gcloud projects add-iam-policy-binding prod-ab-cloud-proj \
+  --member="serviceAccount:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --role="roles/secretmanager.viewer" \
+  --condition=None
+```
+
+**Option B: Scoped access with IAM conditions (recommended for production)**
+
+Use IAM conditions to restrict access to secrets matching a specific pattern (e.g., only secrets for certain workspaces). This requires setting up conditional IAM bindings - consult GCP documentation for details.
+
+**Note:** Without workspace-specific secret access, `fetch-connection-config` will work but `--with-secrets` will fail with a "Permission denied" error when trying to resolve the actual credentials.
+
+To check if the permission is already granted:
+
+```bash
+# List all IAM bindings for the project and filter for the service account
+gcloud projects get-iam-policy prod-ab-cloud-proj \
+  --flatten="bindings[].members" \
+  --filter="bindings.members:YOUR_SERVICE_ACCOUNT@prod-ab-cloud-proj.iam.gserviceaccount.com" \
+  --format="table(bindings.role)"
+```
+
+### Verifying Permissions
+
+To check existing permissions on the secret:
+
+```bash
+gcloud secrets get-iam-policy projects/587336813068/secrets/CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS
+```
+
+### Using Cloud SQL Proxy
+
+When running outside of the VPC (without Tailscale), use the Cloud SQL Auth Proxy:
+
+**Option A: Using the CLI (Recommended)**
+
+Pre-install the CLI tool:
+```bash
+uv tool install airbyte-internal-ops
+airbyte-ops cloud db start-proxy --port=5433
+```
+
+Or use as a single-step command:
+```bash
+uvx --from=airbyte-internal-ops airbyte-ops cloud db start-proxy --port=5433
+```
+
+**Option B: Manual startup**
+
+```bash
+# Start the proxy on port 5433 (avoids conflicts with default PostgreSQL port 5432)
+cloud-sql-proxy prod-ab-cloud-proj:us-west3:prod-pgsql-replica --port=5433
+```
+
+Then set environment variables for your application:
+```bash
+export USE_CLOUD_SQL_PROXY=1
+export DB_PORT=5433
+export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account-key.json
+```
+
+## Regression Test Workflow
+
+The [Connector Regression Test workflow](https://github.com/airbytehq/airbyte-ops-mcp/actions/workflows/connector-regression-test.yml) can be triggered manually or via the `run_regression_tests` MCP tool.
+
+### Workflow Parameters
+
+| Parameter | Required | Description |
+|-----------|----------|-------------|
+| `connector_name` | Yes | Connector name (e.g., `source-amazon-seller-partner`) |
+| `pr` | Yes | PR number from the airbyte repo to build from (e.g., `71055`) |
+| `connection_id` | No | Airbyte Cloud connection ID to fetch config/catalog from |
+| `skip_compare` | No | If true, run single-version tests only (default: false) |
+| `skip_read_action` | No | If true, skip the read action (default: false) |
+| `override_test_image` | No | Override test connector image with tag |
+| `override_control_image` | No | Override control connector image (baseline version) |
+
+### Required CI Secrets
+
+The workflow requires these secrets (configured at the org level):
+- `AIRBYTE_CLOUD_CLIENT_ID` / `AIRBYTE_CLOUD_CLIENT_SECRET` - For Airbyte Cloud API access
+- `GCP_GSM_CREDENTIALS` - For GCP Secret Manager access
+- `GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL` - For prod database access (mapped to `GCP_PROD_DB_ACCESS_CREDENTIALS`)

{airbyte_internal_ops-0.3.0 → airbyte_internal_ops-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airbyte-internal-ops
-Version: 0.3.0
+Version: 0.4.0
 Summary: MCP and API interfaces that let the agents do the admin work
 Author-email: Aaron Steers <aj@airbyte.io>
 Keywords: admin,airbyte,api,mcp

{airbyte_internal_ops-0.3.0 → airbyte_internal_ops-0.4.0}/src/airbyte_ops_mcp/connection_config_retriever/audit_logging.py

@@ -11,9 +11,8 @@ import logging
 import subprocess
 from typing import TYPE_CHECKING, Any, Callable
 
-from google.cloud import logging as gcloud_logging
-
 from airbyte_ops_mcp.constants import GCP_PROJECT_NAME
+from airbyte_ops_mcp.gcp_auth import get_logging_client
 
 if TYPE_CHECKING:
     from airbyte_ops_mcp.connection_config_retriever.retrieval import (
@@ -23,21 +22,18 @@ if TYPE_CHECKING:
 LOGGER = logging.getLogger(__name__)
 
 # Lazy-initialized to avoid import-time GCP calls
-_logging_client: gcloud_logging.Client | None = None
 _airbyte_gcloud_logger: Any = None
 
 
 def _get_logger() -> Any:
     """Get the GCP Cloud Logger, initializing lazily on first use."""
-    global _logging_client, _airbyte_gcloud_logger
+    global _airbyte_gcloud_logger
 
     if _airbyte_gcloud_logger is not None:
         return _airbyte_gcloud_logger
 
-    _logging_client = gcloud_logging.Client(project=GCP_PROJECT_NAME)
-    _airbyte_gcloud_logger = _logging_client.logger(
-        "airbyte-cloud-connection-retriever"
-    )
+    logging_client = get_logging_client(GCP_PROJECT_NAME)
+    _airbyte_gcloud_logger = logging_client.logger("airbyte-cloud-connection-retriever")
     return _airbyte_gcloud_logger