airbyte-internal-ops 0.4.2__tar.gz → 0.5.1__tar.gz

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.1}/.github/workflows/connector-regression-test.yml

@@ -186,36 +186,88 @@ jobs:
             exit 1
           fi
 
-      - name: Build common args
-        id: build-args
+      - name: Setup paths and args
+        id: setup
         run: |
+          echo "=========================================="
+          echo "SETUP: Calculating all paths and args upfront"
+          echo "=========================================="
+          
+          # Define all artifact paths upfront
+          ARTIFACTS_DIR="/tmp/regression_test_artifacts"
+          SPEC_OUTPUT_DIR="$ARTIFACTS_DIR/spec"
+          CHECK_OUTPUT_DIR="$ARTIFACTS_DIR/check"
+          DISCOVER_OUTPUT_DIR="$ARTIFACTS_DIR/discover"
+          READ_OUTPUT_DIR="$ARTIFACTS_DIR/read"
+          
+          # Deterministic path for configured catalog (used by both GSM and connection_id modes)
+          CONFIGURED_CATALOG_PATH="$ARTIFACTS_DIR/configured_catalog.json"
+          
+          # Discover output paths (where catalog.jsonl might be found)
+          DISCOVER_TARGET_CATALOG="$DISCOVER_OUTPUT_DIR/target/airbyte_messages/catalog.jsonl"
+          DISCOVER_ROOT_CATALOG="$DISCOVER_OUTPUT_DIR/airbyte_messages/catalog.jsonl"
+          
+          echo "--- Artifact Paths ---"
+          echo "ARTIFACTS_DIR=$ARTIFACTS_DIR"
+          echo "SPEC_OUTPUT_DIR=$SPEC_OUTPUT_DIR"
+          echo "CHECK_OUTPUT_DIR=$CHECK_OUTPUT_DIR"
+          echo "DISCOVER_OUTPUT_DIR=$DISCOVER_OUTPUT_DIR"
+          echo "READ_OUTPUT_DIR=$READ_OUTPUT_DIR"
+          echo "CONFIGURED_CATALOG_PATH=$CONFIGURED_CATALOG_PATH"
+          echo "DISCOVER_TARGET_CATALOG=$DISCOVER_TARGET_CATALOG"
+          echo "DISCOVER_ROOT_CATALOG=$DISCOVER_ROOT_CATALOG"
+          
+          # Determine catalog mode
+          CONNECTION_ID="${{ inputs.connection_id }}"
+          if [ -n "$CONNECTION_ID" ]; then
+            CATALOG_MODE="connection_id"
+            echo "--- Catalog Mode: connection_id ---"
+            echo "Catalog will be fetched from Airbyte Cloud connection: $CONNECTION_ID"
+          else
+            CATALOG_MODE="gsm"
+            echo "--- Catalog Mode: gsm ---"
+            echo "Catalog will be generated from discover output"
+          fi
+          
+          # Build common args
           SKIP_COMPARE="${{ inputs.skip_compare }}"
           ARGS=""
-
-          # Always add connector_name and repo-root (both are required)
           ARGS+=" --connector-name ${{ inputs.connector_name }}"
           ARGS+=" --repo-root ${{ github.workspace }}/connector-repo"
-
-          # Add connection_id if provided
-          if [ -n "${{ inputs.connection_id }}" ]; then
-            ARGS+=" --connection-id ${{ inputs.connection_id }}"
+          
+          if [ -n "$CONNECTION_ID" ]; then
+            ARGS+=" --connection-id $CONNECTION_ID"
           fi
-
+          
           if [ "$SKIP_COMPARE" = "true" ]; then
-            # Single-version mode: use --skip-compare flag
             ARGS+=" --skip-compare"
-
             if [ -n "${{ inputs.override_test_image }}" ]; then
               ARGS+=" --test-image ${{ inputs.override_test_image }}"
             fi
           else
-            # Comparison mode (default): run target vs control
             if [ -n "${{ inputs.override_control_image }}" ]; then
               ARGS+=" --control-image ${{ inputs.override_control_image }}"
             fi
           fi
-
+          
+          echo "--- Common Args ---"
+          echo "ARGS=$ARGS"
+          
+          # Output all values for use in subsequent steps
+          echo "artifacts_dir=$ARTIFACTS_DIR" >> "$GITHUB_OUTPUT"
+          echo "spec_output_dir=$SPEC_OUTPUT_DIR" >> "$GITHUB_OUTPUT"
+          echo "check_output_dir=$CHECK_OUTPUT_DIR" >> "$GITHUB_OUTPUT"
+          echo "discover_output_dir=$DISCOVER_OUTPUT_DIR" >> "$GITHUB_OUTPUT"
+          echo "read_output_dir=$READ_OUTPUT_DIR" >> "$GITHUB_OUTPUT"
+          echo "configured_catalog_path=$CONFIGURED_CATALOG_PATH" >> "$GITHUB_OUTPUT"
+          echo "discover_target_catalog=$DISCOVER_TARGET_CATALOG" >> "$GITHUB_OUTPUT"
+          echo "discover_root_catalog=$DISCOVER_ROOT_CATALOG" >> "$GITHUB_OUTPUT"
+          echo "catalog_mode=$CATALOG_MODE" >> "$GITHUB_OUTPUT"
           echo "common_args=$ARGS" >> "$GITHUB_OUTPUT"
+          
+          echo "=========================================="
+          echo "SETUP COMPLETE"
+          echo "=========================================="
 
       - name: Run SPEC
         id: run-spec
@@ -225,16 +277,14 @@ jobs:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
           USE_CONNECTION_SECRET_RETRIEVER: "true"
-        run: |
-          echo "Running: uv run airbyte-ops cloud connector regression-test --command spec ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/spec"
-          uv run airbyte-ops cloud connector regression-test --command spec ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/spec
+        run: uv run airbyte-ops cloud connector regression-test --command spec ${{ steps.setup.outputs.common_args }} --output-dir ${{ steps.setup.outputs.spec_output_dir }}
 
       - name: Upload SPEC Artifacts
         if: always()
         uses: actions/upload-artifact@v6
         with:
           name: regression-test-artifacts-spec-${{ github.run_id }}
-          path: /tmp/regression_test_artifacts/spec/
+          path: ${{ steps.setup.outputs.spec_output_dir }}/
           retention-days: 7
 
       - name: Write Context Block to Summary
@@ -259,6 +309,13 @@ jobs:
             fi
           fi
           
+          # Determine PR URL based on repo input
+          REPO="${{ inputs.repo }}"
+          if [ -z "$REPO" ]; then
+            REPO="airbyte"
+          fi
+          PR_URL="https://github.com/airbytehq/${REPO}/pull/${{ inputs.pr }}"
+          
           # Write Context block
           {
             echo "# Regression Test Report"
@@ -267,6 +324,7 @@ jobs:
             echo ""
             echo "- **Test Date:** $(date -u '+%Y-%m-%d %H:%M:%S') UTC"
             echo "- **Connector:** \`${{ inputs.connector_name }}\`"
+            echo "- **PR Under Test:** [#${{ inputs.pr }}](${PR_URL})"
             if [ "${{ inputs.skip_compare }}" != "true" ]; then
               echo "- **Control Version:** \`$CONTROL_IMAGE\`"
               echo "- **Target Version:** \`$TARGET_IMAGE\`"
@@ -285,16 +343,14 @@ jobs:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
           USE_CONNECTION_SECRET_RETRIEVER: "true"
-        run: |
-          echo "Running: uv run airbyte-ops cloud connector regression-test --command check ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/check"
-          uv run airbyte-ops cloud connector regression-test --command check ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/check
+        run: uv run airbyte-ops cloud connector regression-test --command check ${{ steps.setup.outputs.common_args }} --output-dir ${{ steps.setup.outputs.check_output_dir }}
 
       - name: Upload CHECK Artifacts
         if: always()
         uses: actions/upload-artifact@v6
         with:
           name: regression-test-artifacts-check-${{ github.run_id }}
-          path: /tmp/regression_test_artifacts/check/
+          path: ${{ steps.setup.outputs.check_output_dir }}/
           retention-days: 7
 
       - name: Run DISCOVER
@@ -305,18 +361,46 @@ jobs:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
           USE_CONNECTION_SECRET_RETRIEVER: "true"
-        run: |
-          echo "Running: uv run airbyte-ops cloud connector regression-test --command discover ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/discover"
-          uv run airbyte-ops cloud connector regression-test --command discover ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/discover
+        run: uv run airbyte-ops cloud connector regression-test --command discover ${{ steps.setup.outputs.common_args }} --output-dir ${{ steps.setup.outputs.discover_output_dir }}
 
       - name: Upload DISCOVER Artifacts
         if: always()
         uses: actions/upload-artifact@v6
         with:
           name: regression-test-artifacts-discover-${{ github.run_id }}
-          path: /tmp/regression_test_artifacts/discover/
+          path: ${{ steps.setup.outputs.discover_output_dir }}/
           retention-days: 7
 
+      - name: Generate Configured Catalog from Discover Output
+        id: generate-catalog
+        if: inputs.skip_read_action != true && inputs.connection_id == ''
+        run: |
+          # Find catalog file from discover output
+          if [ -f "${{ steps.setup.outputs.discover_target_catalog }}" ]; then
+            CATALOG_FILE="${{ steps.setup.outputs.discover_target_catalog }}"
+          elif [ -f "${{ steps.setup.outputs.discover_root_catalog }}" ]; then
+            CATALOG_FILE="${{ steps.setup.outputs.discover_root_catalog }}"
+          else
+            echo "No catalog file found"; exit 0
+          fi
+          
+          # Generate configured catalog
+          python3 -c "
+          import json, sys
+          with open('${CATALOG_FILE}') as f:
+              for line in f:
+                  msg = json.loads(line.strip()) if line.strip() else {}
+                  if msg.get('type') == 'CATALOG':
+                      catalog = msg['catalog']
+                      break
+              else:
+                  sys.exit('No CATALOG message found')
+          streams = [{'stream': s, 'sync_mode': 'full_refresh' if not s.get('supported_sync_modes') or 'full_refresh' in s.get('supported_sync_modes', []) else s['supported_sync_modes'][0], 'destination_sync_mode': 'overwrite'} for s in catalog.get('streams', [])]
+          with open('${{ steps.setup.outputs.configured_catalog_path }}', 'w') as f:
+              json.dump({'streams': streams}, f, indent=2)
+          print(f'Generated {len(streams)} streams')
+          "
+
       - name: Run READ
         id: run-read
         if: inputs.skip_read_action != true
@@ -327,15 +411,18 @@ jobs:
           GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
           USE_CONNECTION_SECRET_RETRIEVER: "true"
         run: |
-          echo "Running: uv run airbyte-ops cloud connector regression-test --command read ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/read"
-          uv run airbyte-ops cloud connector regression-test --command read ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/read
+          CATALOG_ARG=""
+          if [ "${{ steps.setup.outputs.catalog_mode }}" = "gsm" ] && [ -f "${{ steps.setup.outputs.configured_catalog_path }}" ]; then
+            CATALOG_ARG="--catalog-path ${{ steps.setup.outputs.configured_catalog_path }}"
+          fi
+          uv run airbyte-ops cloud connector regression-test --command read ${{ steps.setup.outputs.common_args }} $CATALOG_ARG --output-dir ${{ steps.setup.outputs.read_output_dir }}
 
       - name: Upload READ Artifacts
         if: always() && inputs.skip_read_action != true
         uses: actions/upload-artifact@v6
         with:
           name: regression-test-artifacts-read-${{ github.run_id }}
-          path: /tmp/regression_test_artifacts/read/
+          path: ${{ steps.setup.outputs.read_output_dir }}/
           retention-days: 7
 
       - name: Append READ Skipped Message to Summary

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.1}/.github/workflows/sentry-apply-command.yml

@@ -136,7 +136,7 @@ jobs:
 
       - name: Generate Sentry provider SDK
         run: |
-          pulumi package add terraform-provider jianyuan/sentry 0.14.7
+          pulumi package add terraform-provider jianyuan/sentry 0.14.8
 
       - name: Install generated SDK
         run: |

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.1}/.github/workflows/sentry-preview.yml

@@ -43,7 +43,7 @@ jobs:
 
       - name: Generate Sentry provider SDK
         run: |
-          pulumi package add terraform-provider jianyuan/sentry 0.14.7
+          pulumi package add terraform-provider jianyuan/sentry 0.14.8
 
       - name: Install generated SDK
         run: |

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airbyte-internal-ops
-Version: 0.4.2
+Version: 0.5.1
 Summary: MCP and API interfaces that let the agents do the admin work
 Author-email: Aaron Steers <aj@airbyte.io>
 Keywords: admin,airbyte,api,mcp
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: <3.13,>=3.11
 Requires-Dist: airbyte-cdk<8.0,>=7.3.9
+Requires-Dist: airbyte-connector-models<1.0,>=0.1.3
 Requires-Dist: airbyte-protocol-models-pdv2>=0.13.0
 Requires-Dist: airbyte==0.35.4
 Requires-Dist: asyncer<1.0,>=0.0.4

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.1}/conftest.py

@@ -3,12 +3,12 @@
 # Each legacy subproject has a corresponding GitHub issue for tracking its migration:
 #   - connector_erd_generator: https://github.com/airbytehq/airbyte-ops-mcp/issues/49
 #   - connector_insights: https://github.com/airbytehq/airbyte-ops-mcp/issues/50
-#   - connector_live_tests: https://github.com/airbytehq/airbyte-ops-mcp/issues/51
 #   - connector_ops: https://github.com/airbytehq/airbyte-ops-mcp/issues/52
 #   - connector_pipelines: https://github.com/airbytehq/airbyte-ops-mcp/issues/53
 #   - connector_qa: https://github.com/airbytehq/airbyte-ops-mcp/issues/54
 #   - metadata_models: https://github.com/airbytehq/airbyte-ops-mcp/issues/55
 #   - metadata_service: https://github.com/airbytehq/airbyte-ops-mcp/issues/56
+# Note: connector_live_tests was migrated to regression_tests/ (issue #51 closed)
 
 collect_ignore_glob = [
     # Legacy airbyte_ci source modules (recursive)

airbyte_internal_ops-0.5.1/infra/sentry/Pulumi.prod.yaml

@@ -0,0 +1,9 @@
+# Production stack configuration for Airbyte Sentry
+#
+# This stack manages multiple Sentry projects for Airbyte.
+#
+# Authentication:
+#   The Sentry auth token is provided via SENTRY_AUTH_TOKEN environment variable
+#   (configured as GitHub secrets: SENTRY_AUTH_TOKEN_READONLY, SENTRY_AUTH_TOKEN_READWRITE)
+#
+# See: https://docs.sentry.io/product/integrations/integration-platform/internal-integration/#auth-tokens

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.1}/infra/sentry/Pulumi.yaml

@@ -12,12 +12,13 @@ description: |
   - Dashboard configurations
   - Organization settings
   - Fingerprint rules and stack trace grouping rules
+  - Ownership rules for alert routing
 
   SETUP: Before running `pulumi up`, generate the Sentry provider SDK:
     pulumi package add terraform-provider jianyuan/sentry
 
-  This uses the Terraform Sentry provider directly (v0.14.5) which supports
-  fingerprinting_rules and grouping_enhancements on Project resources.
+  This uses the Terraform Sentry provider directly (v0.14.8) which supports
+  fingerprinting_rules, grouping_enhancements, and project_ownership on Project resources.
 
   The SDK is generated locally in sdks/sentry/ and is gitignored.
 packages:
@@ -26,4 +27,4 @@ packages:
     version: 1.0.2
     parameters:
       - jianyuan/sentry
-      - 0.14.5
+      - 0.14.8

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.1}/infra/sentry/README.md

@@ -182,14 +182,14 @@ Sentry configuration can be managed as Infrastructure as Code using Pulumi. The
 - **Metric Alerts** - Metric-based alert rules
 - **Dashboards** - Custom dashboards
 - **Keys** - DSN keys
+- **Ownership Rules** - Team assignment rules for alert routing (see [Ownership Rules](#ownership-rules))
+- **Inbound Filters** - Event filtering before ingestion
 
 ### What Is Managed via Sentry UI
 
 These are managed directly in the [Sentry UI](https://sentry.io/settings/airbytehq/):
 
 - **Teams** - Team membership and permissions (referenced by slug in Pulumi, but managed in UI)
-- **Ownership Rules** - Team assignment rules for alert routing
-- **Inbound Filters** - Event filtering before ingestion
 
 ### Getting Started with Pulumi