airbyte-internal-ops 0.4.2__tar.gz → 0.5.0__tar.gz

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.0}/.github/workflows/connector-regression-test.yml

@@ -317,6 +317,93 @@ jobs:
           path: /tmp/regression_test_artifacts/discover/
           retention-days: 7
 
+      - name: Generate Configured Catalog from Discover Output
+        id: generate-catalog
+        if: inputs.skip_read_action != true && inputs.connection_id == ''
+        run: |
+          # When using GSM credentials (no connection_id), we need to generate a configured catalog
+          # from the discovered catalog for the READ step
+          echo "Generating configured catalog from discover output..."
+          
+          # Define paths as variables for maintainability
+          DISCOVER_ARTIFACTS_DIR="/tmp/regression_test_artifacts/discover"
+          TARGET_CATALOG_PATH="$DISCOVER_ARTIFACTS_DIR/target/airbyte_messages/catalog.jsonl"
+          ROOT_CATALOG_PATH="$DISCOVER_ARTIFACTS_DIR/airbyte_messages/catalog.jsonl"
+          GENERATED_CATALOG_PATH="/tmp/regression_test_artifacts/generated_configured_catalog.json"
+          
+          # Find the catalog file from discover output (check both target and root locations)
+          CATALOG_FILE=""
+          if [ -f "$TARGET_CATALOG_PATH" ]; then
+            CATALOG_FILE="$TARGET_CATALOG_PATH"
+          elif [ -f "$ROOT_CATALOG_PATH" ]; then
+            CATALOG_FILE="$ROOT_CATALOG_PATH"
+          fi
+          
+          if [ -z "$CATALOG_FILE" ] || [ ! -f "$CATALOG_FILE" ]; then
+            echo "Warning: No catalog file found from discover output"
+            exit 0
+          fi
+          
+          echo "Found catalog file: $CATALOG_FILE"
+          
+          # Extract the catalog JSON from the JSONL file (it's an AirbyteMessage with type=CATALOG)
+          # and convert it to a ConfiguredAirbyteCatalog with all streams selected
+          python3 -c "
+          import json
+          import sys
+          
+          catalog_file = '${CATALOG_FILE}'
+          output_file = '${GENERATED_CATALOG_PATH}'
+          
+          # Read the catalog from the JSONL file
+          catalog = None
+          with open(catalog_file, 'r') as f:
+              for line in f:
+                  line = line.strip()
+                  if not line:
+                      continue
+                  msg = json.loads(line)
+                  if msg.get('type') == 'CATALOG':
+                      catalog = msg.get('catalog')
+                      break
+          
+          if not catalog:
+              print(f'No CATALOG message found in file: {catalog_file}')
+              sys.exit(0)
+          
+          # Convert AirbyteCatalog to ConfiguredAirbyteCatalog
+          # Select all streams, choosing a sync mode supported by each stream
+          configured_streams = []
+          for stream in catalog.get('streams', []):
+              supported_sync_modes = stream.get('supported_sync_modes') or []
+              if not supported_sync_modes or 'full_refresh' in supported_sync_modes:
+                  sync_mode = 'full_refresh'
+              else:
+                  sync_mode = supported_sync_modes[0]
+              configured_stream = {
+                  'stream': stream,
+                  'sync_mode': sync_mode,
+                  'destination_sync_mode': 'overwrite'
+              }
+              configured_streams.append(configured_stream)
+          
+          configured_catalog = {
+              'streams': configured_streams
+          }
+          
+          # Write the configured catalog to a file
+          with open(output_file, 'w') as f:
+              json.dump(configured_catalog, f, indent=2)
+          
+          print(f'Generated configured catalog with {len(configured_streams)} streams')
+          "
+          
+          if [ -f "$GENERATED_CATALOG_PATH" ]; then
+            echo "Successfully generated configured catalog"
+          else
+            echo "Warning: Failed to generate configured catalog"
+          fi
+
       - name: Run READ
         id: run-read
         if: inputs.skip_read_action != true
@@ -327,8 +414,16 @@ jobs:
           GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
           USE_CONNECTION_SECRET_RETRIEVER: "true"
         run: |
-          echo "Running: uv run airbyte-ops cloud connector regression-test --command read ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/read"
-          uv run airbyte-ops cloud connector regression-test --command read ${{ steps.build-args.outputs.common_args }} --output-dir /tmp/regression_test_artifacts/read
+          # Use generated catalog if available (when using GSM credentials without connection_id)
+          GENERATED_CATALOG_PATH="/tmp/regression_test_artifacts/generated_configured_catalog.json"
+          CATALOG_ARG=""
+          if [ -f "$GENERATED_CATALOG_PATH" ]; then
+            CATALOG_ARG="--catalog-path $GENERATED_CATALOG_PATH"
+            echo "Using generated catalog from discover output"
+          fi
+          
+          echo "Running: uv run airbyte-ops cloud connector regression-test --command read ${{ steps.build-args.outputs.common_args }} ${CATALOG_ARG} --output-dir /tmp/regression_test_artifacts/read"
+          uv run airbyte-ops cloud connector regression-test --command read ${{ steps.build-args.outputs.common_args }} ${CATALOG_ARG} --output-dir /tmp/regression_test_artifacts/read
 
       - name: Upload READ Artifacts
         if: always() && inputs.skip_read_action != true

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airbyte-internal-ops
-Version: 0.4.2
+Version: 0.5.0
 Summary: MCP and API interfaces that let the agents do the admin work
 Author-email: Aaron Steers <aj@airbyte.io>
 Keywords: admin,airbyte,api,mcp

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.0}/conftest.py

@@ -3,12 +3,12 @@
 # Each legacy subproject has a corresponding GitHub issue for tracking its migration:
 #   - connector_erd_generator: https://github.com/airbytehq/airbyte-ops-mcp/issues/49
 #   - connector_insights: https://github.com/airbytehq/airbyte-ops-mcp/issues/50
-#   - connector_live_tests: https://github.com/airbytehq/airbyte-ops-mcp/issues/51
 #   - connector_ops: https://github.com/airbytehq/airbyte-ops-mcp/issues/52
 #   - connector_pipelines: https://github.com/airbytehq/airbyte-ops-mcp/issues/53
 #   - connector_qa: https://github.com/airbytehq/airbyte-ops-mcp/issues/54
 #   - metadata_models: https://github.com/airbytehq/airbyte-ops-mcp/issues/55
 #   - metadata_service: https://github.com/airbytehq/airbyte-ops-mcp/issues/56
+# Note: connector_live_tests was migrated to regression_tests/ (issue #51 closed)
 
 collect_ignore_glob = [
     # Legacy airbyte_ci source modules (recursive)

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.0}/infra/sentry/Pulumi.prod.yaml

@@ -60,6 +60,21 @@ config:
 
         # Broken pipe errors - transient network interruptions
         error.value:"Broken pipe*" -> broken-pipe
+
+        # Statement timeout errors - group by connector name
+        # These occur during T+D (typing/deduping) operations and vary by pg_temp_XX schema numbers
+        # and SQL statement content, causing many separate Sentry issues for the same root cause
+        error.value:"*canceling statement due to statement timeout*" -> statement-timeout, {{tags.connector_name}}
+
+        # PostgreSQL I/O errors - transient connection issues (EOF, connection reset, etc.)
+        # These vary by the specific operation being performed when the connection drops,
+        # causing separate Sentry issues for the same root cause
+        error.value:"*I/O error occurred while sending to the backend*" -> postgres-io-error, {{tags.connector_name}}
+
+        # State checksum mismatch errors - group by connector name
+        # These occur when source state record counts don't match platform tracked counts,
+        # varying by specific record counts and stream names, causing many separate issues
+        error.value:"*state record count*does not equal platform tracked record count*" -> state-checksum-mismatch, {{tags.connector_name}}
       # Grouping enhancements (imported from Sentry 2025-12-22)
       grouping_enhancements: |-
         # java: mark everything from io.airbyte.* as in-app

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.0}/infra/sentry/__main__.py

@@ -542,6 +542,65 @@ def define_connector_incidents_alert_rules(
                 import_=f"{ORGANIZATION}/{CONNECTOR_INCIDENTS_PROJECT_ID}/11478402",
             ),
         ),
+        # Alert: New or Escalating Destination Issues (Slack notification only)
+        # Re-enabled per team-move request to receive alerts for destination issues
+        # without workspace threshold - triggers on new/escalating issues
+        sentry.IssueAlert(
+            "alert-destination-issues-slack",
+            organization=ORGANIZATION,
+            project=CONNECTOR_INCIDENTS_PROJECT_ID,
+            name="Alert on New or Escalating Destination Issues",
+            action_match="any",
+            filter_match="all",
+            frequency=5,  # Minimum allowed value (5 minutes)
+            environment="production",  # Only trigger for production environment
+            owner="team:1199001",  # team-move
+            conditions_v2s=[
+                sentry.IssueAlertConditionsV2Args(
+                    first_seen_event=sentry.IssueAlertConditionsV2FirstSeenEventArgs(),
+                ),
+                sentry.IssueAlertConditionsV2Args(
+                    regression_event=sentry.IssueAlertConditionsV2RegressionEventArgs(),
+                ),
+                sentry.IssueAlertConditionsV2Args(
+                    reappeared_event=sentry.IssueAlertConditionsV2ReappearedEventArgs(),
+                ),
+            ],
+            filters_v2s=[
+                sentry.IssueAlertFiltersV2Args(
+                    tagged_event=sentry.IssueAlertFiltersV2TaggedEventArgs(
+                        key="connector_release_stage",
+                        match="CONTAINS",
+                        value="generally_available",
+                    ),
+                ),
+                sentry.IssueAlertFiltersV2Args(
+                    assigned_to=sentry.IssueAlertFiltersV2AssignedToArgs(
+                        target_type="Team",
+                        target_identifier="1838479",  # team-move
+                    ),
+                ),
+                sentry.IssueAlertFiltersV2Args(
+                    tagged_event=sentry.IssueAlertFiltersV2TaggedEventArgs(
+                        key="failure_origin",
+                        match="CONTAINS",
+                        value="destination",
+                    ),
+                ),
+            ],
+            actions_v2s=[
+                sentry.IssueAlertActionsV2Args(
+                    slack_notify_service=sentry.IssueAlertActionsV2SlackNotifyServiceArgs(
+                        channel="#dev-platform-move-alerts",
+                        workspace="139867",
+                        # Note: channel_id is read-only and auto-resolved by Sentry from channel name
+                    ),
+                ),
+            ],
+            opts=pulumi.ResourceOptions(
+                protect=False,  # New resource, not imported - allow modifications
+            ),
+        ),
     ]
 
 

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.0}/pyproject.toml

@@ -181,3 +181,8 @@ known_first_party = [
 # DEP002: Temporarily ignore unused dependencies - many are only used in legacy modules
 #         which are excluded during the migration. See GitHub issues #49-#56 for tracking.
 ignore = ["DEP004", "DEP002"]
+
+# Per-rule ignores for optional dependencies that are imported conditionally
+[tool.deptry.per_rule_ignores]
+# segment-analytics-python is an optional dependency in the live-tests group
+DEP001 = ["segment"]

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.0}/ruff.toml

@@ -11,7 +11,6 @@ exclude = [
     # Legacy airbyte_ci subprojects - see linked issues for each domain
     "src/airbyte_ops_mcp/_legacy/airbyte_ci/connector_erd_generator/*",    # https://github.com/airbytehq/airbyte-ops-mcp/issues/49
     "src/airbyte_ops_mcp/_legacy/airbyte_ci/connector_insights/*",         # https://github.com/airbytehq/airbyte-ops-mcp/issues/50
-    "src/airbyte_ops_mcp/_legacy/airbyte_ci/connector_live_tests/*",       # https://github.com/airbytehq/airbyte-ops-mcp/issues/51
     "src/airbyte_ops_mcp/_legacy/airbyte_ci/connector_ops/*",              # https://github.com/airbytehq/airbyte-ops-mcp/issues/52
     "src/airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/*",        # https://github.com/airbytehq/airbyte-ops-mcp/issues/53
     "src/airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/*",               # https://github.com/airbytehq/airbyte-ops-mcp/issues/54
@@ -19,9 +18,9 @@ exclude = [
     "src/airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/*",           # https://github.com/airbytehq/airbyte-ops-mcp/issues/56
     # Legacy tests for the above subprojects
     "tests/legacy/airbyte_ci/connector_erd_generator/*",                   # https://github.com/airbytehq/airbyte-ops-mcp/issues/49
-    "tests/legacy/airbyte_ci/connector_live_tests/*",                      # https://github.com/airbytehq/airbyte-ops-mcp/issues/51
     "tests/legacy/airbyte_ci/connector_ops/*",                             # https://github.com/airbytehq/airbyte-ops-mcp/issues/52
     "tests/legacy/airbyte_ci/pipelines/*",                                 # https://github.com/airbytehq/airbyte-ops-mcp/issues/53
+    # Note: connector_live_tests was migrated to regression_tests/ (issue #51 closed)
 ]
 
 [lint]

{airbyte_internal_ops-0.4.2 → airbyte_internal_ops-0.5.0}/src/airbyte_ops_mcp/cli/cloud.py

@@ -82,6 +82,7 @@ from airbyte_ops_mcp.regression_tests.models import (
     ExecutionInputs,
     TargetOrControl,
 )
+from airbyte_ops_mcp.telemetry import track_regression_test
 
 # Path to connectors directory within the airbyte repo
 CONNECTORS_SUBDIR = Path("airbyte-integrations") / "connectors"
@@ -953,6 +954,32 @@ def regression_test(
             f"Failed to pull control connector image: {resolved_control_image}"
         )
 
+    # Track telemetry for the regression test
+    # Extract version from image tag (e.g., "airbyte/source-github:1.0.0" -> "1.0.0")
+    target_version = (
+        resolved_test_image.rsplit(":", 1)[-1]
+        if ":" in resolved_test_image
+        else "unknown"
+    )
+    control_version = None
+    if resolved_control_image and ":" in resolved_control_image:
+        control_version = resolved_control_image.rsplit(":", 1)[-1]
+
+    # Get tester identity from environment (GitHub Actions sets GITHUB_ACTOR)
+    tester = os.getenv("GITHUB_ACTOR") or os.getenv("USER")
+
+    track_regression_test(
+        user_id=tester,
+        connector_image=resolved_test_image,
+        command=command,
+        target_version=target_version,
+        control_version=control_version,
+        additional_properties={
+            "connection_id": connection_id,
+            "skip_compare": skip_compare,
+        },
+    )
+
     # Execute the appropriate mode
     if skip_compare:
         # Single-version mode: run only the connector image