airbyte-internal-ops 0.9.1__tar.gz → 0.10.0__tar.gz

{airbyte_internal_ops-0.9.1 → airbyte_internal_ops-0.10.0}/.github/workflows/pdoc_preview.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v6
       - name: Set up uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
       - name: Set up Python
         uses: actions/setup-python@v6
         with:

{airbyte_internal_ops-0.9.1 → airbyte_internal_ops-0.10.0}/.github/workflows/pdoc_publish.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v6
       - name: Set up uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
       - name: Set up Python
         uses: actions/setup-python@v6
         with:

{airbyte_internal_ops-0.9.1 → airbyte_internal_ops-0.10.0}/.github/workflows/semantic-pr-check.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check semantic PR title
-        uses: amannn/action-semantic-pull-request@v5
+        uses: amannn/action-semantic-pull-request@v6
         if: ${{ github.event.pull_request.draft == false }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -47,7 +47,7 @@ jobs:
 
       - name: Check for "do not merge" in PR title
         if: ${{ github.event.pull_request.draft == false }}
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             const title = context.payload.pull_request.title.toLowerCase();

{airbyte_internal_ops-0.9.1 → airbyte_internal_ops-0.10.0}/.github/workflows/sentry-apply-command.yml

@@ -84,7 +84,7 @@ jobs:
       - name: Get PR info (slash command only)
         if: steps.context.outputs.has_pr == 'true'
         id: pr-info
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             const { data: pr } = await github.rest.pulls.get({
@@ -149,7 +149,7 @@ jobs:
       - name: Download state artifact
         id: download-state
         continue-on-error: true
-        uses: dawidd6/action-download-artifact@v12
+        uses: dawidd6/action-download-artifact@v13
         with:
           name: ${{ env.STATE_ARTIFACT_NAME }}
           path: ${{ env.PULUMI_BACKEND_DIR }}

{airbyte_internal_ops-0.9.1 → airbyte_internal_ops-0.10.0}/.github/workflows/sentry-preview.yml

@@ -62,7 +62,7 @@ jobs:
       - name: Download state artifact
         id: download-state
         continue-on-error: true
-        uses: dawidd6/action-download-artifact@v12
+        uses: dawidd6/action-download-artifact@v13
         with:
           name: ${{ env.STATE_ARTIFACT_NAME }}
           path: ${{ env.PULUMI_BACKEND_DIR }}
@@ -127,7 +127,7 @@ jobs:
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Find existing preview comment
-        uses: peter-evans/find-comment@v3
+        uses: peter-evans/find-comment@v4
         id: find-comment
         with:
           issue-number: ${{ github.event.pull_request.number }}

{airbyte_internal_ops-0.9.1 → airbyte_internal_ops-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airbyte-internal-ops
-Version: 0.9.1
+Version: 0.10.0
 Summary: MCP and API interfaces that let the agents do the admin work
 Author-email: Aaron Steers <aj@airbyte.io>
 Keywords: admin,airbyte,api,mcp

airbyte_internal_ops-0.10.0/docs/progressive-rollouts.md

@@ -0,0 +1,464 @@
+# Connector Progressive Rollouts
+
+This document describes the process for managing connector progressive rollouts in Airbyte Cloud. Progressive rollouts allow gradual migration of customers to new connector versions, enabling monitoring and rollback capabilities before full release.
+
+## Overview
+
+Progressive rollouts provide a controlled way to release new connector versions:
+
+- **Release Candidates (RCs)**: Pre-release versions (e.g., `0.1.0-rc.1`) that can be tested with a subset of customers
+- **Gradual Migration**: Customers are progressively migrated from the stable version to the RC
+- **Monitoring**: Track success/failure rates during the rollout
+- **Rollback Capability**: If issues are detected, the rollout can be paused or rolled back
+
+## Progressive Rollout Lifecycle
+
+1. [Initialize](#1-initialize) - Prepare metadata.yaml and verify rollout status
+2. [Start Workflow](#2-start-workflow) - Begin the rollout workflow
+3. [Manually Pin Actors](#3-manually-pin-actors-optional) - Pin specific actors for early testing (optional)
+4. [Monitor](#4-monitor) - Track rollout progress and performance
+5. [Pause/Resume](#5-pauseresume-optional) - Pause if issues detected (optional)
+6. [Finalize](#6-finalize) - Promote, rollback, or cancel the rollout
+7. [Cleanup](#7-cleanup) - Remove version pins after completion
+
+### 1. Initialize
+
+Prepare the connector for progressive rollout and confirm the rollout is ready to start.
+
+**Steps:**
+
+1. **Mark the PR for progressive rollout**: Use the CLI tool to configure the connector's `metadata.yaml`:
+   ```bash
+   airbyte-ops repo connector setup-progressive-rollout \
+     --name <connector-name> \
+     --repo-path /path/to/airbyte-enterprise \
+     --rc-version <version>  # Optional, defaults to next minor with -rc.1 suffix
+   ```
+   
+   This command:
+   - Sets `dockerImageTag` to the RC version
+   - Adds `registryOverrides.cloud.dockerImageTag` to pin Cloud users to the current stable version
+   - Adds `registryOverrides.oss.dockerImageTag` to pin OSS users to the current stable version
+   - Enables `releases.rolloutConfiguration.enableProgressiveRollout: true`
+
+2. **Merge the PR**: Once the metadata.yaml changes are reviewed and approved, merge the PR.
+
+3. **Verify initialization**: The connector will be automatically published after the PR is merged. Wait for the publish workflow to complete (typically a few minutes), then confirm the rollout appears in the database with "initialized" status:
+   ```bash
+   # Using the MCP tool
+   poe mcp-tool-test query_prod_connector_rollouts '{"actor_definition_id": "<connector-uuid>"}'
+   ```
+   
+   The rollout should show `state: "initialized"`.
+
+### 2. Start Workflow
+
+Start the rollout workflow to begin the progressive migration process.
+
+**Tool:** `start_connector_rollout`
+
+> [!NOTE]
+> This step requires [human approval](#appendix-approval-comment-urls).
+
+```bash
+poe mcp-tool-test start_connector_rollout '{
+  "docker_repository": "airbyte/source-example",
+  "docker_image_tag": "1.0.0-rc.1",
+  "actor_definition_id": "<connector-uuid>",
+  "approval_comment_url": "<github-comment-url>",
+  "rollout_strategy": "manual",
+  "initial_rollout_pct": 25,
+  "final_target_rollout_pct": 50
+}'
+```
+
+**Parameters:**
+
+- `docker_repository`: The docker repository (e.g., `airbyte/source-pokeapi`)
+- `docker_image_tag`: The RC version tag (e.g., `0.3.48-rc.1`)
+- `actor_definition_id`: The connector's UUID
+- `approval_comment_url`: GitHub comment URL from an @airbyte.io user authorizing the action
+- `rollout_strategy`: One of `manual`, `automated`, or `overridden` (default: `manual`)
+- `initial_rollout_pct`: Step size percentage for rollout progression (default: 25%)
+- `final_target_rollout_pct`: Maximum percentage of actors to pin (default: 50%)
+
+**Important: Multiple Calls Supported**
+
+This tool can be called multiple times while the rollout is in `INITIALIZED` state to update the configuration (strategy, percentages). Once the Temporal workflow starts and the state transitions to `WORKFLOW_STARTED`, the configuration is locked and cannot be changed.
+
+<!-- TODO: Determine and document the recommended approach for configuring rollouts
+https://github.com/airbytehq/airbyte-ops-mcp/issues/322 -->
+
+**Authorization:** The start tool requires:
+
+- `AIRBYTE_INTERNAL_ADMIN_FLAG=airbyte.io` environment variable
+- A GitHub comment URL from an @airbyte.io user authorizing the action
+
+After starting, the rollout state transitions to `"workflow_started"` and then `"in_progress"`.
+
+### 3. Manually Pin Actors (Optional)
+
+Before or during the rollout, you can manually pin specific actors (sources/destinations) to the RC version for early testing.
+
+**Tools:** Use existing ops-mcp actor pinning tools:
+
+- `set_workspace_connector_version_override` - Pin all actors in a workspace
+- `set_cloud_connector_version_override` - Set version overrides at the cloud level
+
+> [!NOTE]
+> These tools require [human approval](#appendix-approval-comment-urls).
+
+This is useful for:
+
+- Testing with internal workspaces before broader rollout
+- Targeting specific customers who have agreed to beta test
+- Validating fixes for customer-reported issues
+
+### 4. Monitor
+
+Track the rollout progress and performance of the RC version. This step is performed repeatedly throughout the rollout.
+
+**Tool:** `query_prod_rollout_monitoring_stats`
+
+```bash
+poe mcp-tool-test query_prod_rollout_monitoring_stats '{"rollout_id": "<rollout-uuid>", "days_back": 7}'
+```
+
+**Returns:**
+
+- **Aggregate stats:**
+  - `num_actors`: Total actors using this connector
+  - `num_actors_pinned`: Actors with any version pin (eligible or already pinned)
+  - `num_pinned_to_rollout`: Actors specifically pinned to this rollout's RC version
+  
+- **Per-actor stats:**
+  - `actor_id`: The actor UUID
+  - `num_succeeded`: Number of successful syncs in the lookback period
+  - `num_failed`: Number of failed syncs in the lookback period
+  - `num_connections`: Number of connections using this actor
+
+**What to look for:**
+
+- Compare success/failure rates between RC and stable versions
+- Watch for any actors with unusually high failure rates
+- Monitor the progression of `num_pinned_to_rollout` as the rollout advances
+
+### 5. Pause/Resume (Optional)
+
+If issues are detected during monitoring, the rollout can be paused to prevent further customer migration.
+
+**Tools:** <!-- TODO: Implement pause_rollout and resume_rollout MCP tools - https://github.com/airbytehq/airbyte-ops-mcp/issues/323 -->
+
+```bash
+# TODO: Document pause/resume commands once implemented
+# https://github.com/airbytehq/airbyte-ops-mcp/issues/323
+```
+
+**When to pause:**
+
+- Significant increase in failure rates compared to stable version
+- Customer-reported issues specific to the RC version
+- Infrastructure issues affecting the rollout
+
+### 6. Finalize
+
+Complete the rollout with one of three outcomes:
+
+**Tool:** `finalize_connector_rollout`
+
+> [!NOTE]
+> This step requires [human approval](#appendix-approval-comment-urls).
+
+#### Promote to GA (succeeded)
+
+The RC is performing well and should become the new stable version:
+
+```bash
+poe mcp-tool-test finalize_connector_rollout '{
+  "docker_repository": "airbyte/source-example",
+  "docker_image_tag": "1.0.0-rc.1",
+  "actor_definition_id": "<connector-uuid>",
+  "rollout_id": "<rollout-uuid>",
+  "state": "succeeded",
+  "approval_comment_url": "<github-comment-url>"
+}'
+```
+
+#### Rollback (failed_rolled_back)
+
+Issues were found and the RC should not be released:
+
+```bash
+poe mcp-tool-test finalize_connector_rollout '{
+  "docker_repository": "airbyte/source-example",
+  "docker_image_tag": "1.0.0-rc.1",
+  "actor_definition_id": "<connector-uuid>",
+  "rollout_id": "<rollout-uuid>",
+  "state": "failed_rolled_back",
+  "approval_comment_url": "<github-comment-url>",
+  "error_msg": "High failure rate detected",
+  "failed_reason": "FOUND_ISSUES"
+}'
+```
+
+#### Cancel
+
+The rollout should be stopped without completing:
+
+```bash
+poe mcp-tool-test finalize_connector_rollout '{
+  "docker_repository": "airbyte/source-example",
+  "docker_image_tag": "1.0.0-rc.1",
+  "actor_definition_id": "<connector-uuid>",
+  "rollout_id": "<rollout-uuid>",
+  "state": "canceled",
+  "approval_comment_url": "<github-comment-url>",
+  "retain_pins_on_cancellation": false
+}'
+```
+
+**Authorization:** The finalize tool requires:
+
+- `AIRBYTE_INTERNAL_ADMIN_FLAG=airbyte.io` environment variable
+- A GitHub comment URL from an @airbyte.io user authorizing the action
+
+### 7. Cleanup
+
+After a successful rollout, remove the version pins from metadata.yaml.
+
+**Tool:** `airbyte-ops repo connector cleanup-progressive-rollout`
+
+```bash
+airbyte-ops repo connector cleanup-progressive-rollout \
+  --name source-github \
+  --repo-path /path/to/airbyte
+```
+
+This step:
+
+- Removes `registryOverrides.cloud.dockerImageTag`
+- Removes `registryOverrides.oss.dockerImageTag`
+- Updates `dockerImageTag` to the GA version (removing `-rc.X` suffix)
+- Removes or sets `releases.rolloutConfiguration.enableProgressiveRollout: false`
+
+## Rollout States
+
+| State | Description |
+|-------|-------------|
+| `initialized` | Rollout created, waiting to start |
+| `workflow_started` | Rollout workflow has been triggered |
+| `in_progress` | Actively migrating customers to RC |
+| `paused` | Rollout temporarily stopped |
+| `finalizing` | Completing the rollout |
+| `succeeded` | RC promoted to GA |
+| `failed_rolled_back` | RC rolled back due to issues |
+| `canceled` | Rollout canceled |
+| `errored` | Rollout encountered an error |
+
+## Automated Decision Logic
+
+The progressive rollout backend includes automated failure detection and decision-making. This section describes how the system monitors rollouts and what actions it can take.
+
+### Signal: Sync Success/Failure Counts
+
+The `RolloutProgressionDecider` monitors sync results from pinned connections. For each actor (source/destination) pinned to the RC version, it tracks:
+
+- Number of successful syncs
+- Number of failed syncs
+
+### Decision States
+
+The system can make one of four decisions:
+
+| Decision | Description | Trigger |
+|----------|-------------|---------|
+| `RELEASE` | Promote RC to GA | 100% sync success rate with sufficient data |
+| `ROLLBACK` | Roll back to previous version | Reserved for future use (currently triggers PAUSE) |
+| `PAUSE` | Pause rollout for manual intervention | Any sync failures detected |
+| `INSUFFICIENT_DATA` | Continue waiting | Not enough actors pinned or syncs completed |
+
+### Decision Thresholds
+
+The system uses these thresholds to make decisions:
+
+- **Success Threshold**: 100% - All syncs must succeed for automatic promotion
+- **Completion Threshold**: 50% - At least 50% of pinned actors must have completed syncs before making a final decision
+- **Failure Threshold**: Any (>0) - A single failed sync triggers PAUSE
+
+### Decision Flow
+
+1. **Check for failures**: If any syncs have failed (`nFailedSyncs > 0`), immediately return `PAUSE`
+2. **Check actor coverage**: If fewer than the required percentage of actors have completed syncs, return `INSUFFICIENT_DATA`
+3. **Check success rate**: If 100% of completed syncs succeeded, return `RELEASE`
+4. **Fallback**: If none of the above, return `PAUSE` for manual review
+
+### Automatic Actions
+
+When the workflow receives a decision:
+
+- **RELEASE**: Finalizes the rollout with `SUCCEEDED` state, triggers GA promotion workflow
+- **ROLLBACK**: Finalizes with `FAILED_ROLLED_BACK` state, removes RC pins (path exists but not currently auto-triggered)
+- **PAUSE**: Pauses the workflow, requiring manual intervention via the `resume_rollout` or `finalize_connector_rollout` tools
+- **INSUFFICIENT_DATA**: Continues the rollout loop, pinning more actors and waiting for sync results
+
+### Rollout Expiration
+
+If a rollout runs for too long without reaching a terminal state, it automatically pauses. This prevents rollouts from running indefinitely and ensures human review of stalled rollouts.
+
+### Current Limitations
+
+The system is intentionally conservative:
+
+- **Any failure triggers PAUSE**: Rather than automatically rolling back, the system pauses to allow human judgment. This prevents premature rollbacks due to transient issues or unrelated failures.
+- **100% success required**: The high threshold ensures only well-tested versions are promoted automatically.
+- **Manual intervention for edge cases**: Complex scenarios (partial failures, intermittent issues) require human decision-making.
+
+## metadata.yaml Configuration
+
+### Progressive Rollout Fields
+
+```yaml
+data:
+  # The version to be released (RC version during rollout)
+  dockerImageTag: "1.0.0-rc.1"
+  
+  # Pin existing users to stable version during rollout
+  registryOverrides:
+    cloud:
+      dockerImageTag: "0.9.0"  # Current stable version for Cloud
+    oss:
+      dockerImageTag: "0.9.0"  # Current stable version for OSS
+  
+  # Enable progressive rollout
+  releases:
+    rolloutConfiguration:
+      enableProgressiveRollout: true
+```
+
+### Version Naming Convention
+
+- **Stable versions**: `X.Y.Z` (e.g., `1.0.0`, `0.9.5`)
+- **Release candidates**: `X.Y.Z-rc.N` (e.g., `1.0.0-rc.1`, `1.0.0-rc.2`)
+
+When creating an RC:
+- If the current version is `0.9.5`, the default RC version would be `0.10.0-rc.1`
+- Multiple RCs can be created: `1.0.0-rc.1`, `1.0.0-rc.2`, etc.
+
+## Available Tools
+
+### MCP Tools (airbyte-ops-mcp)
+
+| Tool | Purpose |
+|------|---------|
+| `query_prod_connector_rollouts` | Query rollout status and history |
+| `query_prod_rollout_monitoring_stats` | Get rollout progress and sync stats |
+| `finalize_connector_rollout` | Promote, rollback, or cancel a rollout |
+| `set_workspace_connector_version_override` | Pin actors to specific versions |
+| `start_connector_rollout` | Start the rollout workflow |
+| `pause_rollout` | Pause an in-progress rollout <!-- TODO: Implement pause_rollout tool - https://github.com/airbytehq/airbyte-ops-mcp/issues/323 --> |
+| `resume_rollout` | Resume a paused rollout <!-- TODO: Implement resume_rollout tool - https://github.com/airbytehq/airbyte-ops-mcp/issues/323 --> |
+
+### CLI Tools (airbyte-ops)
+
+| Command | Purpose |
+|---------|---------|
+| `airbyte-ops repo connector setup-progressive-rollout` | Configure metadata.yaml for rollout |
+| `airbyte-ops repo connector cleanup-progressive-rollout` | Remove rollout config after completion |
+
+## Related Resources
+
+- [Airbyte Ops MCP Repository](https://github.com/airbytehq/airbyte-ops-mcp)
+- [Connector Registry](https://connectors.airbyte.com/files/registries/v0/oss_registry.json)
+- <!-- TODO: Add link to internal rollout dashboard if available - https://github.com/airbytehq/airbyte-ops-mcp/issues/322 -->
+- <!-- TODO: Add link to connector publishing documentation - https://github.com/airbytehq/airbyte-ops-mcp/issues/322 -->
+
+## Appendix: Approval Comment URLs
+
+Certain high-impact operations require an "approval comment URL" as an authorization mechanism. This provides an audit trail and ensures that sensitive actions are explicitly approved by an authorized team member.
+
+### How It Works
+
+An approval comment URL is a direct link to a GitHub comment where an @airbyte.io team member has explicitly approved the action. The system validates that the comment author has a verified @airbyte.io email address before allowing the operation to proceed.
+
+**Example approval comment URL:**
+```
+https://github.com/airbytehq/airbyte-ops-mcp/issues/310#issuecomment-3837359087
+```
+
+### Creating an Approval Comment
+
+To create a valid approval comment:
+
+1. Navigate to the relevant GitHub issue or PR tracking the rollout
+2. Post a comment explicitly approving the action (e.g., "Approved for GA promotion")
+3. Copy the permalink URL of that comment (click the timestamp, then copy the URL)
+4. Give this URL to the AI agent, or if calling directly, pass as the `approval_comment_url` parameter
+
+### Actions Requiring Approval
+
+| Action | Tool | Why Approval Is Required |
+|--------|------|--------------------------|
+| Pin workspace actors | `set_workspace_connector_version_override` | Affects all actors in a workspace |
+| Pin cloud actors | `set_cloud_connector_version_override` | Affects actors at the cloud level |
+| Start rollout workflow | `start_connector_rollout` | Initiates customer migration to RC version |
+| Promote RC to GA | `finalize_connector_rollout` (state: `succeeded`) | Affects all customers using the connector |
+| Rollback RC | `finalize_connector_rollout` (state: `failed_rolled_back`) | Reverts customers to previous version |
+| Cancel rollout | `finalize_connector_rollout` (state: `canceled`) | Terminates the rollout process |
+
+### Additional Requirements
+
+In addition to the approval comment URL, these operations also require the `AIRBYTE_INTERNAL_ADMIN_FLAG=airbyte.io` environment variable to be set. This two-factor authorization ensures that both the tool operator and an @airbyte.io team member have approved the action.
+
+## Troubleshooting
+
+### Rollout not appearing after merge
+
+**Symptom:** The connector PR was merged but no rollout appears in the database.
+
+**Possible causes:**
+1. The connector publish workflow hasn't completed yet
+2. `enableProgressiveRollout` is not set to `true` in metadata.yaml
+3. The connector version wasn't properly tagged as an RC
+
+**Resolution:**
+- Wait for the publish workflow to complete (check GitHub Actions)
+- Verify metadata.yaml has `releases.rolloutConfiguration.enableProgressiveRollout: true`
+- Ensure `dockerImageTag` follows the RC naming convention (`X.Y.Z-rc.N`)
+
+### High failure rate during rollout
+
+**Symptom:** Monitoring shows significantly higher failure rates for the RC compared to stable.
+
+**Resolution:**
+1. Pause the rollout to prevent further customer impact
+2. Investigate the failures using job logs
+3. If issues are confirmed, finalize with `failed_rolled_back` state
+4. Fix the issues and create a new RC (`X.Y.Z-rc.2`)
+
+### Cannot finalize rollout
+
+**Symptom:** The finalize tool returns an authorization error.
+
+**Possible causes:**
+1. Missing `AIRBYTE_INTERNAL_ADMIN_FLAG` environment variable
+2. Invalid or expired GitHub comment URL
+3. Comment author is not an @airbyte.io user
+
+**Resolution:**
+- Ensure `AIRBYTE_INTERNAL_ADMIN_FLAG=airbyte.io` is set
+- Get a fresh approval comment from an @airbyte.io team member
+- Verify the comment URL is accessible and the author's email is verified
+
+### Actors not being migrated
+
+**Symptom:** `num_pinned_to_rollout` stays at 0 even after starting the workflow.
+
+**Possible causes:**
+1. Rollout is still in `initialized` state (workflow not started)
+2. Rollout percentage is set to 0
+3. No eligible actors for this connector
+
+**Resolution:**
+- Verify the rollout state is `in_progress`
+- Check the `current_target_rollout_pct` value
+- Confirm there are active actors using this connector (`num_actors` > 0)