airbyte-internal-ops 0.1.3__tar.gz → 0.1.4__tar.gz

{airbyte_internal_ops-0.1.3 → airbyte_internal_ops-0.1.4}/.github/workflows/connector-live-test.yml

@@ -6,8 +6,12 @@ on:
   workflow_dispatch:
     inputs:
       connector_image:
-        description: "Full connector image name with tag (e.g., airbyte/source-github:1.0.0)"
-        required: true
+        description: "Full connector image name with tag (e.g., airbyte/source-github:1.0.0). Optional if connector_name or connection_id is provided."
+        required: false
+        type: string
+      connector_name:
+        description: "Connector name to build from source (e.g., 'source-pokeapi'). If provided, builds the image locally."
+        required: false
         type: string
       command:
         description: "Airbyte command to run"
@@ -20,7 +24,7 @@ on:
           - read
         default: check
       connection_id:
-        description: "Airbyte Cloud connection ID to fetch config/catalog from"
+        description: "Airbyte Cloud connection ID to fetch config/catalog from. If provided, connector_image can be auto-detected."
         required: false
         type: string
       state_path:
@@ -55,7 +59,15 @@ jobs:
           AIRBYTE_CLOUD_CLIENT_ID: ${{ secrets.AIRBYTE_CLOUD_CLIENT_ID }}
           AIRBYTE_CLOUD_CLIENT_SECRET: ${{ secrets.AIRBYTE_CLOUD_CLIENT_SECRET }}
         run: |
-          ARGS=(--connector-image "${{ inputs.connector_image }}" --command "${{ inputs.command }}")
+          ARGS=(--command "${{ inputs.command }}")
+
+          if [ -n "${{ inputs.connector_image }}" ]; then
+            ARGS+=(--connector-image "${{ inputs.connector_image }}")
+          fi
+
+          if [ -n "${{ inputs.connector_name }}" ]; then
+            ARGS+=(--connector-name "${{ inputs.connector_name }}")
+          fi
 
           if [ -n "${{ inputs.connection_id }}" ]; then
             ARGS+=(--connection-id "${{ inputs.connection_id }}")

{airbyte_internal_ops-0.1.3 → airbyte_internal_ops-0.1.4}/.github/workflows/connector-regression-test.yml

@@ -7,12 +7,16 @@ on:
   workflow_dispatch:
     inputs:
       target_image:
-        description: "Target connector image (new version) with tag (e.g., airbyte/source-github:2.0.0)"
-        required: true
+        description: "Target connector image (new version) with tag (e.g., airbyte/source-github:2.0.0). Optional if connector_name is provided."
+        required: false
         type: string
       control_image:
-        description: "Control connector image (baseline version) with tag (e.g., airbyte/source-github:1.0.0)"
-        required: true
+        description: "Control connector image (baseline version) with tag (e.g., airbyte/source-github:1.0.0). Optional if connection_id is provided (auto-detected)."
+        required: false
+        type: string
+      connector_name:
+        description: "Connector name to build target image from source (e.g., 'source-pokeapi'). If provided, builds the target image locally."
+        required: false
         type: string
       command:
         description: "Airbyte command to run"
@@ -25,7 +29,7 @@ on:
           - read
         default: check
       connection_id:
-        description: "Airbyte Cloud connection ID to fetch config/catalog from"
+        description: "Airbyte Cloud connection ID to fetch config/catalog from. If provided, control_image can be auto-detected."
         required: false
         type: string
       state_path:
@@ -36,12 +40,16 @@ on:
   workflow_call:
     inputs:
       target_image:
-        description: "Target connector image (new version) with tag"
-        required: true
+        description: "Target connector image (new version) with tag. Optional if connector_name is provided."
+        required: false
         type: string
       control_image:
-        description: "Control connector image (baseline version) with tag"
-        required: true
+        description: "Control connector image (baseline version) with tag. Optional if connection_id is provided."
+        required: false
+        type: string
+      connector_name:
+        description: "Connector name to build target image from source (e.g., 'source-pokeapi')."
+        required: false
         type: string
       command:
         description: "Airbyte command to run (spec, check, discover, read)"
@@ -49,7 +57,7 @@ on:
         type: string
         default: check
       connection_id:
-        description: "Airbyte Cloud connection ID to fetch config/catalog from"
+        description: "Airbyte Cloud connection ID to fetch config/catalog from. If provided, control_image can be auto-detected."
         required: false
         type: string
       state_path:
@@ -103,7 +111,19 @@ jobs:
           AIRBYTE_CLOUD_CLIENT_ID: ${{ secrets.AIRBYTE_CLOUD_CLIENT_ID }}
           AIRBYTE_CLOUD_CLIENT_SECRET: ${{ secrets.AIRBYTE_CLOUD_CLIENT_SECRET }}
         run: |
-          ARGS=(--target-image "${{ inputs.target_image }}" --control-image "${{ inputs.control_image }}" --command "${{ inputs.command }}")
+          ARGS=(--command "${{ inputs.command }}")
+
+          if [ -n "${{ inputs.target_image }}" ]; then
+            ARGS+=(--target-image "${{ inputs.target_image }}")
+          fi
+
+          if [ -n "${{ inputs.control_image }}" ]; then
+            ARGS+=(--control-image "${{ inputs.control_image }}")
+          fi
+
+          if [ -n "${{ inputs.connector_name }}" ]; then
+            ARGS+=(--connector-name "${{ inputs.connector_name }}")
+          fi
 
           if [ -n "${{ inputs.connection_id }}" ]; then
             ARGS+=(--connection-id "${{ inputs.connection_id }}")

airbyte_internal_ops-0.1.4/.github/workflows/pdoc_preview.yml

@@ -0,0 +1,42 @@
+name: Generate Docs
+
+on:
+  push:
+    branches:
+      - main
+  pull_request: {}
+
+permissions:
+  contents: read
+
+jobs:
+  preview_docs:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Generate documentation
+        run: |
+          set -e
+          uv run poe docs-generate
+          if [ ! -d "docs/generated" ]; then
+            echo "Error: Documentation generation failed - docs/generated directory not found"
+            exit 1
+          fi
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          # Upload generated documentation
+          path: "docs/generated"

airbyte_internal_ops-0.1.4/.github/workflows/pdoc_publish.yml

@@ -0,0 +1,57 @@
+name: Publish Docs
+
+on:
+  push:
+    branches:
+      - main
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  publish_docs:
+    runs-on: ubuntu-24.04
+    environment:
+      name: "github-pages"
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Generate documentation
+        run: |
+          uv run poe docs-generate
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          # Upload entire repository
+          path: "docs/generated"
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

airbyte_internal_ops-0.1.4/.github/workflows/test-connection-retriever.yml

@@ -0,0 +1,84 @@
+name: Test Connection Retriever
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      connection_id:
+        description: "Airbyte Cloud connection ID to test retrieval for"
+        required: true
+        type: string
+
+jobs:
+  test-retriever:
+    name: Test Connection Retriever
+    runs-on: ubuntu-latest
+    env:
+      CI: "true"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Set up GCP credentials
+        env:
+          GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_INTEGRATION_TESTER_CREDENTIALS }}
+        run: |
+          echo "$GCP_PROD_DB_ACCESS_CREDENTIALS" > /tmp/credentials.json
+          echo "GOOGLE_APPLICATION_CREDENTIALS=/tmp/credentials.json" >> $GITHUB_ENV
+
+      - name: Download Cloud SQL Proxy
+        run: |
+          curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.14.3/cloud-sql-proxy.linux.amd64
+          chmod +x cloud-sql-proxy
+
+      - name: Test Connection Retriever
+        env:
+          CONNECTION_ID: ${{ inputs.connection_id }}
+        run: |
+          # Start Cloud SQL Proxy in background
+          ./cloud-sql-proxy prod-ab-cloud-proj:us-west3:prod-pgsql-replica --credentials-file /tmp/credentials.json &
+          proxy_pid=$!
+
+          # Wait for proxy to be ready
+          sleep 5
+
+          # Run the test
+          uv run python -c "
+          from airbyte_ops_mcp.live_tests.connection_secret_retriever import retrieve_unmasked_config
+          import os
+
+          connection_id = os.environ['CONNECTION_ID']
+          print(f'Testing retrieval for connection: {connection_id}')
+
+          config = retrieve_unmasked_config(
+              connection_id=connection_id,
+              retrieval_reason='CI test of connection-retriever integration'
+          )
+
+          # Verify we got a config (don't print it - contains secrets)
+          assert config is not None, 'Failed to retrieve config'
+          assert isinstance(config, dict), 'Config should be a dict'
+          print('Successfully retrieved unmasked config')
+          "
+          test_exit=$?
+
+          # Kill the proxy
+          kill $proxy_pid
+          wait $proxy_pid 2>/dev/null || true
+
+          exit $test_exit

{airbyte_internal_ops-0.1.3 → airbyte_internal_ops-0.1.4}/CONTRIBUTING.md

@@ -32,3 +32,17 @@ First clone the repo, then use something like the following MCP config.
   }
 }
 ```
+
+## Internal Secrets for Live Tests
+
+The live tests feature can retrieve unmasked connection secrets from Airbyte Cloud's internal database. This requires:
+
+- **GCP_PROD_DB_ACCESS_CREDENTIALS** - Access to prod Cloud SQL and Google Secret Manager for DB connection details
+
+To test locally:
+
+1. Set up GCP Application Default Credentials: `gcloud auth application-default login`
+2. Ensure you have access to the `prod-ab-cloud-proj` project
+3. Connect to Tailscale (required for private network access)
+
+In CI, these secrets are available at the org level and a Cloud SQL Auth Proxy handles connectivity.

{airbyte_internal_ops-0.1.3 → airbyte_internal_ops-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airbyte-internal-ops
-Version: 0.1.3
+Version: 0.1.4
 Summary: MCP and API interfaces that let the agents do the admin work
 Author-email: Aaron Steers <aj@airbyte.io>
 Keywords: admin,airbyte,api,mcp
@@ -10,18 +10,20 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Requires-Python: >=3.11
-Requires-Dist: airbyte-cdk<7.0,>=6.0.0
+Requires-Python: <3.13,>=3.11
+Requires-Dist: airbyte-cdk<8.0,>=7.3.9
 Requires-Dist: airbyte-protocol-models-pdv2>=0.13.0
-Requires-Dist: airbyte>=0.1.0
+Requires-Dist: airbyte>=0.34.0
 Requires-Dist: asyncer<1.0,>=0.0.4
 Requires-Dist: click<9.0,>=8.1.3
+Requires-Dist: cloud-sql-python-connector[pg8000]<2.0,>=1.7.0
 Requires-Dist: cyclopts<5.0,>=4.0.0
 Requires-Dist: docker<7.0,>=6.0
 Requires-Dist: dpath<3.0,>=2.1.5
 Requires-Dist: fastmcp<3.0,>=2.12.1
 Requires-Dist: gitpython<4.0,>=3.1.29
+Requires-Dist: google-cloud-logging<4.0,>=3.9.0
+Requires-Dist: google-cloud-secret-manager<3.0,>=2.18.0
 Requires-Dist: jinja2<4.0,>=3.1.2
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: python-dotenv<2.0,>=1.0.0
@@ -30,6 +32,7 @@ Requires-Dist: requests<3.0,>=2.31.0
 Requires-Dist: rich<14.0,>=13.0.0
 Requires-Dist: semver<4.0,>=3.0.1
 Requires-Dist: slack-sdk<4.0,>=3.36.0
+Requires-Dist: sqlalchemy<3.0,>=2.0.0
 Requires-Dist: toml<1.0,>=0.10.2
 Description-Content-Type: text/markdown
 

airbyte_internal_ops-0.1.4/docs/.gitignore

@@ -0,0 +1 @@
+generated

airbyte_internal_ops-0.1.4/docs/generate.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Generate docs for all public modules in airbyte-ops-mcp and save them to docs/generated.
+
+Usage:
+    uv run python docs/generate.py
+
+Or with Poe-the-Poet:
+    poe docs-generate
+    poe docs-preview
+
+"""
+
+from __future__ import annotations
+
+import pathlib
+import shutil
+
+import pdoc
+
+
+def run() -> None:
+    """Generate docs for all public modules in airbyte-ops-mcp and save them to docs/generated."""
+    public_modules = ["airbyte_ops_mcp"]
+
+    # recursively delete the docs/generated folder if it exists
+    if pathlib.Path("docs/generated").exists():
+        shutil.rmtree("docs/generated")
+
+    pdoc.render.configure(
+        template_directory=pathlib.Path("docs/templates"),
+        show_source=True,
+        search=True,
+        logo="https://docs.airbyte.com/img/logo-dark.png",
+        favicon="https://docs.airbyte.com/img/favicon.png",
+        mermaid=True,
+        docformat="google",
+    )
+    pdoc.pdoc(
+        *public_modules,
+        output_directory=pathlib.Path("docs/generated"),
+    )
+
+
+if __name__ == "__main__":
+    run()

airbyte_internal_ops-0.1.4/docs/templates/custom.css

@@ -0,0 +1,268 @@
+/*
+ * Airbyte Ops MCP Documentation - Custom Styling
+ * 
+ * This file applies Airbyte branding and design elements to the API documentation.
+ * 
+ * METHODOLOGY:
+ * - Colors and design tokens are sourced from the main Airbyte documentation site
+ * - Primary source: https://github.com/airbytehq/airbyte/blob/main/docusaurus/src/css/custom.css
+ * - The Airbyte design system uses a purple primary color (#615eff) and Inter font family
+ * - This CSS overrides pdoc's default theme variables to match Airbyte branding
+ * 
+ * UPDATING STYLES:
+ * When Airbyte brand styles are updated, check the following upstream sources:
+ * 1. Main docs CSS: https://github.com/airbytehq/airbyte/blob/main/docusaurus/src/css/custom.css
+ * 2. Docusaurus config: https://github.com/airbytehq/airbyte/blob/main/docusaurus/docusaurus.config.js
+ * 3. Live site: https://docs.airbyte.com (inspect styles in browser dev tools)
+ * 
+ * Key variables to sync:
+ * - --ifm-color-primary and variants (primary brand color)
+ * - --ifm-font-family-base (typography)
+ * - Color palette variables (grey, green, blue scales)
+ * 
+ * PDOC INTEGRATION:
+ * pdoc uses specific CSS variables that we override here:
+ * - --link: Link color (set to Airbyte primary)
+ * - --link-hover: Link hover color (set to Airbyte primary-dark)
+ * - --code: Code background color
+ * - --accent: Accent/highlight color
+ * - --text: Main text color
+ */
+
+@import url("https://fonts.googleapis.com/css2?family=Inter:wght@200;300;400;500;600;700&display=swap");
+
+:root {
+  /* Airbyte Primary Colors - from docusaurus/src/css/custom.css */
+  --airbyte-primary: #615eff;
+  --airbyte-primary-dark: #3f3bff;
+  --airbyte-primary-darker: #2e2aff;
+  --airbyte-primary-darkest: #0500f4;
+  --airbyte-primary-light: #8381ff;
+  --airbyte-primary-lighter: #9492ff;
+  --airbyte-primary-lightest: #c8c7ff;
+  
+  /* Airbyte Color Palette - from docusaurus/src/css/custom.css */
+  --color-white: hsl(0, 0%, 100%);
+  --color-grey-40: hsl(240, 25%, 98%);
+  --color-grey-100: hsl(240, 12%, 92%);
+  --color-grey-400: hsl(239, 10%, 59%);
+  --color-grey-500: hsl(240, 10%, 49%);
+  --color-grey-900: hsl(240, 19%, 18%);
+  --color-green-40: hsl(185, 76%, 97%);
+  --color-green-50: hsl(184, 67%, 92%);
+  --color-green-600: hsl(185, 100%, 35%);
+  --color-green-800: hsl(184, 100%, 26%);
+  --color-blue-30: hsl(240, 100%, 98%);
+  --color-dark-blue-40: hsl(230, 23%, 95%);
+  --color-dark-blue-700: hsl(236, 43%, 31%);
+}
+
+/* Override pdoc's default theme variables - must be in .pdoc selector for specificity */
+.pdoc {
+  --link: #615eff;
+  --link-hover: #3f3bff;
+  --code: hsl(240, 100%, 98%);
+  --accent: hsl(240, 25%, 98%);
+  --text: hsl(240, 19%, 18%);
+  --name: #615eff;
+  --def: #3f3bff;
+}
+
+/* Dark mode colors - from docusaurus/src/css/custom.css */
+@media (prefers-color-scheme: dark) {
+  :root {
+    --airbyte-primary: #7976ff;
+    --airbyte-primary-dark: #3f3bff;
+    --airbyte-primary-darker: #2e2aff;
+    --airbyte-primary-light: #8381ff;
+    --airbyte-primary-lighter: #9492ff;
+    --airbyte-primary-lightest: #c8c7ff;
+    
+    --color-grey-40: hsl(240, 14%, 14%);
+    --color-grey-100: hsl(240, 15%, 15%);
+    --color-grey-400: hsl(240, 13%, 72%);
+    --color-grey-500: hsl(240, 10%, 70%);
+    --color-grey-900: hsl(240, 10%, 90%);
+    --color-green-50: hsl(184, 100%, 12%);
+    --color-blue-30: hsl(252, 25%, 18%);
+    --color-dark-blue-700: hsl(240, 100%, 98%);
+    --color-white: hsl(0, 0%, 16%);
+  }
+  
+  /* Override pdoc's default theme variables for dark mode - must be in .pdoc selector */
+  .pdoc {
+    --link: #9492ff;
+    --link-hover: #c8c7ff;
+    --code: hsl(252, 25%, 18%);
+    --accent: hsl(240, 14%, 14%);
+    --text: hsl(240, 10%, 90%);
+    --name: #9492ff;
+    --def: #8381ff;
+  }
+}
+
+/* Apply Inter font family - Airbyte's brand font */
+body {
+  font-family: "Inter", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+}
+
+/* Logo styling - swap to light logo in dark mode for better visibility */
+@media (prefers-color-scheme: dark) {
+  .logo {
+    /* Replace the dark logo with the light logo in dark mode */
+    content: url("https://docs.airbyte.com/img/logo-light.png");
+  }
+}
+
+/* Enhanced heading styles */
+h1, h2, h3, h4, h5, h6 {
+  font-weight: 600;
+}
+
+/* Enhanced code block styling */
+pre code {
+  border-radius: 8px;
+}
+
+/* Style tables with Airbyte theme */
+table {
+  border-spacing: 0;
+  border-collapse: separate;
+  border-radius: 10px;
+  overflow: hidden;
+}
+
+table th {
+  background-color: var(--airbyte-primary);
+  color: var(--color-white);
+  font-weight: 600;
+  padding: 12px;
+}
+
+table th:first-child {
+  border-top-left-radius: 10px;
+}
+
+table th:last-child {
+  border-top-right-radius: 10px;
+}
+
+table td {
+  padding: 10px 12px;
+  border-bottom: 1px solid var(--color-grey-100);
+}
+
+table tr:hover {
+  background-color: var(--color-grey-40);
+  transition: background-color 0.2s ease;
+}
+
+table tr:last-child td:first-child {
+  border-bottom-left-radius: 10px;
+}
+
+table tr:last-child td:last-child {
+  border-bottom-right-radius: 10px;
+}
+
+/* Style navigation with Airbyte colors - use !important to override pdoc's default nav styles */
+nav a {
+  color: var(--link) !important;
+}
+
+nav a:hover {
+  color: var(--link-hover) !important;
+}
+
+/* Style badges and labels */
+.badge {
+  background-color: var(--color-green-40);
+  border: 1px solid var(--color-green-800);
+  color: var(--color-green-800);
+  font-weight: 500;
+  border-radius: 4px;
+  padding: 2px 8px;
+  font-size: 0.85em;
+}
+
+/* Add rounded corners to images */
+img {
+  max-width: 100%;
+  border-radius: 10px;
+}
+
+/* Style the header/logo area */
+header {
+  border-bottom: 2px solid var(--airbyte-primary-lightest);
+}
+
+/* Style buttons and interactive elements */
+button, .button {
+  background-color: var(--airbyte-primary);
+  color: var(--color-white);
+  border: none;
+  border-radius: 6px;
+  padding: 8px 16px;
+  font-weight: 500;
+  cursor: pointer;
+  transition: background-color 0.2s ease;
+}
+
+button:hover, .button:hover {
+  background-color: var(--airbyte-primary-dark);
+}
+
+/* Improve code syntax highlighting */
+.pdoc-code {
+  border-radius: 8px;
+  border: 1px solid var(--color-grey-100);
+}
+
+/* Style docstring sections */
+.docstring {
+  line-height: 1.6;
+}
+
+/* Add subtle shadow to main content area */
+main {
+  box-shadow: 0 2px 8px rgba(0, 0, 0, 0.05);
+}
+
+/* Style search box if present */
+input[type="search"], input[type="text"] {
+  border: 1px solid var(--color-grey-100);
+  border-radius: 6px;
+  padding: 8px 12px;
+  font-family: inherit;
+}
+
+input[type="search"]:focus, input[type="text"]:focus {
+  outline: none;
+  border-color: var(--airbyte-primary);
+  box-shadow: 0 0 0 3px var(--airbyte-primary-lightest);
+}
+
+/* Improve spacing and readability */
+.pdoc-module-list {
+  line-height: 1.8;
+}
+
+/* Style admonitions/notes */
+.admonition {
+  border-left: 4px solid var(--airbyte-primary);
+  background-color: var(--color-blue-30);
+  padding: 12px 16px;
+  border-radius: 4px;
+  margin: 16px 0;
+}
+
+/* Responsive improvements */
+@media (max-width: 768px) {
+  body {
+    font-size: 14px;
+  }
+  
+  table {
+    font-size: 0.9em;
+  }
+}