airbyte-internal-ops 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

airbyte_ops_mcp/prod_db_access/sql.py

@@ -934,3 +934,79 @@ SELECT_DESTINATION_CONNECTION_STATS = sqlalchemy.text(
     ORDER BY total_connections DESC
     """
 )
+
+# =============================================================================
+# Stream-based Connection Queries
+# =============================================================================
+
+# Query connections by source connector type that have a specific stream enabled
+# The catalog field is JSONB with structure: {"streams": [{"stream": {"name": "..."}, ...}, ...]}
+SELECT_CONNECTIONS_BY_SOURCE_CONNECTOR_AND_STREAM = sqlalchemy.text(
+    """
+    SELECT
+         connection.id AS connection_id,
+         connection.name AS connection_name,
+         connection.source_id,
+         connection.status AS connection_status,
+         workspace.id AS workspace_id,
+         workspace.name AS workspace_name,
+         workspace.organization_id,
+         workspace.dataplane_group_id,
+         dataplane_group.name AS dataplane_name,
+         source_actor.actor_definition_id AS source_definition_id,
+         source_actor.name AS source_name
+    FROM connection
+    JOIN actor AS source_actor
+      ON connection.source_id = source_actor.id
+     AND source_actor.tombstone = false
+    JOIN workspace
+      ON source_actor.workspace_id = workspace.id
+     AND workspace.tombstone = false
+    LEFT JOIN dataplane_group
+      ON workspace.dataplane_group_id = dataplane_group.id
+    WHERE
+         source_actor.actor_definition_id = :connector_definition_id
+     AND connection.status = 'active'
+     AND EXISTS (
+         SELECT 1 FROM jsonb_array_elements(connection.catalog->'streams') AS stream
+         WHERE stream->'stream'->>'name' = :stream_name
+     )
+    LIMIT :limit
+    """
+)
+
+# Query connections by source connector type and stream, filtered by organization
+SELECT_CONNECTIONS_BY_SOURCE_CONNECTOR_AND_STREAM_AND_ORG = sqlalchemy.text(
+    """
+    SELECT
+         connection.id AS connection_id,
+         connection.name AS connection_name,
+         connection.source_id,
+         connection.status AS connection_status,
+         workspace.id AS workspace_id,
+         workspace.name AS workspace_name,
+         workspace.organization_id,
+         workspace.dataplane_group_id,
+         dataplane_group.name AS dataplane_name,
+         source_actor.actor_definition_id AS source_definition_id,
+         source_actor.name AS source_name
+    FROM connection
+    JOIN actor AS source_actor
+      ON connection.source_id = source_actor.id
+     AND source_actor.tombstone = false
+    JOIN workspace
+      ON source_actor.workspace_id = workspace.id
+     AND workspace.tombstone = false
+    LEFT JOIN dataplane_group
+      ON workspace.dataplane_group_id = dataplane_group.id
+    WHERE
+         source_actor.actor_definition_id = :connector_definition_id
+     AND workspace.organization_id = :organization_id
+     AND connection.status = 'active'
+     AND EXISTS (
+         SELECT 1 FROM jsonb_array_elements(connection.catalog->'streams') AS stream
+         WHERE stream->'stream'->>'name' = :stream_name
+     )
+    LIMIT :limit
+    """
+)

airbyte_ops_mcp/regression_tests/connection_fetcher.py

@@ -96,7 +96,7 @@ def fetch_connection_data(
         )
 
     access_token = _get_access_token(client_id, client_secret)
-    api_root = constants.CLOUD_API_ROOT
+    public_api_root = constants.CLOUD_API_ROOT
     headers = {
         "Authorization": f"Bearer {access_token}",
         "Content-Type": "application/json",
@@ -104,7 +104,7 @@ def fetch_connection_data(
 
     # Get connection details
     conn_response = requests.get(
-        f"{api_root}/connections/{connection_id}",
+        f"{public_api_root}/connections/{connection_id}",
         headers=headers,
         timeout=30,
     )
@@ -120,7 +120,7 @@ def fetch_connection_data(
 
     # Get source details (includes config)
     source_response = requests.get(
-        f"{api_root}/sources/{source_id}",
+        f"{public_api_root}/sources/{source_id}",
         headers=headers,
         timeout=30,
     )
@@ -160,7 +160,9 @@ def fetch_connection_data(
     stream_names = [s["name"] for s in streams_config]
 
     # Build Airbyte protocol catalog format
-    catalog = _build_configured_catalog(streams_config, source_id, headers, api_root)
+    catalog = _build_configured_catalog(
+        streams_config, source_id, headers, public_api_root
+    )
 
     return ConnectionData(
         connection_id=connection_id,
@@ -179,12 +181,21 @@ def _build_configured_catalog(
     streams_config: list[dict[str, Any]],
     source_id: str,
     headers: dict[str, str],
-    api_root: str,
+    public_api_root: str,
 ) -> dict[str, Any]:
     """Build a configured catalog from connection stream configuration.
 
     This creates a catalog in the Airbyte protocol format that can be used
     with connector commands.
+
+    Args:
+        streams_config: List of stream configuration dicts from the connection.
+        source_id: The source ID.
+        headers: HTTP headers for API requests.
+        public_api_root: The Public API root URL (e.g., CLOUD_API_ROOT).
+
+    Returns:
+        A configured catalog dict in Airbyte protocol format.
     """
     # For now, create a minimal catalog structure
     # A full implementation would fetch the source's discovered catalog

airbyte_ops_mcp/regression_tests/models.py

@@ -250,7 +250,7 @@ class ExecutionResult:
         messages_by_type: dict[str, list[str]] = defaultdict(list)
         for message in self.airbyte_messages:
             type_name = message.type.value.lower()
-            messages_by_type[type_name].append(message.json())
+            messages_by_type[type_name].append(message.model_dump_json())
 
         for type_name, messages in messages_by_type.items():
             file_path = airbyte_messages_dir / f"{type_name}.jsonl"
@@ -259,7 +259,7 @@ class ExecutionResult:
         # Save configured catalog (input) if available
         if self.configured_catalog is not None:
             catalog_path = output_dir / "configured_catalog.json"
-            catalog_path.write_text(self.configured_catalog.json(indent=2))
+            catalog_path.write_text(self.configured_catalog.model_dump_json(indent=2))
             self.logger.info(f"Saved configured catalog to {catalog_path}")
 
         self.logger.info(f"Artifacts saved to {output_dir}")

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/README.md

@@ -1,91 +0,0 @@
-# Connector Metadata Service Library
-
-This submodule is responsible for managing all the logic related to validating, uploading, and managing connector metadata.
-
-## Installation
-
-To use this submodule, it is recommended that you use Poetry to manage dependencies.
-
-```
-poetry install
-```
-
-### Node.js Requirement
-
-The model generation process also requires Node.js to bundle JSON schemas. Install Node.js:
-
-- On macOS: `brew install node`
-- On Ubuntu/Debian: `sudo apt-get install nodejs npm`
-- On other systems: https://nodejs.org/
-
-Node.js dependencies will be automatically installed when running `poetry run poe generate-models`.
-
-## Generating Models
-
-This submodule includes a tool for generating Python models from JSON Schema specifications. To generate the models, we use the library [datamodel-code-generator](https://github.com/koxudaxi/datamodel-code-generator). The generated models are stored in `models/generated`.
-
-To generate the models, run the following command:
-
-```bash
-poetry run poe generate-models
-
-```
-
-This will read the JSON Schema specifications in `models/src` and generate Python models in `models/generated`.
-
-## Running Tests
-
-```bash
-poetry run pytest
-```
-
-## Changelog
-
-### 0.24.1
-Update Python version requirement from 3.10 to 3.11.
-
-## Validating Metadata Files
-
-To be considered valid, a connector must have a metadata.yaml file which must conform to the [ConnectorMetadataDefinitionV0](./metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml) schema, and a documentation file.
-
-The paths to both files must be passed to the validate command.
-
-```bash
-poetry run metadata_service validate tests/fixtures/metadata_validate/valid/metadata_simple.yaml tests/fixtures/doc.md
-```
-
-## Useful Commands
-
-### Replicate Production Data in your Development Bucket
-
-This will replicate all the production data to your development bucket. This is useful for testing the metadata service with real up to date data.
-
-_💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage/docs/gsutil) installed and have run `gsutil auth login`_
-
-_⚠️ Warning: Its important to know that this will remove ANY files you have in your destination buckets as it calls `gsutil rsync` with `-d` enabled._
-
-```bash
-TARGET_BUCKET=<YOUR-DEV_BUCKET> poetry run poe replicate-prod
-```
-
-### Copy specific connector version to your Development Bucket
-
-This will copy the specified connector version to your development bucket. This is useful for testing the metadata service with a specific version of a connector.
-
-_💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage/docs/gsutil) installed and have run `gsutil auth login`_
-
-```bash
-TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-preview.ea013c8" poetry run poe copy-connector-from-prod
-```
-
-### Promote Connector Version to Latest
-
-This will promote the specified connector version to the latest version in the registry. This is useful for creating a mocked registry in which a prerelease connector is treated as if it was already published.
-
-_💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage/docs/gsutil) installed and have run `gsutil auth login`_
-
-_⚠️ Warning: Its important to know that this will remove ANY existing files in the latest folder that are not in the versioned folder as it calls `gsutil rsync` with `-d` enabled._
-
-```bash
-TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-preview.ea013c8" poetry run poe promote-connector-to-latest
-```

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/bin/bundle-schemas.js

@@ -1,48 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Bundle JSON schemas using @apidevtools/json-schema-ref-parser
- * This script resolves all $ref references in the schema files and creates a single bundled schema.
- */
-
-const $RefParser = require('@apidevtools/json-schema-ref-parser');
-const fs = require('fs');
-const path = require('path');
-
-const YAML_DIR = 'metadata_service/models/src';
-const OUTPUT_DIR = 'metadata_service/models/generated';
-const ENTRY_SCHEMA = path.join(YAML_DIR, 'ConnectorMetadataDefinitionV0.yaml');
-const BUNDLE_OUTPUT = path.join(OUTPUT_DIR, 'ConnectorMetadataDefinitionV0.json');
-
-async function bundleSchemas() {
-  try {
-    console.log('📦 Bundling JSON schemas...');
-    console.log(`   Entry schema: ${ENTRY_SCHEMA}`);
-    console.log(`   Output: ${BUNDLE_OUTPUT}`);
-
-    if (!fs.existsSync(YAML_DIR)) {
-      console.error(`❌ Error: The yaml directory does not exist: ${YAML_DIR}`);
-      process.exit(1);
-    }
-
-    if (!fs.existsSync(OUTPUT_DIR)) {
-      fs.mkdirSync(OUTPUT_DIR, { recursive: true });
-    }
-
-    const schema = await $RefParser.bundle(ENTRY_SCHEMA, {
-      dereference: {
-        circular: 'ignore' // Handle circular references gracefully
-      }
-    });
-
-    fs.writeFileSync(BUNDLE_OUTPUT, JSON.stringify(schema, null, 2));
-
-    console.log('✅ Successfully bundled schema to', BUNDLE_OUTPUT);
-    console.log('   This bundled schema can be used for IDE validation and other tools.');
-  } catch (error) {
-    console.error('❌ Error bundling schemas:', error.message);
-    process.exit(1);
-  }
-}
-
-bundleSchemas();

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/bin/generate-metadata-models.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-
-YAML_DIR=metadata_service/models/src
-OUTPUT_DIR=metadata_service/models/generated
-
-# Ensure the yaml directory exists
-if [ ! -d "$YAML_DIR" ]; then
-  echo "The yaml directory does not exist: $YAML_DIR"
-  exit 1
-fi
-
-
-rm -rf "$OUTPUT_DIR"/*.py
-mkdir -p "$OUTPUT_DIR"
-
-echo "# generated by generate-metadata-models" > "$OUTPUT_DIR"/__init__.py
-
-for f in "$YAML_DIR"/*.yaml; do
-  filename_wo_ext=$(basename "$f" | cut -d . -f 1)
-  echo "from .$filename_wo_ext import *" >> "$OUTPUT_DIR"/__init__.py
-
-  datamodel-codegen \
-    --input "$YAML_DIR/$filename_wo_ext.yaml" \
-    --output "$OUTPUT_DIR/$filename_wo_ext.py" \
-    --use-title-as-name \
-    --use-double-quotes \
-    --enum-field-as-literal all \
-    --disable-timestamp
-done
-
-echo ""
-echo "Generating bundled JSON schema..."
-node bin/bundle-schemas.js

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/generated/ActorDefinitionResourceRequirements.py

@@ -1,54 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  ActorDefinitionResourceRequirements.yaml
-
-from __future__ import annotations
-
-from typing import List, Optional
-
-from pydantic import BaseModel, Extra, Field
-from typing_extensions import Literal
-
-
-class ResourceRequirements(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    cpu_request: Optional[str] = None
-    cpu_limit: Optional[str] = None
-    memory_request: Optional[str] = None
-    memory_limit: Optional[str] = None
-
-
-class JobType(BaseModel):
-    __root__: Literal[
-        "get_spec",
-        "check_connection",
-        "discover_schema",
-        "sync",
-        "reset_connection",
-        "connection_updater",
-        "replicate",
-    ] = Field(
-        ...,
-        description="enum that describes the different types of jobs that the platform runs.",
-        title="JobType",
-    )
-
-
-class JobTypeResourceLimit(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    jobType: JobType
-    resourceRequirements: ResourceRequirements
-
-
-class ActorDefinitionResourceRequirements(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    default: Optional[ResourceRequirements] = Field(
-        None,
-        description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.",
-    )
-    jobSpecific: Optional[List[JobTypeResourceLimit]] = None

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/generated/AirbyteInternal.py

@@ -1,22 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  AirbyteInternal.yaml
-
-from __future__ import annotations
-
-from typing import Optional
-
-from pydantic import BaseModel, Extra, Field
-from typing_extensions import Literal
-
-
-class AirbyteInternal(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    sl: Optional[Literal[0, 100, 200, 300]] = None
-    ql: Optional[Literal[0, 100, 200, 300, 400, 500, 600]] = None
-    isEnterprise: Optional[bool] = False
-    requireVersionIncrementsInPullRequests: Optional[bool] = Field(
-        True,
-        description="When false, version increment checks will be skipped for this connector",
-    )

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/generated/AllowedHosts.py

@@ -1,18 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  AllowedHosts.yaml
-
-from __future__ import annotations
-
-from typing import List, Optional
-
-from pydantic import BaseModel, Extra, Field
-
-
-class AllowedHosts(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    hosts: Optional[List[str]] = Field(
-        None,
-        description="An array of hosts that this connector can connect to.  AllowedHosts not being present for the source or destination means that access to all hosts is allowed.  An empty list here means that no network access is granted.",
-    )

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/generated/ConnectorBreakingChanges.py

@@ -1,65 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  ConnectorBreakingChanges.yaml
-
-from __future__ import annotations
-
-from datetime import date
-from typing import Dict, List, Optional
-
-from pydantic import AnyUrl, BaseModel, Extra, Field, constr
-from typing_extensions import Literal
-
-
-class StreamBreakingChangeScope(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    scopeType: str = Field("stream", const=True)
-    impactedScopes: List[str] = Field(
-        ...,
-        description="List of streams that are impacted by the breaking change.",
-        min_items=1,
-    )
-
-
-class BreakingChangeScope(BaseModel):
-    __root__: StreamBreakingChangeScope = Field(
-        ...,
-        description="A scope that can be used to limit the impact of a breaking change.",
-    )
-
-
-class VersionBreakingChange(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    upgradeDeadline: date = Field(
-        ...,
-        description="The deadline by which to upgrade before the breaking change takes effect.",
-    )
-    message: str = Field(
-        ..., description="Descriptive message detailing the breaking change."
-    )
-    deadlineAction: Optional[Literal["auto_upgrade", "disable"]] = Field(
-        None, description="Action to do when the deadline is reached."
-    )
-    migrationDocumentationUrl: Optional[AnyUrl] = Field(
-        None,
-        description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",
-    )
-    scopedImpact: Optional[List[BreakingChangeScope]] = Field(
-        None,
-        description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.",
-        min_items=1,
-    )
-
-
-class ConnectorBreakingChanges(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field(
-        ...,
-        description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.",
-        title="ConnectorBreakingChanges",
-    )

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/generated/ConnectorBuildOptions.py

@@ -1,15 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  ConnectorBuildOptions.yaml
-
-from __future__ import annotations
-
-from typing import Optional
-
-from pydantic import BaseModel, Extra
-
-
-class ConnectorBuildOptions(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    baseImage: Optional[str] = None

airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/generated/ConnectorIPCOptions.py

@@ -1,25 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  ConnectorIPCOptions.yaml
-
-from __future__ import annotations
-
-from typing import List
-
-from pydantic import BaseModel, Extra
-from typing_extensions import Literal
-
-
-class DataChannel(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    version: str
-    supportedSerialization: List[Literal["JSONL", "PROTOBUF", "FLATBUFFERS"]]
-    supportedTransport: List[Literal["STDIO", "SOCKET"]]
-
-
-class ConnectorIPCOptions(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    dataChannel: DataChannel