airbyte-agent-mcp 0.1.30__py3-none-any.whl

airbyte_agent_mcp/config.py

@@ -0,0 +1,97 @@
+"""Configuration loading and validation."""
+
+import logging
+from pathlib import Path
+
+import yaml
+
+from airbyte_agent_mcp.models import Config, ConnectorType
+
+logger = logging.getLogger(__name__)
+
+
+def load_connector_config(config_path: str = "configured_connectors.yaml") -> Config:
+    """Load and validate configuration from YAML file.
+
+    Args:
+        config_path: Path to configured_connectors.yaml file
+
+    Returns:
+        Validated Config object
+
+    Raises:
+        FileNotFoundError: If config file doesn't exist
+        ValueError: If config is invalid
+    """
+    path = Path(config_path)
+
+    if not path.exists():
+        raise FileNotFoundError(
+            f"Configuration file not found: {config_path}\nCreate a configured_connectors.yaml file with your connector definitions."
+        )
+
+    logger.info(f"Loading configuration from {config_path}")
+
+    with open(path) as f:
+        data = yaml.safe_load(f)
+
+    if not data:
+        raise ValueError(f"Configuration file is empty: {config_path}")
+
+    try:
+        config = Config(**data)
+    except Exception as e:
+        raise ValueError(f"Invalid configuration: {e}") from e
+
+    logger.info(f"Loaded {len(config.connectors)} connector(s): {', '.join(c.id for c in config.connectors)}")
+
+    return config
+
+
+def validate_connectors(config: Config) -> list[str]:
+    """Validate that connectors are available.
+
+    For LOCAL connectors with path: Check if connector.yaml exists
+    For LOCAL connectors with connector_name: Skip validation (validated lazily on first use)
+    For HOSTED connectors: Not implemented
+
+    Args:
+        config: Configuration to validate
+
+    Returns:
+        List of error messages (empty if all valid)
+    """
+    errors = []
+
+    for connector in config.connectors:
+        logger.debug(f"Validating connector: {connector.id}")
+
+        if connector.type == ConnectorType.LOCAL:
+            if connector.path:
+                # Check if connector.yaml exists
+                connector_path = Path(connector.path)
+                if not connector_path.exists():
+                    error = f"LOCAL connector '{connector.id}': File not found: {connector.path}"
+                    errors.append(error)
+                    logger.error(error)
+                else:
+                    logger.debug(f"LOCAL connector '{connector.id}' file found")
+            elif connector.connector_name:
+                # Registry-based connector: validation happens lazily on first use
+                logger.debug(
+                    f"LOCAL connector '{connector.id}' uses registry " f"(connector_name={connector.connector_name}), skipping startup validation"
+                )
+        elif connector.type == ConnectorType.HOSTED:
+            # HOSTED connectors are not implemented yet
+            error = f"HOSTED connector '{connector.id}' - not implemented"
+            errors.append(error)
+            logger.error(error)
+        else:
+            error = f"Unknown connector type '{connector.type}' found"
+            errors.append(error)
+            logger.error(error)
+
+    if not errors:
+        logger.info("All connectors validated successfully")
+
+    return errors

airbyte_agent_mcp/connector_manager.py

@@ -0,0 +1,340 @@
+"""Connector instantiation and execution management."""
+
+import base64
+import inspect
+import logging
+from typing import Any
+
+from ._vendored.connector_sdk import LocalExecutor as ConnectorExecutor
+from ._vendored.connector_sdk.connector_model_loader import load_connector_model
+from ._vendored.connector_sdk.executor.models import ExecutionConfig
+
+from airbyte_agent_mcp.models import Config, ConnectorConfig, ConnectorInfo, ConnectorType, DiscoverConnectorsResponse
+from airbyte_agent_mcp.registry_client import RegistryClient
+from airbyte_agent_mcp.secret_manager import SecretsManager
+
+logger = logging.getLogger(__name__)
+
+
+class ConnectorManager:
+    """Manages connector lifecycle and execution."""
+
+    def __init__(
+        self,
+        config: Config,
+        secrets_manager: SecretsManager,
+        registry_client: RegistryClient | None = None,
+    ):
+        """Initialize manager.
+
+        Args:
+            config: Configuration with connector definitions
+            secrets_manager: Secrets manager for resolving credentials
+            registry_client: Optional registry client for fetching remote connectors
+        """
+        self.config = config
+        self.secrets_manager = secrets_manager
+        self.registry_client = registry_client or RegistryClient()
+
+    async def _get_connector_path(self, connector_config: ConnectorConfig) -> str:
+        """Get path to connector.yaml (local file or downloaded from registry).
+
+        Args:
+            connector_config: The connector configuration
+
+        Returns:
+            Path to the connector.yaml file
+        """
+        if connector_config.path:
+            # Use local path
+            return connector_config.path
+
+        # Download from registry
+        path = await self.registry_client.download_connector(
+            connector_name=connector_config.connector_name,
+            version=connector_config.version,
+        )
+        return str(path)
+
+    async def execute(
+        self,
+        connector_id: str,
+        entity: str,
+        action: str,
+        params: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Execute an operation on a connector.
+
+        This is stateless - creates a fresh connector instance for each call.
+
+        Args:
+            connector_id: Connector ID from config
+            entity: Entity name (e.g., "customers")
+            action: Operation action (e.g., "list", "get", "create")
+            params: Operation parameters (optional)
+
+        Returns:
+            Result from connector execution
+
+        Raises:
+            ValueError: If connector not found
+            Exception: Any error from connector execution
+        """
+        params = params or {}
+
+        logger.info(f"Executing: {connector_id}.{entity}.{action} with params: {list(params.keys())}")
+
+        connector_config = self.config.get_connector(connector_id)
+
+        secrets = {}
+        if connector_config.secrets:
+            secrets = self.secrets_manager.get_secrets(connector_config.secrets)
+
+        # Resolve config_values from environment variables
+        config_values = {}
+        if connector_config.config_values:
+            config_values = self.secrets_manager.get_secrets(connector_config.config_values)
+
+        # Get path (local or from registry)
+        path = await self._get_connector_path(connector_config)
+        logger.info(f"Using connector path: {path}")
+        connector = self._create_yaml_connector(path, secrets, auth_scheme=connector_config.auth_scheme, config_values=config_values)
+
+        logger.debug(f"Calling connector.execute({entity}, {action}, ...)")
+        result = await connector.execute(ExecutionConfig(entity=entity, action=action, params=params))
+
+        # Handle ExecutionResult from SDK
+        if not result.success:
+            raise Exception(result.error or "Execution failed")
+
+        # Handle download operations (data is AsyncIterator[bytes])
+        if inspect.isasyncgen(result.data):
+            return await self._handle_download(result.data)
+
+        logger.info("Execution successful")
+        return result.data
+
+    def _create_yaml_connector(
+        self, path: str, secrets: dict[str, Any], auth_scheme: str | None = None, config_values: dict[str, str] | None = None
+    ) -> Any:
+        """Create a YAML-based connector instance.
+
+        Args:
+            path: Path to connector.yaml file
+            secrets: Resolved secrets dict
+            auth_scheme: Auth scheme for multi-auth connectors (optional)
+            config_values: Non-secret config values like subdomain, region (optional)
+
+        Returns:
+            ConnectorExecutor instance
+        """
+        connector = ConnectorExecutor(
+            config_path=path, auth_config=secrets, auth_scheme=auth_scheme, config_values=config_values, execution_context="mcp"
+        )
+
+        return connector
+
+    async def _handle_download(self, stream: Any) -> dict[str, Any]:
+        """Handle download operations by collecting bytes and base64 encoding.
+
+        Args:
+            stream: AsyncIterator[bytes] from connector download operation
+
+        Returns:
+            Dict with base64-encoded data and metadata
+
+        Raises:
+            Exception: If file exceeds maximum size limit
+        """
+        max_size = 50 * 1024 * 1024  # 50MB limit for MCP downloads
+        chunks: list[bytes] = []
+        total_size = 0
+
+        async for chunk in stream:
+            total_size += len(chunk)
+            if total_size > max_size:
+                raise Exception(
+                    f"Download exceeds maximum size limit ({max_size // (1024 * 1024)}MB). Use the SDK directly for large file downloads."
+                )
+            chunks.append(chunk)
+
+        binary_data = b"".join(chunks)
+
+        logger.info(f"Download successful: {len(binary_data)} bytes")
+
+        return {
+            "data": base64.b64encode(binary_data).decode("utf-8"),
+            "size": len(binary_data),
+            "encoding": "base64",
+        }
+
+    async def describe_connector(self, connector_id: str) -> list[dict[str, Any]]:
+        """List available entities for a connector.
+
+        Args:
+            connector_id: Connector ID from config
+
+        Returns:
+            List of entity info dicts
+        """
+        logger.info(f"Listing entities for: {connector_id}")
+
+        connector_config = self.config.get_connector(connector_id)
+        if connector_config.type == ConnectorType.LOCAL:
+            path = await self._get_connector_path(connector_config)
+            return await self._describe_connector_from_sdk(path)
+
+    async def _describe_connector_from_sdk(self, path: str) -> list[dict[str, Any]]:
+        """List entities from a YAML connector definition.
+
+        Args:
+            path: Path to connector.yaml (OpenAPI spec)
+
+        Returns:
+            List of entity info dicts
+        """
+
+        # Load and parse the connector model using SDK
+        connector_model = load_connector_model(path)
+
+        # Build parameter lookup from OpenAPI spec for full metadata
+        param_lookup = self._build_param_lookup(connector_model)
+
+        entities = []
+        for entity_def in connector_model.entities:
+            description = ""
+            parameters: dict[str, list[dict[str, Any]]] = {}
+
+            # Extract parameters for each action from endpoints
+            if entity_def.endpoints:
+                for action, endpoint in entity_def.endpoints.items():
+                    if not description and endpoint.description:
+                        description = endpoint.description
+
+                    # Get the lookup key for this endpoint
+                    lookup_key = (endpoint.path, endpoint.method.lower())
+                    operation_params = param_lookup.get(lookup_key, {})
+
+                    # Collect all parameters for this action with full metadata
+                    action_params = []
+
+                    # Path params (always required)
+                    for param_name in endpoint.path_params:
+                        param_meta = operation_params.get(param_name, {})
+                        action_params.append(
+                            {
+                                "name": param_name,
+                                "in": "path",
+                                "required": True,  # Path params are always required
+                                "type": param_meta.get("type", "string"),
+                                "description": param_meta.get("description", ""),
+                            }
+                        )
+
+                    # Query params (get required/type/description from spec)
+                    for param_name in endpoint.query_params:
+                        param_meta = operation_params.get(param_name, {})
+                        action_params.append(
+                            {
+                                "name": param_name,
+                                "in": "query",
+                                "required": param_meta.get("required", False),
+                                "type": param_meta.get("type", "string"),
+                                "description": param_meta.get("description", ""),
+                            }
+                        )
+
+                    # Body fields (get required from request schema)
+                    request_schema = endpoint.request_schema or {}
+                    required_fields = request_schema.get("required", [])
+                    properties = request_schema.get("properties", {})
+
+                    for param_name in endpoint.body_fields:
+                        prop = properties.get(param_name, {})
+                        action_params.append(
+                            {
+                                "name": param_name,
+                                "in": "body",
+                                "required": param_name in required_fields,
+                                "type": prop.get("type", "string"),
+                                "description": prop.get("description", ""),
+                            }
+                        )
+
+                    if action_params:
+                        parameters[action.value] = action_params
+
+            # Convert Action enums to strings
+            available_actions = [action.value for action in entity_def.actions]
+
+            entities.append(
+                {
+                    "entity_name": entity_def.name,
+                    "description": description,
+                    "available_actions": available_actions,
+                    "parameters": parameters,
+                }
+            )
+
+        return entities
+
+    def _build_param_lookup(self, connector_model: Any) -> dict[tuple[str, str], dict[str, dict[str, Any]]]:
+        """Build a lookup of parameter metadata from OpenAPI spec.
+
+        Args:
+            connector_model: Loaded connector model with openapi_spec
+
+        Returns:
+            Dict mapping (path, method) -> {param_name -> {type, description, required}}
+        """
+        lookup: dict[tuple[str, str], dict[str, dict[str, Any]]] = {}
+
+        openapi_spec = getattr(connector_model, "openapi_spec", None)
+        if not openapi_spec or not hasattr(openapi_spec, "paths"):
+            return lookup
+
+        for path, path_item in openapi_spec.paths.items():
+            for method_name in ["get", "post", "put", "delete", "patch"]:
+                operation = getattr(path_item, method_name, None)
+                if not operation:
+                    continue
+
+                param_map: dict[str, dict[str, Any]] = {}
+                if operation.parameters:
+                    for param in operation.parameters:
+                        # Extract type from schema if available
+                        param_type = "string"
+                        if param.schema_:
+                            param_type = param.schema_.get("type", "string")
+
+                        param_map[param.name] = {
+                            "type": param_type,
+                            "description": param.description or "",
+                            "required": param.required or False,
+                        }
+
+                lookup[(path, method_name)] = param_map
+
+        return lookup
+
+    def discover_connectors(self) -> dict[str, Any]:
+        """Discover all available configured connectors.
+
+        Returns:
+            Dictionary with list of connector information
+        """
+        logger.info("Discovering connectors from configuration")
+
+        # Build list of connector info
+        connector_infos = [
+            ConnectorInfo(
+                id=connector.id,
+                type=connector.type.value,  # Convert enum to string
+                description=connector.description,
+            )
+            for connector in self.config.connectors
+        ]
+
+        response = DiscoverConnectorsResponse(connectors=connector_infos)
+
+        return response.model_dump()

airbyte_agent_mcp/models.py

@@ -0,0 +1,147 @@
+"""Data models for airbyte-agent-mcp configuration and responses."""
+
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class BackoffType(str, Enum):
+    """Retry backoff strategy."""
+
+    EXPONENTIAL = "exponential"
+    LINEAR = "linear"
+
+
+class ErrorType(str, Enum):
+    """Categorized error types for retry logic."""
+
+    RATE_LIMIT = "rate_limit"
+    TIMEOUT = "timeout"
+    SERVER_ERROR = "server_error"
+    UNKNOWN = "unknown"
+
+
+class ConnectorType(str, Enum):
+    """Type of connector."""
+
+    LOCAL = "local"
+    HOSTED = "hosted"
+
+
+class ConnectorConfig(BaseModel):
+    """Configuration for a single connector."""
+
+    id: str = Field(..., description="Unique connector identifier")
+    type: ConnectorType = Field(..., description="Connector type")
+    path: str | None = Field(None, description="Path to connector.yaml (optional if connector_name provided)")
+    connector_name: str | None = Field(None, description="Connector name in registry")
+    version: str | None = Field(None, description="Version to pin (optional, defaults to latest)")
+    description: str = Field(default="", description="Human-readable description")
+    secrets: dict[str, str] = Field(default_factory=dict, description="Mapping of secret param names to secret keys")
+    config_values: dict[str, str] = Field(default_factory=dict, description="Non-secret config values (e.g., subdomain, region)")
+    auth_scheme: str | None = Field(None, description="Auth scheme for multi-auth connectors (e.g., 'oauth', 'api_token')")
+
+    def model_post_init(self, __context):
+        """Validate connector-type-specific fields."""
+        if self.type == ConnectorType.LOCAL:
+            if not self.path and not self.connector_name:
+                raise ValueError(f"LOCAL connector '{self.id}' must specify either 'path' or 'connector_name'")
+
+            # Version pinning only makes sense with registry-based connectors
+            if self.path and self.version:
+                raise ValueError(
+                    f"LOCAL connector '{self.id}' uses 'path' and 'version'. "
+                    "Version pinning is only supported for registry-based connectors "
+                    "using 'connector_name'. Remove 'version' or switch to 'connector_name'."
+                )
+
+
+class Config(BaseModel):
+    """Root configuration model."""
+
+    connectors: list[ConnectorConfig] = Field(..., min_length=1)
+
+    def get_connector(self, connector_id: str) -> ConnectorConfig:
+        """Look up connector by ID."""
+        for connector in self.connectors:
+            if connector.id == connector_id:
+                return connector
+        raise ValueError(f"Connector not found: {connector_id}")
+
+
+class ExecuteResponse(BaseModel):
+    """Response from execute tool."""
+
+    success: bool
+    data: Any = None
+    error: dict | None = None
+    connector_id: str
+    entity: str
+    action: str
+
+
+class ParameterInfo(BaseModel):
+    """Information about an operation parameter."""
+
+    name: str = Field(..., description="Parameter name")
+    in_: str = Field(..., alias="in", description="Parameter location: path, query, or body")
+    required: bool = Field(default=False, description="Whether the parameter is required")
+    type: str = Field(default="string", description="Parameter type (string, integer, boolean, etc.)")
+    description: str = Field(default="", description="Parameter description")
+
+
+class EntityInfo(BaseModel):
+    """Information about a connector entity."""
+
+    entity_name: str = Field(..., description="Entity name to use in execute() calls")
+    description: str = ""
+    available_actions: list[str] = Field(default_factory=list)
+    parameters: dict[str, list[dict[str, Any]]] = Field(
+        default_factory=dict,
+        description="Parameters for each action, keyed by action name. Each parameter has: name, in (path/query/body), required, type, description",
+    )
+
+
+class ListEntitiesResponse(BaseModel):
+    """Response from list_entities tool."""
+
+    connector_id: str
+    entities: list[EntityInfo]
+
+
+class DescribeEntityResponse(BaseModel):
+    """Response from describe_entity tool."""
+
+    connector_id: str
+    entity: str
+    actions: dict
+    schema: dict
+
+
+class ValidationError(BaseModel):
+    """Validation error detail."""
+
+    field: str
+    message: str
+
+
+class ValidateOperationResponse(BaseModel):
+    """Response from validate_operation tool."""
+
+    valid: bool
+    errors: list[ValidationError] = Field(default_factory=list)
+
+
+class ConnectorInfo(BaseModel):
+    """Information about a configured connector."""
+
+    id: str = Field(..., description="Connector identifier")
+    type: str = Field(..., description="Connector type (local or remote)")
+    description: str = Field(default="", description="Human-readable description")
+
+
+class DiscoverConnectorsResponse(BaseModel):
+    """Response from discover_connectors tool."""
+
+    connectors: list[ConnectorInfo] = Field(..., description="List of configured connectors")

airbyte_agent_mcp/registry_client.py

@@ -0,0 +1,103 @@
+"""Client for fetching connectors from the Airbyte registry."""
+
+import atexit
+import logging
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+REGISTRY_BASE_URL = "https://connectors.airbyte.ai"
+REGISTRY_JSON_URL = f"{REGISTRY_BASE_URL}/registry.json"
+
+
+class RegistryClient:
+    """Client for the Airbyte connector registry.
+
+    Downloads connectors fresh from registry on each use (no caching).
+    Uses temp directory for downloaded connector.yaml files.
+    """
+
+    def __init__(self):
+        self.temp_dir = Path(tempfile.mkdtemp(prefix="airbyte-connectors-"))
+        atexit.register(self._cleanup)
+
+    def _cleanup(self):
+        """Clean up temp directory on exit."""
+        if self.temp_dir.exists():
+            shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+    async def fetch_registry(self) -> dict[str, Any]:
+        """Fetch the registry.json index (always fresh)."""
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.get(REGISTRY_JSON_URL)
+            response.raise_for_status()
+            return response.json()
+
+    async def resolve_connector_url(
+        self,
+        connector_name: str,
+        version: str | None = None,
+    ) -> str:
+        """Resolve connector to a download URL.
+
+        Args:
+            connector_name: Connector name (e.g., "stripe")
+            version: Specific version or None for latest
+
+        Returns:
+            URL to download connector.yaml
+        """
+        registry = await self.fetch_registry()
+
+        # Find connector by name
+        connector = None
+        for c in registry["connectors"]:
+            if c["connector_name"] == connector_name:
+                connector = c
+                break
+
+        if not connector:
+            raise ValueError(f"Connector not found in registry: {connector_name}")
+
+        # Get version URL
+        if version:
+            for v in connector["versions"]:
+                if v["version"] == version:
+                    return v["url"]
+            raise ValueError(f"Version {version} not found for {connector['connector_name']}")
+
+        return connector["latest_url"]
+
+    async def download_connector(
+        self,
+        connector_name: str,
+        version: str | None = None,
+    ) -> Path:
+        """Download connector.yaml and return path (cached within session)."""
+        url = await self.resolve_connector_url(connector_name, version)
+
+        # Create temp path based on URL
+        parts = url.replace(f"{REGISTRY_BASE_URL}/definitions/", "").split("/")
+        name, ver = parts[0], parts[1]
+        temp_path = self.temp_dir / name / ver / "connector.yaml"
+
+        # Return cached version if already downloaded
+        if temp_path.exists():
+            logger.debug(f"Using cached connector: {temp_path}")
+            return temp_path
+
+        # Download fresh
+        logger.info(f"Downloading connector from: {url}")
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.get(url)
+            response.raise_for_status()
+
+            temp_path.parent.mkdir(parents=True, exist_ok=True)
+            temp_path.write_text(response.text)
+
+        return temp_path