airbyte-cdk 6.61.2__tar.gz → 6.61.3__tar.gz

{airbyte_cdk-6.61.2 → airbyte_cdk-6.61.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.61.2
+Version: 6.61.3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -19,6 +19,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Provides-Extra: dev
 Provides-Extra: file-based
+Provides-Extra: manifest-server
 Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
@@ -35,6 +36,7 @@ Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dateparser (>=1.2.2,<2.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastapi (>=0.116.1) ; extra == "manifest-server"
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
 Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
@@ -77,6 +79,7 @@ Requires-Dist: typing-extensions
 Requires-Dist: unidecode (>=1.3.8,<2.0.0)
 Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
 Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: uvicorn (>=0.35.0) ; extra == "manifest-server"
 Requires-Dist: wcmatch (==10.0)
 Requires-Dist: whenever (>=0.6.16,<0.7.0)
 Requires-Dist: xmltodict (>=0.13,<0.15)

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/Dockerfile

@@ -0,0 +1,45 @@
+# Dockerfile for the Airbyte Manifest Server.
+#
+# This Dockerfile should be built from the root of the repository.
+#
+# Example:
+# docker build -f airbyte_cdk/manifest_server/Dockerfile -t airbyte/manifest-server .
+
+FROM python:3.12-slim-bookworm
+
+# Install git (needed for dynamic versioning) and poetry
+RUN apt-get update && \
+    apt-get install -y git && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install poetry==1.8.3
+
+# Configure poetry to not create virtual environments and disable interactive mode
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+
+# Copy poetry files (build from project root)
+COPY pyproject.toml poetry.lock ./
+
+# Copy the project source code (needed for dynamic versioning)
+COPY . /app
+
+# Install dependencies and package directly to system Python
+RUN --mount=type=cache,target=$POETRY_CACHE_DIR \
+    poetry config virtualenvs.create false && \
+    poetry install --extras manifest-server
+
+# Create a non-root user and group
+RUN groupadd --gid 1000 airbyte && \
+    useradd --uid 1000 --gid airbyte --shell /bin/bash --create-home airbyte
+
+# Change ownership
+RUN chown -R airbyte:airbyte /app
+
+# Run app as non-root user
+USER airbyte:airbyte
+
+EXPOSE 8080
+
+CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/README.md

@@ -0,0 +1,142 @@
+# Manifest Server
+
+An HTTP server for running Airbyte declarative connectors via their manifest files.
+
+## Quick Start
+
+### Installation
+
+The manifest server is available as an extra dependency:
+
+```bash
+# Using Poetry (preferred)
+poetry install --extras manifest-server
+
+# Using pip
+pip install airbyte-cdk[manifest-server]
+
+# Using uv
+uv pip install 'airbyte-cdk[manifest-server]'
+```
+
+### Running the Server
+
+```bash
+# Start the server (default port 8000)
+manifest-server start
+
+# Start on a specific port
+manifest-server start --port 8080
+
+# Or using Python module
+python -m airbyte_cdk.manifest_server.cli.run start
+```
+
+The server will start on `http://localhost:8000` by default.
+
+## API Endpoints
+
+### `/v1/manifest/test_read`
+Test reading from a specific stream in the manifest.
+
+**POST** - Test stream reading with configurable limits for records, pages, and slices.
+
+### `/v1/manifest/check`
+Check configuration against a manifest.
+
+**POST** - Validates connector configuration and returns success/failure status with message.
+
+### `/v1/manifest/discover`
+Discover streams from a manifest.
+
+**POST** - Returns the catalog of available streams from the manifest.
+
+### `/v1/manifest/resolve` 
+Resolve a manifest to its final configuration.
+
+**POST** - Returns the resolved manifest without dynamic stream generation.
+
+### `/v1/manifest/full_resolve`
+Fully resolve a manifest including dynamic streams.
+
+**POST** - Generates dynamic streams up to specified limits and includes them in the resolved manifest.
+
+## Custom Components
+
+The manifest server supports custom Python components, but this feature is **disabled by default** for security reasons.
+
+### Enabling Custom Components
+
+To allow custom Python components in your manifest files, set the environment variable:
+```bash
+export AIRBYTE_ENABLE_UNSAFE_CODE=true
+```
+
+## Authentication
+
+The manifest server supports optional JWT bearer token authentication:
+
+### Configuration
+Set the environment variable to enable authentication:
+```bash
+export AB_JWT_SIGNATURE_SECRET="your-jwt-secret-key"
+```
+
+### Usage
+When authentication is enabled, include a valid JWT token in the Authorization header:
+```bash
+curl -H "Authorization: Bearer <your-jwt-token>" \
+  http://localhost:8000/v1/manifest/test_read
+```
+
+### Behavior
+- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through 
+- **With `AB_JWT_SIGNATURE_SECRET`**: Requires valid JWT bearer token using HS256 algorithm
+
+## OpenAPI Specification
+
+The manifest server provides an OpenAPI specification for API client generation:
+
+### Generating the OpenAPI Spec
+```bash
+# Generate OpenAPI YAML (default location)
+manifest-server generate-openapi
+
+# Generate to custom location
+manifest-server generate-openapi --output /path/to/openapi.yaml
+```
+
+The generated OpenAPI specification is consumed by other applications and tools to:
+- Generate API clients in various programming languages
+- Create SDK bindings for the manifest server
+- Provide API documentation and validation
+- Enable integration with API development tools
+
+### Interactive API Documentation
+
+When running, interactive API documentation is available at:
+- Swagger UI: `http://localhost:8000/docs`
+- ReDoc: `http://localhost:8000/redoc`
+
+## Testing
+
+Run the manifest server tests from the repository root:
+
+```bash
+# Run all manifest server tests
+poetry run pytest unit_tests/manifest_server/ -v
+```
+
+## Docker
+
+The manifest server can be containerized using the included Dockerfile. Build from the repository root:
+
+```bash
+# Build from repository root (not from manifest_server subdirectory)
+docker build -f airbyte_cdk/manifest_server/Dockerfile -t manifest-server .
+
+# Run the container
+docker run -p 8080:8080 manifest-server
+```
+
+Note: The container runs on port 8080 by default.

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/__init__.py

@@ -0,0 +1,3 @@
+"""
+.. include:: ./README.md
+"""

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/api_models/__init__.py

@@ -0,0 +1,49 @@
+"""
+API Models for the Manifest Server Service.
+
+This package contains all Pydantic models used for API requests and responses.
+"""
+
+from .dicts import ConnectorConfig, Manifest
+from .manifest import (
+    CheckRequest,
+    CheckResponse,
+    DiscoverRequest,
+    DiscoverResponse,
+    FullResolveRequest,
+    ManifestResponse,
+    ResolveRequest,
+    StreamTestReadRequest,
+)
+from .stream import (
+    AuxiliaryRequest,
+    HttpRequest,
+    HttpResponse,
+    LogMessage,
+    StreamReadPages,
+    StreamReadResponse,
+    StreamReadSlices,
+)
+
+__all__ = [
+    # Typed Dicts
+    "ConnectorConfig",
+    "Manifest",
+    # Manifest request/response models
+    "FullResolveRequest",
+    "ManifestResponse",
+    "StreamTestReadRequest",
+    "ResolveRequest",
+    "CheckRequest",
+    "CheckResponse",
+    "DiscoverRequest",
+    "DiscoverResponse",
+    # Stream models
+    "AuxiliaryRequest",
+    "HttpRequest",
+    "HttpResponse",
+    "LogMessage",
+    "StreamReadResponse",
+    "StreamReadPages",
+    "StreamReadSlices",
+]

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/api_models/capabilities.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class CapabilitiesResponse(BaseModel):
+    """Capabilities of the manifest server."""
+
+    custom_code_execution: bool

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/api_models/dicts.py

@@ -0,0 +1,17 @@
+"""
+Common API models shared across different endpoints.
+"""
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Manifest(BaseModel):
+    """Base manifest model. Allows client generation to replace with proper JsonNode types."""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class ConnectorConfig(BaseModel):
+    """Base connector configuration model. Allows client generation to replace with proper JsonNode types."""
+
+    model_config = ConfigDict(extra="allow")

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/api_models/manifest.py

@@ -0,0 +1,73 @@
+"""
+Manifest-related API models.
+
+These models define the request and response structures for manifest operations
+like reading, resolving, and full resolution.
+"""
+
+from typing import Any, List, Optional
+
+from airbyte_protocol_dataclasses.models import AirbyteCatalog
+from pydantic import BaseModel, Field
+
+from .dicts import ConnectorConfig, Manifest
+
+
+class StreamTestReadRequest(BaseModel):
+    """Request to test read from a specific stream."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+    stream_name: str
+    state: List[Any] = Field(default_factory=list)
+    custom_components_code: Optional[str] = None
+    record_limit: int = Field(default=100, ge=1, le=5000)
+    page_limit: int = Field(default=5, ge=1, le=20)
+    slice_limit: int = Field(default=5, ge=1, le=20)
+
+
+class CheckRequest(BaseModel):
+    """Request to check a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+
+
+class CheckResponse(BaseModel):
+    """Response to check a manifest."""
+
+    success: bool
+    message: Optional[str] = None
+
+
+class DiscoverRequest(BaseModel):
+    """Request to discover a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+
+
+class DiscoverResponse(BaseModel):
+    """Response to discover a manifest."""
+
+    catalog: AirbyteCatalog
+
+
+class ResolveRequest(BaseModel):
+    """Request to resolve a manifest."""
+
+    manifest: Manifest
+
+
+class ManifestResponse(BaseModel):
+    """Response containing a manifest."""
+
+    manifest: Manifest
+
+
+class FullResolveRequest(BaseModel):
+    """Request to fully resolve a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+    stream_limit: int = Field(default=100, ge=1, le=100)

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/api_models/stream.py

@@ -0,0 +1,76 @@
+"""
+Stream-related API models.
+
+These models define the structure for stream reading operations and responses.
+They accurately reflect the runtime types returned by the CDK, particularly
+fixing type mismatches like slice_descriptor being a string rather than an object.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel
+
+
+class HttpRequest(BaseModel):
+    """HTTP request details."""
+
+    url: str
+    headers: Optional[Dict[str, Any]]
+    http_method: str
+    body: Optional[str] = None
+
+
+class HttpResponse(BaseModel):
+    """HTTP response details."""
+
+    status: int
+    body: Optional[str] = None
+    headers: Optional[Dict[str, Any]] = None
+
+
+class LogMessage(BaseModel):
+    """Log message from stream processing."""
+
+    message: str
+    level: str
+    internal_message: Optional[str] = None
+    stacktrace: Optional[str] = None
+
+
+class AuxiliaryRequest(BaseModel):
+    """Auxiliary HTTP request made during stream processing."""
+
+    title: str
+    type: str
+    description: str
+    request: HttpRequest
+    response: HttpResponse
+
+
+class StreamReadPages(BaseModel):
+    """Pages of data read from a stream slice."""
+
+    records: List[object]
+    request: Optional[HttpRequest] = None
+    response: Optional[HttpResponse] = None
+
+
+class StreamReadSlices(BaseModel):
+    """Slices of data read from a stream."""
+
+    pages: List[StreamReadPages]
+    slice_descriptor: Optional[str]  # This is actually a string at runtime, not Dict[str, Any]
+    state: Optional[List[Dict[str, Any]]] = None
+    auxiliary_requests: Optional[List[AuxiliaryRequest]] = None
+
+
+class StreamReadResponse(BaseModel):
+    """Complete stream read response with properly typed fields."""
+
+    logs: List[LogMessage]
+    slices: List[StreamReadSlices]
+    test_read_limit_reached: bool
+    auxiliary_requests: List[AuxiliaryRequest]
+    inferred_schema: Optional[Dict[str, Any]]
+    inferred_datetime_formats: Optional[Dict[str, str]]
+    latest_config_update: Optional[Dict[str, Any]]

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/app.py

@@ -0,0 +1,17 @@
+from fastapi import FastAPI
+
+from .routers import capabilities, health, manifest
+
+app = FastAPI(
+    title="Manifest Server",
+    description="A service for running low-code Airbyte connectors",
+    version="0.1.0",
+    contact={
+        "name": "Airbyte",
+        "url": "https://airbyte.com",
+    },
+)
+
+app.include_router(health.router)
+app.include_router(capabilities.router)
+app.include_router(manifest.router, prefix="/v1")

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/auth.py

@@ -0,0 +1,43 @@
+import os
+from typing import Optional
+
+import jwt
+from fastapi import HTTPException, Security, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
+security = HTTPBearer(auto_error=False)
+
+
+def verify_jwt_token(
+    credentials: Optional[HTTPAuthorizationCredentials] = Security(security),
+) -> None:
+    """
+    Verify JWT token if AB_JWT_SIGNATURE_SECRET is set, otherwise allow through.
+
+    Args:
+        credentials: Bearer token credentials from request header
+
+    Raises:
+        HTTPException: If token is invalid or missing when secret is configured
+    """
+    jwt_secret = os.getenv("AB_JWT_SIGNATURE_SECRET")
+
+    # If no secret is configured, allow all requests through
+    if not jwt_secret:
+        return
+
+    if not credentials:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Bearer token required",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    try:
+        jwt.decode(credentials.credentials, jwt_secret, algorithms=["HS256"])
+    except jwt.InvalidTokenError:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/cli/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""The `airbyte_cdk.manifest_server.cli` module provides a standalone CLI for the Airbyte CDK Manifest Server.
+
+This CLI enables running a FastAPI server for managing and executing Airbyte declarative manifests.
+"""

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/cli/_common.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Common utilities for manifest server CLI commands."""
+
+import sys
+
+import rich_click as click
+
+# Import server dependencies with graceful fallback
+try:
+    import fastapi  # noqa: F401
+    import uvicorn  # noqa: F401
+
+    FASTAPI_AVAILABLE = True
+except ImportError:
+    FASTAPI_AVAILABLE = False
+
+
+def check_manifest_server_dependencies() -> None:
+    """Check if manifest-server dependencies are installed."""
+    if not FASTAPI_AVAILABLE:
+        click.echo(
+            "❌ Manifest runner dependencies not found. Please install with:\n\n"
+            "  pip install airbyte-cdk[manifest-server]\n"
+            "  # or\n"
+            "  poetry install --extras manifest-server\n",
+            err=True,
+        )
+        sys.exit(1)

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/cli/_info.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Info command for the manifest server CLI."""
+
+from typing import Any, Optional
+
+import rich_click as click
+
+# Import server dependencies with graceful fallback
+fastapi: Optional[Any] = None
+uvicorn: Optional[Any] = None
+
+try:
+    import fastapi  # type: ignore[no-redef]
+    import uvicorn  # type: ignore[no-redef]
+
+    FASTAPI_AVAILABLE = True
+except ImportError:
+    FASTAPI_AVAILABLE = False
+
+
+@click.command()
+def info() -> None:
+    """Show manifest server information and status."""
+    if FASTAPI_AVAILABLE and fastapi is not None and uvicorn is not None:
+        click.echo("✅ Manifest runner dependencies are installed")
+        click.echo(f"   FastAPI version: {fastapi.__version__}")
+        click.echo(f"   Uvicorn version: {uvicorn.__version__}")
+    else:
+        click.echo("❌ Manifest runner dependencies not installed")
+        click.echo("   Install with: pip install airbyte-cdk[manifest-server]")

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/cli/_openapi.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Generate OpenAPI command for the manifest server CLI."""
+
+from pathlib import Path
+
+import rich_click as click
+from yaml import dump
+
+from ._common import check_manifest_server_dependencies
+
+
+@click.command("generate-openapi")
+@click.option(
+    "--output",
+    "-o",
+    default="airbyte_cdk/manifest_server/openapi.yaml",
+    help="Output path for the OpenAPI YAML file",
+    show_default=True,
+)
+def generate_openapi(output: str) -> None:
+    """Generate OpenAPI YAML specification for the manifest server."""
+    check_manifest_server_dependencies()
+
+    # Import the FastAPI app
+    from airbyte_cdk.manifest_server.app import app
+
+    # Get OpenAPI schema
+    openapi_schema = app.openapi()
+
+    # Ensure output directory exists
+    output_path = Path(output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write YAML file with header comment
+    with open(output_path, "w") as f:
+        f.write("# This file is auto-generated. Do not edit manually.\n")
+        f.write("# To regenerate, run: manifest-server generate-openapi\n")
+        f.write("\n")
+        dump(openapi_schema, f, default_flow_style=False, sort_keys=False)
+
+    click.echo(f"✅ OpenAPI YAML generated at: {output_path}")
+    click.echo(f"   Title: {openapi_schema.get('info', {}).get('title', 'N/A')}")
+    click.echo(f"   Version: {openapi_schema.get('info', {}).get('version', 'N/A')}")

airbyte_cdk-6.61.3/airbyte_cdk/manifest_server/cli/_start.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Start command for the manifest server CLI."""
+
+import rich_click as click
+
+from ._common import check_manifest_server_dependencies
+
+
+@click.command()
+@click.option(
+    "--host",
+    default="127.0.0.1",
+    help="Host to bind the server to",
+    show_default=True,
+)
+@click.option(
+    "--port",
+    default=8000,
+    help="Port to bind the server to",
+    show_default=True,
+)
+@click.option(
+    "--reload",
+    is_flag=True,
+    help="Enable auto-reload for development",
+)
+def start(host: str, port: int, reload: bool) -> None:
+    """Start the FastAPI manifest server server."""
+    check_manifest_server_dependencies()
+
+    # Import and use the main server function
+    from airbyte_cdk.manifest_server.main import run_server
+
+    run_server(
+        host=host,
+        port=port,
+        reload=reload,
+    )