airbyte-cdk 6.60.15__py3-none-any.whl → 6.60.16.post40.dev17219503797__py3-none-any.whl

airbyte_cdk/manifest_server/cli/_common.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Common utilities for manifest server CLI commands."""
+
+import sys
+
+import rich_click as click
+
+# Import server dependencies with graceful fallback
+try:
+    import fastapi  # noqa: F401
+    import uvicorn  # noqa: F401
+
+    FASTAPI_AVAILABLE = True
+except ImportError:
+    FASTAPI_AVAILABLE = False
+
+
+def check_manifest_server_dependencies() -> None:
+    """Check if manifest-server dependencies are installed."""
+    if not FASTAPI_AVAILABLE:
+        click.echo(
+            "❌ Manifest runner dependencies not found. Please install with:\n\n"
+            "  pip install airbyte-cdk[manifest-server]\n"
+            "  # or\n"
+            "  poetry install --extras manifest-server\n",
+            err=True,
+        )
+        sys.exit(1)

airbyte_cdk/manifest_server/cli/_info.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Info command for the manifest server CLI."""
+
+from typing import Any, Optional
+
+import rich_click as click
+
+# Import server dependencies with graceful fallback
+fastapi: Optional[Any] = None
+uvicorn: Optional[Any] = None
+
+try:
+    import fastapi  # type: ignore[no-redef]
+    import uvicorn  # type: ignore[no-redef]
+
+    FASTAPI_AVAILABLE = True
+except ImportError:
+    FASTAPI_AVAILABLE = False
+
+
+@click.command()
+def info() -> None:
+    """Show manifest server information and status."""
+    if FASTAPI_AVAILABLE and fastapi is not None and uvicorn is not None:
+        click.echo("✅ Manifest runner dependencies are installed")
+        click.echo(f"   FastAPI version: {fastapi.__version__}")
+        click.echo(f"   Uvicorn version: {uvicorn.__version__}")
+    else:
+        click.echo("❌ Manifest runner dependencies not installed")
+        click.echo("   Install with: pip install airbyte-cdk[manifest-server]")

airbyte_cdk/manifest_server/cli/_openapi.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Generate OpenAPI command for the manifest server CLI."""
+
+from pathlib import Path
+
+import rich_click as click
+from yaml import dump
+
+from ._common import check_manifest_server_dependencies
+
+
+@click.command("generate-openapi")
+@click.option(
+    "--output",
+    "-o",
+    default="airbyte_cdk/manifest_server/openapi.yaml",
+    help="Output path for the OpenAPI YAML file",
+    show_default=True,
+)
+def generate_openapi(output: str) -> None:
+    """Generate OpenAPI YAML specification for the manifest server."""
+    check_manifest_server_dependencies()
+
+    # Import the FastAPI app
+    from airbyte_cdk.manifest_server.app import app
+
+    # Get OpenAPI schema
+    openapi_schema = app.openapi()
+
+    # Ensure output directory exists
+    output_path = Path(output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write YAML file with header comment
+    with open(output_path, "w") as f:
+        f.write("# This file is auto-generated. Do not edit manually.\n")
+        f.write("# To regenerate, run: manifest-server generate-openapi\n")
+        f.write("\n")
+        dump(openapi_schema, f, default_flow_style=False, sort_keys=False)
+
+    click.echo(f"✅ OpenAPI YAML generated at: {output_path}")
+    click.echo(f"   Title: {openapi_schema.get('info', {}).get('title', 'N/A')}")
+    click.echo(f"   Version: {openapi_schema.get('info', {}).get('version', 'N/A')}")

airbyte_cdk/manifest_server/cli/_start.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Start command for the manifest server CLI."""
+
+import rich_click as click
+
+from ._common import check_manifest_server_dependencies
+
+
+@click.command()
+@click.option(
+    "--host",
+    default="127.0.0.1",
+    help="Host to bind the server to",
+    show_default=True,
+)
+@click.option(
+    "--port",
+    default=8000,
+    help="Port to bind the server to",
+    show_default=True,
+)
+@click.option(
+    "--reload",
+    is_flag=True,
+    help="Enable auto-reload for development",
+)
+def start(host: str, port: int, reload: bool) -> None:
+    """Start the FastAPI manifest server server."""
+    check_manifest_server_dependencies()
+
+    # Import and use the main server function
+    from airbyte_cdk.manifest_server.main import run_server
+
+    run_server(
+        host=host,
+        port=port,
+        reload=reload,
+    )

airbyte_cdk/manifest_server/cli/run.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Standalone CLI for the Airbyte CDK Manifest Server.
+
+This CLI provides commands for running and managing the manifest server server.
+
+**Installation:**
+
+To use the manifest-server functionality, install the CDK with the manifest-server extra:
+
+```bash
+pip install airbyte-cdk[manifest-server]
+# or
+poetry install --extras manifest-server
+```
+
+**Usage:**
+
+```bash
+manifest-server start --port 8000
+manifest-server info
+manifest-server --help
+```
+"""
+
+import rich_click as click
+
+from ._info import info
+from ._openapi import generate_openapi
+from ._start import start
+
+
+@click.group(
+    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown)
+    invoke_without_command=True,
+)
+@click.pass_context
+def cli(
+    ctx: click.Context,
+) -> None:
+    """Airbyte Manifest Server CLI."""
+
+    if ctx.invoked_subcommand is None:
+        # If no subcommand is provided, show the help message.
+        click.echo(ctx.get_help())
+        ctx.exit()
+
+
+cli.add_command(start)
+cli.add_command(info)
+cli.add_command(generate_openapi)
+
+
+def run() -> None:
+    """Entry point for the manifest-server CLI."""
+    cli()
+
+
+if __name__ == "__main__":
+    run()

airbyte_cdk/manifest_server/command_processor/processor.py

@@ -0,0 +1,151 @@
+import logging
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple
+
+from airbyte_protocol_dataclasses.models import (
+    AirbyteCatalog,
+    AirbyteConnectionStatus,
+    AirbyteMessage,
+    Status,
+    TraceType,
+)
+from airbyte_protocol_dataclasses.models import Type as AirbyteMessageType
+from fastapi import HTTPException
+
+from airbyte_cdk.connector_builder.models import StreamRead
+from airbyte_cdk.connector_builder.test_reader import TestReader
+from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.models import (
+    AirbyteStateMessage,
+    ConfiguredAirbyteCatalog,
+)
+from airbyte_cdk.sources.declarative.manifest_declarative_source import (
+    ManifestDeclarativeSource,
+)
+
+
+class ManifestCommandProcessor:
+    _source: ManifestDeclarativeSource
+    _logger = logging.getLogger("airbyte.manifest-server")
+
+    def __init__(self, source: ManifestDeclarativeSource) -> None:
+        self._source = source
+
+    def test_read(
+        self,
+        config: Mapping[str, Any],
+        catalog: ConfiguredAirbyteCatalog,
+        state: List[AirbyteStateMessage],
+        record_limit: int,
+        page_limit: int,
+        slice_limit: int,
+    ) -> StreamRead:
+        """
+        Test the read method of the source.
+        """
+
+        test_read_handler = TestReader(
+            max_pages_per_slice=page_limit,
+            max_slices=slice_limit,
+            max_record_limit=record_limit,
+        )
+
+        stream_read = test_read_handler.run_test_read(
+            source=self._source,
+            config=config,
+            configured_catalog=catalog,
+            state=state,
+            stream_name=catalog.streams[0].stream.name,
+            record_limit=record_limit,
+        )
+
+        return stream_read
+
+    def check_connection(
+        self,
+        config: Mapping[str, Any],
+    ) -> Tuple[bool, str]:
+        """
+        Check the connection to the source.
+        """
+
+        spec = self._source.spec(self._logger)
+        messages = AirbyteEntrypoint(source=self._source).check(spec, config)
+        messages_by_type = self._get_messages_by_type(messages)
+        self._raise_on_trace_message(messages_by_type)
+        connection_status = self._get_connection_status(messages_by_type)
+
+        if connection_status:
+            return connection_status.status == Status.SUCCEEDED, connection_status.message
+        return False, "Connection check failed"
+
+    def discover(
+        self,
+        config: Mapping[str, Any],
+    ) -> Optional[AirbyteCatalog]:
+        """
+        Discover the catalog from the source.
+        """
+        spec = self._source.spec(self._logger)
+        messages = AirbyteEntrypoint(source=self._source).discover(spec, config)
+        messages_by_type = self._get_messages_by_type(messages)
+        self._raise_on_trace_message(messages_by_type)
+        return self._get_catalog(messages_by_type)
+
+    def _get_messages_by_type(
+        self,
+        messages: Iterable[AirbyteMessage],
+    ) -> Dict[str, List[AirbyteMessage]]:
+        """
+        Group messages by type.
+        """
+        grouped: Dict[str, List[AirbyteMessage]] = {}
+        for message in messages:
+            msg_type = message.type
+            if msg_type not in grouped:
+                grouped[msg_type] = []
+            grouped[msg_type].append(message)
+        return grouped
+
+    def _get_connection_status(
+        self,
+        messages_by_type: Mapping[str, List[AirbyteMessage]],
+    ) -> Optional[AirbyteConnectionStatus]:
+        """
+        Get the connection status from the messages.
+        """
+        messages = messages_by_type.get(AirbyteMessageType.CONNECTION_STATUS)
+        return messages[-1].connectionStatus if messages else None
+
+    def _get_catalog(
+        self,
+        messages_by_type: Mapping[str, List[AirbyteMessage]],
+    ) -> Optional[AirbyteCatalog]:
+        """
+        Get the catalog from the messages.
+        """
+        messages = messages_by_type.get(AirbyteMessageType.CATALOG)
+        return messages[-1].catalog if messages else None
+
+    def _raise_on_trace_message(
+        self,
+        messages_by_type: Mapping[str, List[AirbyteMessage]],
+    ) -> None:
+        """
+        Raise an exception if a trace message is found.
+        """
+        messages = [
+            message
+            for message in messages_by_type.get(AirbyteMessageType.TRACE, [])
+            if message.trace.type == TraceType.ERROR
+        ]
+        if messages:
+            error_message = messages[-1].trace.error.message
+            self._logger.warning(
+                "Error response from CDK: %s\n%s",
+                error_message,
+                messages[-1].trace.error.stack_trace,
+            )
+            raise HTTPException(
+                status_code=422,
+                detail=f"AirbyteTraceMessage response from CDK: {error_message}",
+            )

airbyte_cdk/manifest_server/command_processor/utils.py

@@ -0,0 +1,76 @@
+from typing import Any, Mapping, Optional
+
+from airbyte_cdk.models import (
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+    SyncMode,
+)
+from airbyte_cdk.sources.declarative.manifest_declarative_source import (
+    ManifestDeclarativeSource,
+)
+from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
+    ModelToComponentFactory,
+)
+
+SHOULD_NORMALIZE_KEY = "__should_normalize"
+SHOULD_MIGRATE_KEY = "__should_migrate"
+
+
+def build_catalog(stream_name: str) -> ConfiguredAirbyteCatalog:
+    return ConfiguredAirbyteCatalog(
+        streams=[
+            ConfiguredAirbyteStream(
+                stream=AirbyteStream(
+                    name=stream_name,
+                    json_schema={},
+                    supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental],
+                ),
+                sync_mode=SyncMode.incremental,
+                destination_sync_mode=DestinationSyncMode.overwrite,
+            )
+        ]
+    )
+
+
+def should_migrate_manifest(manifest: Mapping[str, Any]) -> bool:
+    """
+    Determines whether the manifest should be migrated,
+    based on the presence of the "__should_migrate" key.
+
+    This flag is set by the UI.
+    """
+    return manifest.get(SHOULD_MIGRATE_KEY, False)
+
+
+def should_normalize_manifest(manifest: Mapping[str, Any]) -> bool:
+    """
+    Determines whether the manifest should be normalized,
+    based on the presence of the "__should_normalize" key.
+
+    This flag is set by the UI.
+    """
+    return manifest.get(SHOULD_NORMALIZE_KEY, False)
+
+
+def build_source(
+    manifest: Mapping[str, Any],
+    config: Mapping[str, Any],
+    page_limit: Optional[int] = None,
+    slice_limit: Optional[int] = None,
+) -> ManifestDeclarativeSource:
+    return ManifestDeclarativeSource(
+        source_config=manifest,
+        config=config,
+        normalize_manifest=should_normalize_manifest(manifest),
+        migrate_manifest=should_migrate_manifest(manifest),
+        emit_connector_builder_messages=True,
+        component_factory=ModelToComponentFactory(
+            emit_connector_builder_messages=True,
+            limit_pages_fetched_per_slice=page_limit,
+            limit_slices_fetched=slice_limit,
+            disable_retries=True,
+            disable_cache=True,
+        ),
+    )

airbyte_cdk/manifest_server/main.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Main entry point for the Airbyte Manifest Server server."""
+
+import uvicorn
+
+
+def run_server(
+    host: str = "127.0.0.1", port: int = 8000, reload: bool = False, log_level: str = "info"
+) -> None:
+    """Run the FastAPI server."""
+
+    print(f"🚀 Starting Airbyte CDK Manifest Server on {host}:{port}")
+
+    uvicorn.run(
+        "airbyte_cdk.manifest_server.app:app",
+        host=host,
+        port=port,
+        reload=reload,
+        log_level=log_level,
+    )
+
+
+if __name__ == "__main__":
+    run_server()