airbyte-cdk 6.61.1__py3-none-any.whl → 6.61.3__py3-none-any.whl

airbyte_cdk/__init__.py

@@ -107,7 +107,6 @@ from .sources.declarative.extractors.record_extractor import RecordExtractor
 from .sources.declarative.extractors.record_filter import RecordFilter
 from .sources.declarative.incremental import DatetimeBasedCursor
 from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
-from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
     LegacyToPerPartitionStateMigration,
 )
@@ -253,7 +252,6 @@ __all__ = [
     "JsonDecoder",
     "JsonFileSchemaLoader",
     "LegacyToPerPartitionStateMigration",
-    "ManifestDeclarativeSource",
     "MinMaxDatetime",
     "NoAuth",
     "OffsetIncrement",

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -19,8 +19,6 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
     ConcurrentDeclarativeSource,
     TestLimits,
 )
-from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
-from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
@@ -90,7 +88,7 @@ def create_source(
 
 
 def read_stream(
-    source: DeclarativeSource,
+    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
     config: Mapping[str, Any],
     configured_catalog: ConfiguredAirbyteCatalog,
     state: List[AirbyteStateMessage],
@@ -128,7 +126,9 @@ def read_stream(
         return error.as_airbyte_message()
 
 
-def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
+def resolve_manifest(
+    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+) -> AirbyteMessage:
     try:
         return AirbyteMessage(
             type=Type.RECORD,
@@ -145,7 +145,9 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
         return error.as_airbyte_message()
 
 
-def full_resolve_manifest(source: ManifestDeclarativeSource, limits: TestLimits) -> AirbyteMessage:
+def full_resolve_manifest(
+    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]], limits: TestLimits
+) -> AirbyteMessage:
     try:
         manifest = {**source.resolved_manifest}
         streams = manifest.get("streams", [])

airbyte_cdk/connector_builder/main.py

@@ -25,7 +25,9 @@ from airbyte_cdk.models import (
     ConfiguredAirbyteCatalog,
     ConfiguredAirbyteCatalogSerializer,
 )
-from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
 from airbyte_cdk.sources.source import Source
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
@@ -68,7 +70,7 @@ def get_config_and_catalog_from_args(
 
 
 def handle_connector_builder_request(
-    source: ManifestDeclarativeSource,
+    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
     command: str,
     config: Mapping[str, Any],
     catalog: Optional[ConfiguredAirbyteCatalog],

airbyte_cdk/connector_builder/test_reader/reader.py

@@ -23,7 +23,9 @@ from airbyte_cdk.models import (
     ConfiguredAirbyteCatalog,
     TraceType,
 )
-from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
 from airbyte_cdk.utils import AirbyteTracedException
 from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
 from airbyte_cdk.utils.schema_inferrer import (
@@ -55,7 +57,7 @@ class TestReader:
             that contains slices of data, log messages, auxiliary requests, and any inferred schema or datetime formats.
 
             Parameters:
-                source (DeclarativeSource): The data source to read from.
+                source (ConcurrentDeclarativeSource): The data source to read from.
                 config (Mapping[str, Any]): Configuration parameters for the source.
                 configured_catalog (ConfiguredAirbyteCatalog): Catalog containing stream configuration.
                 state (List[AirbyteStateMessage]): Current state information for the read.
@@ -83,7 +85,7 @@ class TestReader:
 
     def run_test_read(
         self,
-        source: DeclarativeSource,
+        source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
         stream_name: str,
@@ -94,7 +96,7 @@ class TestReader:
         Run a test read for the connector by reading from a single stream and inferring schema and datetime formats.
 
         Parameters:
-            source (DeclarativeSource): The source instance providing the streams.
+            source (ConcurrentDeclarativeSource): The source instance providing the streams.
             config (Mapping[str, Any]): The configuration settings to use for reading.
             configured_catalog (ConfiguredAirbyteCatalog): The catalog specifying the stream configuration.
             state (List[AirbyteStateMessage]): A list of state messages to resume the read.
@@ -126,7 +128,7 @@ class TestReader:
             if stream
             else None,
             self._cursor_field_to_nested_and_composite_field(stream.cursor_field)
-            if stream
+            if stream and stream.cursor_field
             else None,
         )
         datetime_format_inferrer = DatetimeFormatInferrer()
@@ -381,13 +383,13 @@ class TestReader:
 
     def _read_stream(
         self,
-        source: DeclarativeSource,
+        source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
         state: List[AirbyteStateMessage],
     ) -> Iterator[AirbyteMessage]:
         """
-        Reads messages from the given DeclarativeSource using an AirbyteEntrypoint.
+        Reads messages from the given ConcurrentDeclarativeSource using an AirbyteEntrypoint.
 
         This method attempts to yield messages from the source's read generator. If the generator
         raises an AirbyteTracedException, it checks whether the exception message indicates a non-actionable
@@ -396,7 +398,7 @@ class TestReader:
         wrapped into an AirbyteTracedException, and yielded as an AirbyteMessage.
 
         Parameters:
-            source (DeclarativeSource): The source object that provides data reading logic.
+            source (ConcurrentDeclarativeSource): The source object that provides data reading logic.
             config (Mapping[str, Any]): The configuration dictionary for the source.
             configured_catalog (ConfiguredAirbyteCatalog): The catalog defining the streams and their configurations.
             state (List[AirbyteStateMessage]): A list representing the current state for incremental sync.

airbyte_cdk/legacy/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.

airbyte_cdk/legacy/sources/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.

airbyte_cdk/legacy/sources/declarative/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.

airbyte_cdk/legacy/sources/declarative/incremental/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.

airbyte_cdk/{sources → legacy/sources}/declarative/manifest_declarative_source.py

@@ -20,6 +20,7 @@ from airbyte_cdk.config_observation import create_connector_config_control_messa
 from airbyte_cdk.connector_builder.models import (
     LogMessage as ConnectorBuilderLogMessage,
 )
+from airbyte_cdk.legacy.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.manifest_migrations.migration_handler import (
     ManifestMigrationHandler,
 )
@@ -34,7 +35,6 @@ from airbyte_cdk.models import (
 from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
 from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
-from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ConditionalStreams as ConditionalStreamsModel,

airbyte_cdk/manifest_server/Dockerfile

@@ -0,0 +1,45 @@
+# Dockerfile for the Airbyte Manifest Server.
+#
+# This Dockerfile should be built from the root of the repository.
+#
+# Example:
+# docker build -f airbyte_cdk/manifest_server/Dockerfile -t airbyte/manifest-server .
+
+FROM python:3.12-slim-bookworm
+
+# Install git (needed for dynamic versioning) and poetry
+RUN apt-get update && \
+    apt-get install -y git && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install poetry==1.8.3
+
+# Configure poetry to not create virtual environments and disable interactive mode
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+
+# Copy poetry files (build from project root)
+COPY pyproject.toml poetry.lock ./
+
+# Copy the project source code (needed for dynamic versioning)
+COPY . /app
+
+# Install dependencies and package directly to system Python
+RUN --mount=type=cache,target=$POETRY_CACHE_DIR \
+    poetry config virtualenvs.create false && \
+    poetry install --extras manifest-server
+
+# Create a non-root user and group
+RUN groupadd --gid 1000 airbyte && \
+    useradd --uid 1000 --gid airbyte --shell /bin/bash --create-home airbyte
+
+# Change ownership
+RUN chown -R airbyte:airbyte /app
+
+# Run app as non-root user
+USER airbyte:airbyte
+
+EXPOSE 8080
+
+CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]

airbyte_cdk/manifest_server/README.md

@@ -0,0 +1,142 @@
+# Manifest Server
+
+An HTTP server for running Airbyte declarative connectors via their manifest files.
+
+## Quick Start
+
+### Installation
+
+The manifest server is available as an extra dependency:
+
+```bash
+# Using Poetry (preferred)
+poetry install --extras manifest-server
+
+# Using pip
+pip install airbyte-cdk[manifest-server]
+
+# Using uv
+uv pip install 'airbyte-cdk[manifest-server]'
+```
+
+### Running the Server
+
+```bash
+# Start the server (default port 8000)
+manifest-server start
+
+# Start on a specific port
+manifest-server start --port 8080
+
+# Or using Python module
+python -m airbyte_cdk.manifest_server.cli.run start
+```
+
+The server will start on `http://localhost:8000` by default.
+
+## API Endpoints
+
+### `/v1/manifest/test_read`
+Test reading from a specific stream in the manifest.
+
+**POST** - Test stream reading with configurable limits for records, pages, and slices.
+
+### `/v1/manifest/check`
+Check configuration against a manifest.
+
+**POST** - Validates connector configuration and returns success/failure status with message.
+
+### `/v1/manifest/discover`
+Discover streams from a manifest.
+
+**POST** - Returns the catalog of available streams from the manifest.
+
+### `/v1/manifest/resolve` 
+Resolve a manifest to its final configuration.
+
+**POST** - Returns the resolved manifest without dynamic stream generation.
+
+### `/v1/manifest/full_resolve`
+Fully resolve a manifest including dynamic streams.
+
+**POST** - Generates dynamic streams up to specified limits and includes them in the resolved manifest.
+
+## Custom Components
+
+The manifest server supports custom Python components, but this feature is **disabled by default** for security reasons.
+
+### Enabling Custom Components
+
+To allow custom Python components in your manifest files, set the environment variable:
+```bash
+export AIRBYTE_ENABLE_UNSAFE_CODE=true
+```
+
+## Authentication
+
+The manifest server supports optional JWT bearer token authentication:
+
+### Configuration
+Set the environment variable to enable authentication:
+```bash
+export AB_JWT_SIGNATURE_SECRET="your-jwt-secret-key"
+```
+
+### Usage
+When authentication is enabled, include a valid JWT token in the Authorization header:
+```bash
+curl -H "Authorization: Bearer <your-jwt-token>" \
+  http://localhost:8000/v1/manifest/test_read
+```
+
+### Behavior
+- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through 
+- **With `AB_JWT_SIGNATURE_SECRET`**: Requires valid JWT bearer token using HS256 algorithm
+
+## OpenAPI Specification
+
+The manifest server provides an OpenAPI specification for API client generation:
+
+### Generating the OpenAPI Spec
+```bash
+# Generate OpenAPI YAML (default location)
+manifest-server generate-openapi
+
+# Generate to custom location
+manifest-server generate-openapi --output /path/to/openapi.yaml
+```
+
+The generated OpenAPI specification is consumed by other applications and tools to:
+- Generate API clients in various programming languages
+- Create SDK bindings for the manifest server
+- Provide API documentation and validation
+- Enable integration with API development tools
+
+### Interactive API Documentation
+
+When running, interactive API documentation is available at:
+- Swagger UI: `http://localhost:8000/docs`
+- ReDoc: `http://localhost:8000/redoc`
+
+## Testing
+
+Run the manifest server tests from the repository root:
+
+```bash
+# Run all manifest server tests
+poetry run pytest unit_tests/manifest_server/ -v
+```
+
+## Docker
+
+The manifest server can be containerized using the included Dockerfile. Build from the repository root:
+
+```bash
+# Build from repository root (not from manifest_server subdirectory)
+docker build -f airbyte_cdk/manifest_server/Dockerfile -t manifest-server .
+
+# Run the container
+docker run -p 8080:8080 manifest-server
+```
+
+Note: The container runs on port 8080 by default.

airbyte_cdk/manifest_server/__init__.py

@@ -0,0 +1,3 @@
+"""
+.. include:: ./README.md
+"""

airbyte_cdk/manifest_server/api_models/__init__.py

@@ -0,0 +1,49 @@
+"""
+API Models for the Manifest Server Service.
+
+This package contains all Pydantic models used for API requests and responses.
+"""
+
+from .dicts import ConnectorConfig, Manifest
+from .manifest import (
+    CheckRequest,
+    CheckResponse,
+    DiscoverRequest,
+    DiscoverResponse,
+    FullResolveRequest,
+    ManifestResponse,
+    ResolveRequest,
+    StreamTestReadRequest,
+)
+from .stream import (
+    AuxiliaryRequest,
+    HttpRequest,
+    HttpResponse,
+    LogMessage,
+    StreamReadPages,
+    StreamReadResponse,
+    StreamReadSlices,
+)
+
+__all__ = [
+    # Typed Dicts
+    "ConnectorConfig",
+    "Manifest",
+    # Manifest request/response models
+    "FullResolveRequest",
+    "ManifestResponse",
+    "StreamTestReadRequest",
+    "ResolveRequest",
+    "CheckRequest",
+    "CheckResponse",
+    "DiscoverRequest",
+    "DiscoverResponse",
+    # Stream models
+    "AuxiliaryRequest",
+    "HttpRequest",
+    "HttpResponse",
+    "LogMessage",
+    "StreamReadResponse",
+    "StreamReadPages",
+    "StreamReadSlices",
+]

airbyte_cdk/manifest_server/api_models/capabilities.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class CapabilitiesResponse(BaseModel):
+    """Capabilities of the manifest server."""
+
+    custom_code_execution: bool

airbyte_cdk/manifest_server/api_models/dicts.py

@@ -0,0 +1,17 @@
+"""
+Common API models shared across different endpoints.
+"""
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Manifest(BaseModel):
+    """Base manifest model. Allows client generation to replace with proper JsonNode types."""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class ConnectorConfig(BaseModel):
+    """Base connector configuration model. Allows client generation to replace with proper JsonNode types."""
+
+    model_config = ConfigDict(extra="allow")

airbyte_cdk/manifest_server/api_models/manifest.py

@@ -0,0 +1,73 @@
+"""
+Manifest-related API models.
+
+These models define the request and response structures for manifest operations
+like reading, resolving, and full resolution.
+"""
+
+from typing import Any, List, Optional
+
+from airbyte_protocol_dataclasses.models import AirbyteCatalog
+from pydantic import BaseModel, Field
+
+from .dicts import ConnectorConfig, Manifest
+
+
+class StreamTestReadRequest(BaseModel):
+    """Request to test read from a specific stream."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+    stream_name: str
+    state: List[Any] = Field(default_factory=list)
+    custom_components_code: Optional[str] = None
+    record_limit: int = Field(default=100, ge=1, le=5000)
+    page_limit: int = Field(default=5, ge=1, le=20)
+    slice_limit: int = Field(default=5, ge=1, le=20)
+
+
+class CheckRequest(BaseModel):
+    """Request to check a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+
+
+class CheckResponse(BaseModel):
+    """Response to check a manifest."""
+
+    success: bool
+    message: Optional[str] = None
+
+
+class DiscoverRequest(BaseModel):
+    """Request to discover a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+
+
+class DiscoverResponse(BaseModel):
+    """Response to discover a manifest."""
+
+    catalog: AirbyteCatalog
+
+
+class ResolveRequest(BaseModel):
+    """Request to resolve a manifest."""
+
+    manifest: Manifest
+
+
+class ManifestResponse(BaseModel):
+    """Response containing a manifest."""
+
+    manifest: Manifest
+
+
+class FullResolveRequest(BaseModel):
+    """Request to fully resolve a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+    stream_limit: int = Field(default=100, ge=1, le=100)

airbyte_cdk/manifest_server/api_models/stream.py

@@ -0,0 +1,76 @@
+"""
+Stream-related API models.
+
+These models define the structure for stream reading operations and responses.
+They accurately reflect the runtime types returned by the CDK, particularly
+fixing type mismatches like slice_descriptor being a string rather than an object.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel
+
+
+class HttpRequest(BaseModel):
+    """HTTP request details."""
+
+    url: str
+    headers: Optional[Dict[str, Any]]
+    http_method: str
+    body: Optional[str] = None
+
+
+class HttpResponse(BaseModel):
+    """HTTP response details."""
+
+    status: int
+    body: Optional[str] = None
+    headers: Optional[Dict[str, Any]] = None
+
+
+class LogMessage(BaseModel):
+    """Log message from stream processing."""
+
+    message: str
+    level: str
+    internal_message: Optional[str] = None
+    stacktrace: Optional[str] = None
+
+
+class AuxiliaryRequest(BaseModel):
+    """Auxiliary HTTP request made during stream processing."""
+
+    title: str
+    type: str
+    description: str
+    request: HttpRequest
+    response: HttpResponse
+
+
+class StreamReadPages(BaseModel):
+    """Pages of data read from a stream slice."""
+
+    records: List[object]
+    request: Optional[HttpRequest] = None
+    response: Optional[HttpResponse] = None
+
+
+class StreamReadSlices(BaseModel):
+    """Slices of data read from a stream."""
+
+    pages: List[StreamReadPages]
+    slice_descriptor: Optional[str]  # This is actually a string at runtime, not Dict[str, Any]
+    state: Optional[List[Dict[str, Any]]] = None
+    auxiliary_requests: Optional[List[AuxiliaryRequest]] = None
+
+
+class StreamReadResponse(BaseModel):
+    """Complete stream read response with properly typed fields."""
+
+    logs: List[LogMessage]
+    slices: List[StreamReadSlices]
+    test_read_limit_reached: bool
+    auxiliary_requests: List[AuxiliaryRequest]
+    inferred_schema: Optional[Dict[str, Any]]
+    inferred_datetime_formats: Optional[Dict[str, str]]
+    latest_config_update: Optional[Dict[str, Any]]

airbyte_cdk/manifest_server/app.py

@@ -0,0 +1,17 @@
+from fastapi import FastAPI
+
+from .routers import capabilities, health, manifest
+
+app = FastAPI(
+    title="Manifest Server",
+    description="A service for running low-code Airbyte connectors",
+    version="0.1.0",
+    contact={
+        "name": "Airbyte",
+        "url": "https://airbyte.com",
+    },
+)
+
+app.include_router(health.router)
+app.include_router(capabilities.router)
+app.include_router(manifest.router, prefix="/v1")

airbyte_cdk/manifest_server/auth.py

@@ -0,0 +1,43 @@
+import os
+from typing import Optional
+
+import jwt
+from fastapi import HTTPException, Security, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
+security = HTTPBearer(auto_error=False)
+
+
+def verify_jwt_token(
+    credentials: Optional[HTTPAuthorizationCredentials] = Security(security),
+) -> None:
+    """
+    Verify JWT token if AB_JWT_SIGNATURE_SECRET is set, otherwise allow through.
+
+    Args:
+        credentials: Bearer token credentials from request header
+
+    Raises:
+        HTTPException: If token is invalid or missing when secret is configured
+    """
+    jwt_secret = os.getenv("AB_JWT_SIGNATURE_SECRET")
+
+    # If no secret is configured, allow all requests through
+    if not jwt_secret:
+        return
+
+    if not credentials:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Bearer token required",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    try:
+        jwt.decode(credentials.credentials, jwt_secret, algorithms=["HS256"])
+    except jwt.InvalidTokenError:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )

airbyte_cdk/manifest_server/cli/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""The `airbyte_cdk.manifest_server.cli` module provides a standalone CLI for the Airbyte CDK Manifest Server.
+
+This CLI enables running a FastAPI server for managing and executing Airbyte declarative manifests.
+"""