airbyte-cdk 6.60.16__tar.gz → 6.60.16.post40.dev17219503797__tar.gz

{airbyte_cdk-6.60.16 → airbyte_cdk-6.60.16.post40.dev17219503797}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.60.16
+Version: 6.60.16.post40.dev17219503797
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -19,6 +19,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Provides-Extra: dev
 Provides-Extra: file-based
+Provides-Extra: manifest-server
 Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
@@ -35,6 +36,7 @@ Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dateparser (>=1.2.2,<2.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastapi (>=0.116.1) ; extra == "manifest-server"
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
 Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
@@ -77,6 +79,7 @@ Requires-Dist: typing-extensions
 Requires-Dist: unidecode (>=1.3.8,<2.0.0)
 Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
 Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: uvicorn (>=0.35.0) ; extra == "manifest-server"
 Requires-Dist: wcmatch (==10.0)
 Requires-Dist: whenever (>=0.6.16,<0.7.0)
 Requires-Dist: xmltodict (>=0.13,<0.15)

{airbyte_cdk-6.60.16 → airbyte_cdk-6.60.16.post40.dev17219503797}/airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -3,8 +3,8 @@
 #
 
 
-from dataclasses import asdict, dataclass, field
-from typing import Any, ClassVar, Dict, List, Mapping
+from dataclasses import asdict
+from typing import Any, Dict, List, Mapping, Optional
 
 from airbyte_cdk.connector_builder.test_reader import TestReader
 from airbyte_cdk.models import (
@@ -15,45 +15,32 @@ from airbyte_cdk.models import (
     Type,
 )
 from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+    TestLimits,
+)
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
-    ModelToComponentFactory,
-)
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
-DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
-DEFAULT_MAXIMUM_NUMBER_OF_SLICES = 5
-DEFAULT_MAXIMUM_RECORDS = 100
-DEFAULT_MAXIMUM_STREAMS = 100
-
 MAX_PAGES_PER_SLICE_KEY = "max_pages_per_slice"
 MAX_SLICES_KEY = "max_slices"
 MAX_RECORDS_KEY = "max_records"
 MAX_STREAMS_KEY = "max_streams"
 
 
-@dataclass
-class TestLimits:
-    __test__: ClassVar[bool] = False  # Tell Pytest this is not a Pytest class, despite its name
-
-    max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
-    max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
-    max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)
-    max_streams: int = field(default=DEFAULT_MAXIMUM_STREAMS)
-
-
 def get_limits(config: Mapping[str, Any]) -> TestLimits:
     command_config = config.get("__test_read_config", {})
-    max_pages_per_slice = (
-        command_config.get(MAX_PAGES_PER_SLICE_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE
+    return TestLimits(
+        max_records=command_config.get(MAX_RECORDS_KEY, TestLimits.DEFAULT_MAX_RECORDS),
+        max_pages_per_slice=command_config.get(
+            MAX_PAGES_PER_SLICE_KEY, TestLimits.DEFAULT_MAX_PAGES_PER_SLICE
+        ),
+        max_slices=command_config.get(MAX_SLICES_KEY, TestLimits.DEFAULT_MAX_SLICES),
+        max_streams=command_config.get(MAX_STREAMS_KEY, TestLimits.DEFAULT_MAX_STREAMS),
     )
-    max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES
-    max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS
-    max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS
-    return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
 
 def should_migrate_manifest(config: Mapping[str, Any]) -> bool:
@@ -75,21 +62,30 @@ def should_normalize_manifest(config: Mapping[str, Any]) -> bool:
     return config.get("__should_normalize", False)
 
 
-def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource:
+def create_source(
+    config: Mapping[str, Any],
+    limits: TestLimits,
+    catalog: Optional[ConfiguredAirbyteCatalog],
+    state: Optional[List[AirbyteStateMessage]],
+) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
     manifest = config["__injected_declarative_manifest"]
-    return ManifestDeclarativeSource(
+
+    # We enforce a concurrency level of 1 so that the stream is processed on a single thread
+    # to retain ordering for the grouping of the builder message responses.
+    if "concurrency_level" in manifest:
+        manifest["concurrency_level"]["default_concurrency"] = 1
+    else:
+        manifest["concurrency_level"] = {"type": "ConcurrencyLevel", "default_concurrency": 1}
+
+    return ConcurrentDeclarativeSource(
+        catalog=catalog,
         config=config,
-        emit_connector_builder_messages=True,
+        state=state,
         source_config=manifest,
+        emit_connector_builder_messages=True,
         migrate_manifest=should_migrate_manifest(config),
         normalize_manifest=should_normalize_manifest(config),
-        component_factory=ModelToComponentFactory(
-            emit_connector_builder_messages=True,
-            limit_pages_fetched_per_slice=limits.max_pages_per_slice,
-            limit_slices_fetched=limits.max_slices,
-            disable_retries=True,
-            disable_cache=True,
-        ),
+        limits=limits,
     )
 
 

{airbyte_cdk-6.60.16 → airbyte_cdk-6.60.16.post40.dev17219503797}/airbyte_cdk/connector_builder/main.py

@@ -91,12 +91,12 @@ def handle_connector_builder_request(
 def handle_request(args: List[str]) -> str:
     command, config, catalog, state = get_config_and_catalog_from_args(args)
     limits = get_limits(config)
-    source = create_source(config, limits)
-    return orjson.dumps(
+    source = create_source(config=config, limits=limits, catalog=catalog, state=state)
+    return orjson.dumps(  # type: ignore[no-any-return] # Serializer.dump() always returns AirbyteMessage
         AirbyteMessageSerializer.dump(
             handle_connector_builder_request(source, command, config, catalog, state, limits)
         )
-    ).decode()  # type: ignore[no-any-return] # Serializer.dump() always returns AirbyteMessage
+    ).decode()
 
 
 if __name__ == "__main__":

{airbyte_cdk-6.60.16 → airbyte_cdk-6.60.16.post40.dev17219503797}/airbyte_cdk/connector_builder/test_reader/helpers.py

@@ -5,7 +5,7 @@
 import json
 from copy import deepcopy
 from json import JSONDecodeError
-from typing import Any, Dict, List, Mapping, Optional
+from typing import Any, Dict, List, Mapping, Optional, Union
 
 from airbyte_cdk.connector_builder.models import (
     AuxiliaryRequest,
@@ -17,6 +17,8 @@ from airbyte_cdk.connector_builder.models import (
 from airbyte_cdk.models import (
     AirbyteLogMessage,
     AirbyteMessage,
+    AirbyteStateBlob,
+    AirbyteStateMessage,
     OrchestratorType,
     TraceType,
 )
@@ -466,7 +468,7 @@ def handle_current_slice(
     return StreamReadSlices(
         pages=current_slice_pages,
         slice_descriptor=current_slice_descriptor,
-        state=[latest_state_message] if latest_state_message else [],
+        state=[convert_state_blob_to_mapping(latest_state_message)] if latest_state_message else [],
         auxiliary_requests=auxiliary_requests if auxiliary_requests else [],
     )
 
@@ -718,3 +720,23 @@ def get_auxiliary_request_type(stream: dict, http: dict) -> str:  # type: ignore
     Determines the type of the auxiliary request based on the stream and HTTP properties.
     """
     return "PARENT_STREAM" if stream.get("is_substream", False) else str(http.get("type", None))
+
+
+def convert_state_blob_to_mapping(
+    state_message: Union[AirbyteStateMessage, Dict[str, Any]],
+) -> Dict[str, Any]:
+    """
+    The AirbyteStreamState stores state as an AirbyteStateBlob which deceivingly is not
+    a dictionary, but rather a list of kwargs fields. This in turn causes it to not be
+    properly turned into a dictionary when translating this back into response output
+    by the connector_builder_handler using asdict()
+    """
+
+    if isinstance(state_message, AirbyteStateMessage) and state_message.stream:
+        state_value = state_message.stream.stream_state
+        if isinstance(state_value, AirbyteStateBlob):
+            state_value_mapping = {k: v for k, v in state_value.__dict__.items()}
+            state_message.stream.stream_state = state_value_mapping  # type: ignore  # we intentionally set this as a Dict so that StreamReadSlices is translated properly in the resulting HTTP response
+        return state_message  # type: ignore  # See above, but when this is an AirbyteStateMessage we must convert AirbyteStateBlob to a Dict
+    else:
+        return state_message  # type: ignore  # This is guaranteed to be a Dict since we check isinstance AirbyteStateMessage above

{airbyte_cdk-6.60.16 → airbyte_cdk-6.60.16.post40.dev17219503797}/airbyte_cdk/connector_builder/test_reader/message_grouper.py

@@ -95,7 +95,7 @@ def get_message_groups(
     latest_state_message: Optional[Dict[str, Any]] = None
     slice_auxiliary_requests: List[AuxiliaryRequest] = []
 
-    while records_count < limit and (message := next(messages, None)):
+    while message := next(messages, None):
         json_message = airbyte_message_to_json(message)
 
         if is_page_http_request_for_different_stream(json_message, stream_name):

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/Dockerfile

@@ -0,0 +1,45 @@
+# Dockerfile for the Airbyte Manifest Server.
+#
+# This Dockerfile should be built from the root of the repository.
+#
+# Example:
+# docker build -f airbyte_cdk/manifest_server/Dockerfile -t airbyte/manifest-server .
+
+FROM python:3.12-slim-bookworm
+
+# Install git (needed for dynamic versioning) and poetry
+RUN apt-get update && \
+    apt-get install -y git && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install poetry==1.8.3
+
+# Configure poetry to not create virtual environments and disable interactive mode
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+
+# Copy poetry files (build from project root)
+COPY pyproject.toml poetry.lock ./
+
+# Copy the project source code (needed for dynamic versioning)
+COPY . /app
+
+# Install dependencies and package directly to system Python
+RUN --mount=type=cache,target=$POETRY_CACHE_DIR \
+    poetry config virtualenvs.create false && \
+    poetry install --extras manifest-server
+
+# Create a non-root user and group
+RUN groupadd --gid 1000 airbyte && \
+    useradd --uid 1000 --gid airbyte --shell /bin/bash --create-home airbyte
+
+# Change ownership
+RUN chown -R airbyte:airbyte /app
+
+# Run app as non-root user
+USER airbyte:airbyte
+
+EXPOSE 8080
+
+CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/README.md

@@ -0,0 +1,142 @@
+# Manifest Server
+
+An HTTP server for running Airbyte declarative connectors via their manifest files.
+
+## Quick Start
+
+### Installation
+
+The manifest server is available as an extra dependency:
+
+```bash
+# Using Poetry (preferred)
+poetry install --extras manifest-server
+
+# Using pip
+pip install airbyte-cdk[manifest-server]
+
+# Using uv
+uv pip install 'airbyte-cdk[manifest-runner]'
+```
+
+### Running the Server
+
+```bash
+# Start the server (default port 8000)
+manifest-server start
+
+# Start on a specific port
+manifest-server start --port 8080
+
+# Or using Python module
+python -m airbyte_cdk.manifest_server.cli.run start
+```
+
+The server will start on `http://localhost:8000` by default.
+
+## API Endpoints
+
+### `/manifest/test_read`
+Test reading from a specific stream in the manifest.
+
+**POST** - Test stream reading with configurable limits for records, pages, and slices.
+
+### `/manifest/check`
+Check configuration against a manifest.
+
+**POST** - Validates connector configuration and returns success/failure status with message.
+
+### `/manifest/discover`
+Discover streams from a manifest.
+
+**POST** - Returns the catalog of available streams from the manifest.
+
+### `/manifest/resolve` 
+Resolve a manifest to its final configuration.
+
+**POST** - Returns the resolved manifest without dynamic stream generation.
+
+### `/manifest/full_resolve`
+Fully resolve a manifest including dynamic streams.
+
+**POST** - Generates dynamic streams up to specified limits and includes them in the resolved manifest.
+
+## Custom Components
+
+The manifest server supports custom Python components, but this feature is **disabled by default** for security reasons.
+
+### Enabling Custom Components
+
+To allow custom Python components in your manifest files, set the environment variable:
+```bash
+export AIRBYTE_ENABLE_UNSAFE_CODE=true
+```
+
+## Authentication
+
+The manifest server supports optional JWT bearer token authentication:
+
+### Configuration
+Set the environment variable to enable authentication:
+```bash
+export AB_JWT_SIGNATURE_SECRET="your-jwt-secret-key"
+```
+
+### Usage
+When authentication is enabled, include a valid JWT token in the Authorization header:
+```bash
+curl -H "Authorization: Bearer <your-jwt-token>" \
+  http://localhost:8000/manifest/test_read
+```
+
+### Behavior
+- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through 
+- **With `AB_JWT_SIGNATURE_SECRET`**: Requires valid JWT bearer token using HS256 algorithm
+
+## OpenAPI Specification
+
+The manifest server provides an OpenAPI specification for API client generation:
+
+### Generating the OpenAPI Spec
+```bash
+# Generate OpenAPI YAML (default location)
+manifest-server generate-openapi
+
+# Generate to custom location
+manifest-server generate-openapi --output /path/to/openapi.yaml
+```
+
+The generated OpenAPI specification is consumed by other applications and tools to:
+- Generate API clients in various programming languages
+- Create SDK bindings for the manifest server
+- Provide API documentation and validation
+- Enable integration with API development tools
+
+### Interactive API Documentation
+
+When running, interactive API documentation is available at:
+- Swagger UI: `http://localhost:8000/docs`
+- ReDoc: `http://localhost:8000/redoc`
+
+## Testing
+
+Run the manifest server tests from the repository root:
+
+```bash
+# Run all manifest server tests
+poetry run pytest unit_tests/manifest_server/ -v
+```
+
+## Docker
+
+The manifest server can be containerized using the included Dockerfile. Build from the repository root:
+
+```bash
+# Build from repository root (not from manifest_server subdirectory)
+docker build -f airbyte_cdk/manifest_server/Dockerfile -t manifest-server .
+
+# Run the container
+docker run -p 8080:8080 manifest-server
+```
+
+Note: The container runs on port 8080 by default.

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/__init__.py

@@ -0,0 +1,3 @@
+"""
+.. include:: ./README.md
+"""

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/api_models/__init__.py

@@ -0,0 +1,41 @@
+"""
+API Models for the Manifest Server Service.
+
+This package contains all Pydantic models used for API requests and responses.
+"""
+
+from .dicts import ConnectorConfig, Manifest
+from .manifest import (
+    FullResolveRequest,
+    ManifestResponse,
+    ResolveRequest,
+    StreamTestReadRequest,
+)
+from .stream import (
+    AuxiliaryRequest,
+    HttpRequest,
+    HttpResponse,
+    LogMessage,
+    StreamRead,
+    StreamReadPages,
+    StreamReadSlices,
+)
+
+__all__ = [
+    # Typed Dicts
+    "ConnectorConfig",
+    "Manifest",
+    # Manifest request/response models
+    "FullResolveRequest",
+    "ManifestResponse",
+    "StreamTestReadRequest",
+    "ResolveRequest",
+    # Stream models
+    "AuxiliaryRequest",
+    "HttpRequest",
+    "HttpResponse",
+    "LogMessage",
+    "StreamRead",
+    "StreamReadPages",
+    "StreamReadSlices",
+]

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/api_models/capabilities.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class CapabilitiesResponse(BaseModel):
+    """Capabilities of the manifest server."""
+
+    custom_code_execution: bool

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/api_models/dicts.py

@@ -0,0 +1,17 @@
+"""
+Common API models shared across different endpoints.
+"""
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Manifest(BaseModel):
+    """Base manifest model. Allows client generation to replace with proper JsonNode types."""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class ConnectorConfig(BaseModel):
+    """Base connector configuration model. Allows client generation to replace with proper JsonNode types."""
+
+    model_config = ConfigDict(extra="allow")

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/api_models/manifest.py

@@ -0,0 +1,73 @@
+"""
+Manifest-related API models.
+
+These models define the request and response structures for manifest operations
+like reading, resolving, and full resolution.
+"""
+
+from typing import Any, List, Optional
+
+from airbyte_protocol_dataclasses.models import AirbyteCatalog
+from pydantic import BaseModel, Field
+
+from .dicts import ConnectorConfig, Manifest
+
+
+class StreamTestReadRequest(BaseModel):
+    """Request to test read from a specific stream."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+    stream_name: str
+    state: List[Any] = []
+    custom_components_code: Optional[str] = None
+    record_limit: int = Field(default=100, ge=1, le=5000)
+    page_limit: int = Field(default=5, ge=1, le=20)
+    slice_limit: int = Field(default=5, ge=1, le=20)
+
+
+class CheckRequest(BaseModel):
+    """Request to check a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+
+
+class CheckResponse(BaseModel):
+    """Response to check a manifest."""
+
+    success: bool
+    message: Optional[str] = None
+
+
+class DiscoverRequest(BaseModel):
+    """Request to discover a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+
+
+class DiscoverResponse(BaseModel):
+    """Response to discover a manifest."""
+
+    catalog: AirbyteCatalog
+
+
+class ResolveRequest(BaseModel):
+    """Request to resolve a manifest."""
+
+    manifest: Manifest
+
+
+class ManifestResponse(BaseModel):
+    """Response containing a manifest."""
+
+    manifest: Manifest
+
+
+class FullResolveRequest(BaseModel):
+    """Request to fully resolve a manifest."""
+
+    manifest: Manifest
+    config: ConnectorConfig
+    stream_limit: int = Field(default=100, ge=1, le=100)

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/api_models/stream.py

@@ -0,0 +1,76 @@
+"""
+Stream-related API models.
+
+These models define the structure for stream reading operations and responses.
+They accurately reflect the runtime types returned by the CDK, particularly
+fixing type mismatches like slice_descriptor being a string rather than an object.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel
+
+
+class HttpRequest(BaseModel):
+    """HTTP request details."""
+
+    url: str
+    headers: Optional[Dict[str, Any]]
+    http_method: str
+    body: Optional[str] = None
+
+
+class HttpResponse(BaseModel):
+    """HTTP response details."""
+
+    status: int
+    body: Optional[str] = None
+    headers: Optional[Dict[str, Any]] = None
+
+
+class LogMessage(BaseModel):
+    """Log message from stream processing."""
+
+    message: str
+    level: str
+    internal_message: Optional[str] = None
+    stacktrace: Optional[str] = None
+
+
+class AuxiliaryRequest(BaseModel):
+    """Auxiliary HTTP request made during stream processing."""
+
+    title: str
+    type: str
+    description: str
+    request: HttpRequest
+    response: HttpResponse
+
+
+class StreamReadPages(BaseModel):
+    """Pages of data read from a stream slice."""
+
+    records: List[object]
+    request: Optional[HttpRequest] = None
+    response: Optional[HttpResponse] = None
+
+
+class StreamReadSlices(BaseModel):
+    """Slices of data read from a stream."""
+
+    pages: List[StreamReadPages]
+    slice_descriptor: Optional[str]  # This is actually a string at runtime, not Dict[str, Any]
+    state: Optional[List[Dict[str, Any]]] = None
+    auxiliary_requests: Optional[List[AuxiliaryRequest]] = None
+
+
+class StreamRead(BaseModel):
+    """Complete stream read response with properly typed fields."""
+
+    logs: List[LogMessage]
+    slices: List[StreamReadSlices]
+    test_read_limit_reached: bool
+    auxiliary_requests: List[AuxiliaryRequest]
+    inferred_schema: Optional[Dict[str, Any]]
+    inferred_datetime_formats: Optional[Dict[str, str]]
+    latest_config_update: Optional[Dict[str, Any]]

airbyte_cdk-6.60.16.post40.dev17219503797/airbyte_cdk/manifest_server/app.py

@@ -0,0 +1,17 @@
+from fastapi import FastAPI
+
+from ..manifest_server.routers import capabilities, health, manifest
+
+app = FastAPI(
+    title="Manifest Server Service",
+    description="A service for running low-code Airbyte connectors",
+    version="0.1.0",
+    contact={
+        "name": "Airbyte",
+        "url": "https://airbyte.com",
+    },
+)
+
+app.include_router(health.router)
+app.include_router(capabilities.router)
+app.include_router(manifest.router, prefix="/v1")