airbyte-cdk 7.3.5__tar.gz → 7.3.7__tar.gz

{airbyte_cdk-7.3.5 → airbyte_cdk-7.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.3.5
+Version: 7.3.7
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-7.3.5 → airbyte_cdk-7.3.7}/airbyte_cdk/manifest_server/api_models/__init__.py

@@ -10,6 +10,7 @@ from .manifest import (
     CheckResponse,
     DiscoverRequest,
     DiscoverResponse,
+    ErrorResponse,
     FullResolveRequest,
     ManifestResponse,
     RequestContext,
@@ -40,6 +41,7 @@ __all__ = [
     "CheckResponse",
     "DiscoverRequest",
     "DiscoverResponse",
+    "ErrorResponse",
     # Stream models
     "AuxiliaryRequest",
     "HttpRequest",

{airbyte_cdk-7.3.5 → airbyte_cdk-7.3.7}/airbyte_cdk/manifest_server/api_models/manifest.py

@@ -83,3 +83,9 @@ class FullResolveRequest(BaseModel):
     config: ConnectorConfig
     stream_limit: int = Field(default=100, ge=1, le=100)
     context: Optional[RequestContext] = None
+
+
+class ErrorResponse(BaseModel):
+    """Error response for API requests."""
+
+    detail: str

{airbyte_cdk-7.3.5 → airbyte_cdk-7.3.7}/airbyte_cdk/manifest_server/app.py

@@ -11,7 +11,7 @@ from .routers import capabilities, health, manifest
 app = FastAPI(
     title="Manifest Server",
     description="A service for running low-code Airbyte connectors",
-    version="0.1.0",
+    version="0.2.0",
     contact={
         "name": "Airbyte",
         "url": "https://airbyte.com",

{airbyte_cdk-7.3.5 → airbyte_cdk-7.3.7}/airbyte_cdk/manifest_server/command_processor/utils.py

@@ -64,16 +64,7 @@ def build_source(
     page_limit: Optional[int] = None,
     slice_limit: Optional[int] = None,
 ) -> ConcurrentDeclarativeSource:
-    # We enforce a concurrency level of 1 so that the stream is processed on a single thread
-    # to retain ordering for the grouping of the builder message responses.
     definition = copy.deepcopy(manifest)
-    if "concurrency_level" in definition:
-        definition["concurrency_level"]["default_concurrency"] = 1
-    else:
-        definition["concurrency_level"] = {
-            "type": "ConcurrencyLevel",
-            "default_concurrency": 1,
-        }
 
     should_normalize = should_normalize_manifest(manifest)
     if should_normalize:

{airbyte_cdk-7.3.5 → airbyte_cdk-7.3.7}/airbyte_cdk/manifest_server/openapi.yaml

@@ -8,7 +8,7 @@ info:
   contact:
     name: Airbyte
     url: https://airbyte.com/
-  version: 0.1.0
+  version: 0.2.0
 paths:
   /health/:
     get:
@@ -62,6 +62,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/StreamReadResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -90,6 +96,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/CheckResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -118,6 +130,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/DiscoverResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -146,6 +164,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ManifestResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -180,6 +204,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ManifestResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -353,6 +383,16 @@ components:
       - catalog
       title: DiscoverResponse
       description: Response to discover a manifest.
+    ErrorResponse:
+      properties:
+        detail:
+          type: string
+          title: Detail
+      type: object
+      required:
+      - detail
+      title: ErrorResponse
+      description: Error response for API requests.
     FullResolveRequest:
       properties:
         manifest:

airbyte_cdk-7.3.7/airbyte_cdk/manifest_server/routers/manifest.py

@@ -0,0 +1,264 @@
+import hashlib
+from dataclasses import asdict
+from typing import Any, Dict, List, Mapping, Optional
+
+import jsonschema
+from airbyte_protocol_dataclasses.models import AirbyteStateMessage, ConfiguredAirbyteCatalog
+from fastapi import APIRouter, Depends, HTTPException
+
+from airbyte_cdk.models import AirbyteStateMessageSerializer
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
+    INJECTED_COMPONENTS_PY,
+    INJECTED_COMPONENTS_PY_CHECKSUMS,
+)
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
+
+from ..api_models import (
+    CheckRequest,
+    CheckResponse,
+    DiscoverRequest,
+    DiscoverResponse,
+    ErrorResponse,
+    FullResolveRequest,
+    Manifest,
+    ManifestResponse,
+    ResolveRequest,
+    StreamReadResponse,
+    StreamTestReadRequest,
+)
+from ..command_processor.processor import ManifestCommandProcessor
+from ..command_processor.utils import build_catalog, build_source
+from ..helpers.auth import verify_jwt_token
+from ..helpers.tracing import apply_trace_tags_from_context
+
+
+def safe_build_source(
+    manifest_dict: Dict[str, Any],
+    config_dict: Mapping[str, Any],
+    catalog: Optional[ConfiguredAirbyteCatalog] = None,
+    state: Optional[List[AirbyteStateMessage]] = None,
+    page_limit: Optional[int] = None,
+    slice_limit: Optional[int] = None,
+    record_limit: Optional[int] = None,
+) -> ConcurrentDeclarativeSource:
+    """Wrapper around build_source that converts ValidationError to HTTPException."""
+    try:
+        return build_source(
+            manifest_dict,
+            catalog,
+            config_dict,
+            state,
+            record_limit,
+            page_limit,
+            slice_limit,
+        )
+    except jsonschema.exceptions.ValidationError as e:
+        raise HTTPException(status_code=400, detail=f"Invalid manifest: {e.message}")
+
+
+router = APIRouter(
+    prefix="/manifest",
+    tags=["manifest"],
+    dependencies=[Depends(verify_jwt_token)],
+)
+
+
+@router.post(
+    "/test_read",
+    operation_id="testRead",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
+def test_read(request: StreamTestReadRequest) -> StreamReadResponse:
+    """
+    Test reading from a specific stream in the manifest.
+    """
+    # Apply trace tags from context if provided
+    if request.context:
+        apply_trace_tags_from_context(
+            workspace_id=request.context.workspace_id,
+            project_id=request.context.project_id,
+        )
+
+    config_dict = request.config.model_dump()
+
+    catalog = build_catalog(request.stream_name)
+    converted_state = [AirbyteStateMessageSerializer.load(state) for state in request.state]
+
+    if request.custom_components_code:
+        config_dict[INJECTED_COMPONENTS_PY] = request.custom_components_code
+        config_dict[INJECTED_COMPONENTS_PY_CHECKSUMS] = {
+            "md5": hashlib.md5(request.custom_components_code.encode()).hexdigest()
+        }
+
+    # We enforce a concurrency level of 1 so that the stream is processed on a single thread
+    # to retain ordering for the grouping of the builder message responses.
+    manifest = request.manifest.model_dump()
+    if "concurrency_level" in manifest:
+        manifest["concurrency_level"]["default_concurrency"] = 1
+    else:
+        manifest["concurrency_level"] = {
+            "type": "ConcurrencyLevel",
+            "default_concurrency": 1,
+        }
+
+    source = safe_build_source(
+        manifest,
+        config_dict,
+        catalog,
+        converted_state,
+        request.page_limit,
+        request.slice_limit,
+        request.record_limit,
+    )
+
+    runner = ManifestCommandProcessor(source)
+    try:
+        cdk_result = runner.test_read(
+            config_dict,
+            catalog,
+            converted_state,
+            request.record_limit,
+            request.page_limit,
+            request.slice_limit,
+        )
+        return StreamReadResponse.model_validate(asdict(cdk_result))
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error reading stream: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)
+
+
+@router.post(
+    "/check",
+    operation_id="check",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
+def check(request: CheckRequest) -> CheckResponse:
+    """Check configuration against a manifest"""
+    # Apply trace tags from context if provided
+    if request.context:
+        apply_trace_tags_from_context(
+            workspace_id=request.context.workspace_id,
+            project_id=request.context.project_id,
+        )
+
+    try:
+        source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+        runner = ManifestCommandProcessor(source)
+        success, message = runner.check_connection(request.config.model_dump())
+        return CheckResponse(success=success, message=message)
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error checking connection: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)
+
+
+@router.post(
+    "/discover",
+    operation_id="discover",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
+def discover(request: DiscoverRequest) -> DiscoverResponse:
+    """Discover streams from a manifest"""
+    # Apply trace tags from context if provided
+    if request.context:
+        apply_trace_tags_from_context(
+            workspace_id=request.context.workspace_id,
+            project_id=request.context.project_id,
+        )
+
+    try:
+        source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+        runner = ManifestCommandProcessor(source)
+        catalog = runner.discover(request.config.model_dump())
+        if catalog is None:
+            raise HTTPException(
+                status_code=422, detail="Connector did not return a discovered catalog"
+            )
+        return DiscoverResponse(catalog=catalog)
+    except HTTPException:
+        # Re-raise HTTPExceptions as-is (like the catalog None check above)
+        raise
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error discovering streams: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)
+
+
+@router.post(
+    "/resolve",
+    operation_id="resolve",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
+def resolve(request: ResolveRequest) -> ManifestResponse:
+    """Resolve a manifest to its final configuration."""
+    # Apply trace tags from context if provided
+    if request.context:
+        apply_trace_tags_from_context(
+            workspace_id=request.context.workspace_id,
+            project_id=request.context.project_id,
+        )
+
+    try:
+        source = safe_build_source(request.manifest.model_dump(), {})
+        return ManifestResponse(manifest=Manifest(**source.resolved_manifest))
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error resolving manifest: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)
+
+
+@router.post(
+    "/full_resolve",
+    operation_id="fullResolve",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
+def full_resolve(request: FullResolveRequest) -> ManifestResponse:
+    """
+    Fully resolve a manifest, including dynamic streams.
+
+    This is a similar operation to resolve, but has an extra step which generates streams from dynamic stream templates if the manifest contains any. This is used when a user clicks the generate streams button on a stream template in the Builder UI
+    """
+    # Apply trace tags from context if provided
+    if request.context:
+        apply_trace_tags_from_context(
+            workspace_id=request.context.workspace_id,
+            project_id=request.context.project_id,
+        )
+
+    try:
+        source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+        manifest = {**source.resolved_manifest}
+        streams = manifest.get("streams", [])
+        for stream in streams:
+            stream["dynamic_stream_name"] = None
+
+        mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
+        for stream in source.dynamic_streams:
+            generated_streams = mapped_streams.setdefault(stream["dynamic_stream_name"], [])
+
+            if len(generated_streams) < request.stream_limit:
+                generated_streams += [stream]
+
+        for generated_streams_list in mapped_streams.values():
+            streams.extend(generated_streams_list)
+
+        manifest["streams"] = streams
+        return ManifestResponse(manifest=Manifest(**manifest))
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error full resolving manifest: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)

{airbyte_cdk-7.3.5 → airbyte_cdk-7.3.7}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -384,10 +384,7 @@ class SimpleRetriever(Retriever):
 
             response = None
             try:
-                if (
-                    self.additional_query_properties
-                    and self.additional_query_properties.property_chunking
-                ):
+                if self.additional_query_properties:
                     for properties in self.additional_query_properties.get_request_property_chunks(
                         stream_slice=stream_slice
                     ):
@@ -401,15 +398,19 @@ class SimpleRetriever(Retriever):
                         )
 
                         for current_record in records_generator_fn(response):
-                            merge_key = (
-                                self.additional_query_properties.property_chunking.get_merge_key(
+                            if self.additional_query_properties.property_chunking:
+                                merge_key = self.additional_query_properties.property_chunking.get_merge_key(
                                     current_record
                                 )
-                            )
-                            if merge_key:
-                                _deep_merge(merged_records[merge_key], current_record)
+                                if merge_key:
+                                    _deep_merge(merged_records[merge_key], current_record)
+                                else:
+                                    # We should still emit records even if the record did not have a merge key
+                                    pagination_tracker.observe(current_record)
+                                    last_page_size += 1
+                                    last_record = current_record
+                                    yield current_record
                             else:
-                                # We should still emit records even if the record did not have a merge key
                                 pagination_tracker.observe(current_record)
                                 last_page_size += 1
                                 last_record = current_record

{airbyte_cdk-7.3.5 → airbyte_cdk-7.3.7}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "7.3.5"  # Version will be calculated dynamically.
+version = "7.3.7"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false

airbyte_cdk-7.3.5/airbyte_cdk/manifest_server/routers/manifest.py

@@ -1,191 +0,0 @@
-import hashlib
-from dataclasses import asdict
-from typing import Any, Dict, List, Mapping, Optional
-
-import jsonschema
-from airbyte_protocol_dataclasses.models import AirbyteStateMessage, ConfiguredAirbyteCatalog
-from fastapi import APIRouter, Depends, HTTPException
-
-from airbyte_cdk.models import AirbyteStateMessageSerializer
-from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
-    ConcurrentDeclarativeSource,
-)
-from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
-    INJECTED_COMPONENTS_PY,
-    INJECTED_COMPONENTS_PY_CHECKSUMS,
-)
-
-from ..api_models import (
-    CheckRequest,
-    CheckResponse,
-    DiscoverRequest,
-    DiscoverResponse,
-    FullResolveRequest,
-    Manifest,
-    ManifestResponse,
-    ResolveRequest,
-    StreamReadResponse,
-    StreamTestReadRequest,
-)
-from ..command_processor.processor import ManifestCommandProcessor
-from ..command_processor.utils import build_catalog, build_source
-from ..helpers.auth import verify_jwt_token
-from ..helpers.tracing import apply_trace_tags_from_context
-
-
-def safe_build_source(
-    manifest_dict: Dict[str, Any],
-    config_dict: Mapping[str, Any],
-    catalog: Optional[ConfiguredAirbyteCatalog] = None,
-    state: Optional[List[AirbyteStateMessage]] = None,
-    page_limit: Optional[int] = None,
-    slice_limit: Optional[int] = None,
-    record_limit: Optional[int] = None,
-) -> ConcurrentDeclarativeSource:
-    """Wrapper around build_source that converts ValidationError to HTTPException."""
-    try:
-        return build_source(
-            manifest_dict,
-            catalog,
-            config_dict,
-            state,
-            record_limit,
-            page_limit,
-            slice_limit,
-        )
-    except jsonschema.exceptions.ValidationError as e:
-        raise HTTPException(status_code=400, detail=f"Invalid manifest: {e.message}")
-
-
-router = APIRouter(
-    prefix="/manifest",
-    tags=["manifest"],
-    dependencies=[Depends(verify_jwt_token)],
-)
-
-
-@router.post("/test_read", operation_id="testRead")
-def test_read(request: StreamTestReadRequest) -> StreamReadResponse:
-    """
-    Test reading from a specific stream in the manifest.
-    """
-    # Apply trace tags from context if provided
-    if request.context:
-        apply_trace_tags_from_context(
-            workspace_id=request.context.workspace_id,
-            project_id=request.context.project_id,
-        )
-
-    config_dict = request.config.model_dump()
-
-    catalog = build_catalog(request.stream_name)
-    converted_state = [AirbyteStateMessageSerializer.load(state) for state in request.state]
-
-    if request.custom_components_code:
-        config_dict[INJECTED_COMPONENTS_PY] = request.custom_components_code
-        config_dict[INJECTED_COMPONENTS_PY_CHECKSUMS] = {
-            "md5": hashlib.md5(request.custom_components_code.encode()).hexdigest()
-        }
-
-    source = safe_build_source(
-        request.manifest.model_dump(),
-        config_dict,
-        catalog,
-        converted_state,
-        request.page_limit,
-        request.slice_limit,
-        request.record_limit,
-    )
-
-    runner = ManifestCommandProcessor(source)
-    cdk_result = runner.test_read(
-        config_dict,
-        catalog,
-        converted_state,
-        request.record_limit,
-        request.page_limit,
-        request.slice_limit,
-    )
-    return StreamReadResponse.model_validate(asdict(cdk_result))
-
-
-@router.post("/check", operation_id="check")
-def check(request: CheckRequest) -> CheckResponse:
-    """Check configuration against a manifest"""
-    # Apply trace tags from context if provided
-    if request.context:
-        apply_trace_tags_from_context(
-            workspace_id=request.context.workspace_id,
-            project_id=request.context.project_id,
-        )
-
-    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
-    runner = ManifestCommandProcessor(source)
-    success, message = runner.check_connection(request.config.model_dump())
-    return CheckResponse(success=success, message=message)
-
-
-@router.post("/discover", operation_id="discover")
-def discover(request: DiscoverRequest) -> DiscoverResponse:
-    """Discover streams from a manifest"""
-    # Apply trace tags from context if provided
-    if request.context:
-        apply_trace_tags_from_context(
-            workspace_id=request.context.workspace_id,
-            project_id=request.context.project_id,
-        )
-
-    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
-    runner = ManifestCommandProcessor(source)
-    catalog = runner.discover(request.config.model_dump())
-    if catalog is None:
-        raise HTTPException(status_code=422, detail="Connector did not return a discovered catalog")
-    return DiscoverResponse(catalog=catalog)
-
-
-@router.post("/resolve", operation_id="resolve")
-def resolve(request: ResolveRequest) -> ManifestResponse:
-    """Resolve a manifest to its final configuration."""
-    # Apply trace tags from context if provided
-    if request.context:
-        apply_trace_tags_from_context(
-            workspace_id=request.context.workspace_id,
-            project_id=request.context.project_id,
-        )
-
-    source = safe_build_source(request.manifest.model_dump(), {})
-    return ManifestResponse(manifest=Manifest(**source.resolved_manifest))
-
-
-@router.post("/full_resolve", operation_id="fullResolve")
-def full_resolve(request: FullResolveRequest) -> ManifestResponse:
-    """
-    Fully resolve a manifest, including dynamic streams.
-
-    This is a similar operation to resolve, but has an extra step which generates streams from dynamic stream templates if the manifest contains any. This is used when a user clicks the generate streams button on a stream template in the Builder UI
-    """
-    # Apply trace tags from context if provided
-    if request.context:
-        apply_trace_tags_from_context(
-            workspace_id=request.context.workspace_id,
-            project_id=request.context.project_id,
-        )
-
-    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
-    manifest = {**source.resolved_manifest}
-    streams = manifest.get("streams", [])
-    for stream in streams:
-        stream["dynamic_stream_name"] = None
-
-    mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
-    for stream in source.dynamic_streams:
-        generated_streams = mapped_streams.setdefault(stream["dynamic_stream_name"], [])
-
-        if len(generated_streams) < request.stream_limit:
-            generated_streams += [stream]
-
-    for generated_streams_list in mapped_streams.values():
-        streams.extend(generated_streams_list)
-
-    manifest["streams"] = streams
-    return ManifestResponse(manifest=Manifest(**manifest))