airbyte-cdk 7.3.6__tar.gz → 7.3.7.post1.dev18600150715__tar.gz

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.3.6
+Version: 7.3.7.post1.dev18600150715
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -35,7 +35,7 @@ Requires-Dist: click (>=8.1.8,<9.0.0)
 Requires-Dist: cohere (>=4.21,<6.0.0) ; extra == "vector-db-based"
 Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dateparser (>=1.2.2,<2.0.0)
-Requires-Dist: ddtrace (>=3.12.3,<4.0.0) ; extra == "manifest-server"
+Requires-Dist: ddtrace (>=3.12.0,<4.0.0) ; extra == "manifest-server"
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastapi (>=0.116.1) ; extra == "manifest-server"
@@ -43,13 +43,12 @@ Requires-Dist: fastavro (>=1.11.0,<2.0.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
 Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
-Requires-Dist: jsonref (>=0.2,<0.3)
+Requires-Dist: jsonref (>=1,<2)
 Requires-Dist: jsonschema (>=4.17.3,<5.0)
 Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
 Requires-Dist: langchain_core (==0.1.42)
 Requires-Dist: markdown ; extra == "file-based"
 Requires-Dist: nltk (==3.9.1)
-Requires-Dist: numpy (<2)
 Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
 Requires-Dist: orjson (>=3.10.7,<4.0.0)
 Requires-Dist: packaging

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/airbyte_cdk/manifest_server/api_models/__init__.py

@@ -10,6 +10,7 @@ from .manifest import (
     CheckResponse,
     DiscoverRequest,
     DiscoverResponse,
+    ErrorResponse,
     FullResolveRequest,
     ManifestResponse,
     RequestContext,
@@ -40,6 +41,7 @@ __all__ = [
     "CheckResponse",
     "DiscoverRequest",
     "DiscoverResponse",
+    "ErrorResponse",
     # Stream models
     "AuxiliaryRequest",
     "HttpRequest",

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/airbyte_cdk/manifest_server/api_models/manifest.py

@@ -83,3 +83,9 @@ class FullResolveRequest(BaseModel):
     config: ConnectorConfig
     stream_limit: int = Field(default=100, ge=1, le=100)
     context: Optional[RequestContext] = None
+
+
+class ErrorResponse(BaseModel):
+    """Error response for API requests."""
+
+    detail: str

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/airbyte_cdk/manifest_server/app.py

@@ -11,7 +11,7 @@ from .routers import capabilities, health, manifest
 app = FastAPI(
     title="Manifest Server",
     description="A service for running low-code Airbyte connectors",
-    version="0.1.0",
+    version="0.2.0",
     contact={
         "name": "Airbyte",
         "url": "https://airbyte.com",

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/airbyte_cdk/manifest_server/openapi.yaml

@@ -8,7 +8,7 @@ info:
   contact:
     name: Airbyte
     url: https://airbyte.com/
-  version: 0.1.0
+  version: 0.2.0
 paths:
   /health/:
     get:
@@ -62,6 +62,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/StreamReadResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -90,6 +96,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/CheckResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -118,6 +130,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/DiscoverResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -146,6 +164,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ManifestResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -180,6 +204,12 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ManifestResponse'
+        '400':
+          description: Bad Request - Error processing request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorResponse'
         '422':
           description: Validation Error
           content:
@@ -353,6 +383,16 @@ components:
       - catalog
       title: DiscoverResponse
       description: Response to discover a manifest.
+    ErrorResponse:
+      properties:
+        detail:
+          type: string
+          title: Detail
+      type: object
+      required:
+      - detail
+      title: ErrorResponse
+      description: Error response for API requests.
     FullResolveRequest:
       properties:
         manifest:

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/airbyte_cdk/manifest_server/routers/manifest.py

@@ -14,12 +14,14 @@ from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
     INJECTED_COMPONENTS_PY,
     INJECTED_COMPONENTS_PY_CHECKSUMS,
 )
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 
 from ..api_models import (
     CheckRequest,
     CheckResponse,
     DiscoverRequest,
     DiscoverResponse,
+    ErrorResponse,
     FullResolveRequest,
     Manifest,
     ManifestResponse,
@@ -64,7 +66,13 @@ router = APIRouter(
 )
 
 
-@router.post("/test_read", operation_id="testRead")
+@router.post(
+    "/test_read",
+    operation_id="testRead",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
 def test_read(request: StreamTestReadRequest) -> StreamReadResponse:
     """
     Test reading from a specific stream in the manifest.
@@ -109,18 +117,29 @@ def test_read(request: StreamTestReadRequest) -> StreamReadResponse:
     )
 
     runner = ManifestCommandProcessor(source)
-    cdk_result = runner.test_read(
-        config_dict,
-        catalog,
-        converted_state,
-        request.record_limit,
-        request.page_limit,
-        request.slice_limit,
-    )
-    return StreamReadResponse.model_validate(asdict(cdk_result))
-
-
-@router.post("/check", operation_id="check")
+    try:
+        cdk_result = runner.test_read(
+            config_dict,
+            catalog,
+            converted_state,
+            request.record_limit,
+            request.page_limit,
+            request.slice_limit,
+        )
+        return StreamReadResponse.model_validate(asdict(cdk_result))
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error reading stream: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)
+
+
+@router.post(
+    "/check",
+    operation_id="check",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
 def check(request: CheckRequest) -> CheckResponse:
     """Check configuration against a manifest"""
     # Apply trace tags from context if provided
@@ -130,13 +149,24 @@ def check(request: CheckRequest) -> CheckResponse:
             project_id=request.context.project_id,
         )
 
-    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
-    runner = ManifestCommandProcessor(source)
-    success, message = runner.check_connection(request.config.model_dump())
-    return CheckResponse(success=success, message=message)
-
-
-@router.post("/discover", operation_id="discover")
+    try:
+        source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+        runner = ManifestCommandProcessor(source)
+        success, message = runner.check_connection(request.config.model_dump())
+        return CheckResponse(success=success, message=message)
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error checking connection: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)
+
+
+@router.post(
+    "/discover",
+    operation_id="discover",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
 def discover(request: DiscoverRequest) -> DiscoverResponse:
     """Discover streams from a manifest"""
     # Apply trace tags from context if provided
@@ -146,15 +176,31 @@ def discover(request: DiscoverRequest) -> DiscoverResponse:
             project_id=request.context.project_id,
         )
 
-    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
-    runner = ManifestCommandProcessor(source)
-    catalog = runner.discover(request.config.model_dump())
-    if catalog is None:
-        raise HTTPException(status_code=422, detail="Connector did not return a discovered catalog")
-    return DiscoverResponse(catalog=catalog)
-
-
-@router.post("/resolve", operation_id="resolve")
+    try:
+        source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+        runner = ManifestCommandProcessor(source)
+        catalog = runner.discover(request.config.model_dump())
+        if catalog is None:
+            raise HTTPException(
+                status_code=422, detail="Connector did not return a discovered catalog"
+            )
+        return DiscoverResponse(catalog=catalog)
+    except HTTPException:
+        # Re-raise HTTPExceptions as-is (like the catalog None check above)
+        raise
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error discovering streams: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)
+
+
+@router.post(
+    "/resolve",
+    operation_id="resolve",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
 def resolve(request: ResolveRequest) -> ManifestResponse:
     """Resolve a manifest to its final configuration."""
     # Apply trace tags from context if provided
@@ -164,11 +210,22 @@ def resolve(request: ResolveRequest) -> ManifestResponse:
             project_id=request.context.project_id,
         )
 
-    source = safe_build_source(request.manifest.model_dump(), {})
-    return ManifestResponse(manifest=Manifest(**source.resolved_manifest))
-
-
-@router.post("/full_resolve", operation_id="fullResolve")
+    try:
+        source = safe_build_source(request.manifest.model_dump(), {})
+        return ManifestResponse(manifest=Manifest(**source.resolved_manifest))
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error resolving manifest: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)
+
+
+@router.post(
+    "/full_resolve",
+    operation_id="fullResolve",
+    responses={
+        400: {"description": "Bad Request - Error processing request", "model": ErrorResponse}
+    },
+)
 def full_resolve(request: FullResolveRequest) -> ManifestResponse:
     """
     Fully resolve a manifest, including dynamic streams.
@@ -182,21 +239,26 @@ def full_resolve(request: FullResolveRequest) -> ManifestResponse:
             project_id=request.context.project_id,
         )
 
-    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
-    manifest = {**source.resolved_manifest}
-    streams = manifest.get("streams", [])
-    for stream in streams:
-        stream["dynamic_stream_name"] = None
-
-    mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
-    for stream in source.dynamic_streams:
-        generated_streams = mapped_streams.setdefault(stream["dynamic_stream_name"], [])
-
-        if len(generated_streams) < request.stream_limit:
-            generated_streams += [stream]
-
-    for generated_streams_list in mapped_streams.values():
-        streams.extend(generated_streams_list)
-
-    manifest["streams"] = streams
-    return ManifestResponse(manifest=Manifest(**manifest))
+    try:
+        source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+        manifest = {**source.resolved_manifest}
+        streams = manifest.get("streams", [])
+        for stream in streams:
+            stream["dynamic_stream_name"] = None
+
+        mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
+        for stream in source.dynamic_streams:
+            generated_streams = mapped_streams.setdefault(stream["dynamic_stream_name"], [])
+
+            if len(generated_streams) < request.stream_limit:
+                generated_streams += [stream]
+
+        for generated_streams_list in mapped_streams.values():
+            streams.extend(generated_streams_list)
+
+        manifest["streams"] = streams
+        return ManifestResponse(manifest=Manifest(**manifest))
+    except Exception as exc:
+        # Filter secrets from error message before returning to client
+        sanitized_message = filter_secrets(f"Error full resolving manifest: {str(exc)}")
+        raise HTTPException(status_code=400, detail=sanitized_message)

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py

@@ -7,11 +7,11 @@ import uuid
 import zlib
 from contextlib import closing
 from dataclasses import InitVar, dataclass
+from math import nan
 from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
 
 import pandas as pd
 import requests
-from numpy import nan
 
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -384,10 +384,7 @@ class SimpleRetriever(Retriever):
 
             response = None
             try:
-                if (
-                    self.additional_query_properties
-                    and self.additional_query_properties.property_chunking
-                ):
+                if self.additional_query_properties:
                     for properties in self.additional_query_properties.get_request_property_chunks(
                         stream_slice=stream_slice
                     ):
@@ -401,15 +398,19 @@ class SimpleRetriever(Retriever):
                         )
 
                         for current_record in records_generator_fn(response):
-                            merge_key = (
-                                self.additional_query_properties.property_chunking.get_merge_key(
+                            if self.additional_query_properties.property_chunking:
+                                merge_key = self.additional_query_properties.property_chunking.get_merge_key(
                                     current_record
                                 )
-                            )
-                            if merge_key:
-                                _deep_merge(merged_records[merge_key], current_record)
+                                if merge_key:
+                                    _deep_merge(merged_records[merge_key], current_record)
+                                else:
+                                    # We should still emit records even if the record did not have a merge key
+                                    pagination_tracker.observe(current_record)
+                                    last_page_size += 1
+                                    last_record = current_record
+                                    yield current_record
                             else:
-                                # We should still emit records even if the record did not have a merge key
                                 pagination_tracker.observe(current_record)
                                 last_page_size += 1
                                 last_record = current_record

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/airbyte_cdk/sources/file_based/file_types/excel_parser.py

@@ -9,8 +9,6 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
 
 import orjson
 import pandas as pd
-from numpy import datetime64, issubdtype
-from numpy import dtype as dtype_
 from pydantic.v1 import BaseModel
 
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
@@ -141,7 +139,7 @@ class ExcelParser(FileTypeParser):
     @staticmethod
     def dtype_to_json_type(
         current_type: Optional[str],
-        dtype: dtype_,  # type: ignore [type-arg]
+        dtype: Any,  # pandas dtype
     ) -> str:
         """
         Convert Pandas DataFrame types to Airbyte Types.
@@ -163,7 +161,7 @@ class ExcelParser(FileTypeParser):
             return "number"
         if dtype == "bool" and (not current_type or current_type == "boolean"):
             return "boolean"
-        if issubdtype(dtype, datetime64):
+        if pd.api.types.is_datetime64_any_dtype(dtype):
             return "date-time"
         return "string"
 

{airbyte_cdk-7.3.6 → airbyte_cdk-7.3.7.post1.dev18600150715}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "7.3.6"  # Version will be calculated dynamically.
+version = "7.3.7.post1.dev18600150715"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -42,7 +42,7 @@ genson = "1.3.0"
 google-cloud-secret-manager = "^2.17.0"
 isodate = "~0.6.1"
 Jinja2 = "~3.1.2"
-jsonref = "~0.2"
+jsonref = "^1"
 jsonschema = ">=4.17.3,<5.0"
 packaging = "*"  # Transitive dependency used directly in code
 referencing = ">=0.36.2"
@@ -76,9 +76,6 @@ python-calamine = { version = "0.2.3", optional = true }  # TODO: Remove if unus
 python-snappy = { version = "0.7.3", optional = true }  # TODO: remove if unused
 tiktoken = { version = "0.8.0", optional = true }
 nltk = { version = "3.9.1", optional = true }
-# This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen.
-# airbyte-ci will try to install latest version when --use-local-cdk is used, resulting in the conflict.
-numpy = "<2"
 unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true }
 "unstructured.pytesseract" = { version = ">=0.3.12", optional = true }
 pyjwt = "^2.8.0"
@@ -90,7 +87,7 @@ serpyco-rs = "^1.10.2"
 sqlalchemy = {version = "^2.0,!=2.0.36", optional = true }
 fastapi = { version = ">=0.116.1", optional = true }
 uvicorn = { version = ">=0.35.0", optional = true}
-ddtrace = { version = "^3.12.3", optional = true }
+ddtrace = { version = "^3.12.0", optional = true }
 xmltodict = ">=0.13,<0.15"
 anyascii = "^0.3.2"
 whenever = ">=0.7.3,<0.9.0"