airbyte-cdk 6.61.2__py3-none-any.whl → 6.61.3.post2.dev17299502224__py3-none-any.whl

airbyte_cdk/manifest_server/routers/manifest.py

@@ -0,0 +1,155 @@
+import hashlib
+from dataclasses import asdict
+from typing import Any, Dict, List, Mapping, Optional
+
+import jsonschema
+from airbyte_protocol_dataclasses.models import AirbyteStateMessage, ConfiguredAirbyteCatalog
+from fastapi import APIRouter, Depends, HTTPException
+
+from airbyte_cdk.models import AirbyteStateMessageSerializer
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
+    INJECTED_COMPONENTS_PY,
+    INJECTED_COMPONENTS_PY_CHECKSUMS,
+)
+
+from ..api_models import (
+    CheckRequest,
+    CheckResponse,
+    DiscoverRequest,
+    DiscoverResponse,
+    FullResolveRequest,
+    Manifest,
+    ManifestResponse,
+    ResolveRequest,
+    StreamReadResponse,
+    StreamTestReadRequest,
+)
+from ..auth import verify_jwt_token
+from ..command_processor.processor import ManifestCommandProcessor
+from ..command_processor.utils import build_catalog, build_source
+
+
+def safe_build_source(
+    manifest_dict: Dict[str, Any],
+    config_dict: Mapping[str, Any],
+    catalog: Optional[ConfiguredAirbyteCatalog] = None,
+    state: Optional[List[AirbyteStateMessage]] = None,
+    page_limit: Optional[int] = None,
+    slice_limit: Optional[int] = None,
+    record_limit: Optional[int] = None,
+) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
+    """Wrapper around build_source that converts ValidationError to HTTPException."""
+    try:
+        return build_source(
+            manifest_dict,
+            catalog,
+            config_dict,
+            state,
+            record_limit,
+            page_limit,
+            slice_limit,
+        )
+    except jsonschema.exceptions.ValidationError as e:
+        raise HTTPException(status_code=400, detail=f"Invalid manifest: {e.message}")
+
+
+router = APIRouter(
+    prefix="/manifest",
+    tags=["manifest"],
+    dependencies=[Depends(verify_jwt_token)],
+)
+
+
+@router.post("/test_read", operation_id="testRead")
+def test_read(request: StreamTestReadRequest) -> StreamReadResponse:
+    """
+    Test reading from a specific stream in the manifest.
+    """
+    config_dict = request.config.model_dump()
+
+    catalog = build_catalog(request.stream_name)
+    converted_state = [AirbyteStateMessageSerializer.load(state) for state in request.state]
+
+    if request.custom_components_code:
+        config_dict[INJECTED_COMPONENTS_PY] = request.custom_components_code
+        config_dict[INJECTED_COMPONENTS_PY_CHECKSUMS] = {
+            "md5": hashlib.md5(request.custom_components_code.encode()).hexdigest()
+        }
+
+    source = safe_build_source(
+        request.manifest.model_dump(),
+        config_dict,
+        catalog,
+        converted_state,
+        request.page_limit,
+        request.slice_limit,
+        request.record_limit,
+    )
+
+    runner = ManifestCommandProcessor(source)
+    cdk_result = runner.test_read(
+        config_dict,
+        catalog,
+        converted_state,
+        request.record_limit,
+        request.page_limit,
+        request.slice_limit,
+    )
+    return StreamReadResponse.model_validate(asdict(cdk_result))
+
+
+@router.post("/check", operation_id="check")
+def check(request: CheckRequest) -> CheckResponse:
+    """Check configuration against a manifest"""
+    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+    runner = ManifestCommandProcessor(source)
+    success, message = runner.check_connection(request.config.model_dump())
+    return CheckResponse(success=success, message=message)
+
+
+@router.post("/discover", operation_id="discover")
+def discover(request: DiscoverRequest) -> DiscoverResponse:
+    """Discover streams from a manifest"""
+    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+    runner = ManifestCommandProcessor(source)
+    catalog = runner.discover(request.config.model_dump())
+    if catalog is None:
+        raise HTTPException(status_code=422, detail="Connector did not return a discovered catalog")
+    return DiscoverResponse(catalog=catalog)
+
+
+@router.post("/resolve", operation_id="resolve")
+def resolve(request: ResolveRequest) -> ManifestResponse:
+    """Resolve a manifest to its final configuration."""
+    source = safe_build_source(request.manifest.model_dump(), {})
+    return ManifestResponse(manifest=Manifest(**source.resolved_manifest))
+
+
+@router.post("/full_resolve", operation_id="fullResolve")
+def full_resolve(request: FullResolveRequest) -> ManifestResponse:
+    """
+    Fully resolve a manifest, including dynamic streams.
+
+    This is a similar operation to resolve, but has an extra step which generates streams from dynamic stream templates if the manifest contains any. This is used when a user clicks the generate streams button on a stream template in the Builder UI
+    """
+    source = safe_build_source(request.manifest.model_dump(), request.config.model_dump())
+    manifest = {**source.resolved_manifest}
+    streams = manifest.get("streams", [])
+    for stream in streams:
+        stream["dynamic_stream_name"] = None
+
+    mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
+    for stream in source.dynamic_streams:
+        generated_streams = mapped_streams.setdefault(stream["dynamic_stream_name"], [])
+
+        if len(generated_streams) < request.stream_limit:
+            generated_streams += [stream]
+
+    for generated_streams_list in mapped_streams.values():
+        streams.extend(generated_streams_list)
+
+    manifest["streams"] = streams
+    return ManifestResponse(manifest=Manifest(**manifest))

airbyte_cdk/sources/file_based/file_types/excel_parser.py

@@ -118,9 +118,9 @@ class ExcelParser(FileTypeParser):
                 # DataFrame.to_dict() method returns datetime values in pandas.Timestamp values, which are not serializable by orjson
                 # DataFrame.to_json() returns string with datetime values serialized to iso8601 with microseconds to align with pydantic behavior
                 # see PR description: https://github.com/airbytehq/airbyte/pull/44444/
-                yield from orjson.loads(
-                    df.to_json(orient="records", date_format="iso", date_unit="us")
-                )
+                for index, row in df.iterrows():
+                    # Convert each row (as a Series) to a JSON string
+                    yield orjson.loads(row.to_json(date_format="iso", date_unit="us"))
 
         except Exception as exc:
             # Raise a RecordParseError if any exception occurs during parsing

{airbyte_cdk-6.61.2.dist-info → airbyte_cdk-6.61.3.post2.dev17299502224.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.61.2
+Version: 6.61.3.post2.dev17299502224
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -19,6 +19,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Provides-Extra: dev
 Provides-Extra: file-based
+Provides-Extra: manifest-server
 Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
@@ -35,6 +36,7 @@ Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dateparser (>=1.2.2,<2.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastapi (>=0.116.1) ; extra == "manifest-server"
 Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
 Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
@@ -77,6 +79,7 @@ Requires-Dist: typing-extensions
 Requires-Dist: unidecode (>=1.3.8,<2.0.0)
 Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
 Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: uvicorn (>=0.35.0) ; extra == "manifest-server"
 Requires-Dist: wcmatch (==10.0)
 Requires-Dist: whenever (>=0.6.16,<0.7.0)
 Requires-Dist: xmltodict (>=0.13,<0.15)

{airbyte_cdk-6.61.2.dist-info → airbyte_cdk-6.61.3.post2.dev17299502224.dist-info}/RECORD

@@ -55,6 +55,31 @@ airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data
 airbyte_cdk/manifest_migrations/migrations/http_requester_url_base_to_url.py,sha256=EX1MVYVpoWypA28qoH48wA0SYZjGdlR8bcSixTDzfgo,1346
 airbyte_cdk/manifest_migrations/migrations/registry.yaml,sha256=F-hdapvl_vZnsI7CQsV00Rb7g7j4Nt2zaM83-Tbwgbg,956
 airbyte_cdk/manifest_migrations/migrations_registry.py,sha256=zly2fwaOxDukqC7eowzrDlvhA2v71FjW74kDzvRXhSY,2619
+airbyte_cdk/manifest_server/Dockerfile,sha256=hNgnUguE9jA5XFkpLmzvbsQbsZTuwOxJgE7qYNkPueo,1316
+airbyte_cdk/manifest_server/README.md,sha256=W2zHnGM4pOynkfm4gSzFGHfEkou4NgDDDRFQGiJhFZ8,3772
+airbyte_cdk/manifest_server/__init__.py,sha256=EE54nk2dbtExIEEvLPCTUkJ_ESV5OYP4B2rBJlDpJ5g,33
+airbyte_cdk/manifest_server/api_models/__init__.py,sha256=1UFpMsJm1sjNBjgEasbOU98BgZxLthlPM1KdUpGfC4I,1015
+airbyte_cdk/manifest_server/api_models/capabilities.py,sha256=eL88UxojIewHt97wMeCvyt92Hzh95UvLvVH6MSlsj5o,152
+airbyte_cdk/manifest_server/api_models/dicts.py,sha256=Rm10IeV745MY8bLyrYyG7a9NGNrZBlnfkXct8gi7OTI,467
+airbyte_cdk/manifest_server/api_models/manifest.py,sha256=cqxA4hU7Q29R8w6fqRDxtM7bVcFcADU2eQXZYKbib3M,1673
+airbyte_cdk/manifest_server/api_models/stream.py,sha256=Jtz4TbqLf6Xbnu1KtTEQhzi_1rqClB2n22pueK8xgrI,2001
+airbyte_cdk/manifest_server/app.py,sha256=kPJVHIq8twI5KqlCuyV1fT01sFYQoSBdGBj87rfLcFE,429
+airbyte_cdk/manifest_server/auth.py,sha256=kKET6zg4olyf8p_UMcTnm7vkAtowxK6loSf8o83SdEM,1264
+airbyte_cdk/manifest_server/cli/__init__.py,sha256=YfCEfXq3Jr7z3GOKMA6vF-D-7Y66BNHUwBLNM9UQbiQ,273
+airbyte_cdk/manifest_server/cli/_common.py,sha256=5hfwKjkB5IQ4SGI54DBKMEbzLMsgN2Itv1wlQBzj8ts,799
+airbyte_cdk/manifest_server/cli/_info.py,sha256=-h8U1bSD1umqLuoXfg4yDNWFR1BZuxcXxLhVKfLWB0Y,982
+airbyte_cdk/manifest_server/cli/_openapi.py,sha256=rNghqbBMMT118auUIsxr89Mu6L6hqoQAifxyDbv1ZqQ,1445
+airbyte_cdk/manifest_server/cli/_start.py,sha256=SgkW_dQcpCIrXYNZ6qF95oYIVaCszm9tFEbYiAZo4EQ,876
+airbyte_cdk/manifest_server/cli/run.py,sha256=-Yv_Jv_hDeMBVZdzuyeJJ_JBT1WUhCyUQn4f4mA21Ds,1224
+airbyte_cdk/manifest_server/command_processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airbyte_cdk/manifest_server/command_processor/processor.py,sha256=qaQdmF1SaM7gr0B2D53eaooZ_cOv70hSlHL4c_iV8hg,3830
+airbyte_cdk/manifest_server/command_processor/utils.py,sha256=f_CkN2nURTTp07Twr9vZfFj5j-jTFtezoOUu2i402W4,3221
+airbyte_cdk/manifest_server/main.py,sha256=iSgL7x8ifBpGpXi7Dq7017QjeC20l_CYBAIsumiyJn0,573
+airbyte_cdk/manifest_server/openapi.yaml,sha256=iTHsQLmZLKYj4hWns0vgDr8KFnK1MbiOO76kz1SEcUk,16526
+airbyte_cdk/manifest_server/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airbyte_cdk/manifest_server/routers/capabilities.py,sha256=UeZQzbqEuDN23vizgDb_N7KEXR-YZUPnW5AkXeXy1hA,727
+airbyte_cdk/manifest_server/routers/health.py,sha256=akBUaHUGN-jmN82lQ3kj_3-wdS6gsZx3iSD2KMI5dW8,200
+airbyte_cdk/manifest_server/routers/manifest.py,sha256=6UT8bD-og-Eg-cfxuXkJXzzj0z3Kko6vOfsGjOAaPks,5570
 airbyte_cdk/models/__init__.py,sha256=Et9wJWs5VOWynGbb-3aJRhsdAHAiLkNNLxdwqJAuqkw,2114
 airbyte_cdk/models/airbyte_protocol.py,sha256=oZdKsZ7yPjUt9hvxdWNpxCtgjSV2RWhf4R9Np03sqyY,3613
 airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=Dq4ry_Wwvzsos6neDiaOZkY6riQYC33ZlPNWpfIIB1E,1926
@@ -275,7 +300,7 @@ airbyte_cdk/sources/file_based/file_record_data.py,sha256=Vkr5AyZzlsOezjVCLhFrm_
 airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
 airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=USEYqiICXBWpDV443VtNOCmUA-GINzY_Zah74_5w3qQ,10860
 airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=QlCXB-ry3np67Q_VerQEPoWDOTcPTB6Go4ydZxY9ae4,20445
-airbyte_cdk/sources/file_based/file_types/excel_parser.py,sha256=BeplCq0hmojELU6bZCvvpRLpQ9us81TqbGYwrhd3INo,7188
+airbyte_cdk/sources/file_based/file_types/excel_parser.py,sha256=bpAtkDs_QyHAUYcWRkF3ROkQB4wbGe3TzY2jwQgG664,7251
 airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=5l2Jo6bp6neDmgM427PrZMZeqU0hCIZVWnzUZ_7BT10,1100
 airbyte_cdk/sources/file_based/file_types/file_type_parser.py,sha256=JgpH21PrbRqwK92BJklZWvh2TndA6xZ-eP1LPMo44oQ,2832
 airbyte_cdk/sources/file_based/file_types/jsonl_parser.py,sha256=GwyNyxmST4RX-XpXy7xVH0D-znYWWBmGv_pVAu95oHQ,5886
@@ -429,9 +454,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.61.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.61.2.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.61.2.dist-info/METADATA,sha256=jrR3TXXz1BFLlHY2ls02vKhGHZRKFAqSfJ7en-6zfHY,6477
-airbyte_cdk-6.61.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.61.2.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.61.2.dist-info/RECORD,,
+airbyte_cdk-6.61.3.post2.dev17299502224.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.61.3.post2.dev17299502224.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.61.3.post2.dev17299502224.dist-info/METADATA,sha256=B0FyXybZxYX0Y7NO06p5xim0SwSEENo58CGuaCUmRTs,6657
+airbyte_cdk-6.61.3.post2.dev17299502224.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.61.3.post2.dev17299502224.dist-info/entry_points.txt,sha256=eLZ2UYvJZGm1s07Pplcs--1Gim60YhZWTb53j_dghwU,195
+airbyte_cdk-6.61.3.post2.dev17299502224.dist-info/RECORD,,

{airbyte_cdk-6.61.2.dist-info → airbyte_cdk-6.61.3.post2.dev17299502224.dist-info}/entry_points.txt

@@ -1,4 +1,5 @@
 [console_scripts]
 airbyte-cdk=airbyte_cdk.cli.airbyte_cdk:cli
+manifest-server=airbyte_cdk.manifest_server.cli.run:run
 source-declarative-manifest=airbyte_cdk.cli.source_declarative_manifest:run