airbyte-cdk 6.61.3.post2.dev17299502224__tar.gz → 6.61.5__tar.gz

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.61.3.post2.dev17299502224
+Version: 6.61.5
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.10,<3.13
+Requires-Python: >=3.10,<3.14
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Provides-Extra: dev
@@ -31,13 +32,14 @@ Requires-Dist: backoff
 Requires-Dist: boltons (>=25.0.0,<26.0.0)
 Requires-Dist: cachetools
 Requires-Dist: click (>=8.1.8,<9.0.0)
-Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cohere (>=4.21,<6.0.0) ; extra == "vector-db-based"
 Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dateparser (>=1.2.2,<2.0.0)
+Requires-Dist: ddtrace (>=3.12.3,<4.0.0) ; extra == "manifest-server"
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastapi (>=0.116.1) ; extra == "manifest-server"
-Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: fastavro (>=1.11.0,<2.0.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
 Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
@@ -51,7 +53,7 @@ Requires-Dist: numpy (<2)
 Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
 Requires-Dist: orjson (>=3.10.7,<4.0.0)
 Requires-Dist: packaging
-Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pandas (==2.2.3)
 Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
 Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
 Requires-Dist: psutil (==6.1.0)

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/airbyte_cdk/manifest_server/README.md

@@ -37,26 +37,31 @@ The server will start on `http://localhost:8000` by default.
 ## API Endpoints
 
 ### `/v1/manifest/test_read`
+
 Test reading from a specific stream in the manifest.
 
 **POST** - Test stream reading with configurable limits for records, pages, and slices.
 
 ### `/v1/manifest/check`
+
 Check configuration against a manifest.
 
 **POST** - Validates connector configuration and returns success/failure status with message.
 
 ### `/v1/manifest/discover`
+
 Discover streams from a manifest.
 
 **POST** - Returns the catalog of available streams from the manifest.
 
-### `/v1/manifest/resolve` 
+### `/v1/manifest/resolve`
+
 Resolve a manifest to its final configuration.
 
 **POST** - Returns the resolved manifest without dynamic stream generation.
 
 ### `/v1/manifest/full_resolve`
+
 Fully resolve a manifest including dynamic streams.
 
 **POST** - Generates dynamic streams up to specified limits and includes them in the resolved manifest.
@@ -68,6 +73,7 @@ The manifest server supports custom Python components, but this feature is **dis
 ### Enabling Custom Components
 
 To allow custom Python components in your manifest files, set the environment variable:
+
 ```bash
 export AIRBYTE_ENABLE_UNSAFE_CODE=true
 ```
@@ -77,20 +83,25 @@ export AIRBYTE_ENABLE_UNSAFE_CODE=true
 The manifest server supports optional JWT bearer token authentication:
 
 ### Configuration
+
 Set the environment variable to enable authentication:
+
 ```bash
 export AB_JWT_SIGNATURE_SECRET="your-jwt-secret-key"
 ```
 
 ### Usage
+
 When authentication is enabled, include a valid JWT token in the Authorization header:
+
 ```bash
 curl -H "Authorization: Bearer <your-jwt-token>" \
   http://localhost:8000/v1/manifest/test_read
 ```
 
 ### Behavior
-- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through 
+
+- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through
 - **With `AB_JWT_SIGNATURE_SECRET`**: Requires valid JWT bearer token using HS256 algorithm
 
 ## OpenAPI Specification
@@ -98,6 +109,7 @@ curl -H "Authorization: Bearer <your-jwt-token>" \
 The manifest server provides an OpenAPI specification for API client generation:
 
 ### Generating the OpenAPI Spec
+
 ```bash
 # Generate OpenAPI YAML (default location)
 manifest-server generate-openapi
@@ -107,6 +119,7 @@ manifest-server generate-openapi --output /path/to/openapi.yaml
 ```
 
 The generated OpenAPI specification is consumed by other applications and tools to:
+
 - Generate API clients in various programming languages
 - Create SDK bindings for the manifest server
 - Provide API documentation and validation
@@ -115,6 +128,7 @@ The generated OpenAPI specification is consumed by other applications and tools
 ### Interactive API Documentation
 
 When running, interactive API documentation is available at:
+
 - Swagger UI: `http://localhost:8000/docs`
 - ReDoc: `http://localhost:8000/redoc`
 
@@ -139,4 +153,26 @@ docker build -f airbyte_cdk/manifest_server/Dockerfile -t manifest-server .
 docker run -p 8080:8080 manifest-server
 ```
 
-Note: The container runs on port 8080 by default.
+Note: The container runs on port 8080 by default.
+
+## Datadog APM
+
+The manifest server supports Datadog APM tracing for monitoring and observability:
+
+### Configuration
+
+To enable Datadog tracing, set the environment variable:
+
+```bash
+export DD_ENABLED=true
+```
+
+This requires the `ddtrace` dependency, which is included in the `manifest-server` extra. For additional configuration options via environment variables, see [ddtrace configuration](https://ddtrace.readthedocs.io/en/stable/configuration.html).
+
+### Usage
+
+```bash
+# Run with Datadog tracing enabled
+DD_ENABLED=true manifest-server start
+```
+

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/airbyte_cdk/manifest_server/app.py

@@ -1,3 +1,9 @@
+import os
+
+if os.getenv("DD_ENABLED", "false").lower() == "true":
+    # Auto-instrumentation should be imported as early as possible.
+    import ddtrace.auto  # noqa: F401
+
 from fastapi import FastAPI
 
 from .routers import capabilities, health, manifest

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/airbyte_cdk/manifest_server/cli/_common.py

@@ -7,6 +7,7 @@ import rich_click as click
 
 # Import server dependencies with graceful fallback
 try:
+    import ddtrace  # noqa: F401
     import fastapi  # noqa: F401
     import uvicorn  # noqa: F401
 

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/airbyte_cdk/manifest_server/command_processor/processor.py

@@ -41,7 +41,6 @@ class ManifestCommandProcessor:
         """
         Test the read method of the source.
         """
-
         test_read_handler = TestReader(
             max_pages_per_slice=page_limit,
             max_slices=slice_limit,

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/airbyte_cdk/manifest_server/openapi.yaml

@@ -61,7 +61,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/StreamRead'
+                $ref: '#/components/schemas/StreamReadResponse'
         '422':
           description: Validation Error
           content:
@@ -159,12 +159,13 @@ paths:
       tags:
       - manifest
       summary: Full Resolve
-      description: 'Fully resolve a manifest including dynamic streams.
+      description: 'Fully resolve a manifest, including dynamic streams.
 
 
-        Generates dynamic streams up to the specified limit and includes
-
-        them in the resolved manifest.'
+        This is a similar operation to resolve, but has an extra step which generates
+        streams from dynamic stream templates if the manifest contains any. This is
+        used when a user clicks the generate streams button on a stream template in
+        the Builder UI'
       operationId: fullResolve
       requestBody:
         content:
@@ -465,7 +466,26 @@ components:
       - manifest
       title: ResolveRequest
       description: Request to resolve a manifest.
-    StreamRead:
+    StreamReadPages:
+      properties:
+        records:
+          items: {}
+          type: array
+          title: Records
+        request:
+          anyOf:
+          - $ref: '#/components/schemas/HttpRequest'
+          - type: 'null'
+        response:
+          anyOf:
+          - $ref: '#/components/schemas/HttpResponse'
+          - type: 'null'
+      type: object
+      required:
+      - records
+      title: StreamReadPages
+      description: Pages of data read from a stream slice.
+    StreamReadResponse:
       properties:
         logs:
           items:
@@ -511,27 +531,8 @@ components:
       - inferred_schema
       - inferred_datetime_formats
       - latest_config_update
-      title: StreamRead
+      title: StreamReadResponse
       description: Complete stream read response with properly typed fields.
-    StreamReadPages:
-      properties:
-        records:
-          items: {}
-          type: array
-          title: Records
-        request:
-          anyOf:
-          - $ref: '#/components/schemas/HttpRequest'
-          - type: 'null'
-        response:
-          anyOf:
-          - $ref: '#/components/schemas/HttpResponse'
-          - type: 'null'
-      type: object
-      required:
-      - records
-      title: StreamReadPages
-      description: Pages of data read from a stream slice.
     StreamReadSlices:
       properties:
         pages:
@@ -577,7 +578,6 @@ components:
           items: {}
           type: array
           title: State
-          default: []
         custom_components_code:
           anyOf:
           - type: string

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py

@@ -149,6 +149,7 @@ class CartesianProductStreamSlicer(PartitionRouter):
         for stream_slice_tuple in product:
             partition = dict(ChainMap(*[s.partition for s in stream_slice_tuple]))  # type: ignore # ChainMap expects a MutableMapping[Never, Never] for reasons
             cursor_slices = [s.cursor_slice for s in stream_slice_tuple if s.cursor_slice]
+            extra_fields = dict(ChainMap(*[s.extra_fields for s in stream_slice_tuple]))  # type: ignore # ChainMap expects a MutableMapping[Never, Never] for reasons
             if len(cursor_slices) > 1:
                 raise ValueError(
                     f"There should only be a single cursor slice. Found {cursor_slices}"
@@ -157,7 +158,9 @@ class CartesianProductStreamSlicer(PartitionRouter):
                 cursor_slice = cursor_slices[0]
             else:
                 cursor_slice = {}
-            yield StreamSlice(partition=partition, cursor_slice=cursor_slice)
+            yield StreamSlice(
+                partition=partition, cursor_slice=cursor_slice, extra_fields=extra_fields
+            )
 
     def set_initial_state(self, stream_state: StreamState) -> None:
         """

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -14,10 +14,21 @@ from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import Stre
 from airbyte_cdk.sources.types import Record, StreamSlice
 from airbyte_cdk.utils.slice_hasher import SliceHasher
 
+
 # For Connector Builder test read operations, we track the total number of records
-# read for the stream at the global level so that we can stop reading early if we
-# exceed the record limit
-total_record_counter = 0
+# read for the stream so that we can stop reading early if we exceed the record limit.
+class RecordCounter:
+    def __init__(self) -> None:
+        self.total_record_counter = 0
+
+    def increment(self) -> None:
+        self.total_record_counter += 1
+
+    def reset(self) -> None:
+        self.total_record_counter = 0
+
+    def get_total_records(self) -> int:
+        return self.total_record_counter
 
 
 class SchemaLoaderCachingDecorator(SchemaLoader):
@@ -51,6 +62,7 @@ class DeclarativePartitionFactory:
         self._retriever = retriever
         self._message_repository = message_repository
         self._max_records_limit = max_records_limit
+        self._record_counter = RecordCounter()
 
     def create(self, stream_slice: StreamSlice) -> Partition:
         return DeclarativePartition(
@@ -60,6 +72,7 @@ class DeclarativePartitionFactory:
             message_repository=self._message_repository,
             max_records_limit=self._max_records_limit,
             stream_slice=stream_slice,
+            record_counter=self._record_counter,
         )
 
 
@@ -72,6 +85,7 @@ class DeclarativePartition(Partition):
         message_repository: MessageRepository,
         max_records_limit: Optional[int],
         stream_slice: StreamSlice,
+        record_counter: RecordCounter,
     ):
         self._stream_name = stream_name
         self._schema_loader = schema_loader
@@ -80,17 +94,17 @@ class DeclarativePartition(Partition):
         self._max_records_limit = max_records_limit
         self._stream_slice = stream_slice
         self._hash = SliceHasher.hash(self._stream_name, self._stream_slice)
+        self._record_counter = record_counter
 
     def read(self) -> Iterable[Record]:
         if self._max_records_limit is not None:
-            global total_record_counter
-            if total_record_counter >= self._max_records_limit:
+            if self._record_counter.get_total_records() >= self._max_records_limit:
                 return
         for stream_data in self._retriever.read_records(
             self._schema_loader.get_json_schema(), self._stream_slice
         ):
             if self._max_records_limit is not None:
-                if total_record_counter >= self._max_records_limit:
+                if self._record_counter.get_total_records() >= self._max_records_limit:
                     break
 
             if isinstance(stream_data, Mapping):
@@ -108,7 +122,7 @@ class DeclarativePartition(Partition):
                 self._message_repository.emit_message(stream_data)
 
             if self._max_records_limit is not None:
-                total_record_counter += 1
+                self._record_counter.increment()
 
     def to_slice(self) -> Optional[Mapping[str, Any]]:
         return self._stream_slice

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/airbyte_cdk/sources/file_based/file_types/excel_parser.py

@@ -118,9 +118,9 @@ class ExcelParser(FileTypeParser):
                 # DataFrame.to_dict() method returns datetime values in pandas.Timestamp values, which are not serializable by orjson
                 # DataFrame.to_json() returns string with datetime values serialized to iso8601 with microseconds to align with pydantic behavior
                 # see PR description: https://github.com/airbytehq/airbyte/pull/44444/
-                for index, row in df.iterrows():
-                    # Convert each row (as a Series) to a JSON string
-                    yield orjson.loads(row.to_json(date_format="iso", date_unit="us"))
+                yield from orjson.loads(
+                    df.to_json(orient="records", date_format="iso", date_unit="us")
+                )
 
         except Exception as exc:
             # Raise a RecordParseError if any exception occurs during parsing

{airbyte_cdk-6.61.3.post2.dev17299502224 → airbyte_cdk-6.61.5}/pyproject.toml

@@ -23,13 +23,13 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.61.3.post2.dev17299502224"  # Version will be calculated dynamically.
+version = "6.61.5"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
 
 [tool.poetry.dependencies]
-python = ">=3.10,<3.13"
+python = ">=3.10,<3.14"
 airbyte-protocol-models-dataclasses = "^0.17.1"
 
 backoff = "*"
@@ -45,7 +45,7 @@ Jinja2 = "~3.1.2"
 jsonref = "~0.2"
 jsonschema = "~4.17.3"  # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0
 packaging = "*"  # Transitive dependency used directly in code
-pandas = "2.2.2"
+pandas = "2.2.3"
 psutil = "6.1.0" # TODO: Remove if unused
 pydantic = "^2.7"
 pyrate-limiter = "~3.1.0"
@@ -61,8 +61,8 @@ typing-extensions = "*"  # Transitive dependency used directly in code
 wcmatch = "10.0"
 # Extras depedencies
 avro = { version = ">=1.11.2,<1.13.0", optional = true }  # TODO: Move into dev dependencies if only used in tests
-cohere = { version = "4.21", optional = true }
-fastavro = { version = "~1.8.0", optional = true }
+cohere = { version = ">=4.21,<6.0.0", optional = true }
+fastavro = { version = ">=1.11.0,<2.0.0", optional = true }
 langchain = { version = "0.1.16", optional = true }
 langchain_core = { version = "0.1.42", optional = true }
 markdown = { version = "*", optional = true }  # TODO: Remove if unused
@@ -89,6 +89,7 @@ serpyco-rs = "^1.10.2"
 sqlalchemy = {version = "^2.0,!=2.0.36", optional = true }
 fastapi = { version = ">=0.116.1", optional = true }
 uvicorn = { version = ">=0.35.0", optional = true}
+ddtrace = { version = "^3.12.3", optional = true }
 xmltodict = ">=0.13,<0.15"
 anyascii = "^0.3.2"
 whenever = "^0.6.16"
@@ -124,7 +125,7 @@ file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfmi
 vector-db-based = ["langchain", "openai", "cohere", "tiktoken"]
 sql = ["sqlalchemy"]
 dev = ["pytest"]
-manifest-server = ["fastapi", "uvicorn"]
+manifest-server = ["fastapi", "uvicorn", "ddtrace"]
 
 [tool.poetry.scripts]
 
@@ -208,7 +209,12 @@ ignore = [
 # NOTE: PyTest options moved to dedicated `pytest.ini`
 
 [tool.airbyte_ci]
-python_versions = ["3.10", "3.11"]
+python_versions = [
+  "3.10",
+  "3.11",
+  "3.12",
+  "3.13",
+]
 optional_poetry_groups = ["dev"]
 poetry_extras = ["file-based", "vector-db-based", "manifest-server"]
 poe_tasks = ["check-ci"]