airbyte-cdk 6.61.4__tar.gz → 6.62.0.dev0__tar.gz

{airbyte_cdk-6.61.4 → airbyte_cdk-6.62.0.dev0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.61.4
+Version: 6.62.0.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.10,<3.13
+Requires-Python: >=3.10,<3.14
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Provides-Extra: dev
@@ -31,13 +32,13 @@ Requires-Dist: backoff
 Requires-Dist: boltons (>=25.0.0,<26.0.0)
 Requires-Dist: cachetools
 Requires-Dist: click (>=8.1.8,<9.0.0)
-Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cohere (>=4.21,<6.0.0) ; extra == "vector-db-based"
 Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dateparser (>=1.2.2,<2.0.0)
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastapi (>=0.116.1) ; extra == "manifest-server"
-Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: fastavro (>=1.11.0,<2.0.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
 Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
@@ -51,7 +52,7 @@ Requires-Dist: numpy (<2)
 Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
 Requires-Dist: orjson (>=3.10.7,<4.0.0)
 Requires-Dist: packaging
-Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pandas (==2.2.3)
 Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
 Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
 Requires-Dist: psutil (==6.1.0)

{airbyte_cdk-6.61.4 → airbyte_cdk-6.62.0.dev0}/airbyte_cdk/legacy/sources/declarative/incremental/per_partition_cursor.py

@@ -146,8 +146,10 @@ class PerPartitionCursor(DeclarativeCursor):
             if "state" in stream_state:
                 self._state_to_migrate_from = stream_state["state"]
 
-        # Set parent state for partition routers based on parent streams
-        self._partition_router.set_initial_state(stream_state)
+        # We used to set the parent state through this method but since moving the SubstreamPartitionRouter to the
+        # Concurrent CDK/AbstractStream, the state is passed at the __init__ stage and this does not need to be called.
+        # We are still keeping this line as a comment to be explicit about the past behavior.
+        # self._partition_router.set_initial_state(stream_state)
 
     def observe(self, stream_slice: StreamSlice, record: Record) -> None:
         self._cursor_per_partition[self._to_partition_key(stream_slice.partition)].observe(

{airbyte_cdk-6.61.4 → airbyte_cdk-6.62.0.dev0}/airbyte_cdk/manifest_server/README.md

@@ -37,26 +37,31 @@ The server will start on `http://localhost:8000` by default.
 ## API Endpoints
 
 ### `/v1/manifest/test_read`
+
 Test reading from a specific stream in the manifest.
 
 **POST** - Test stream reading with configurable limits for records, pages, and slices.
 
 ### `/v1/manifest/check`
+
 Check configuration against a manifest.
 
 **POST** - Validates connector configuration and returns success/failure status with message.
 
 ### `/v1/manifest/discover`
+
 Discover streams from a manifest.
 
 **POST** - Returns the catalog of available streams from the manifest.
 
-### `/v1/manifest/resolve` 
+### `/v1/manifest/resolve`
+
 Resolve a manifest to its final configuration.
 
 **POST** - Returns the resolved manifest without dynamic stream generation.
 
 ### `/v1/manifest/full_resolve`
+
 Fully resolve a manifest including dynamic streams.
 
 **POST** - Generates dynamic streams up to specified limits and includes them in the resolved manifest.
@@ -68,6 +73,7 @@ The manifest server supports custom Python components, but this feature is **dis
 ### Enabling Custom Components
 
 To allow custom Python components in your manifest files, set the environment variable:
+
 ```bash
 export AIRBYTE_ENABLE_UNSAFE_CODE=true
 ```
@@ -77,20 +83,25 @@ export AIRBYTE_ENABLE_UNSAFE_CODE=true
 The manifest server supports optional JWT bearer token authentication:
 
 ### Configuration
+
 Set the environment variable to enable authentication:
+
 ```bash
 export AB_JWT_SIGNATURE_SECRET="your-jwt-secret-key"
 ```
 
 ### Usage
+
 When authentication is enabled, include a valid JWT token in the Authorization header:
+
 ```bash
 curl -H "Authorization: Bearer <your-jwt-token>" \
   http://localhost:8000/v1/manifest/test_read
 ```
 
 ### Behavior
-- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through 
+
+- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through
 - **With `AB_JWT_SIGNATURE_SECRET`**: Requires valid JWT bearer token using HS256 algorithm
 
 ## OpenAPI Specification
@@ -98,6 +109,7 @@ curl -H "Authorization: Bearer <your-jwt-token>" \
 The manifest server provides an OpenAPI specification for API client generation:
 
 ### Generating the OpenAPI Spec
+
 ```bash
 # Generate OpenAPI YAML (default location)
 manifest-server generate-openapi
@@ -107,6 +119,7 @@ manifest-server generate-openapi --output /path/to/openapi.yaml
 ```
 
 The generated OpenAPI specification is consumed by other applications and tools to:
+
 - Generate API clients in various programming languages
 - Create SDK bindings for the manifest server
 - Provide API documentation and validation
@@ -115,6 +128,7 @@ The generated OpenAPI specification is consumed by other applications and tools
 ### Interactive API Documentation
 
 When running, interactive API documentation is available at:
+
 - Swagger UI: `http://localhost:8000/docs`
 - ReDoc: `http://localhost:8000/redoc`
 
@@ -139,4 +153,4 @@ docker build -f airbyte_cdk/manifest_server/Dockerfile -t manifest-server .
 docker run -p 8080:8080 manifest-server
 ```
 
-Note: The container runs on port 8080 by default.
+Note: The container runs on port 8080 by default.

{airbyte_cdk-6.61.4 → airbyte_cdk-6.62.0.dev0}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -703,7 +703,7 @@ class ConcurrentDeclarativeSource(AbstractSource, Generic[TState]):
                             stream_slicer=declarative_stream.retriever.stream_slicer,
                             slice_limit=self._limits.max_slices
                             if self._limits
-                            else None,  # technically not needed because create_declarative_stream() -> create_simple_retriever() will apply the decorator. But for consistency and depending how we build create_default_stream, this may be needed later
+                            else None,  # technically not needed because create_default_stream() -> create_simple_retriever() will apply the decorator. But for consistency and depending how we build create_default_stream, this may be needed later
                         )
                     else:
                         if (
@@ -772,7 +772,7 @@ class ConcurrentDeclarativeSource(AbstractSource, Generic[TState]):
                         declarative_stream.retriever.stream_slicer,
                         slice_limit=self._limits.max_slices
                         if self._limits
-                        else None,  # technically not needed because create_declarative_stream() -> create_simple_retriever() will apply the decorator. But for consistency and depending how we build create_default_stream, this may be needed later
+                        else None,  # technically not needed because create_default_stream() -> create_simple_retriever() will apply the decorator. But for consistency and depending how we build create_default_stream, this may be needed later
                     )
 
                     final_state_cursor = FinalStateCursor(

{airbyte_cdk-6.61.4 → airbyte_cdk-6.62.0.dev0}/airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -11,6 +11,13 @@ from copy import deepcopy
 from datetime import timedelta
 from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional
 
+from airbyte_cdk.models import (
+    AirbyteStateBlob,
+    AirbyteStateMessage,
+    AirbyteStateType,
+    AirbyteStreamState,
+    StreamDescriptor,
+)
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import (
     Timer,
@@ -48,7 +55,7 @@ class ConcurrentPerPartitionCursor(Cursor):
     Manages state per partition when a stream has many partitions, preventing data loss or duplication.
 
     Attributes:
-        DEFAULT_MAX_PARTITIONS_NUMBER (int): Maximum number of partitions to retain in memory (default is 10,000).
+        DEFAULT_MAX_PARTITIONS_NUMBER (int): Maximum number of partitions to retain in memory (default is 10,000). This limit needs to be higher than the number of threads we might enqueue (which is represented by ThreadPoolManager.DEFAULT_MAX_QUEUE_SIZE). If not, we could have partitions that have been generated and submitted to the ThreadPool but got deleted from the ConcurrentPerPartitionCursor and when closing them, it will generate KeyError.
 
     - **Partition Limitation Logic**
       Ensures the number of tracked partitions does not exceed the specified limit to prevent memory overuse. Oldest partitions are removed when the limit is reached.
@@ -128,6 +135,7 @@ class ConcurrentPerPartitionCursor(Cursor):
 
         # FIXME this is a temporary field the time of the migration from declarative cursors to concurrent ones
         self._attempt_to_create_cursor_if_not_provided = attempt_to_create_cursor_if_not_provided
+        self._synced_some_data = False
 
     @property
     def cursor_field(self) -> CursorField:
@@ -168,8 +176,8 @@ class ConcurrentPerPartitionCursor(Cursor):
         with self._lock:
             self._semaphore_per_partition[partition_key].acquire()
             if not self._use_global_cursor:
-                self._cursor_per_partition[partition_key].close_partition(partition=partition)
                 cursor = self._cursor_per_partition[partition_key]
+                cursor.close_partition(partition=partition)
                 if (
                     partition_key in self._partitions_done_generating_stream_slices
                     and self._semaphore_per_partition[partition_key]._value == 0
@@ -213,8 +221,10 @@ class ConcurrentPerPartitionCursor(Cursor):
         if not any(
             semaphore_item[1]._value for semaphore_item in self._semaphore_per_partition.items()
         ):
-            self._global_cursor = self._new_global_cursor
-            self._lookback_window = self._timer.finish()
+            if self._synced_some_data:
+                # we only update those if we actually synced some data
+                self._global_cursor = self._new_global_cursor
+                self._lookback_window = self._timer.finish()
             self._parent_state = self._partition_router.get_stream_state()
         self._emit_state_message(throttle=False)
 
@@ -422,9 +432,6 @@ class ConcurrentPerPartitionCursor(Cursor):
         if stream_state.get("parent_state"):
             self._parent_state = stream_state["parent_state"]
 
-        # Set parent state for partition routers based on parent streams
-        self._partition_router.set_initial_state(stream_state)
-
     def _set_global_state(self, stream_state: Mapping[str, Any]) -> None:
         """
         Initializes the global cursor state from the provided stream state.
@@ -458,6 +465,7 @@ class ConcurrentPerPartitionCursor(Cursor):
         except ValueError:
             return
 
+        self._synced_some_data = True
         record_cursor = self._connector_state_converter.output_format(
             self._connector_state_converter.parse_value(record_cursor_value)
         )
@@ -541,3 +549,45 @@ class ConcurrentPerPartitionCursor(Cursor):
 
     def limit_reached(self) -> bool:
         return self._number_of_partitions > self.SWITCH_TO_GLOBAL_LIMIT
+
+    @staticmethod
+    def get_parent_state(
+        stream_state: Optional[StreamState], parent_stream_name: str
+    ) -> Optional[AirbyteStateMessage]:
+        if not stream_state:
+            return None
+
+        if "parent_state" not in stream_state:
+            logger.warning(
+                f"Trying to get_parent_state for stream `{parent_stream_name}` when there are not parent state in the state"
+            )
+            return None
+        elif parent_stream_name not in stream_state["parent_state"]:
+            logger.info(
+                f"Could not find parent state for stream `{parent_stream_name}`. On parents available are {list(stream_state['parent_state'].keys())}"
+            )
+            return None
+
+        return AirbyteStateMessage(
+            type=AirbyteStateType.STREAM,
+            stream=AirbyteStreamState(
+                stream_descriptor=StreamDescriptor(parent_stream_name, None),
+                stream_state=AirbyteStateBlob(stream_state["parent_state"][parent_stream_name]),
+            ),
+        )
+
+    @staticmethod
+    def get_global_state(
+        stream_state: Optional[StreamState], parent_stream_name: str
+    ) -> Optional[AirbyteStateMessage]:
+        return (
+            AirbyteStateMessage(
+                type=AirbyteStateType.STREAM,
+                stream=AirbyteStreamState(
+                    stream_descriptor=StreamDescriptor(parent_stream_name, None),
+                    stream_state=AirbyteStateBlob(stream_state["state"]),
+                ),
+            )
+            if stream_state and "state" in stream_state
+            else None
+        )

{airbyte_cdk-6.61.4 → airbyte_cdk-6.62.0.dev0}/airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py

@@ -192,8 +192,10 @@ class GlobalSubstreamCursor(DeclarativeCursor):
             # Example: {"global_state_format_key": "global_state_format_value"}
             self._stream_cursor.set_initial_state(stream_state)
 
-        # Set parent state for partition routers based on parent streams
-        self._partition_router.set_initial_state(stream_state)
+        # We used to set the parent state through this method but since moving the SubstreamPartitionRouter to the
+        # Concurrent CDK/AbstractStream, the state is passed at the __init__ stage and this does not need to be called.
+        # We are still keeping this line as a comment to be explicit about the past behavior.
+        # self._partition_router.set_initial_state(stream_state)
 
     def _inject_lookback_into_stream_cursor(self, lookback_window: int) -> None:
         """