airbyte-cdk 6.27.0__py3-none-any.whl → 6.27.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -95,6 +95,7 @@ class ConcurrentPerPartitionCursor(Cursor):
         self._lookback_window: int = 0
         self._parent_state: Optional[StreamState] = None
         self._over_limit: int = 0
+        self._use_global_cursor: bool = False
         self._partition_serializer = PerPartitionKeySerializer()
 
         self._set_initial_state(stream_state)
@@ -105,16 +106,18 @@ class ConcurrentPerPartitionCursor(Cursor):
 
     @property
     def state(self) -> MutableMapping[str, Any]:
-        states = []
-        for partition_tuple, cursor in self._cursor_per_partition.items():
-            if cursor.state:
-                states.append(
-                    {
-                        "partition": self._to_dict(partition_tuple),
-                        "cursor": copy.deepcopy(cursor.state),
-                    }
-                )
-        state: dict[str, Any] = {self._PERPARTITION_STATE_KEY: states}
+        state: dict[str, Any] = {"use_global_cursor": self._use_global_cursor}
+        if not self._use_global_cursor:
+            states = []
+            for partition_tuple, cursor in self._cursor_per_partition.items():
+                if cursor.state:
+                    states.append(
+                        {
+                            "partition": self._to_dict(partition_tuple),
+                            "cursor": copy.deepcopy(cursor.state),
+                        }
+                    )
+            state[self._PERPARTITION_STATE_KEY] = states
 
         if self._global_cursor:
             state[self._GLOBAL_STATE_KEY] = self._global_cursor
@@ -147,7 +150,8 @@ class ConcurrentPerPartitionCursor(Cursor):
                     < cursor.state[self.cursor_field.cursor_field_key]
                 ):
                     self._new_global_cursor = copy.deepcopy(cursor.state)
-            self._emit_state_message()
+            if not self._use_global_cursor:
+                self._emit_state_message()
 
     def ensure_at_least_one_state_emitted(self) -> None:
         """
@@ -225,14 +229,15 @@ class ConcurrentPerPartitionCursor(Cursor):
         """
         with self._lock:
             while len(self._cursor_per_partition) > self.DEFAULT_MAX_PARTITIONS_NUMBER - 1:
+                self._over_limit += 1
                 # Try removing finished partitions first
                 for partition_key in list(self._cursor_per_partition.keys()):
-                    if partition_key in self._finished_partitions:
+                    if partition_key in self._finished_partitions and self._semaphore_per_partition[partition_key]._value == 0:
                         oldest_partition = self._cursor_per_partition.pop(
                             partition_key
                         )  # Remove the oldest partition
                         logger.warning(
-                            f"The maximum number of partitions has been reached. Dropping the oldest partition: {oldest_partition}. Over limit: {self._over_limit}."
+                            f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._over_limit}."
                         )
                         break
                 else:
@@ -297,6 +302,8 @@ class ConcurrentPerPartitionCursor(Cursor):
             self._new_global_cursor = deepcopy(stream_state)
 
         else:
+            self._use_global_cursor = stream_state.get("use_global_cursor", False)
+
             self._lookback_window = int(stream_state.get("lookback_window", 0))
 
             for state in stream_state.get(self._PERPARTITION_STATE_KEY, []):
@@ -320,6 +327,9 @@ class ConcurrentPerPartitionCursor(Cursor):
         self._partition_router.set_initial_state(stream_state)
 
     def observe(self, record: Record) -> None:
+        if not self._use_global_cursor and self.limit_reached():
+            self._use_global_cursor = True
+
         if not record.associated_slice:
             raise ValueError(
                 "Invalid state as stream slices that are emitted should refer to an existing cursor"
@@ -358,3 +368,6 @@ class ConcurrentPerPartitionCursor(Cursor):
             )
         cursor = self._cursor_per_partition[partition_key]
         return cursor
+
+    def limit_reached(self) -> bool:
+        return self._over_limit > self.DEFAULT_MAX_PARTITIONS_NUMBER

{airbyte_cdk-6.27.0.dist-info → airbyte_cdk-6.27.0.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.27.0
+Version: 6.27.0.dev0
 Summary: A framework for writing Airbyte Connectors.
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk

{airbyte_cdk-6.27.0.dist-info → airbyte_cdk-6.27.0.dev0.dist-info}/RECORD

@@ -88,7 +88,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=tjNwcURmlyD
 airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=LhqGDfX06_dDYLKsIVnwQ_nAWCln-v8PV7Wgt_QVeTI,6533
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
-airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=4Z8qZ5DccF0fw163KR5fWW83O-3-84AlaZKPajZ0ZZI,15945
+airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=TnlwShqF_3JxY8j6mUHeUO3rm1PtkC1N8nL7189Dohs,16588
 airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=_UzUnSIUsDbRgbFTXgSyZEFb4ws-KdhdQPWO8mFbV7U,22028
 airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
 airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=9HO-QbL9akvjq2NP7l498RwLA4iQZlBMQW1tZbt34I8,15943
@@ -350,8 +350,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.27.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.27.0.dist-info/METADATA,sha256=JLRDR-Ds7n-AEubbCyoOdpqY2g8pTQkcfZA2dN9HDmU,5996
-airbyte_cdk-6.27.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-airbyte_cdk-6.27.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.27.0.dist-info/RECORD,,
+airbyte_cdk-6.27.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.27.0.dev0.dist-info/METADATA,sha256=uVSlbLjqWio7gWy4AiOJ82IrCtjS1e8Fd2ZCcCj-zvI,6001
+airbyte_cdk-6.27.0.dev0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+airbyte_cdk-6.27.0.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.27.0.dev0.dist-info/RECORD,,