airbyte-cdk 6.46.1__py3-none-any.whl → 6.47.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -65,6 +65,7 @@ class ConcurrentPerPartitionCursor(Cursor):
     _NO_CURSOR_STATE: Mapping[str, Any] = {}
     _GLOBAL_STATE_KEY = "state"
     _PERPARTITION_STATE_KEY = "states"
+    _IS_PARTITION_DUPLICATION_LOGGED = False
     _KEY = 0
     _VALUE = 1
 
@@ -279,7 +280,13 @@ class ConcurrentPerPartitionCursor(Cursor):
             with self._lock:
                 self._number_of_partitions += 1
                 self._cursor_per_partition[partition_key] = cursor
-        self._semaphore_per_partition[partition_key] = threading.Semaphore(0)
+
+        if partition_key in self._semaphore_per_partition:
+            if not self._IS_PARTITION_DUPLICATION_LOGGED:
+                logger.warning(f"Partition duplication detected for stream {self._stream_name}")
+                self._IS_PARTITION_DUPLICATION_LOGGED = True
+        else:
+            self._semaphore_per_partition[partition_key] = threading.Semaphore(0)
 
         with self._lock:
             if (

airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -1,6 +1,6 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
-from typing import Any, Hashable, Iterable, Mapping, Optional
+from typing import Any, Iterable, Mapping, Optional
 
 from airbyte_cdk.sources.declarative.retrievers import Retriever
 from airbyte_cdk.sources.message import MessageRepository
@@ -89,21 +89,5 @@ class StreamSlicerPartitionGenerator(PartitionGenerator):
         self._stream_slicer = stream_slicer
 
     def generate(self) -> Iterable[Partition]:
-        # Yield partitions for unique stream slices, avoiding duplicates
-        seen_slices: set[Hashable] = set()
         for stream_slice in self._stream_slicer.stream_slices():
-            slice_key = self._make_hashable(stream_slice)
-            if slice_key in seen_slices:
-                continue
-            seen_slices.add(slice_key)
             yield self._partition_factory.create(stream_slice)
-
-    @staticmethod
-    def _make_hashable(obj: Any) -> Any:
-        if isinstance(obj, dict):
-            return frozenset(
-                (k, StreamSlicerPartitionGenerator._make_hashable(v)) for k, v in obj.items()
-            )
-        if isinstance(obj, list):
-            return tuple(StreamSlicerPartitionGenerator._make_hashable(i) for i in obj)
-        return obj

{airbyte_cdk-6.46.1.dist-info → airbyte_cdk-6.47.0.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.46.1
+Version: 6.47.0.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.46.1.dist-info → airbyte_cdk-6.47.0.dev0.dist-info}/RECORD

@@ -100,7 +100,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=Xg4e0exJ5Tq
 airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
-airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=Mrx5XY6G8ZT-imsjUggpjzWo-Po_Wvi1WpylEW1ohIQ,22263
+airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=4OSvZWRhKNORXJUchlvXKsWMbZKSC20AKDzt1hQnDz8,22602
 airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
 airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
 airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
@@ -199,7 +199,7 @@ airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLn
 airbyte_cdk/sources/declarative/spec/__init__.py,sha256=H0UwoRhgucbKBIzg85AXrifybVmfpwWpPdy22vZKVuo,141
 airbyte_cdk/sources/declarative/spec/spec.py,sha256=ODSNUgkDOhnLQnwLjgSaME6R3kNeywjROvbNrWEnsgU,1876
 airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
-airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=V_WXPrEzn3mlS_calo5GHUIkugGKcrDwHp06uaQLUMg,4247
+airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=cjKGm4r438dd1GxrFHJ4aYrdzG2bkncnwaWxAwlXR3M,3585
 airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
 airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
 airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=Eg1jQtRObgzxbtySTQs5uEZIjEklsoHFxYSPf78x6Ng,5420
@@ -390,9 +390,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.46.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.46.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.46.1.dist-info/METADATA,sha256=7LlsiUvkE6tU3sQz44kb011La3QmPw8wYWSfcuSeqSc,6323
-airbyte_cdk-6.46.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.46.1.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.46.1.dist-info/RECORD,,
+airbyte_cdk-6.47.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.47.0.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.47.0.dev0.dist-info/METADATA,sha256=w15gBn52dA5wRUmZ_w1t2ViE2CDJimBiS4PWfjnK2vg,6328
+airbyte_cdk-6.47.0.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.47.0.dev0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.47.0.dev0.dist-info/RECORD,,