airbyte-cdk 6.51.0.post4.dev15422454178__tar.gz → 6.51.0.post7.dev15491071971__tar.gz

{airbyte_cdk-6.51.0.post4.dev15422454178 → airbyte_cdk-6.51.0.post7.dev15491071971}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.51.0.post4.dev15422454178
+Version: 6.51.0.post7.dev15491071971
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.51.0.post4.dev15422454178 → airbyte_cdk-6.51.0.post7.dev15491071971}/airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -27,6 +27,7 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_sta
     AbstractStreamStateConverter,
 )
 from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
+import gc
 
 logger = logging.getLogger("airbyte")
 
@@ -59,8 +60,8 @@ class ConcurrentPerPartitionCursor(Cursor):
     CurrentPerPartitionCursor expects the state of the ConcurrentCursor to follow the format {cursor_field: cursor_value}.
     """
 
-    DEFAULT_MAX_PARTITIONS_NUMBER = 25_000
-    SWITCH_TO_GLOBAL_LIMIT = 10_000
+    DEFAULT_MAX_PARTITIONS_NUMBER = 250
+    SWITCH_TO_GLOBAL_LIMIT = 100
     _NO_STATE: Mapping[str, Any] = {}
     _NO_CURSOR_STATE: Mapping[str, Any] = {}
     _GLOBAL_STATE_KEY = "state"
@@ -179,6 +180,16 @@ class ConcurrentPerPartitionCursor(Cursor):
 
             self._emit_state_message()
 
+            gc.collect()
+            alive = [o for o in gc.get_objects() if isinstance(o, ConcurrentCursor)]
+            print(f"ConcurrentCursor {len(alive)=}")
+
+            alive = [o for o in gc.get_objects() if isinstance(o, threading.Semaphore)]
+            print(f"Semaphores {len(alive)=}")
+
+            alive = [o for o in gc.get_objects() if isinstance(o, dict)]
+            print(f"Dictionaries {len(alive)=}")
+
     def _check_and_update_parent_state(self) -> None:
         last_closed_state = None
 
@@ -295,6 +306,7 @@ class ConcurrentPerPartitionCursor(Cursor):
             ):
                 self._partition_parent_state_map[partition_key] = (deepcopy(parent_state), seq)
 
+        # try:
         for cursor_slice, is_last_slice, _ in iterate_with_last_flag_and_state(
             cursor.stream_slices(),
             lambda: None,
@@ -306,6 +318,7 @@ class ConcurrentPerPartitionCursor(Cursor):
                 partition=partition, cursor_slice=cursor_slice, extra_fields=partition.extra_fields
             )
 
+
     def _ensure_partition_limit(self) -> None:
         """
         Ensure the maximum number of partitions does not exceed the predefined limit.
@@ -492,11 +505,10 @@ class ConcurrentPerPartitionCursor(Cursor):
     def _create_cursor(
         self, cursor_state: Any, runtime_lookback_window: int = 0
     ) -> ConcurrentCursor:
-        cursor = self._cursor_factory.create(
+        return self._cursor_factory.create(
             stream_state=deepcopy(cursor_state),
             runtime_lookback_window=timedelta(seconds=runtime_lookback_window),
         )
-        return cursor
 
     def should_be_synced(self, record: Record) -> bool:
         return self._get_cursor(record).should_be_synced(record)
@@ -511,8 +523,7 @@ class ConcurrentPerPartitionCursor(Cursor):
             raise ValueError(
                 "Invalid state as stream slices that are emitted should refer to an existing cursor"
             )
-        cursor = self._cursor_per_partition[partition_key]
-        return cursor
+        return self._cursor_per_partition[partition_key]
 
     def limit_reached(self) -> bool:
         return self._number_of_partitions > self.SWITCH_TO_GLOBAL_LIMIT

{airbyte_cdk-6.51.0.post4.dev15422454178 → airbyte_cdk-6.51.0.post7.dev15491071971}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.51.0.post4.dev15422454178"  # Version will be calculated dynamically.
+version = "6.51.0.post7.dev15491071971"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false