airbyte-cdk 6.51.0.post8.dev15492684389__py3-none-any.whl → 6.51.0.post9.dev15582628322__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -27,7 +27,6 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_sta
     AbstractStreamStateConverter,
 )
 from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
-import gc
 
 logger = logging.getLogger("airbyte")
 
@@ -60,8 +59,8 @@ class ConcurrentPerPartitionCursor(Cursor):
     CurrentPerPartitionCursor expects the state of the ConcurrentCursor to follow the format {cursor_field: cursor_value}.
     """
 
-    DEFAULT_MAX_PARTITIONS_NUMBER = 250
-    SWITCH_TO_GLOBAL_LIMIT = 100
+    DEFAULT_MAX_PARTITIONS_NUMBER = 25_000
+    SWITCH_TO_GLOBAL_LIMIT = 10_000
     _NO_STATE: Mapping[str, Any] = {}
     _NO_CURSOR_STATE: Mapping[str, Any] = {}
     _GLOBAL_STATE_KEY = "state"
@@ -180,16 +179,6 @@ class ConcurrentPerPartitionCursor(Cursor):
 
             self._emit_state_message()
 
-            gc.collect()
-            alive = [o for o in gc.get_objects() if isinstance(o, ConcurrentCursor)]
-            print(f"ConcurrentCursor {len(alive)=}")
-
-            alive = [o for o in gc.get_objects() if isinstance(o, threading.Semaphore)]
-            print(f"Semaphores {len(alive)=}")
-
-            alive = [o for o in gc.get_objects() if isinstance(o, dict)]
-            print(f"Dictionaries {len(alive)=}")
-
     def _check_and_update_parent_state(self) -> None:
         last_closed_state = None
 
@@ -306,7 +295,6 @@ class ConcurrentPerPartitionCursor(Cursor):
             ):
                 self._partition_parent_state_map[partition_key] = (deepcopy(parent_state), seq)
 
-        # try:
         for cursor_slice, is_last_slice, _ in iterate_with_last_flag_and_state(
             cursor.stream_slices(),
             lambda: None,
@@ -318,7 +306,6 @@ class ConcurrentPerPartitionCursor(Cursor):
                 partition=partition, cursor_slice=cursor_slice, extra_fields=partition.extra_fields
             )
 
-
     def _ensure_partition_limit(self) -> None:
         """
         Ensure the maximum number of partitions does not exceed the predefined limit.
@@ -348,7 +335,7 @@ class ConcurrentPerPartitionCursor(Cursor):
                         oldest_partition = self._cursor_per_partition.pop(
                             partition_key
                         )  # Remove the oldest partition
-                        logger.warning(
+                        logger.debug(
                             f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._number_of_partitions - self.DEFAULT_MAX_PARTITIONS_NUMBER}."
                         )
                         break
@@ -505,10 +492,11 @@ class ConcurrentPerPartitionCursor(Cursor):
     def _create_cursor(
         self, cursor_state: Any, runtime_lookback_window: int = 0
     ) -> ConcurrentCursor:
-        return self._cursor_factory.create(
+        cursor = self._cursor_factory.create(
             stream_state=deepcopy(cursor_state),
             runtime_lookback_window=timedelta(seconds=runtime_lookback_window),
         )
+        return cursor
 
     def should_be_synced(self, record: Record) -> bool:
         return self._get_cursor(record).should_be_synced(record)
@@ -523,7 +511,8 @@ class ConcurrentPerPartitionCursor(Cursor):
             raise ValueError(
                 "Invalid state as stream slices that are emitted should refer to an existing cursor"
             )
-        return self._cursor_per_partition[partition_key]
+        cursor = self._cursor_per_partition[partition_key]
+        return cursor
 
     def limit_reached(self) -> bool:
         return self._number_of_partitions > self.SWITCH_TO_GLOBAL_LIMIT

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -145,7 +145,7 @@ class JinjaInterpolation(Interpolation):
             # It can be returned as is
             return s
 
-    # @cache
+    @cache
     def _find_undeclared_variables(self, s: Optional[str]) -> Set[str]:
         """
         Find undeclared variables and cache them
@@ -153,7 +153,7 @@ class JinjaInterpolation(Interpolation):
         ast = _ENVIRONMENT.parse(s)  # type: ignore # parse is able to handle None
         return meta.find_undeclared_variables(ast)
 
-    # @cache
+    @cache
     def _compile(self, s: str) -> Template:
         """
         We must cache the Jinja Template ourselves because we're using `from_string` instead of a template loader

{airbyte_cdk-6.51.0.post8.dev15492684389.dist-info → airbyte_cdk-6.51.0.post9.dev15582628322.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.51.0.post8.dev15492684389
+Version: 6.51.0.post9.dev15582628322
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.51.0.post8.dev15492684389.dist-info → airbyte_cdk-6.51.0.post9.dev15582628322.dist-info}/RECORD

@@ -111,7 +111,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=vCpwX1PVRFP
 airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
-airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=AjS24owY5oNP4Z6GbG7VlKIvKKlaf6IiFrQqoDZm6Xg,22988
+airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=araWk039M89c6lQHEUltfM1VI_xGw9gZIDXRWWF6SkM,22591
 airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
 airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
 airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
@@ -125,7 +125,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py,sha256=h36
 airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha256=myVaNtFqxOAwrbp93rgd1dhkqyuvXvET9rsimQ89ktc,1873
 airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkHqGlfa87G6VYMtBAQWin7ECKpfMdrDcg0JO5_rhc,3212
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
-airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=y-m7Iu1EQGEPUkmtXYcSdJTucQj67tcXvV3QJyNTQWU,7186
+airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=UYSJ5gW7TkHALYnNvUnRP3RlyGwGuRMObF3BHuNzjJM,5320
 airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=AVb86oWIAxAOrE3KAyNr81bF5Dd2QjKdQ6E4Jl4yrQw,24719
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -419,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.51.0.post8.dev15492684389.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.51.0.post8.dev15492684389.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.51.0.post8.dev15492684389.dist-info/METADATA,sha256=QS6CSTvQH-DFLtv0_nul6yIIuO6hH3iB8UC9xMUNS4A,6364
-airbyte_cdk-6.51.0.post8.dev15492684389.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.51.0.post8.dev15492684389.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.51.0.post8.dev15492684389.dist-info/RECORD,,
+airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/METADATA,sha256=fLMVooGtQA4iha7ZEeM69ZueOJ05qTP_0cKuMdgB1Vo,6364
+airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/RECORD,,