airbyte-cdk 6.51.0.post7.dev15492779494__py3-none-any.whl → 6.51.0.post9.dev15582628322__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +15 -17
- airbyte_cdk/sources/declarative/interpolation/jinja.py +2 -2
- {airbyte_cdk-6.51.0.post7.dev15492779494.dist-info → airbyte_cdk-6.51.0.post9.dev15582628322.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.51.0.post7.dev15492779494.dist-info → airbyte_cdk-6.51.0.post9.dev15582628322.dist-info}/RECORD +8 -8
- {airbyte_cdk-6.51.0.post7.dev15492779494.dist-info → airbyte_cdk-6.51.0.post9.dev15582628322.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.51.0.post7.dev15492779494.dist-info → airbyte_cdk-6.51.0.post9.dev15582628322.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.51.0.post7.dev15492779494.dist-info → airbyte_cdk-6.51.0.post9.dev15582628322.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.51.0.post7.dev15492779494.dist-info → airbyte_cdk-6.51.0.post9.dev15582628322.dist-info}/entry_points.txt +0 -0
@@ -295,20 +295,16 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
295
295
|
):
|
296
296
|
self._partition_parent_state_map[partition_key] = (deepcopy(parent_state), seq)
|
297
297
|
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
)
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
)
|
309
|
-
finally:
|
310
|
-
del cursor
|
311
|
-
del partition
|
298
|
+
for cursor_slice, is_last_slice, _ in iterate_with_last_flag_and_state(
|
299
|
+
cursor.stream_slices(),
|
300
|
+
lambda: None,
|
301
|
+
):
|
302
|
+
self._semaphore_per_partition[partition_key].release()
|
303
|
+
if is_last_slice:
|
304
|
+
self._partitions_done_generating_stream_slices.add(partition_key)
|
305
|
+
yield StreamSlice(
|
306
|
+
partition=partition, cursor_slice=cursor_slice, extra_fields=partition.extra_fields
|
307
|
+
)
|
312
308
|
|
313
309
|
def _ensure_partition_limit(self) -> None:
|
314
310
|
"""
|
@@ -339,7 +335,7 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
339
335
|
oldest_partition = self._cursor_per_partition.pop(
|
340
336
|
partition_key
|
341
337
|
) # Remove the oldest partition
|
342
|
-
logger.
|
338
|
+
logger.debug(
|
343
339
|
f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._number_of_partitions - self.DEFAULT_MAX_PARTITIONS_NUMBER}."
|
344
340
|
)
|
345
341
|
break
|
@@ -496,10 +492,11 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
496
492
|
def _create_cursor(
|
497
493
|
self, cursor_state: Any, runtime_lookback_window: int = 0
|
498
494
|
) -> ConcurrentCursor:
|
499
|
-
|
495
|
+
cursor = self._cursor_factory.create(
|
500
496
|
stream_state=deepcopy(cursor_state),
|
501
497
|
runtime_lookback_window=timedelta(seconds=runtime_lookback_window),
|
502
498
|
)
|
499
|
+
return cursor
|
503
500
|
|
504
501
|
def should_be_synced(self, record: Record) -> bool:
|
505
502
|
return self._get_cursor(record).should_be_synced(record)
|
@@ -514,7 +511,8 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
514
511
|
raise ValueError(
|
515
512
|
"Invalid state as stream slices that are emitted should refer to an existing cursor"
|
516
513
|
)
|
517
|
-
|
514
|
+
cursor = self._cursor_per_partition[partition_key]
|
515
|
+
return cursor
|
518
516
|
|
519
517
|
def limit_reached(self) -> bool:
|
520
518
|
return self._number_of_partitions > self.SWITCH_TO_GLOBAL_LIMIT
|
@@ -145,7 +145,7 @@ class JinjaInterpolation(Interpolation):
|
|
145
145
|
# It can be returned as is
|
146
146
|
return s
|
147
147
|
|
148
|
-
|
148
|
+
@cache
|
149
149
|
def _find_undeclared_variables(self, s: Optional[str]) -> Set[str]:
|
150
150
|
"""
|
151
151
|
Find undeclared variables and cache them
|
@@ -153,7 +153,7 @@ class JinjaInterpolation(Interpolation):
|
|
153
153
|
ast = _ENVIRONMENT.parse(s) # type: ignore # parse is able to handle None
|
154
154
|
return meta.find_undeclared_variables(ast)
|
155
155
|
|
156
|
-
|
156
|
+
@cache
|
157
157
|
def _compile(self, s: str) -> Template:
|
158
158
|
"""
|
159
159
|
We must cache the Jinja Template ourselves because we're using `from_string` instead of a template loader
|
@@ -111,7 +111,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=vCpwX1PVRFP
|
|
111
111
|
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
|
112
112
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
113
113
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
114
|
-
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=
|
114
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=araWk039M89c6lQHEUltfM1VI_xGw9gZIDXRWWF6SkM,22591
|
115
115
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
|
116
116
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
117
117
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
|
@@ -125,7 +125,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py,sha256=h36
|
|
125
125
|
airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha256=myVaNtFqxOAwrbp93rgd1dhkqyuvXvET9rsimQ89ktc,1873
|
126
126
|
airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkHqGlfa87G6VYMtBAQWin7ECKpfMdrDcg0JO5_rhc,3212
|
127
127
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
|
128
|
-
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=
|
128
|
+
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
|
129
129
|
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=UYSJ5gW7TkHALYnNvUnRP3RlyGwGuRMObF3BHuNzjJM,5320
|
130
130
|
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=AVb86oWIAxAOrE3KAyNr81bF5Dd2QjKdQ6E4Jl4yrQw,24719
|
131
131
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -419,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
419
419
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
420
420
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
421
421
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
422
|
-
airbyte_cdk-6.51.0.
|
423
|
-
airbyte_cdk-6.51.0.
|
424
|
-
airbyte_cdk-6.51.0.
|
425
|
-
airbyte_cdk-6.51.0.
|
426
|
-
airbyte_cdk-6.51.0.
|
427
|
-
airbyte_cdk-6.51.0.
|
422
|
+
airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
423
|
+
airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
424
|
+
airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/METADATA,sha256=fLMVooGtQA4iha7ZEeM69ZueOJ05qTP_0cKuMdgB1Vo,6364
|
425
|
+
airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
426
|
+
airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
|
427
|
+
airbyte_cdk-6.51.0.post9.dev15582628322.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|