airbyte-cdk 6.26.1__py3-none-any.whl → 6.27.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +57 -32
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +26 -13
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +0 -3
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +28 -3
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +1 -1
- airbyte_cdk/sources/types.py +4 -2
- airbyte_cdk/utils/slice_hasher.py +8 -1
- {airbyte_cdk-6.26.1.dist-info → airbyte_cdk-6.27.0.dev0.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.26.1.dist-info → airbyte_cdk-6.27.0.dev0.dist-info}/RECORD +12 -12
- {airbyte_cdk-6.26.1.dist-info → airbyte_cdk-6.27.0.dev0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.26.1.dist-info → airbyte_cdk-6.27.0.dev0.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.26.1.dist-info → airbyte_cdk-6.27.0.dev0.dist-info}/entry_points.txt +0 -0
@@ -34,8 +34,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
34
34
|
from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
|
35
35
|
ModelToComponentFactory,
|
36
36
|
)
|
37
|
+
from airbyte_cdk.sources.declarative.partition_routers import AsyncJobPartitionRouter
|
37
38
|
from airbyte_cdk.sources.declarative.requesters import HttpRequester
|
38
|
-
from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
|
39
|
+
from airbyte_cdk.sources.declarative.retrievers import AsyncRetriever, Retriever, SimpleRetriever
|
39
40
|
from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
|
40
41
|
DeclarativePartitionFactory,
|
41
42
|
StreamSlicerPartitionGenerator,
|
@@ -48,7 +49,7 @@ from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStrea
|
|
48
49
|
from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
|
49
50
|
AlwaysAvailableAvailabilityStrategy,
|
50
51
|
)
|
51
|
-
from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
|
52
|
+
from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, FinalStateCursor
|
52
53
|
from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
|
53
54
|
from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
|
54
55
|
|
@@ -69,6 +70,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
69
70
|
component_factory: Optional[ModelToComponentFactory] = None,
|
70
71
|
**kwargs: Any,
|
71
72
|
) -> None:
|
73
|
+
# todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
|
74
|
+
# no longer needs to store the original incoming state. But maybe there's an edge case?
|
75
|
+
self._connector_state_manager = ConnectorStateManager(state=state) # type: ignore # state is always in the form of List[AirbyteStateMessage]. The ConnectorStateManager should use generics, but this can be done later
|
76
|
+
|
72
77
|
# To reduce the complexity of the concurrent framework, we are not enabling RFR with synthetic
|
73
78
|
# cursors. We do this by no longer automatically instantiating RFR cursors when converting
|
74
79
|
# the declarative models into runtime components. Concurrent sources will continue to checkpoint
|
@@ -76,6 +81,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
76
81
|
component_factory = component_factory or ModelToComponentFactory(
|
77
82
|
emit_connector_builder_messages=emit_connector_builder_messages,
|
78
83
|
disable_resumable_full_refresh=True,
|
84
|
+
connector_state_manager=self._connector_state_manager,
|
79
85
|
)
|
80
86
|
|
81
87
|
super().__init__(
|
@@ -86,10 +92,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
86
92
|
component_factory=component_factory,
|
87
93
|
)
|
88
94
|
|
89
|
-
# todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
|
90
|
-
# no longer needs to store the original incoming state. But maybe there's an edge case?
|
91
|
-
self._state = state
|
92
|
-
|
93
95
|
concurrency_level_from_manifest = self._source_config.get("concurrency_level")
|
94
96
|
if concurrency_level_from_manifest:
|
95
97
|
concurrency_level_component = self._constructor.create_component(
|
@@ -179,8 +181,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
179
181
|
concurrent_streams: List[AbstractStream] = []
|
180
182
|
synchronous_streams: List[Stream] = []
|
181
183
|
|
182
|
-
state_manager = ConnectorStateManager(state=self._state) # type: ignore # state is always in the form of List[AirbyteStateMessage]. The ConnectorStateManager should use generics, but this can be done later
|
183
|
-
|
184
184
|
# Combine streams and dynamic_streams. Note: both cannot be empty at the same time,
|
185
185
|
# and this is validated during the initialization of the source.
|
186
186
|
streams = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
|
@@ -220,31 +220,52 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
220
220
|
if self._is_datetime_incremental_without_partition_routing(
|
221
221
|
declarative_stream, incremental_sync_component_definition
|
222
222
|
):
|
223
|
-
stream_state =
|
223
|
+
stream_state = self._connector_state_manager.get_stream_state(
|
224
224
|
stream_name=declarative_stream.name, namespace=declarative_stream.namespace
|
225
225
|
)
|
226
226
|
|
227
|
-
cursor = self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
|
228
|
-
state_manager=state_manager,
|
229
|
-
model_type=DatetimeBasedCursorModel,
|
230
|
-
component_definition=incremental_sync_component_definition, # type: ignore # Not None because of the if condition above
|
231
|
-
stream_name=declarative_stream.name,
|
232
|
-
stream_namespace=declarative_stream.namespace,
|
233
|
-
config=config or {},
|
234
|
-
stream_state=stream_state,
|
235
|
-
)
|
236
|
-
|
237
227
|
retriever = self._get_retriever(declarative_stream, stream_state)
|
238
228
|
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
229
|
+
if isinstance(declarative_stream.retriever, AsyncRetriever) and isinstance(
|
230
|
+
declarative_stream.retriever.stream_slicer, AsyncJobPartitionRouter
|
231
|
+
):
|
232
|
+
cursor = declarative_stream.retriever.stream_slicer.stream_slicer
|
233
|
+
|
234
|
+
if not isinstance(cursor, ConcurrentCursor):
|
235
|
+
# This should never happen since we instantiate ConcurrentCursor in
|
236
|
+
# model_to_component_factory.py
|
237
|
+
raise ValueError(
|
238
|
+
f"Expected AsyncJobPartitionRouter stream_slicer to be of type ConcurrentCursor, but received{cursor.__class__}"
|
239
|
+
)
|
240
|
+
|
241
|
+
partition_generator = StreamSlicerPartitionGenerator(
|
242
|
+
partition_factory=DeclarativePartitionFactory(
|
243
|
+
declarative_stream.name,
|
244
|
+
declarative_stream.get_json_schema(),
|
245
|
+
retriever,
|
246
|
+
self.message_repository,
|
247
|
+
),
|
248
|
+
stream_slicer=declarative_stream.retriever.stream_slicer,
|
249
|
+
)
|
250
|
+
else:
|
251
|
+
cursor = (
|
252
|
+
self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
|
253
|
+
model_type=DatetimeBasedCursorModel,
|
254
|
+
component_definition=incremental_sync_component_definition, # type: ignore # Not None because of the if condition above
|
255
|
+
stream_name=declarative_stream.name,
|
256
|
+
stream_namespace=declarative_stream.namespace,
|
257
|
+
config=config or {},
|
258
|
+
)
|
259
|
+
)
|
260
|
+
partition_generator = StreamSlicerPartitionGenerator(
|
261
|
+
partition_factory=DeclarativePartitionFactory(
|
262
|
+
declarative_stream.name,
|
263
|
+
declarative_stream.get_json_schema(),
|
264
|
+
retriever,
|
265
|
+
self.message_repository,
|
266
|
+
),
|
267
|
+
stream_slicer=cursor,
|
268
|
+
)
|
248
269
|
|
249
270
|
concurrent_streams.append(
|
250
271
|
DefaultStream(
|
@@ -306,14 +327,14 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
306
327
|
declarative_stream.retriever.stream_slicer, PerPartitionWithGlobalCursor
|
307
328
|
)
|
308
329
|
):
|
309
|
-
stream_state =
|
330
|
+
stream_state = self._connector_state_manager.get_stream_state(
|
310
331
|
stream_name=declarative_stream.name, namespace=declarative_stream.namespace
|
311
332
|
)
|
312
333
|
partition_router = declarative_stream.retriever.stream_slicer._partition_router
|
313
334
|
|
314
335
|
perpartition_cursor = (
|
315
336
|
self._constructor.create_concurrent_cursor_from_perpartition_cursor(
|
316
|
-
state_manager=
|
337
|
+
state_manager=self._connector_state_manager,
|
317
338
|
model_type=DatetimeBasedCursorModel,
|
318
339
|
component_definition=incremental_sync_component_definition,
|
319
340
|
stream_name=declarative_stream.name,
|
@@ -369,7 +390,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
369
390
|
declarative_stream=declarative_stream
|
370
391
|
)
|
371
392
|
and hasattr(declarative_stream.retriever, "stream_slicer")
|
372
|
-
and
|
393
|
+
and (
|
394
|
+
isinstance(declarative_stream.retriever.stream_slicer, DatetimeBasedCursor)
|
395
|
+
or isinstance(declarative_stream.retriever.stream_slicer, AsyncJobPartitionRouter)
|
396
|
+
)
|
373
397
|
)
|
374
398
|
|
375
399
|
def _stream_supports_concurrent_partition_processing(
|
@@ -438,8 +462,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
438
462
|
return False
|
439
463
|
return True
|
440
464
|
|
465
|
+
@staticmethod
|
441
466
|
def _get_retriever(
|
442
|
-
|
467
|
+
declarative_stream: DeclarativeStream, stream_state: Mapping[str, Any]
|
443
468
|
) -> Retriever:
|
444
469
|
retriever = declarative_stream.retriever
|
445
470
|
|
@@ -95,6 +95,7 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
95
95
|
self._lookback_window: int = 0
|
96
96
|
self._parent_state: Optional[StreamState] = None
|
97
97
|
self._over_limit: int = 0
|
98
|
+
self._use_global_cursor: bool = False
|
98
99
|
self._partition_serializer = PerPartitionKeySerializer()
|
99
100
|
|
100
101
|
self._set_initial_state(stream_state)
|
@@ -105,16 +106,18 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
105
106
|
|
106
107
|
@property
|
107
108
|
def state(self) -> MutableMapping[str, Any]:
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
109
|
+
state: dict[str, Any] = {"use_global_cursor": self._use_global_cursor}
|
110
|
+
if not self._use_global_cursor:
|
111
|
+
states = []
|
112
|
+
for partition_tuple, cursor in self._cursor_per_partition.items():
|
113
|
+
if cursor.state:
|
114
|
+
states.append(
|
115
|
+
{
|
116
|
+
"partition": self._to_dict(partition_tuple),
|
117
|
+
"cursor": copy.deepcopy(cursor.state),
|
118
|
+
}
|
119
|
+
)
|
120
|
+
state[self._PERPARTITION_STATE_KEY] = states
|
118
121
|
|
119
122
|
if self._global_cursor:
|
120
123
|
state[self._GLOBAL_STATE_KEY] = self._global_cursor
|
@@ -147,7 +150,8 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
147
150
|
< cursor.state[self.cursor_field.cursor_field_key]
|
148
151
|
):
|
149
152
|
self._new_global_cursor = copy.deepcopy(cursor.state)
|
150
|
-
self.
|
153
|
+
if not self._use_global_cursor:
|
154
|
+
self._emit_state_message()
|
151
155
|
|
152
156
|
def ensure_at_least_one_state_emitted(self) -> None:
|
153
157
|
"""
|
@@ -225,14 +229,15 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
225
229
|
"""
|
226
230
|
with self._lock:
|
227
231
|
while len(self._cursor_per_partition) > self.DEFAULT_MAX_PARTITIONS_NUMBER - 1:
|
232
|
+
self._over_limit += 1
|
228
233
|
# Try removing finished partitions first
|
229
234
|
for partition_key in list(self._cursor_per_partition.keys()):
|
230
|
-
if partition_key in self._finished_partitions:
|
235
|
+
if partition_key in self._finished_partitions and self._semaphore_per_partition[partition_key]._value == 0:
|
231
236
|
oldest_partition = self._cursor_per_partition.pop(
|
232
237
|
partition_key
|
233
238
|
) # Remove the oldest partition
|
234
239
|
logger.warning(
|
235
|
-
f"The maximum number of partitions has been reached. Dropping the oldest partition: {oldest_partition}. Over limit: {self._over_limit}."
|
240
|
+
f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._over_limit}."
|
236
241
|
)
|
237
242
|
break
|
238
243
|
else:
|
@@ -297,6 +302,8 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
297
302
|
self._new_global_cursor = deepcopy(stream_state)
|
298
303
|
|
299
304
|
else:
|
305
|
+
self._use_global_cursor = stream_state.get("use_global_cursor", False)
|
306
|
+
|
300
307
|
self._lookback_window = int(stream_state.get("lookback_window", 0))
|
301
308
|
|
302
309
|
for state in stream_state.get(self._PERPARTITION_STATE_KEY, []):
|
@@ -320,6 +327,9 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
320
327
|
self._partition_router.set_initial_state(stream_state)
|
321
328
|
|
322
329
|
def observe(self, record: Record) -> None:
|
330
|
+
if not self._use_global_cursor and self.limit_reached():
|
331
|
+
self._use_global_cursor = True
|
332
|
+
|
323
333
|
if not record.associated_slice:
|
324
334
|
raise ValueError(
|
325
335
|
"Invalid state as stream slices that are emitted should refer to an existing cursor"
|
@@ -358,3 +368,6 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
358
368
|
)
|
359
369
|
cursor = self._cursor_per_partition[partition_key]
|
360
370
|
return cursor
|
371
|
+
|
372
|
+
def limit_reached(self) -> bool:
|
373
|
+
return self._over_limit > self.DEFAULT_MAX_PARTITIONS_NUMBER
|
@@ -26,9 +26,6 @@ from airbyte_cdk.models import (
|
|
26
26
|
from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
|
27
27
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
28
28
|
from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
|
29
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
30
|
-
CheckStream as CheckStreamModel,
|
31
|
-
)
|
32
29
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
33
30
|
DeclarativeStream as DeclarativeStreamModel,
|
34
31
|
)
|
@@ -507,6 +507,7 @@ class ModelToComponentFactory:
|
|
507
507
|
disable_cache: bool = False,
|
508
508
|
disable_resumable_full_refresh: bool = False,
|
509
509
|
message_repository: Optional[MessageRepository] = None,
|
510
|
+
connector_state_manager: Optional[ConnectorStateManager] = None,
|
510
511
|
):
|
511
512
|
self._init_mappings()
|
512
513
|
self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice
|
@@ -518,6 +519,7 @@ class ModelToComponentFactory:
|
|
518
519
|
self._message_repository = message_repository or InMemoryMessageRepository(
|
519
520
|
self._evaluate_log_level(emit_connector_builder_messages)
|
520
521
|
)
|
522
|
+
self._connector_state_manager = connector_state_manager or ConnectorStateManager()
|
521
523
|
|
522
524
|
def _init_mappings(self) -> None:
|
523
525
|
self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = {
|
@@ -927,17 +929,24 @@ class ModelToComponentFactory:
|
|
927
929
|
|
928
930
|
def create_concurrent_cursor_from_datetime_based_cursor(
|
929
931
|
self,
|
930
|
-
state_manager: ConnectorStateManager,
|
931
932
|
model_type: Type[BaseModel],
|
932
933
|
component_definition: ComponentDefinition,
|
933
934
|
stream_name: str,
|
934
935
|
stream_namespace: Optional[str],
|
935
936
|
config: Config,
|
936
|
-
stream_state: MutableMapping[str, Any],
|
937
937
|
message_repository: Optional[MessageRepository] = None,
|
938
938
|
runtime_lookback_window: Optional[datetime.timedelta] = None,
|
939
939
|
**kwargs: Any,
|
940
940
|
) -> ConcurrentCursor:
|
941
|
+
# Per-partition incremental streams can dynamically create child cursors which will pass their current
|
942
|
+
# state via the stream_state keyword argument. Incremental syncs without parent streams use the
|
943
|
+
# incoming state and connector_state_manager that is initialized when the component factory is created
|
944
|
+
stream_state = (
|
945
|
+
self._connector_state_manager.get_stream_state(stream_name, stream_namespace)
|
946
|
+
if "stream_state" not in kwargs
|
947
|
+
else kwargs["stream_state"]
|
948
|
+
)
|
949
|
+
|
941
950
|
component_type = component_definition.get("type")
|
942
951
|
if component_definition.get("type") != model_type.__name__:
|
943
952
|
raise ValueError(
|
@@ -1131,7 +1140,7 @@ class ModelToComponentFactory:
|
|
1131
1140
|
stream_namespace=stream_namespace,
|
1132
1141
|
stream_state=stream_state,
|
1133
1142
|
message_repository=message_repository or self._message_repository,
|
1134
|
-
connector_state_manager=
|
1143
|
+
connector_state_manager=self._connector_state_manager,
|
1135
1144
|
connector_state_converter=connector_state_converter,
|
1136
1145
|
cursor_field=cursor_field,
|
1137
1146
|
slice_boundary_fields=slice_boundary_fields,
|
@@ -1681,6 +1690,22 @@ class ModelToComponentFactory:
|
|
1681
1690
|
stream_cursor=cursor_component,
|
1682
1691
|
)
|
1683
1692
|
elif model.incremental_sync:
|
1693
|
+
if model.retriever.type == "AsyncRetriever":
|
1694
|
+
if model.incremental_sync.type != "DatetimeBasedCursor":
|
1695
|
+
# We are currently in a transition to the Concurrent CDK and AsyncRetriever can only work with the support or unordered slices (for example, when we trigger reports for January and February, the report in February can be completed first). Once we have support for custom concurrent cursor or have a new implementation available in the CDK, we can enable more cursors here.
|
1696
|
+
raise ValueError(
|
1697
|
+
"AsyncRetriever with cursor other than DatetimeBasedCursor is not supported yet"
|
1698
|
+
)
|
1699
|
+
if model.retriever.partition_router:
|
1700
|
+
# Note that this development is also done in parallel to the per partition development which once merged we could support here by calling `create_concurrent_cursor_from_perpartition_cursor`
|
1701
|
+
raise ValueError("Per partition state is not supported yet for AsyncRetriever")
|
1702
|
+
return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing
|
1703
|
+
model_type=DatetimeBasedCursorModel,
|
1704
|
+
component_definition=model.incremental_sync.__dict__,
|
1705
|
+
stream_name=model.name or "",
|
1706
|
+
stream_namespace=None,
|
1707
|
+
config=config or {},
|
1708
|
+
)
|
1684
1709
|
return (
|
1685
1710
|
self._create_component_from_model(model=model.incremental_sync, config=config)
|
1686
1711
|
if model.incremental_sync
|
@@ -75,7 +75,7 @@ class AsyncRetriever(Retriever):
|
|
75
75
|
"""
|
76
76
|
if not isinstance(stream_slice, StreamSlice) or "partition" not in stream_slice.partition:
|
77
77
|
raise AirbyteTracedException(
|
78
|
-
message="Invalid arguments to
|
78
|
+
message="Invalid arguments to AsyncRetriever.read_records: stream_slice is not optional. Please contact Airbyte Support",
|
79
79
|
failure_type=FailureType.system_error,
|
80
80
|
)
|
81
81
|
return stream_slice["partition"] # type: ignore # stream_slice["partition"] has been added as an AsyncPartition as part of stream_slices
|
airbyte_cdk/sources/types.py
CHANGED
@@ -6,7 +6,7 @@ from __future__ import annotations
|
|
6
6
|
|
7
7
|
from typing import Any, ItemsView, Iterator, KeysView, List, Mapping, Optional, ValuesView
|
8
8
|
|
9
|
-
import
|
9
|
+
from airbyte_cdk.utils.slice_hasher import SliceHasher
|
10
10
|
|
11
11
|
# A FieldPointer designates a path to a field inside a mapping. For example, retrieving ["k1", "k1.2"] in the object {"k1" :{"k1.2":
|
12
12
|
# "hello"}] returns "hello"
|
@@ -151,7 +151,9 @@ class StreamSlice(Mapping[str, Any]):
|
|
151
151
|
return self._stream_slice
|
152
152
|
|
153
153
|
def __hash__(self) -> int:
|
154
|
-
return hash(
|
154
|
+
return SliceHasher.hash(
|
155
|
+
stream_slice=self._stream_slice
|
156
|
+
) # no need to provide stream_name here as this is used for slicing the cursor
|
155
157
|
|
156
158
|
def __bool__(self) -> bool:
|
157
159
|
return bool(self._stream_slice) or bool(self._extra_fields)
|
@@ -16,7 +16,14 @@ class SliceHasher:
|
|
16
16
|
_ENCODING: Final = "utf-8"
|
17
17
|
|
18
18
|
@classmethod
|
19
|
-
def hash(
|
19
|
+
def hash(
|
20
|
+
cls,
|
21
|
+
stream_name: str = "<stream name not provided>",
|
22
|
+
stream_slice: Optional[Mapping[str, Any]] = None,
|
23
|
+
) -> int:
|
24
|
+
"""
|
25
|
+
Note that streams partition with the same slicing value but with different names might collapse if stream name is not provided
|
26
|
+
"""
|
20
27
|
if stream_slice:
|
21
28
|
try:
|
22
29
|
s = json.dumps(stream_slice, sort_keys=True, cls=SliceEncoder)
|
@@ -63,7 +63,7 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQr
|
|
63
63
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
64
64
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
65
65
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
66
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
66
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=uhSXYhXD56E-Wr9qRqzgLmIW1sFHj_2sIXk0enOtpms,27305
|
67
67
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
|
68
68
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
|
69
69
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
@@ -88,7 +88,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=tjNwcURmlyD
|
|
88
88
|
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=LhqGDfX06_dDYLKsIVnwQ_nAWCln-v8PV7Wgt_QVeTI,6533
|
89
89
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
90
90
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
91
|
-
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=
|
91
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=TnlwShqF_3JxY8j6mUHeUO3rm1PtkC1N8nL7189Dohs,16588
|
92
92
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=_UzUnSIUsDbRgbFTXgSyZEFb4ws-KdhdQPWO8mFbV7U,22028
|
93
93
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
94
94
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=9HO-QbL9akvjq2NP7l498RwLA4iQZlBMQW1tZbt34I8,15943
|
@@ -104,7 +104,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=LYEZ
|
|
104
104
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=-V5UddGm69UKEB6o_O1EIES9kfY8FV_X4Ji8w1yOuSA,981
|
105
105
|
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=BtsY_jtT4MihFqeQgc05HXj3Ndt-e2ESQgGwbg3Sdxc,6430
|
106
106
|
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=Y5AWYxbJTUtJ_Jm7DV9qrZDiymFR9LST7fBt4piT2-U,4585
|
107
|
-
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=
|
107
|
+
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=_luZAtjKx7nF_rNM0_QTcbtR_JqOEKRKcOZB_aEdCv0,16716
|
108
108
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
109
109
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
110
110
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
@@ -115,7 +115,7 @@ airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=958MMX6_Z
|
|
115
115
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
116
116
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
|
117
117
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
118
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
118
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=zo1FbRgLL-wA7O0rpyJ7IaRkQ6OhyP6Lq4KZGqU4ijw,125265
|
119
119
|
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
|
120
120
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=n82J15S8bjeMZ5uROu--P3hnbQoxkY5v7RPHYx7g7ro,2929
|
121
121
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
@@ -165,7 +165,7 @@ airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb
|
|
165
165
|
airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
|
166
166
|
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
|
167
167
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
|
168
|
-
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=
|
168
|
+
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=aIxZUhDgq1-fuWKkSrElHyuQaxLsKhbbZM6bWO_VFTQ,3694
|
169
169
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
170
170
|
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=kgnhVQxRlFqJs2-rDu2-QH-p-GzQU3nKmSp6_aq8u0s,24550
|
171
171
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
|
@@ -300,7 +300,7 @@ airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y
|
|
300
300
|
airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=Zse4ve1MvPJBx-7CDtTBTmPuT6b9koGLMGpmd5188Y8,16698
|
301
301
|
airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
|
302
302
|
airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
303
|
-
airbyte_cdk/sources/types.py,sha256=
|
303
|
+
airbyte_cdk/sources/types.py,sha256=aFPGI4t2K1vHz2oFSUIYUyDN7kw-vcYq4D7aD2zgfAU,5128
|
304
304
|
airbyte_cdk/sources/utils/__init__.py,sha256=TTN6VUxVy6Is8BhYQZR5pxJGQh8yH4duXh4O1TiMiEY,118
|
305
305
|
airbyte_cdk/sources/utils/casing.py,sha256=QC-gV1O4e8DR4-bhdXieUPKm_JamzslVyfABLYYRSXA,256
|
306
306
|
airbyte_cdk/sources/utils/record_helper.py,sha256=jeB0mucudzna7Zvj-pCBbwFrbLJ36SlAWZTh5O4Fb9Y,2168
|
@@ -346,12 +346,12 @@ airbyte_cdk/utils/message_utils.py,sha256=OTzbkwN7AdMDA3iKYq1LKwfPFxpyEDfdgEF9BE
|
|
346
346
|
airbyte_cdk/utils/oneof_option_config.py,sha256=N8EmWdYdwt0FM7fuShh6H8nj_r4KEL9tb2DJJtwsPow,1180
|
347
347
|
airbyte_cdk/utils/print_buffer.py,sha256=PhMOi0C4Z91kWKrSvCQXcp8qRh1uCimpIdvrg6voZIA,2810
|
348
348
|
airbyte_cdk/utils/schema_inferrer.py,sha256=_jLzL9PzE4gfR44OSavkIqZNFM9t08c3LuRrkR7PZbk,9861
|
349
|
-
airbyte_cdk/utils/slice_hasher.py,sha256
|
349
|
+
airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7GrKcg,1264
|
350
350
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
351
351
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
352
352
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
353
|
-
airbyte_cdk-6.
|
354
|
-
airbyte_cdk-6.
|
355
|
-
airbyte_cdk-6.
|
356
|
-
airbyte_cdk-6.
|
357
|
-
airbyte_cdk-6.
|
353
|
+
airbyte_cdk-6.27.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
354
|
+
airbyte_cdk-6.27.0.dev0.dist-info/METADATA,sha256=uVSlbLjqWio7gWy4AiOJ82IrCtjS1e8Fd2ZCcCj-zvI,6001
|
355
|
+
airbyte_cdk-6.27.0.dev0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
|
356
|
+
airbyte_cdk-6.27.0.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
357
|
+
airbyte_cdk-6.27.0.dev0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|