airbyte-cdk 6.37.0.dev1__py3-none-any.whl → 6.37.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/models.py +16 -14
- airbyte_cdk/connector_builder/test_reader/helpers.py +120 -22
- airbyte_cdk/connector_builder/test_reader/message_grouper.py +16 -3
- airbyte_cdk/connector_builder/test_reader/types.py +9 -1
- airbyte_cdk/sources/declarative/auth/token_provider.py +1 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +43 -7
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +7 -1
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +67 -46
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +13 -2
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +83 -17
- airbyte_cdk/sources/declarative/interpolation/macros.py +2 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +30 -45
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +18 -4
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +171 -70
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +0 -4
- airbyte_cdk/sources/declarative/requesters/README.md +5 -5
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +60 -17
- airbyte_cdk/sources/declarative/requesters/http_requester.py +7 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +10 -3
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +2 -2
- airbyte_cdk/sources/http_logger.py +3 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +2 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/METADATA +2 -2
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/RECORD +31 -31
- airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py +0 -136
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/entry_points.txt +0 -0
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
from abc import ABC, abstractmethod
|
6
6
|
from enum import Enum
|
7
|
-
from typing import TYPE_CHECKING, Any, List, MutableMapping, Optional, Tuple
|
7
|
+
from typing import TYPE_CHECKING, Any, Callable, List, MutableMapping, Optional, Tuple
|
8
8
|
|
9
9
|
if TYPE_CHECKING:
|
10
10
|
from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
|
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
|
|
12
12
|
|
13
13
|
class ConcurrencyCompatibleStateType(Enum):
|
14
14
|
date_range = "date-range"
|
15
|
+
integer = "integer"
|
15
16
|
|
16
17
|
|
17
18
|
class AbstractStreamStateConverter(ABC):
|
airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from typing import Any, Callable, MutableMapping, Optional, Tuple
|
6
|
+
|
7
|
+
from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
|
8
|
+
from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import (
|
9
|
+
AbstractStreamStateConverter,
|
10
|
+
ConcurrencyCompatibleStateType,
|
11
|
+
)
|
12
|
+
|
13
|
+
|
14
|
+
class IncrementingCountStreamStateConverter(AbstractStreamStateConverter):
|
15
|
+
def _from_state_message(self, value: Any) -> Any:
|
16
|
+
return value
|
17
|
+
|
18
|
+
def _to_state_message(self, value: Any) -> Any:
|
19
|
+
return value
|
20
|
+
|
21
|
+
@classmethod
|
22
|
+
def get_end_provider(cls) -> Callable[[], float]:
|
23
|
+
return lambda: float("inf")
|
24
|
+
|
25
|
+
def convert_from_sequential_state(
|
26
|
+
self,
|
27
|
+
cursor_field: "CursorField", # to deprecate as it is only needed for sequential state
|
28
|
+
stream_state: MutableMapping[str, Any],
|
29
|
+
start: Optional[Any],
|
30
|
+
) -> Tuple[Any, MutableMapping[str, Any]]:
|
31
|
+
"""
|
32
|
+
Convert the state message to the format required by the ConcurrentCursor.
|
33
|
+
|
34
|
+
e.g.
|
35
|
+
{
|
36
|
+
"state_type": ConcurrencyCompatibleStateType.date_range.value,
|
37
|
+
"metadata": { … },
|
38
|
+
"slices": [
|
39
|
+
{"start": "10", "end": "2021-01-18T21:18:20.000+00:00"},
|
40
|
+
]
|
41
|
+
}
|
42
|
+
"""
|
43
|
+
sync_start = self._get_sync_start(cursor_field, stream_state, start)
|
44
|
+
if self.is_state_message_compatible(stream_state):
|
45
|
+
return sync_start, stream_state
|
46
|
+
|
47
|
+
# Create a slice to represent the records synced during prior syncs.
|
48
|
+
# The start and end are the same to avoid confusion as to whether the records for this slice
|
49
|
+
# were actually synced
|
50
|
+
slices = [
|
51
|
+
{
|
52
|
+
self.START_KEY: start if start is not None else sync_start,
|
53
|
+
self.END_KEY: sync_start, # this may not be relevant anymore
|
54
|
+
self.MOST_RECENT_RECORD_KEY: sync_start,
|
55
|
+
}
|
56
|
+
]
|
57
|
+
|
58
|
+
return sync_start, {
|
59
|
+
"state_type": ConcurrencyCompatibleStateType.integer.value,
|
60
|
+
"slices": slices,
|
61
|
+
"legacy": stream_state,
|
62
|
+
}
|
63
|
+
|
64
|
+
def parse_value(self, value: int) -> int:
|
65
|
+
return value
|
66
|
+
|
67
|
+
@property
|
68
|
+
def zero_value(self) -> int:
|
69
|
+
return 0
|
70
|
+
|
71
|
+
def increment(self, value: int) -> int:
|
72
|
+
return value + 1
|
73
|
+
|
74
|
+
def output_format(self, value: int) -> int:
|
75
|
+
return value
|
76
|
+
|
77
|
+
def _get_sync_start(
|
78
|
+
self,
|
79
|
+
cursor_field: CursorField,
|
80
|
+
stream_state: MutableMapping[str, Any],
|
81
|
+
start: Optional[int],
|
82
|
+
) -> int:
|
83
|
+
sync_start = start if start is not None else self.zero_value
|
84
|
+
prev_sync_low_water_mark: Optional[int] = (
|
85
|
+
stream_state[cursor_field.cursor_field_key]
|
86
|
+
if cursor_field.cursor_field_key in stream_state
|
87
|
+
else None
|
88
|
+
)
|
89
|
+
if prev_sync_low_water_mark and prev_sync_low_water_mark >= sync_start:
|
90
|
+
return prev_sync_low_water_mark
|
91
|
+
else:
|
92
|
+
return sync_start
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: airbyte-cdk
|
3
|
-
Version: 6.37.
|
3
|
+
Version: 6.37.1
|
4
4
|
Summary: A framework for writing Airbyte Connectors.
|
5
5
|
Home-page: https://airbyte.com
|
6
6
|
License: MIT
|
@@ -22,8 +22,8 @@ Provides-Extra: sql
|
|
22
22
|
Provides-Extra: vector-db-based
|
23
23
|
Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
|
24
24
|
Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
|
25
|
-
Requires-Dist: Unidecode (>=1.3,<2.0)
|
26
25
|
Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
|
26
|
+
Requires-Dist: anyascii (>=0.3.2,<0.4.0)
|
27
27
|
Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
|
28
28
|
Requires-Dist: backoff
|
29
29
|
Requires-Dist: cachetools
|
@@ -9,12 +9,12 @@ airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz
|
|
9
9
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
10
10
|
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=BntqkP63RBPvGCtB3CrLLtYplfSlBR42kwXyyk4YGas,4268
|
11
11
|
airbyte_cdk/connector_builder/main.py,sha256=ubAPE0Oo5gjZOa-KMtLLJQkc8_inUpFR3sIb2DEh2No,3722
|
12
|
-
airbyte_cdk/connector_builder/models.py,sha256=
|
12
|
+
airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
|
13
13
|
airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
|
14
|
-
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=
|
15
|
-
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=
|
14
|
+
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
|
15
|
+
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3WCfO14WNvh_q7OsjGgDt0q1FTu8eW-w,6918
|
16
16
|
airbyte_cdk/connector_builder/test_reader/reader.py,sha256=GurMB4ITO_PntvhIHSJkXbhynLilI4DObY5A2axavXo,20667
|
17
|
-
airbyte_cdk/connector_builder/test_reader/types.py,sha256=
|
17
|
+
airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
|
18
18
|
airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
|
19
19
|
airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
|
20
20
|
airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
|
@@ -60,22 +60,22 @@ airbyte_cdk/sources/declarative/auth/jwt.py,sha256=SICqNsN2Cn_EgKadIgWuZpQxuMHyz
|
|
60
60
|
airbyte_cdk/sources/declarative/auth/oauth.py,sha256=SUfib1oSzlyRRnOSg8Bui73mfyrcyr9OssdchbKdu4s,14162
|
61
61
|
airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
|
62
62
|
airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
|
63
|
-
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=
|
63
|
+
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
|
64
64
|
airbyte_cdk/sources/declarative/checks/__init__.py,sha256=nsVV5Bo0E_tBNd8A4Xdsdb-75PpcLo5RQu2RQ_Gv-ME,806
|
65
65
|
airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=HUktywjI8pqOeED08UGqponUSwxs2TOAECTowlWlrRE,2138
|
66
66
|
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQrilWCfJmncBzXCZ18ptRNip3XA,2139
|
67
67
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
68
68
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
69
69
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
70
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
70
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=rAp-sgld4n8Tmybz-51m7VcYXqKwzKDpCJVr1elmkRc,26824
|
71
71
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
|
72
|
-
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=
|
72
|
+
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=0qs4hhmh_XOy2B4MHCn2qVMM79C6MizIBqnvpZj1aSE,2923
|
73
73
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
74
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
74
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=Vsem7b0YL_kaLeTwY_kX-EqHzuBDjik0lBN7e3srXT4,147126
|
75
75
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
|
76
76
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
|
77
77
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
78
|
-
airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=
|
78
|
+
airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=DJbWaaJ5LHCBpyWz-4bEw8rqtJYqabEYZtxnfRtWFE0,4946
|
79
79
|
airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
|
80
80
|
airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=BdWpXXPhEGf_zknggJmhojLosmxuw51RBVTS0jvdCPc,2080
|
81
81
|
airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
|
@@ -89,10 +89,10 @@ airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8V
|
|
89
89
|
airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
|
90
90
|
airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
|
91
91
|
airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=HCqx7IyENM_aRF4it2zJN26_vDu6WeP8XgCxQWHUvcY,6934
|
92
|
-
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=
|
92
|
+
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=L6hQV7bdwEp8y-TBMeQY-xmrNyRggL14lKXdWnzYFfA,6622
|
93
93
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
94
94
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
95
|
-
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=
|
95
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=MT5JbdEbnPzk3VWZGGvThe4opoX5dHhSXFrnTRYC6dg,22210
|
96
96
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
|
97
97
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
98
98
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
|
@@ -107,28 +107,27 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha
|
|
107
107
|
airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkHqGlfa87G6VYMtBAQWin7ECKpfMdrDcg0JO5_rhc,3212
|
108
108
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
|
109
109
|
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
|
110
|
-
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=
|
110
|
+
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=uuXBZUWDWM-sPcUKjNSPRN657QhNQCx_hnhTuJj2zOA,5129
|
111
111
|
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=TN6GCgLXaWDONTaJwQ3A5ELqC-sxwKz-UYSraJYB-dI,17078
|
112
112
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
113
113
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
114
114
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
115
115
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
116
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
116
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=Kd8HvvXqvGWZBey99eQzbK5u2k1ItnRAi2h7C7UNwBQ,103225
|
117
117
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
118
|
-
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=
|
118
|
+
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=jDw_TttD3_hpfevXOH-0Ws0eRuqt6wvED0BqosGPRjI,5938
|
119
119
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
120
120
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
|
121
121
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
122
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
123
|
-
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=
|
122
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Mx0KJGbqIZeUWduKy-UvpVH-DRm0pzXDcz203r69oNY,140619
|
123
|
+
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
|
124
124
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
125
125
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
126
|
-
airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py,sha256=PFoY931wC1i7Elphrd7LCFUPYKOTPEovLXC-mvkQow0,5531
|
127
126
|
airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=tmGGpMoOBmaMfhVZq53AEWxoHm2lmNVi6hA2_IVEnAA,4882
|
128
127
|
airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
|
129
128
|
airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=SKzKjSyfccq4dxGIh-J6ejrgkCHzaiTIazmbmeQiRD4,1942
|
130
129
|
airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py,sha256=LlWj-Ofs-xfjlqmDzH8OYpyblP2Pb8bPDdR9g1UZyt0,17693
|
131
|
-
airbyte_cdk/sources/declarative/requesters/README.md,sha256=
|
130
|
+
airbyte_cdk/sources/declarative/requesters/README.md,sha256=DQll2qsIzzTiiP35kJp16ONpr7cFeUQNgPfhl5krB24,2675
|
132
131
|
airbyte_cdk/sources/declarative/requesters/__init__.py,sha256=d7a3OoHbqaJDyyPli3nqqJ2yAW_SLX6XDaBAKOwvpxw,364
|
133
132
|
airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py,sha256=SkEDcJxlT1683rNx93K9whoS0OyUukkuOfToGtgpF58,776
|
134
133
|
airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py,sha256=1WZdpFmWL6W_Dko0qjflTaKIWeqt8jHT-D6HcujIp3s,884
|
@@ -143,8 +142,8 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.
|
|
143
142
|
airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py,sha256=q0YkeYUUWO6iErUy0vjqiOkhg8_9d5YcCmtlpXAJJ9E,1314
|
144
143
|
airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
|
145
144
|
airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
|
146
|
-
airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=
|
147
|
-
airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=
|
145
|
+
airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=4wpP0ZNTMLugi-Rc1OFdFaxWfRZSl45nzhHqMFCE8SQ,11924
|
146
|
+
airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=Sie8IyntFu66UoJASwpWV0WrRDBr9lpHWSOws7vZfM0,15228
|
148
147
|
airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
|
149
148
|
airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=ZW4lwWNAzb4zL0jKc-HjowP5-y0Zg9xi0YlK6tkx_XY,12057
|
150
149
|
airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=j6j9QRPaTbKQ2N661RFVKthhkWiodEp6ut0tKeEd0Ng,2019
|
@@ -170,7 +169,7 @@ airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb
|
|
170
169
|
airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
|
171
170
|
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
|
172
171
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
|
173
|
-
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=
|
172
|
+
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=dwYZ70eg9DKHEqZydHhMFPkEILbNcXu7E-djOCikNgI,3530
|
174
173
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
175
174
|
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=bOAKQLgMv1Vca-ozMPRVAg1V5nkyUoPwqC02lKpnLiM,24575
|
176
175
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
|
@@ -190,7 +189,7 @@ airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=1
|
|
190
189
|
airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=yT3owG6rMKaRX-LJ_T-jSTnh1B5NoAHyH4YZN9yOvE8,1758
|
191
190
|
airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py,sha256=vbIn6ump-Ut6g20yMub7PFoPBhOKVtrHSAUdcOUdLfw,1999
|
192
191
|
airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py,sha256=RTs5KX4V3hM7A6QN1WlGF21YccTIyNH6qQI9IMb__hw,670
|
193
|
-
airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=
|
192
|
+
airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=_3ldEbsA7tQK-zzeU_cG86D1_1SY3wAo1vHE0zXrOck,2265
|
194
193
|
airbyte_cdk/sources/declarative/transformations/remove_fields.py,sha256=EwUP0SZ2p4GRJ6Q8CUzlz9dcUeEidEFDlI2IBye2tlc,2745
|
195
194
|
airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=4sXtx9cNY2EHUPq-xHvDs8GQEBUy3Eo6TkRLKHPXx68,1161
|
196
195
|
airbyte_cdk/sources/declarative/types.py,sha256=yqx0xlZv_76tkC7fqJKefmvl4GJJ8mXbeddwVV8XRJU,778
|
@@ -251,7 +250,7 @@ airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=DwgNU-jDp5vZ_W
|
|
251
250
|
airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=i0Jn0zuAPomLa4pHSu9TQ3gAN5xXhNzPTYVwUDiDEyE,3523
|
252
251
|
airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
|
253
252
|
airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
|
254
|
-
airbyte_cdk/sources/http_logger.py,sha256=
|
253
|
+
airbyte_cdk/sources/http_logger.py,sha256=H93kPAujHhPmXNX0JSFG3D-SL6yEFA5PtKot9Hu3TYA,1690
|
255
254
|
airbyte_cdk/sources/message/__init__.py,sha256=y98fzHsQBwXwp2zEa4K5mxGFqjnx9lDn9O0pTk-VS4U,395
|
256
255
|
airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv14_zIB0SH8AIc,4763
|
257
256
|
airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
|
@@ -285,8 +284,9 @@ airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=
|
|
285
284
|
airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py,sha256=nbdkkHoN0NFeSs7YUFfzY1Lg5Jrt8fWY_ln3YrhY-Ko,544
|
286
285
|
airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=frPVvHtY7vLxpGEbMQzNvF1Y52ZVyct9f1DDhGoRjwY,1166
|
287
286
|
airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
288
|
-
airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=
|
287
|
+
airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=JCRanZBAC8a0pPuzHCHiJ1irHgFkDd83l2K5jA1xRkU,6853
|
289
288
|
airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=x8MLm1pTMfLNHvMF3P1ixYkYt_xjpbaIwnvhY_ofdBo,8076
|
289
|
+
airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py,sha256=bC6L82nsErXcFSPlxcdp4SneJ7qFuqCelP3-8svEh5E,3054
|
290
290
|
airbyte_cdk/sources/streams/core.py,sha256=jiYW6w8cjNjzXMd8U8Gt-02fYYU7b0ciXSSSnGvFRak,32219
|
291
291
|
airbyte_cdk/sources/streams/http/__init__.py,sha256=AGiEZ5B1Joi9ZnFpkJLT7F3QLpCAaBgAeVWy-1znmZw,311
|
292
292
|
airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=sovoGFThZr-doMN9vJvTuJBrvkwQVIO0qTQO64pGZPY,2428
|
@@ -304,7 +304,7 @@ airbyte_cdk/sources/streams/http/http.py,sha256=0uariNq8OFnlX7iqOHwBhecxA-Hfd5hS
|
|
304
304
|
airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
|
305
305
|
airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
|
306
306
|
airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
|
307
|
-
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=
|
307
|
+
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=P9U8vtcrZ3m0InSG2W0H4gTYTxjQxkIe6mhF9xvO8Ug,18824
|
308
308
|
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
|
309
309
|
airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d-3KgHO3NGxIiFdfASjHOtsd6g_LWPYOAs,20311
|
310
310
|
airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
|
@@ -361,9 +361,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
361
361
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
362
362
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
363
363
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
364
|
-
airbyte_cdk-6.37.
|
365
|
-
airbyte_cdk-6.37.
|
366
|
-
airbyte_cdk-6.37.
|
367
|
-
airbyte_cdk-6.37.
|
368
|
-
airbyte_cdk-6.37.
|
369
|
-
airbyte_cdk-6.37.
|
364
|
+
airbyte_cdk-6.37.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
365
|
+
airbyte_cdk-6.37.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
366
|
+
airbyte_cdk-6.37.1.dist-info/METADATA,sha256=yxQpCYYozV1B9r8Ir2NI9I4H7LixbVvsJ-SIZziflbE,6013
|
367
|
+
airbyte_cdk-6.37.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
368
|
+
airbyte_cdk-6.37.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
369
|
+
airbyte_cdk-6.37.1.dist-info/RECORD,,
|
@@ -1,136 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from dataclasses import dataclass
|
6
|
-
from typing import Any, Iterable, Mapping, Optional
|
7
|
-
|
8
|
-
from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
|
9
|
-
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
10
|
-
|
11
|
-
|
12
|
-
@dataclass
|
13
|
-
class GroupingPartitionRouter(PartitionRouter):
|
14
|
-
"""
|
15
|
-
A partition router that groups partitions from an underlying partition router into batches of a specified size.
|
16
|
-
This is useful for APIs that support filtering by multiple partition keys in a single request.
|
17
|
-
|
18
|
-
Attributes:
|
19
|
-
group_size (int): The number of partitions to include in each group.
|
20
|
-
underlying_partition_router (PartitionRouter): The partition router whose output will be grouped.
|
21
|
-
deduplicate (bool): If True, ensures unique partitions within each group by removing duplicates based on the partition key.
|
22
|
-
config (Config): The connector configuration.
|
23
|
-
parameters (Mapping[str, Any]): Additional parameters for interpolation and configuration.
|
24
|
-
"""
|
25
|
-
|
26
|
-
group_size: int
|
27
|
-
underlying_partition_router: PartitionRouter
|
28
|
-
config: Config
|
29
|
-
deduplicate: bool = True
|
30
|
-
|
31
|
-
def stream_slices(self) -> Iterable[StreamSlice]:
|
32
|
-
"""
|
33
|
-
Lazily groups partitions from the underlying partition router into batches of size `group_size`.
|
34
|
-
|
35
|
-
This method processes partitions one at a time from the underlying router, maintaining a batch buffer.
|
36
|
-
When the buffer reaches `group_size` or the underlying router is exhausted, it yields a grouped slice.
|
37
|
-
If deduplication is enabled, it tracks seen partition keys to ensure uniqueness within the current batch.
|
38
|
-
|
39
|
-
Yields:
|
40
|
-
Iterable[StreamSlice]: An iterable of StreamSlice objects, where each slice contains a batch of partition values.
|
41
|
-
"""
|
42
|
-
batch = []
|
43
|
-
seen_keys = set()
|
44
|
-
|
45
|
-
# Iterate over partitions lazily from the underlying router
|
46
|
-
for partition in self.underlying_partition_router.stream_slices():
|
47
|
-
# Extract the partition key (assuming single key-value pair, e.g., {"board_ids": value})
|
48
|
-
key = next(iter(partition.partition.values()), None)
|
49
|
-
|
50
|
-
# Skip duplicates if deduplication is enabled
|
51
|
-
if self.deduplicate and key in seen_keys:
|
52
|
-
continue
|
53
|
-
|
54
|
-
# Add partition to the batch
|
55
|
-
batch.append(partition)
|
56
|
-
if self.deduplicate:
|
57
|
-
seen_keys.add(key)
|
58
|
-
|
59
|
-
# Yield the batch when it reaches the group_size
|
60
|
-
if len(batch) == self.group_size:
|
61
|
-
yield self._create_grouped_slice(batch)
|
62
|
-
batch = [] # Reset the batch
|
63
|
-
|
64
|
-
# Yield any remaining partitions if the batch isn't empty
|
65
|
-
if batch:
|
66
|
-
yield self._create_grouped_slice(batch)
|
67
|
-
|
68
|
-
def _create_grouped_slice(self, batch: list[StreamSlice]) -> StreamSlice:
|
69
|
-
"""
|
70
|
-
Creates a grouped StreamSlice from a batch of partitions, aggregating extra fields into a dictionary with list values.
|
71
|
-
|
72
|
-
Args:
|
73
|
-
batch (list[StreamSlice]): A list of StreamSlice objects to group.
|
74
|
-
|
75
|
-
Returns:
|
76
|
-
StreamSlice: A single StreamSlice with combined partition and extra field values.
|
77
|
-
"""
|
78
|
-
# Combine partition values into a single dict with lists
|
79
|
-
grouped_partition = {
|
80
|
-
key: [p.partition.get(key) for p in batch] for key in batch[0].partition.keys()
|
81
|
-
}
|
82
|
-
|
83
|
-
# Aggregate extra fields into a dict with list values
|
84
|
-
extra_fields_dict = (
|
85
|
-
{
|
86
|
-
key: [p.extra_fields.get(key) for p in batch]
|
87
|
-
for key in set().union(*(p.extra_fields.keys() for p in batch if p.extra_fields))
|
88
|
-
}
|
89
|
-
if any(p.extra_fields for p in batch)
|
90
|
-
else {}
|
91
|
-
)
|
92
|
-
return StreamSlice(
|
93
|
-
partition=grouped_partition,
|
94
|
-
cursor_slice={}, # Cursor is managed by the underlying router or incremental sync
|
95
|
-
extra_fields=extra_fields_dict,
|
96
|
-
)
|
97
|
-
|
98
|
-
def get_request_params(
|
99
|
-
self,
|
100
|
-
stream_state: Optional[StreamState] = None,
|
101
|
-
stream_slice: Optional[StreamSlice] = None,
|
102
|
-
next_page_token: Optional[Mapping[str, Any]] = None,
|
103
|
-
) -> Mapping[str, Any]:
|
104
|
-
return {}
|
105
|
-
|
106
|
-
def get_request_headers(
|
107
|
-
self,
|
108
|
-
stream_state: Optional[StreamState] = None,
|
109
|
-
stream_slice: Optional[StreamSlice] = None,
|
110
|
-
next_page_token: Optional[Mapping[str, Any]] = None,
|
111
|
-
) -> Mapping[str, Any]:
|
112
|
-
return {}
|
113
|
-
|
114
|
-
def get_request_body_data(
|
115
|
-
self,
|
116
|
-
stream_state: Optional[StreamState] = None,
|
117
|
-
stream_slice: Optional[StreamSlice] = None,
|
118
|
-
next_page_token: Optional[Mapping[str, Any]] = None,
|
119
|
-
) -> Mapping[str, Any]:
|
120
|
-
return {}
|
121
|
-
|
122
|
-
def get_request_body_json(
|
123
|
-
self,
|
124
|
-
stream_state: Optional[StreamState] = None,
|
125
|
-
stream_slice: Optional[StreamSlice] = None,
|
126
|
-
next_page_token: Optional[Mapping[str, Any]] = None,
|
127
|
-
) -> Mapping[str, Any]:
|
128
|
-
return {}
|
129
|
-
|
130
|
-
def set_initial_state(self, stream_state: StreamState) -> None:
|
131
|
-
"""Delegate state initialization to the underlying partition router."""
|
132
|
-
self.underlying_partition_router.set_initial_state(stream_state)
|
133
|
-
|
134
|
-
def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
|
135
|
-
"""Delegate state retrieval to the underlying partition router."""
|
136
|
-
return self.underlying_partition_router.get_stream_state()
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|