airbyte-cdk 6.37.0.dev1__py3-none-any.whl → 6.37.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. airbyte_cdk/connector_builder/models.py +16 -14
  2. airbyte_cdk/connector_builder/test_reader/helpers.py +120 -22
  3. airbyte_cdk/connector_builder/test_reader/message_grouper.py +16 -3
  4. airbyte_cdk/connector_builder/test_reader/types.py +9 -1
  5. airbyte_cdk/sources/declarative/auth/token_provider.py +1 -0
  6. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +43 -7
  7. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +7 -1
  8. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +67 -46
  9. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +13 -2
  10. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -0
  11. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +83 -17
  12. airbyte_cdk/sources/declarative/interpolation/macros.py +2 -0
  13. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +30 -45
  14. airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +18 -4
  15. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +171 -70
  16. airbyte_cdk/sources/declarative/partition_routers/__init__.py +0 -4
  17. airbyte_cdk/sources/declarative/requesters/README.md +5 -5
  18. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +60 -17
  19. airbyte_cdk/sources/declarative/requesters/http_requester.py +7 -1
  20. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +10 -3
  21. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +2 -2
  22. airbyte_cdk/sources/http_logger.py +3 -0
  23. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +2 -1
  24. airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
  25. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -0
  26. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/METADATA +2 -2
  27. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/RECORD +31 -31
  28. airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py +0 -136
  29. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/LICENSE.txt +0 -0
  30. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/LICENSE_SHORT +0 -0
  31. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/WHEEL +0 -0
  32. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.1.dist-info}/entry_points.txt +0 -0
@@ -4,7 +4,7 @@
4
4
 
5
5
  from abc import ABC, abstractmethod
6
6
  from enum import Enum
7
- from typing import TYPE_CHECKING, Any, List, MutableMapping, Optional, Tuple
7
+ from typing import TYPE_CHECKING, Any, Callable, List, MutableMapping, Optional, Tuple
8
8
 
9
9
  if TYPE_CHECKING:
10
10
  from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
12
12
 
13
13
  class ConcurrencyCompatibleStateType(Enum):
14
14
  date_range = "date-range"
15
+ integer = "integer"
15
16
 
16
17
 
17
18
  class AbstractStreamStateConverter(ABC):
@@ -0,0 +1,92 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from typing import Any, Callable, MutableMapping, Optional, Tuple
6
+
7
+ from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
8
+ from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import (
9
+ AbstractStreamStateConverter,
10
+ ConcurrencyCompatibleStateType,
11
+ )
12
+
13
+
14
+ class IncrementingCountStreamStateConverter(AbstractStreamStateConverter):
15
+ def _from_state_message(self, value: Any) -> Any:
16
+ return value
17
+
18
+ def _to_state_message(self, value: Any) -> Any:
19
+ return value
20
+
21
+ @classmethod
22
+ def get_end_provider(cls) -> Callable[[], float]:
23
+ return lambda: float("inf")
24
+
25
+ def convert_from_sequential_state(
26
+ self,
27
+ cursor_field: "CursorField", # to deprecate as it is only needed for sequential state
28
+ stream_state: MutableMapping[str, Any],
29
+ start: Optional[Any],
30
+ ) -> Tuple[Any, MutableMapping[str, Any]]:
31
+ """
32
+ Convert the state message to the format required by the ConcurrentCursor.
33
+
34
+ e.g.
35
+ {
36
+ "state_type": ConcurrencyCompatibleStateType.date_range.value,
37
+ "metadata": { … },
38
+ "slices": [
39
+ {"start": "10", "end": "2021-01-18T21:18:20.000+00:00"},
40
+ ]
41
+ }
42
+ """
43
+ sync_start = self._get_sync_start(cursor_field, stream_state, start)
44
+ if self.is_state_message_compatible(stream_state):
45
+ return sync_start, stream_state
46
+
47
+ # Create a slice to represent the records synced during prior syncs.
48
+ # The start and end are the same to avoid confusion as to whether the records for this slice
49
+ # were actually synced
50
+ slices = [
51
+ {
52
+ self.START_KEY: start if start is not None else sync_start,
53
+ self.END_KEY: sync_start, # this may not be relevant anymore
54
+ self.MOST_RECENT_RECORD_KEY: sync_start,
55
+ }
56
+ ]
57
+
58
+ return sync_start, {
59
+ "state_type": ConcurrencyCompatibleStateType.integer.value,
60
+ "slices": slices,
61
+ "legacy": stream_state,
62
+ }
63
+
64
+ def parse_value(self, value: int) -> int:
65
+ return value
66
+
67
+ @property
68
+ def zero_value(self) -> int:
69
+ return 0
70
+
71
+ def increment(self, value: int) -> int:
72
+ return value + 1
73
+
74
+ def output_format(self, value: int) -> int:
75
+ return value
76
+
77
+ def _get_sync_start(
78
+ self,
79
+ cursor_field: CursorField,
80
+ stream_state: MutableMapping[str, Any],
81
+ start: Optional[int],
82
+ ) -> int:
83
+ sync_start = start if start is not None else self.zero_value
84
+ prev_sync_low_water_mark: Optional[int] = (
85
+ stream_state[cursor_field.cursor_field_key]
86
+ if cursor_field.cursor_field_key in stream_state
87
+ else None
88
+ )
89
+ if prev_sync_low_water_mark and prev_sync_low_water_mark >= sync_start:
90
+ return prev_sync_low_water_mark
91
+ else:
92
+ return sync_start
@@ -396,6 +396,7 @@ class AbstractOauth2Authenticator(AuthBase):
396
396
  "Obtains access token",
397
397
  self._NO_STREAM_NAME,
398
398
  is_auxiliary=True,
399
+ type="AUTH",
399
400
  ),
400
401
  )
401
402
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: airbyte-cdk
3
- Version: 6.37.0.dev1
3
+ Version: 6.37.1
4
4
  Summary: A framework for writing Airbyte Connectors.
5
5
  Home-page: https://airbyte.com
6
6
  License: MIT
@@ -22,8 +22,8 @@ Provides-Extra: sql
22
22
  Provides-Extra: vector-db-based
23
23
  Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
24
24
  Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
25
- Requires-Dist: Unidecode (>=1.3,<2.0)
26
25
  Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
26
+ Requires-Dist: anyascii (>=0.3.2,<0.4.0)
27
27
  Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
28
28
  Requires-Dist: backoff
29
29
  Requires-Dist: cachetools
@@ -9,12 +9,12 @@ airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz
9
9
  airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
10
10
  airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=BntqkP63RBPvGCtB3CrLLtYplfSlBR42kwXyyk4YGas,4268
11
11
  airbyte_cdk/connector_builder/main.py,sha256=ubAPE0Oo5gjZOa-KMtLLJQkc8_inUpFR3sIb2DEh2No,3722
12
- airbyte_cdk/connector_builder/models.py,sha256=uCHpOdJx2PyZtIqk-mt9eSVuFMQoEqrW-9sjCz0Z-AQ,1500
12
+ airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
13
13
  airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
14
- airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=niu_EhzwVXnvtzj2Rf_unrxqLRC4Twbe-t17HyNoRJY,23662
15
- airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=L0xKpWQhfRecKblI3uYO9twPTPJYJzlOhK2P8zHWXRU,6487
14
+ airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
15
+ airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3WCfO14WNvh_q7OsjGgDt0q1FTu8eW-w,6918
16
16
  airbyte_cdk/connector_builder/test_reader/reader.py,sha256=GurMB4ITO_PntvhIHSJkXbhynLilI4DObY5A2axavXo,20667
17
- airbyte_cdk/connector_builder/test_reader/types.py,sha256=jP28aOlCS8Q6V7jMksfJKsuAJ-m2dNYiXaUjYvb0DBA,2404
17
+ airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
18
18
  airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
19
19
  airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
20
20
  airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
@@ -60,22 +60,22 @@ airbyte_cdk/sources/declarative/auth/jwt.py,sha256=SICqNsN2Cn_EgKadIgWuZpQxuMHyz
60
60
  airbyte_cdk/sources/declarative/auth/oauth.py,sha256=SUfib1oSzlyRRnOSg8Bui73mfyrcyr9OssdchbKdu4s,14162
61
61
  airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
62
62
  airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
63
- airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=9CuSsmOoHkvlc4k-oZ3Jx5luAgfTMm1I_5HOZxw7wMU,3075
63
+ airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
64
64
  airbyte_cdk/sources/declarative/checks/__init__.py,sha256=nsVV5Bo0E_tBNd8A4Xdsdb-75PpcLo5RQu2RQ_Gv-ME,806
65
65
  airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=HUktywjI8pqOeED08UGqponUSwxs2TOAECTowlWlrRE,2138
66
66
  airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQrilWCfJmncBzXCZ18ptRNip3XA,2139
67
67
  airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
68
68
  airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
69
69
  airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
70
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=MRnIdGeKPk1dO9-4eWRHa7mI6Ay_7szGo9H1RJSZDb8,24453
70
+ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=rAp-sgld4n8Tmybz-51m7VcYXqKwzKDpCJVr1elmkRc,26824
71
71
  airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
72
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
72
+ airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=0qs4hhmh_XOy2B4MHCn2qVMM79C6MizIBqnvpZj1aSE,2923
73
73
  airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
74
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=d8jbuP86iBhEhtbtw5dGg9b8U93KkDACgAO4EYW2lzg,146344
74
+ airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=Vsem7b0YL_kaLeTwY_kX-EqHzuBDjik0lBN7e3srXT4,147126
75
75
  airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
76
76
  airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
77
77
  airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
78
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=kg5_kNlhXj8p9GZFztlbG16pk1XcMtP9ezqulq1VNg4,4545
78
+ airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=DJbWaaJ5LHCBpyWz-4bEw8rqtJYqabEYZtxnfRtWFE0,4946
79
79
  airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
80
80
  airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=BdWpXXPhEGf_zknggJmhojLosmxuw51RBVTS0jvdCPc,2080
81
81
  airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
@@ -89,10 +89,10 @@ airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8V
89
89
  airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
90
90
  airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
91
91
  airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=HCqx7IyENM_aRF4it2zJN26_vDu6WeP8XgCxQWHUvcY,6934
92
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=LhqGDfX06_dDYLKsIVnwQ_nAWCln-v8PV7Wgt_QVeTI,6533
92
+ airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=L6hQV7bdwEp8y-TBMeQY-xmrNyRggL14lKXdWnzYFfA,6622
93
93
  airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
94
94
  airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
95
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=Pg2phEFT9T8AzUjK6hVhn0rgR3yY6JPF-Dfv0g1m5dQ,19191
95
+ airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=MT5JbdEbnPzk3VWZGGvThe4opoX5dHhSXFrnTRYC6dg,22210
96
96
  airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
97
97
  airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
98
98
  airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
@@ -107,28 +107,27 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha
107
107
  airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkHqGlfa87G6VYMtBAQWin7ECKpfMdrDcg0JO5_rhc,3212
108
108
  airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
109
109
  airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
110
- airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=ajgVJT9sZBUFZUDLjzyPWupCNXt1HvzbCq-4yv9aY3c,5042
110
+ airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=uuXBZUWDWM-sPcUKjNSPRN657QhNQCx_hnhTuJj2zOA,5129
111
111
  airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=TN6GCgLXaWDONTaJwQ3A5ELqC-sxwKz-UYSraJYB-dI,17078
112
112
  airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
113
113
  airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
114
114
  airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
115
115
  airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
116
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=Uk8sJRXs763ym_UCBzW2YGDzLPwTvzMQt_J0a3M-5zA,103309
116
+ airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=Kd8HvvXqvGWZBey99eQzbK5u2k1ItnRAi2h7C7UNwBQ,103225
117
117
  airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
118
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=958MMX6_ZOJUlDDdNr9Krosgi2bCKGx2Z765M2Woz18,5505
118
+ airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=jDw_TttD3_hpfevXOH-0Ws0eRuqt6wvED0BqosGPRjI,5938
119
119
  airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
120
120
  airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
121
121
  airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
122
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=nY_s3xidDFJTuxlxbLVfjlZIxMhaXtwAg7tnxdkgdXg,135237
123
- airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
122
+ airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Mx0KJGbqIZeUWduKy-UvpVH-DRm0pzXDcz203r69oNY,140619
123
+ airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
124
124
  airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
125
125
  airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
126
- airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py,sha256=PFoY931wC1i7Elphrd7LCFUPYKOTPEovLXC-mvkQow0,5531
127
126
  airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=tmGGpMoOBmaMfhVZq53AEWxoHm2lmNVi6hA2_IVEnAA,4882
128
127
  airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
129
128
  airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=SKzKjSyfccq4dxGIh-J6ejrgkCHzaiTIazmbmeQiRD4,1942
130
129
  airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py,sha256=LlWj-Ofs-xfjlqmDzH8OYpyblP2Pb8bPDdR9g1UZyt0,17693
131
- airbyte_cdk/sources/declarative/requesters/README.md,sha256=eL1I4iLkxaw7hJi9S9d18_XcRl-R8lUSjqBVJJzvXmg,2656
130
+ airbyte_cdk/sources/declarative/requesters/README.md,sha256=DQll2qsIzzTiiP35kJp16ONpr7cFeUQNgPfhl5krB24,2675
132
131
  airbyte_cdk/sources/declarative/requesters/__init__.py,sha256=d7a3OoHbqaJDyyPli3nqqJ2yAW_SLX6XDaBAKOwvpxw,364
133
132
  airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py,sha256=SkEDcJxlT1683rNx93K9whoS0OyUukkuOfToGtgpF58,776
134
133
  airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py,sha256=1WZdpFmWL6W_Dko0qjflTaKIWeqt8jHT-D6HcujIp3s,884
@@ -143,8 +142,8 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.
143
142
  airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py,sha256=q0YkeYUUWO6iErUy0vjqiOkhg8_9d5YcCmtlpXAJJ9E,1314
144
143
  airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
145
144
  airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
146
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=3GtOefPH08evlSUxaILkiKLTHbIspFY4qd5B3ZqNE60,10063
147
- airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=pR2uR5b9eGyvYIOYwus3mz3OaqRu1ozwja_ys1SE7hc,14952
145
+ airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=4wpP0ZNTMLugi-Rc1OFdFaxWfRZSl45nzhHqMFCE8SQ,11924
146
+ airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=Sie8IyntFu66UoJASwpWV0WrRDBr9lpHWSOws7vZfM0,15228
148
147
  airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
149
148
  airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=ZW4lwWNAzb4zL0jKc-HjowP5-y0Zg9xi0YlK6tkx_XY,12057
150
149
  airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=j6j9QRPaTbKQ2N661RFVKthhkWiodEp6ut0tKeEd0Ng,2019
@@ -170,7 +169,7 @@ airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb
170
169
  airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
171
170
  airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
172
171
  airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
173
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=2oQn_vo7uJKp4pdMnsF5CG5Iwc9rkPeEOLoAm_9bcus,3222
172
+ airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=dwYZ70eg9DKHEqZydHhMFPkEILbNcXu7E-djOCikNgI,3530
174
173
  airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
175
174
  airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=bOAKQLgMv1Vca-ozMPRVAg1V5nkyUoPwqC02lKpnLiM,24575
176
175
  airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
@@ -190,7 +189,7 @@ airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=1
190
189
  airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=yT3owG6rMKaRX-LJ_T-jSTnh1B5NoAHyH4YZN9yOvE8,1758
191
190
  airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py,sha256=vbIn6ump-Ut6g20yMub7PFoPBhOKVtrHSAUdcOUdLfw,1999
192
191
  airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py,sha256=RTs5KX4V3hM7A6QN1WlGF21YccTIyNH6qQI9IMb__hw,670
193
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=43zwe6_F5ba5C4eY0RgXxPz7ndPKZfXGChHepFn-2lk,2263
192
+ airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=_3ldEbsA7tQK-zzeU_cG86D1_1SY3wAo1vHE0zXrOck,2265
194
193
  airbyte_cdk/sources/declarative/transformations/remove_fields.py,sha256=EwUP0SZ2p4GRJ6Q8CUzlz9dcUeEidEFDlI2IBye2tlc,2745
195
194
  airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=4sXtx9cNY2EHUPq-xHvDs8GQEBUy3Eo6TkRLKHPXx68,1161
196
195
  airbyte_cdk/sources/declarative/types.py,sha256=yqx0xlZv_76tkC7fqJKefmvl4GJJ8mXbeddwVV8XRJU,778
@@ -251,7 +250,7 @@ airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=DwgNU-jDp5vZ_W
251
250
  airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=i0Jn0zuAPomLa4pHSu9TQ3gAN5xXhNzPTYVwUDiDEyE,3523
252
251
  airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
253
252
  airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
254
- airbyte_cdk/sources/http_logger.py,sha256=l_1fk5YwdonZ1wvAsTwjj6d36fj2WrVraIAMj5jTQdM,1575
253
+ airbyte_cdk/sources/http_logger.py,sha256=H93kPAujHhPmXNX0JSFG3D-SL6yEFA5PtKot9Hu3TYA,1690
255
254
  airbyte_cdk/sources/message/__init__.py,sha256=y98fzHsQBwXwp2zEa4K5mxGFqjnx9lDn9O0pTk-VS4U,395
256
255
  airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv14_zIB0SH8AIc,4763
257
256
  airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
@@ -285,8 +284,9 @@ airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=
285
284
  airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py,sha256=nbdkkHoN0NFeSs7YUFfzY1Lg5Jrt8fWY_ln3YrhY-Ko,544
286
285
  airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=frPVvHtY7vLxpGEbMQzNvF1Y52ZVyct9f1DDhGoRjwY,1166
287
286
  airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
288
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=CXHUMOhndu-LOKgsnNTItv5s5qrKpmJDeHOzlH1nBy8,6819
287
+ airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=JCRanZBAC8a0pPuzHCHiJ1irHgFkDd83l2K5jA1xRkU,6853
289
288
  airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=x8MLm1pTMfLNHvMF3P1ixYkYt_xjpbaIwnvhY_ofdBo,8076
289
+ airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py,sha256=bC6L82nsErXcFSPlxcdp4SneJ7qFuqCelP3-8svEh5E,3054
290
290
  airbyte_cdk/sources/streams/core.py,sha256=jiYW6w8cjNjzXMd8U8Gt-02fYYU7b0ciXSSSnGvFRak,32219
291
291
  airbyte_cdk/sources/streams/http/__init__.py,sha256=AGiEZ5B1Joi9ZnFpkJLT7F3QLpCAaBgAeVWy-1znmZw,311
292
292
  airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=sovoGFThZr-doMN9vJvTuJBrvkwQVIO0qTQO64pGZPY,2428
@@ -304,7 +304,7 @@ airbyte_cdk/sources/streams/http/http.py,sha256=0uariNq8OFnlX7iqOHwBhecxA-Hfd5hS
304
304
  airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
305
305
  airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
306
306
  airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
307
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=cM5CM1mnbTEMiY6gKHblGXr9KTS5VEziGoc-TXC302k,18791
307
+ airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=P9U8vtcrZ3m0InSG2W0H4gTYTxjQxkIe6mhF9xvO8Ug,18824
308
308
  airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
309
309
  airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d-3KgHO3NGxIiFdfASjHOtsd6g_LWPYOAs,20311
310
310
  airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
@@ -361,9 +361,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
361
361
  airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
362
362
  airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
363
363
  airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
364
- airbyte_cdk-6.37.0.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
365
- airbyte_cdk-6.37.0.dev1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
366
- airbyte_cdk-6.37.0.dev1.dist-info/METADATA,sha256=9grGzb0aDmQhE2MU2nXEKIYJFWcdpJpMTrSW5kTML3M,6015
367
- airbyte_cdk-6.37.0.dev1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
368
- airbyte_cdk-6.37.0.dev1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
369
- airbyte_cdk-6.37.0.dev1.dist-info/RECORD,,
364
+ airbyte_cdk-6.37.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
365
+ airbyte_cdk-6.37.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
366
+ airbyte_cdk-6.37.1.dist-info/METADATA,sha256=yxQpCYYozV1B9r8Ir2NI9I4H7LixbVvsJ-SIZziflbE,6013
367
+ airbyte_cdk-6.37.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
368
+ airbyte_cdk-6.37.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
369
+ airbyte_cdk-6.37.1.dist-info/RECORD,,
@@ -1,136 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from dataclasses import dataclass
6
- from typing import Any, Iterable, Mapping, Optional
7
-
8
- from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
9
- from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
10
-
11
-
12
- @dataclass
13
- class GroupingPartitionRouter(PartitionRouter):
14
- """
15
- A partition router that groups partitions from an underlying partition router into batches of a specified size.
16
- This is useful for APIs that support filtering by multiple partition keys in a single request.
17
-
18
- Attributes:
19
- group_size (int): The number of partitions to include in each group.
20
- underlying_partition_router (PartitionRouter): The partition router whose output will be grouped.
21
- deduplicate (bool): If True, ensures unique partitions within each group by removing duplicates based on the partition key.
22
- config (Config): The connector configuration.
23
- parameters (Mapping[str, Any]): Additional parameters for interpolation and configuration.
24
- """
25
-
26
- group_size: int
27
- underlying_partition_router: PartitionRouter
28
- config: Config
29
- deduplicate: bool = True
30
-
31
- def stream_slices(self) -> Iterable[StreamSlice]:
32
- """
33
- Lazily groups partitions from the underlying partition router into batches of size `group_size`.
34
-
35
- This method processes partitions one at a time from the underlying router, maintaining a batch buffer.
36
- When the buffer reaches `group_size` or the underlying router is exhausted, it yields a grouped slice.
37
- If deduplication is enabled, it tracks seen partition keys to ensure uniqueness within the current batch.
38
-
39
- Yields:
40
- Iterable[StreamSlice]: An iterable of StreamSlice objects, where each slice contains a batch of partition values.
41
- """
42
- batch = []
43
- seen_keys = set()
44
-
45
- # Iterate over partitions lazily from the underlying router
46
- for partition in self.underlying_partition_router.stream_slices():
47
- # Extract the partition key (assuming single key-value pair, e.g., {"board_ids": value})
48
- key = next(iter(partition.partition.values()), None)
49
-
50
- # Skip duplicates if deduplication is enabled
51
- if self.deduplicate and key in seen_keys:
52
- continue
53
-
54
- # Add partition to the batch
55
- batch.append(partition)
56
- if self.deduplicate:
57
- seen_keys.add(key)
58
-
59
- # Yield the batch when it reaches the group_size
60
- if len(batch) == self.group_size:
61
- yield self._create_grouped_slice(batch)
62
- batch = [] # Reset the batch
63
-
64
- # Yield any remaining partitions if the batch isn't empty
65
- if batch:
66
- yield self._create_grouped_slice(batch)
67
-
68
- def _create_grouped_slice(self, batch: list[StreamSlice]) -> StreamSlice:
69
- """
70
- Creates a grouped StreamSlice from a batch of partitions, aggregating extra fields into a dictionary with list values.
71
-
72
- Args:
73
- batch (list[StreamSlice]): A list of StreamSlice objects to group.
74
-
75
- Returns:
76
- StreamSlice: A single StreamSlice with combined partition and extra field values.
77
- """
78
- # Combine partition values into a single dict with lists
79
- grouped_partition = {
80
- key: [p.partition.get(key) for p in batch] for key in batch[0].partition.keys()
81
- }
82
-
83
- # Aggregate extra fields into a dict with list values
84
- extra_fields_dict = (
85
- {
86
- key: [p.extra_fields.get(key) for p in batch]
87
- for key in set().union(*(p.extra_fields.keys() for p in batch if p.extra_fields))
88
- }
89
- if any(p.extra_fields for p in batch)
90
- else {}
91
- )
92
- return StreamSlice(
93
- partition=grouped_partition,
94
- cursor_slice={}, # Cursor is managed by the underlying router or incremental sync
95
- extra_fields=extra_fields_dict,
96
- )
97
-
98
- def get_request_params(
99
- self,
100
- stream_state: Optional[StreamState] = None,
101
- stream_slice: Optional[StreamSlice] = None,
102
- next_page_token: Optional[Mapping[str, Any]] = None,
103
- ) -> Mapping[str, Any]:
104
- return {}
105
-
106
- def get_request_headers(
107
- self,
108
- stream_state: Optional[StreamState] = None,
109
- stream_slice: Optional[StreamSlice] = None,
110
- next_page_token: Optional[Mapping[str, Any]] = None,
111
- ) -> Mapping[str, Any]:
112
- return {}
113
-
114
- def get_request_body_data(
115
- self,
116
- stream_state: Optional[StreamState] = None,
117
- stream_slice: Optional[StreamSlice] = None,
118
- next_page_token: Optional[Mapping[str, Any]] = None,
119
- ) -> Mapping[str, Any]:
120
- return {}
121
-
122
- def get_request_body_json(
123
- self,
124
- stream_state: Optional[StreamState] = None,
125
- stream_slice: Optional[StreamSlice] = None,
126
- next_page_token: Optional[Mapping[str, Any]] = None,
127
- ) -> Mapping[str, Any]:
128
- return {}
129
-
130
- def set_initial_state(self, stream_state: StreamState) -> None:
131
- """Delegate state initialization to the underlying partition router."""
132
- self.underlying_partition_router.set_initial_state(stream_state)
133
-
134
- def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
135
- """Delegate state retrieval to the underlying partition router."""
136
- return self.underlying_partition_router.get_stream_state()