airbyte-cdk 6.33.0__py3-none-any.whl → 6.33.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/auth/token.py +8 -3
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +2 -13
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +212 -15
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +6 -7
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +4 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +169 -10
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +171 -34
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +2 -4
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +9 -3
- airbyte_cdk/sources/declarative/requesters/http_requester.py +4 -5
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +5 -6
- airbyte_cdk/sources/declarative/requesters/request_option.py +4 -83
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +6 -7
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +1 -4
- airbyte_cdk/sources/streams/call_rate.py +84 -71
- airbyte_cdk/utils/mapping_helpers.py +27 -86
- {airbyte_cdk-6.33.0.dist-info → airbyte_cdk-6.33.0.dev0.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.33.0.dist-info → airbyte_cdk-6.33.0.dev0.dist-info}/RECORD +22 -22
- {airbyte_cdk-6.33.0.dist-info → airbyte_cdk-6.33.0.dev0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.33.0.dist-info → airbyte_cdk-6.33.0.dev0.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.33.0.dist-info → airbyte_cdk-6.33.0.dev0.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.33.0.dist-info → airbyte_cdk-6.33.0.dev0.dist-info}/entry_points.txt +0 -0
@@ -112,6 +112,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
112
112
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
113
113
|
AddFields as AddFieldsModel,
|
114
114
|
)
|
115
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
116
|
+
APIBudget as APIBudgetModel,
|
117
|
+
)
|
115
118
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
116
119
|
ApiKeyAuthenticator as ApiKeyAuthenticatorModel,
|
117
120
|
)
|
@@ -226,6 +229,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
226
229
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
227
230
|
ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
|
228
231
|
)
|
232
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
233
|
+
FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel,
|
234
|
+
)
|
229
235
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
230
236
|
FlattenFields as FlattenFieldsModel,
|
231
237
|
)
|
@@ -235,12 +241,18 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
235
241
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
236
242
|
GzipParser as GzipParserModel,
|
237
243
|
)
|
244
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
245
|
+
HTTPAPIBudget as HTTPAPIBudgetModel,
|
246
|
+
)
|
238
247
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
239
248
|
HttpComponentsResolver as HttpComponentsResolverModel,
|
240
249
|
)
|
241
250
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
242
251
|
HttpRequester as HttpRequesterModel,
|
243
252
|
)
|
253
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
254
|
+
HttpRequestMatcher as HttpRequestMatcherModel,
|
255
|
+
)
|
244
256
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
245
257
|
HttpResponseFilter as HttpResponseFilterModel,
|
246
258
|
)
|
@@ -295,6 +307,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
295
307
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
296
308
|
MinMaxDatetime as MinMaxDatetimeModel,
|
297
309
|
)
|
310
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
311
|
+
MovingWindowCallRatePolicy as MovingWindowCallRatePolicyModel,
|
312
|
+
)
|
298
313
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
299
314
|
NoAuth as NoAuthModel,
|
300
315
|
)
|
@@ -313,6 +328,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
313
328
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
314
329
|
ParentStreamConfig as ParentStreamConfigModel,
|
315
330
|
)
|
331
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
332
|
+
Rate as RateModel,
|
333
|
+
)
|
316
334
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
317
335
|
RecordFilter as RecordFilterModel,
|
318
336
|
)
|
@@ -356,6 +374,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
356
374
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
357
375
|
TypesMap as TypesMapModel,
|
358
376
|
)
|
377
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
378
|
+
UnlimitedCallRatePolicy as UnlimitedCallRatePolicyModel,
|
379
|
+
)
|
359
380
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
|
360
381
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
361
382
|
WaitTimeFromHeader as WaitTimeFromHeaderModel,
|
@@ -469,6 +490,15 @@ from airbyte_cdk.sources.message import (
|
|
469
490
|
MessageRepository,
|
470
491
|
NoopMessageRepository,
|
471
492
|
)
|
493
|
+
from airbyte_cdk.sources.streams.call_rate import (
|
494
|
+
APIBudget,
|
495
|
+
FixedWindowCallRatePolicy,
|
496
|
+
HttpAPIBudget,
|
497
|
+
HttpRequestMatcher,
|
498
|
+
MovingWindowCallRatePolicy,
|
499
|
+
Rate,
|
500
|
+
UnlimitedCallRatePolicy,
|
501
|
+
)
|
472
502
|
from airbyte_cdk.sources.streams.concurrent.clamping import (
|
473
503
|
ClampingEndProvider,
|
474
504
|
ClampingStrategy,
|
@@ -520,6 +550,7 @@ class ModelToComponentFactory:
|
|
520
550
|
self._evaluate_log_level(emit_connector_builder_messages)
|
521
551
|
)
|
522
552
|
self._connector_state_manager = connector_state_manager or ConnectorStateManager()
|
553
|
+
self._api_budget: Optional[Union[APIBudget, HttpAPIBudget]] = None
|
523
554
|
|
524
555
|
def _init_mappings(self) -> None:
|
525
556
|
self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = {
|
@@ -607,6 +638,13 @@ class ModelToComponentFactory:
|
|
607
638
|
StreamConfigModel: self.create_stream_config,
|
608
639
|
ComponentMappingDefinitionModel: self.create_components_mapping_definition,
|
609
640
|
ZipfileDecoderModel: self.create_zipfile_decoder,
|
641
|
+
APIBudgetModel: self.create_api_budget,
|
642
|
+
HTTPAPIBudgetModel: self.create_http_api_budget,
|
643
|
+
FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
|
644
|
+
MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
|
645
|
+
UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
|
646
|
+
RateModel: self.create_rate,
|
647
|
+
HttpRequestMatcherModel: self.create_http_request_matcher,
|
610
648
|
}
|
611
649
|
|
612
650
|
# Needed for the case where we need to perform a second parse on the fields of a custom component
|
@@ -733,8 +771,8 @@ class ModelToComponentFactory:
|
|
733
771
|
}
|
734
772
|
return names_to_types[value_type]
|
735
773
|
|
774
|
+
@staticmethod
|
736
775
|
def create_api_key_authenticator(
|
737
|
-
self,
|
738
776
|
model: ApiKeyAuthenticatorModel,
|
739
777
|
config: Config,
|
740
778
|
token_provider: Optional[TokenProvider] = None,
|
@@ -756,8 +794,10 @@ class ModelToComponentFactory:
|
|
756
794
|
)
|
757
795
|
|
758
796
|
request_option = (
|
759
|
-
|
760
|
-
model.inject_into,
|
797
|
+
RequestOption(
|
798
|
+
inject_into=RequestOptionType(model.inject_into.inject_into.value),
|
799
|
+
field_name=model.inject_into.field_name,
|
800
|
+
parameters=model.parameters or {},
|
761
801
|
)
|
762
802
|
if model.inject_into
|
763
803
|
else RequestOption(
|
@@ -766,7 +806,6 @@ class ModelToComponentFactory:
|
|
766
806
|
parameters=model.parameters or {},
|
767
807
|
)
|
768
808
|
)
|
769
|
-
|
770
809
|
return ApiKeyAuthenticator(
|
771
810
|
token_provider=(
|
772
811
|
token_provider
|
@@ -848,7 +887,7 @@ class ModelToComponentFactory:
|
|
848
887
|
token_provider=token_provider,
|
849
888
|
)
|
850
889
|
else:
|
851
|
-
return
|
890
|
+
return ModelToComponentFactory.create_api_key_authenticator(
|
852
891
|
ApiKeyAuthenticatorModel(
|
853
892
|
type="ApiKeyAuthenticator",
|
854
893
|
api_token="",
|
@@ -1488,15 +1527,19 @@ class ModelToComponentFactory:
|
|
1488
1527
|
)
|
1489
1528
|
|
1490
1529
|
end_time_option = (
|
1491
|
-
|
1492
|
-
model.end_time_option,
|
1530
|
+
RequestOption(
|
1531
|
+
inject_into=RequestOptionType(model.end_time_option.inject_into.value),
|
1532
|
+
field_name=model.end_time_option.field_name,
|
1533
|
+
parameters=model.parameters or {},
|
1493
1534
|
)
|
1494
1535
|
if model.end_time_option
|
1495
1536
|
else None
|
1496
1537
|
)
|
1497
1538
|
start_time_option = (
|
1498
|
-
|
1499
|
-
model.start_time_option,
|
1539
|
+
RequestOption(
|
1540
|
+
inject_into=RequestOptionType(model.start_time_option.inject_into.value),
|
1541
|
+
field_name=model.start_time_option.field_name,
|
1542
|
+
parameters=model.parameters or {},
|
1500
1543
|
)
|
1501
1544
|
if model.start_time_option
|
1502
1545
|
else None
|
@@ -1567,15 +1610,19 @@ class ModelToComponentFactory:
|
|
1567
1610
|
cursor_model = model.incremental_sync
|
1568
1611
|
|
1569
1612
|
end_time_option = (
|
1570
|
-
|
1571
|
-
cursor_model.end_time_option,
|
1613
|
+
RequestOption(
|
1614
|
+
inject_into=RequestOptionType(cursor_model.end_time_option.inject_into.value),
|
1615
|
+
field_name=cursor_model.end_time_option.field_name,
|
1616
|
+
parameters=cursor_model.parameters or {},
|
1572
1617
|
)
|
1573
1618
|
if cursor_model.end_time_option
|
1574
1619
|
else None
|
1575
1620
|
)
|
1576
1621
|
start_time_option = (
|
1577
|
-
|
1578
|
-
cursor_model.start_time_option,
|
1622
|
+
RequestOption(
|
1623
|
+
inject_into=RequestOptionType(cursor_model.start_time_option.inject_into.value),
|
1624
|
+
field_name=cursor_model.start_time_option.field_name,
|
1625
|
+
parameters=cursor_model.parameters or {},
|
1579
1626
|
)
|
1580
1627
|
if cursor_model.start_time_option
|
1581
1628
|
else None
|
@@ -1902,6 +1949,8 @@ class ModelToComponentFactory:
|
|
1902
1949
|
)
|
1903
1950
|
)
|
1904
1951
|
|
1952
|
+
api_budget = self._api_budget
|
1953
|
+
|
1905
1954
|
request_options_provider = InterpolatedRequestOptionsProvider(
|
1906
1955
|
request_body_data=model.request_body_data,
|
1907
1956
|
request_body_json=model.request_body_json,
|
@@ -1922,6 +1971,7 @@ class ModelToComponentFactory:
|
|
1922
1971
|
path=model.path,
|
1923
1972
|
authenticator=authenticator,
|
1924
1973
|
error_handler=error_handler,
|
1974
|
+
api_budget=api_budget,
|
1925
1975
|
http_method=HttpMethod[model.http_method.value],
|
1926
1976
|
request_options_provider=request_options_provider,
|
1927
1977
|
config=config,
|
@@ -2141,11 +2191,16 @@ class ModelToComponentFactory:
|
|
2141
2191
|
additional_jwt_payload=model.additional_jwt_payload,
|
2142
2192
|
)
|
2143
2193
|
|
2194
|
+
@staticmethod
|
2144
2195
|
def create_list_partition_router(
|
2145
|
-
|
2196
|
+
model: ListPartitionRouterModel, config: Config, **kwargs: Any
|
2146
2197
|
) -> ListPartitionRouter:
|
2147
2198
|
request_option = (
|
2148
|
-
|
2199
|
+
RequestOption(
|
2200
|
+
inject_into=RequestOptionType(model.request_option.inject_into.value),
|
2201
|
+
field_name=model.request_option.field_name,
|
2202
|
+
parameters=model.parameters or {},
|
2203
|
+
)
|
2149
2204
|
if model.request_option
|
2150
2205
|
else None
|
2151
2206
|
)
|
@@ -2341,25 +2396,7 @@ class ModelToComponentFactory:
|
|
2341
2396
|
model: RequestOptionModel, config: Config, **kwargs: Any
|
2342
2397
|
) -> RequestOption:
|
2343
2398
|
inject_into = RequestOptionType(model.inject_into.value)
|
2344
|
-
|
2345
|
-
[
|
2346
|
-
InterpolatedString.create(segment, parameters=kwargs.get("parameters", {}))
|
2347
|
-
for segment in model.field_path
|
2348
|
-
]
|
2349
|
-
if model.field_path
|
2350
|
-
else None
|
2351
|
-
)
|
2352
|
-
field_name = (
|
2353
|
-
InterpolatedString.create(model.field_name, parameters=kwargs.get("parameters", {}))
|
2354
|
-
if model.field_name
|
2355
|
-
else None
|
2356
|
-
)
|
2357
|
-
return RequestOption(
|
2358
|
-
field_name=field_name,
|
2359
|
-
field_path=field_path,
|
2360
|
-
inject_into=inject_into,
|
2361
|
-
parameters=kwargs.get("parameters", {}),
|
2362
|
-
)
|
2399
|
+
return RequestOption(field_name=model.field_name, inject_into=inject_into, parameters={})
|
2363
2400
|
|
2364
2401
|
def create_record_selector(
|
2365
2402
|
self,
|
@@ -2923,3 +2960,103 @@ class ModelToComponentFactory:
|
|
2923
2960
|
return isinstance(parser.inner_parser, JsonParser)
|
2924
2961
|
else:
|
2925
2962
|
return False
|
2963
|
+
|
2964
|
+
def create_api_budget(self, model: APIBudgetModel, config: Config, **kwargs: Any) -> APIBudget:
|
2965
|
+
policies = [
|
2966
|
+
self._create_component_from_model(model=policy, config=config)
|
2967
|
+
for policy in model.policies
|
2968
|
+
]
|
2969
|
+
|
2970
|
+
return APIBudget(
|
2971
|
+
policies=policies,
|
2972
|
+
maximum_attempts_to_acquire=model.maximum_attempts_to_acquire or 100000,
|
2973
|
+
)
|
2974
|
+
|
2975
|
+
def create_http_api_budget(
|
2976
|
+
self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any
|
2977
|
+
) -> HttpAPIBudget:
|
2978
|
+
policies = [
|
2979
|
+
self._create_component_from_model(model=policy, config=config)
|
2980
|
+
for policy in model.policies
|
2981
|
+
]
|
2982
|
+
|
2983
|
+
return HttpAPIBudget(
|
2984
|
+
policies=policies,
|
2985
|
+
maximum_attempts_to_acquire=model.maximum_attempts_to_acquire or 100000,
|
2986
|
+
ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset",
|
2987
|
+
ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining",
|
2988
|
+
status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or (429,),
|
2989
|
+
)
|
2990
|
+
|
2991
|
+
def create_fixed_window_call_rate_policy(
|
2992
|
+
self, model: FixedWindowCallRatePolicyModel, config: Config, **kwargs: Any
|
2993
|
+
) -> FixedWindowCallRatePolicy:
|
2994
|
+
matchers = [
|
2995
|
+
self._create_component_from_model(model=matcher, config=config)
|
2996
|
+
for matcher in model.matchers
|
2997
|
+
]
|
2998
|
+
return FixedWindowCallRatePolicy(
|
2999
|
+
next_reset_ts=model.next_reset_ts,
|
3000
|
+
period=parse_duration(model.period),
|
3001
|
+
call_limit=model.call_limit,
|
3002
|
+
matchers=matchers,
|
3003
|
+
)
|
3004
|
+
|
3005
|
+
def create_moving_window_call_rate_policy(
|
3006
|
+
self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
|
3007
|
+
) -> MovingWindowCallRatePolicy:
|
3008
|
+
rates = [
|
3009
|
+
self._create_component_from_model(model=rate, config=config) for rate in model.rates
|
3010
|
+
]
|
3011
|
+
matchers = [
|
3012
|
+
self._create_component_from_model(model=matcher, config=config)
|
3013
|
+
for matcher in model.matchers
|
3014
|
+
]
|
3015
|
+
return MovingWindowCallRatePolicy(
|
3016
|
+
rates=rates,
|
3017
|
+
matchers=matchers,
|
3018
|
+
)
|
3019
|
+
|
3020
|
+
def create_unlimited_call_rate_policy(
|
3021
|
+
self, model: UnlimitedCallRatePolicyModel, config: Config, **kwargs: Any
|
3022
|
+
) -> UnlimitedCallRatePolicy:
|
3023
|
+
matchers = [
|
3024
|
+
self._create_component_from_model(model=matcher, config=config)
|
3025
|
+
for matcher in model.matchers
|
3026
|
+
]
|
3027
|
+
|
3028
|
+
return UnlimitedCallRatePolicy(
|
3029
|
+
matchers=matchers,
|
3030
|
+
)
|
3031
|
+
|
3032
|
+
def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate:
|
3033
|
+
return Rate(
|
3034
|
+
limit=model.limit,
|
3035
|
+
interval=model.interval,
|
3036
|
+
)
|
3037
|
+
|
3038
|
+
def create_http_request_matcher(
|
3039
|
+
self, model: HttpRequestMatcherModel, config: Config, **kwargs: Any
|
3040
|
+
) -> HttpRequestMatcher:
|
3041
|
+
return HttpRequestMatcher(
|
3042
|
+
method=model.method,
|
3043
|
+
url_base=model.url_base,
|
3044
|
+
url_path_pattern=model.url_path_pattern,
|
3045
|
+
params=model.params,
|
3046
|
+
headers=model.headers,
|
3047
|
+
)
|
3048
|
+
|
3049
|
+
def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None:
|
3050
|
+
model_str = component_definition.get("type")
|
3051
|
+
if model_str == "APIBudget":
|
3052
|
+
# Annotate model_type as a type that is a subclass of BaseModel
|
3053
|
+
model_type: Union[Type[APIBudgetModel], Type[HTTPAPIBudgetModel]] = APIBudgetModel
|
3054
|
+
elif model_str == "HTTPAPIBudget":
|
3055
|
+
model_type = HTTPAPIBudgetModel
|
3056
|
+
else:
|
3057
|
+
raise ValueError(f"Unknown API Budget type: {model_str}")
|
3058
|
+
|
3059
|
+
# create_component expects a type[BaseModel] and returns an instance of that model.
|
3060
|
+
self._api_budget = self.create_component(
|
3061
|
+
model_type=model_type, component_definition=component_definition, config=config
|
3062
|
+
)
|
@@ -3,7 +3,7 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass
|
6
|
-
from typing import Any, Iterable, List, Mapping,
|
6
|
+
from typing import Any, Iterable, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
9
9
|
from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
|
@@ -100,9 +100,7 @@ class ListPartitionRouter(PartitionRouter):
|
|
100
100
|
):
|
101
101
|
slice_value = stream_slice.get(self._cursor_field.eval(self.config))
|
102
102
|
if slice_value:
|
103
|
-
|
104
|
-
self.request_option.inject_into_request(options, slice_value, self.config)
|
105
|
-
return options
|
103
|
+
return {self.request_option.field_name.eval(self.config): slice_value} # type: ignore # field_name is always casted to InterpolatedString
|
106
104
|
else:
|
107
105
|
return {}
|
108
106
|
else:
|
@@ -4,7 +4,7 @@
|
|
4
4
|
import copy
|
5
5
|
import logging
|
6
6
|
from dataclasses import InitVar, dataclass
|
7
|
-
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping,
|
7
|
+
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
10
|
|
@@ -118,7 +118,7 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
118
118
|
def _get_request_option(
|
119
119
|
self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
|
120
120
|
) -> Mapping[str, Any]:
|
121
|
-
params
|
121
|
+
params = {}
|
122
122
|
if stream_slice:
|
123
123
|
for parent_config in self.parent_stream_configs:
|
124
124
|
if (
|
@@ -128,7 +128,13 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
128
128
|
key = parent_config.partition_field.eval(self.config) # type: ignore # partition_field is always casted to an interpolated string
|
129
129
|
value = stream_slice.get(key)
|
130
130
|
if value:
|
131
|
-
|
131
|
+
params.update(
|
132
|
+
{
|
133
|
+
parent_config.request_option.field_name.eval( # type: ignore [union-attr]
|
134
|
+
config=self.config
|
135
|
+
): value
|
136
|
+
}
|
137
|
+
)
|
132
138
|
return params
|
133
139
|
|
134
140
|
def stream_slices(self) -> Iterable[StreamSlice]:
|
@@ -22,6 +22,7 @@ from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_req
|
|
22
22
|
)
|
23
23
|
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester
|
24
24
|
from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
|
25
|
+
from airbyte_cdk.sources.streams.call_rate import APIBudget
|
25
26
|
from airbyte_cdk.sources.streams.http import HttpClient
|
26
27
|
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
|
27
28
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
@@ -55,6 +56,7 @@ class HttpRequester(Requester):
|
|
55
56
|
http_method: Union[str, HttpMethod] = HttpMethod.GET
|
56
57
|
request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None
|
57
58
|
error_handler: Optional[ErrorHandler] = None
|
59
|
+
api_budget: Optional[APIBudget] = None
|
58
60
|
disable_retries: bool = False
|
59
61
|
message_repository: MessageRepository = NoopMessageRepository()
|
60
62
|
use_cache: bool = False
|
@@ -91,6 +93,7 @@ class HttpRequester(Requester):
|
|
91
93
|
name=self.name,
|
92
94
|
logger=self.logger,
|
93
95
|
error_handler=self.error_handler,
|
96
|
+
api_budget=self.api_budget,
|
94
97
|
authenticator=self._authenticator,
|
95
98
|
use_cache=self.use_cache,
|
96
99
|
backoff_strategy=backoff_strategies,
|
@@ -199,9 +202,6 @@ class HttpRequester(Requester):
|
|
199
202
|
Raise a ValueError if there's a key collision
|
200
203
|
Returned merged mapping otherwise
|
201
204
|
"""
|
202
|
-
|
203
|
-
is_body_json = requester_method.__name__ == "get_request_body_json"
|
204
|
-
|
205
205
|
return combine_mappings(
|
206
206
|
[
|
207
207
|
requester_method(
|
@@ -211,8 +211,7 @@ class HttpRequester(Requester):
|
|
211
211
|
),
|
212
212
|
auth_options_method(),
|
213
213
|
extra_options,
|
214
|
-
]
|
215
|
-
allow_same_value_merge=is_body_json,
|
214
|
+
]
|
216
215
|
)
|
217
216
|
|
218
217
|
def _request_headers(
|
@@ -187,7 +187,7 @@ class DefaultPaginator(Paginator):
|
|
187
187
|
def _get_request_options(
|
188
188
|
self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]]
|
189
189
|
) -> MutableMapping[str, Any]:
|
190
|
-
options
|
190
|
+
options = {}
|
191
191
|
|
192
192
|
token = next_page_token.get("next_page_token") if next_page_token else None
|
193
193
|
if (
|
@@ -196,16 +196,15 @@ class DefaultPaginator(Paginator):
|
|
196
196
|
and isinstance(self.page_token_option, RequestOption)
|
197
197
|
and self.page_token_option.inject_into == option_type
|
198
198
|
):
|
199
|
-
self.page_token_option.
|
200
|
-
|
199
|
+
options[self.page_token_option.field_name.eval(config=self.config)] = token # type: ignore # field_name is always cast to an interpolated string
|
201
200
|
if (
|
202
201
|
self.page_size_option
|
203
202
|
and self.pagination_strategy.get_page_size()
|
204
203
|
and self.page_size_option.inject_into == option_type
|
205
204
|
):
|
206
|
-
|
207
|
-
|
208
|
-
|
205
|
+
options[self.page_size_option.field_name.eval(config=self.config)] = ( # type: ignore [union-attr]
|
206
|
+
self.pagination_strategy.get_page_size()
|
207
|
+
) # type: ignore # field_name is always cast to an interpolated string
|
209
208
|
return options
|
210
209
|
|
211
210
|
|
@@ -4,10 +4,9 @@
|
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass
|
6
6
|
from enum import Enum
|
7
|
-
from typing import Any,
|
7
|
+
from typing import Any, Mapping, Union
|
8
8
|
|
9
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
|
-
from airbyte_cdk.sources.types import Config
|
11
10
|
|
12
11
|
|
13
12
|
class RequestOptionType(Enum):
|
@@ -27,91 +26,13 @@ class RequestOption:
|
|
27
26
|
Describes an option to set on a request
|
28
27
|
|
29
28
|
Attributes:
|
30
|
-
field_name (str): Describes the name of the parameter to inject
|
31
|
-
field_path (list(str)): Describes the path to a nested field as a list of field names.
|
32
|
-
Only valid for body_json injection type, and mutually exclusive with field_name.
|
29
|
+
field_name (str): Describes the name of the parameter to inject
|
33
30
|
inject_into (RequestOptionType): Describes where in the HTTP request to inject the parameter
|
34
31
|
"""
|
35
32
|
|
33
|
+
field_name: Union[InterpolatedString, str]
|
36
34
|
inject_into: RequestOptionType
|
37
35
|
parameters: InitVar[Mapping[str, Any]]
|
38
|
-
field_name: Optional[Union[InterpolatedString, str]] = None
|
39
|
-
field_path: Optional[List[Union[InterpolatedString, str]]] = None
|
40
36
|
|
41
37
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
42
|
-
|
43
|
-
if self.field_name is None and self.field_path is None:
|
44
|
-
raise ValueError("RequestOption requires either a field_name or field_path")
|
45
|
-
|
46
|
-
if self.field_name is not None and self.field_path is not None:
|
47
|
-
raise ValueError(
|
48
|
-
"Only one of field_name or field_path can be provided to RequestOption"
|
49
|
-
)
|
50
|
-
|
51
|
-
# Nested field injection is only supported for body JSON injection
|
52
|
-
if self.field_path is not None and self.inject_into != RequestOptionType.body_json:
|
53
|
-
raise ValueError(
|
54
|
-
"Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
|
55
|
-
)
|
56
|
-
|
57
|
-
# Convert field_name and field_path into InterpolatedString objects if they are strings
|
58
|
-
if self.field_name is not None:
|
59
|
-
self.field_name = InterpolatedString.create(self.field_name, parameters=parameters)
|
60
|
-
elif self.field_path is not None:
|
61
|
-
self.field_path = [
|
62
|
-
InterpolatedString.create(segment, parameters=parameters)
|
63
|
-
for segment in self.field_path
|
64
|
-
]
|
65
|
-
|
66
|
-
@property
|
67
|
-
def _is_field_path(self) -> bool:
|
68
|
-
"""Returns whether this option is a field path (ie, a nested field)"""
|
69
|
-
return self.field_path is not None
|
70
|
-
|
71
|
-
def inject_into_request(
|
72
|
-
self,
|
73
|
-
target: MutableMapping[str, Any],
|
74
|
-
value: Any,
|
75
|
-
config: Config,
|
76
|
-
) -> None:
|
77
|
-
"""
|
78
|
-
Inject a request option value into a target request structure using either field_name or field_path.
|
79
|
-
For non-body-json injection, only top-level field names are supported.
|
80
|
-
For body-json injection, both field names and nested field paths are supported.
|
81
|
-
|
82
|
-
Args:
|
83
|
-
target: The request structure to inject the value into
|
84
|
-
value: The value to inject
|
85
|
-
config: The config object to use for interpolation
|
86
|
-
"""
|
87
|
-
if self._is_field_path:
|
88
|
-
if self.inject_into != RequestOptionType.body_json:
|
89
|
-
raise ValueError(
|
90
|
-
"Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
|
91
|
-
)
|
92
|
-
|
93
|
-
assert self.field_path is not None # for type checker
|
94
|
-
current = target
|
95
|
-
# Convert path segments into strings, evaluating any interpolated segments
|
96
|
-
# Example: ["data", "{{ config[user_type] }}", "id"] -> ["data", "admin", "id"]
|
97
|
-
*path_parts, final_key = [
|
98
|
-
str(
|
99
|
-
segment.eval(config=config)
|
100
|
-
if isinstance(segment, InterpolatedString)
|
101
|
-
else segment
|
102
|
-
)
|
103
|
-
for segment in self.field_path
|
104
|
-
]
|
105
|
-
|
106
|
-
# Build a nested dictionary structure and set the final value at the deepest level
|
107
|
-
for part in path_parts:
|
108
|
-
current = current.setdefault(part, {})
|
109
|
-
current[final_key] = value
|
110
|
-
else:
|
111
|
-
# For non-nested fields, evaluate the field name if it's an interpolated string
|
112
|
-
key = (
|
113
|
-
self.field_name.eval(config=config)
|
114
|
-
if isinstance(self.field_name, InterpolatedString)
|
115
|
-
else self.field_name
|
116
|
-
)
|
117
|
-
target[str(key)] = value
|
38
|
+
self.field_name = InterpolatedString.create(self.field_name, parameters=parameters)
|
@@ -80,13 +80,12 @@ class DatetimeBasedRequestOptionsProvider(RequestOptionsProvider):
|
|
80
80
|
options: MutableMapping[str, Any] = {}
|
81
81
|
if not stream_slice:
|
82
82
|
return options
|
83
|
-
|
84
83
|
if self.start_time_option and self.start_time_option.inject_into == option_type:
|
85
|
-
|
86
|
-
|
87
|
-
|
84
|
+
options[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get( # type: ignore # field_name is always casted to an interpolated string
|
85
|
+
self._partition_field_start.eval(self.config)
|
86
|
+
)
|
88
87
|
if self.end_time_option and self.end_time_option.inject_into == option_type:
|
89
|
-
|
90
|
-
|
91
|
-
|
88
|
+
options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get( # type: ignore [union-attr]
|
89
|
+
self._partition_field_end.eval(self.config)
|
90
|
+
)
|
92
91
|
return options
|
@@ -128,9 +128,6 @@ class SimpleRetriever(Retriever):
|
|
128
128
|
Returned merged mapping otherwise
|
129
129
|
"""
|
130
130
|
# FIXME we should eventually remove the usage of stream_state as part of the interpolation
|
131
|
-
|
132
|
-
is_body_json = paginator_method.__name__ == "get_request_body_json"
|
133
|
-
|
134
131
|
mappings = [
|
135
132
|
paginator_method(
|
136
133
|
stream_state=stream_state,
|
@@ -146,7 +143,7 @@ class SimpleRetriever(Retriever):
|
|
146
143
|
next_page_token=next_page_token,
|
147
144
|
)
|
148
145
|
)
|
149
|
-
return combine_mappings(mappings
|
146
|
+
return combine_mappings(mappings)
|
150
147
|
|
151
148
|
def _request_headers(
|
152
149
|
self,
|