airbyte-cdk 6.45.0__py3-none-any.whl → 6.45.0.dev4100__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/connector_builder_handler.py +6 -45
- airbyte_cdk/connector_builder/main.py +2 -5
- airbyte_cdk/models/__init__.py +1 -0
- airbyte_cdk/models/airbyte_protocol.py +1 -3
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +1 -1
- airbyte_cdk/sources/declarative/async_job/job.py +0 -6
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +18 -18
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +6 -22
- airbyte_cdk/sources/declarative/checks/__init__.py +2 -5
- airbyte_cdk/sources/declarative/checks/check_stream.py +11 -113
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +8 -0
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +50 -210
- airbyte_cdk/sources/declarative/extractors/record_selector.py +6 -1
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +1 -2
- airbyte_cdk/sources/declarative/interpolation/macros.py +4 -8
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +2 -23
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +43 -142
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +4 -16
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +50 -263
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +0 -4
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +1 -5
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +2 -25
- airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +30 -101
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +9 -4
- airbyte_cdk/sources/declarative/transformations/add_fields.py +1 -3
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +9 -9
- airbyte_cdk/sources/file_based/file_record_data.py +24 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +8 -15
- airbyte_cdk/sources/file_based/schema_helpers.py +11 -1
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +0 -1
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +16 -31
- airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +1 -3
- airbyte_cdk/sources/streams/concurrent/default_stream.py +3 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +0 -4
- airbyte_cdk/sources/types.py +11 -2
- airbyte_cdk/sources/utils/files_directory.py +15 -0
- airbyte_cdk/sources/utils/record_helper.py +8 -8
- {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4100.dist-info}/METADATA +2 -2
- {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4100.dist-info}/RECORD +44 -50
- airbyte_cdk/models/file_transfer_record_message.py +0 -13
- airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py +0 -150
- airbyte_cdk/sources/declarative/requesters/query_properties/__init__.py +0 -13
- airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +0 -40
- airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py +0 -69
- airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +0 -58
- airbyte_cdk/sources/declarative/requesters/query_properties/strategies/__init__.py +0 -10
- airbyte_cdk/sources/declarative/requesters/query_properties/strategies/group_by_key.py +0 -33
- airbyte_cdk/sources/declarative/requesters/query_properties/strategies/merge_strategy.py +0 -19
- {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4100.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4100.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4100.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4100.dist-info}/entry_points.txt +0 -0
@@ -1,5 +1,5 @@
|
|
1
1
|
#
|
2
|
-
# Copyright (c)
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
5
|
from __future__ import annotations
|
@@ -54,11 +54,7 @@ from airbyte_cdk.sources.declarative.auth.token_provider import (
|
|
54
54
|
SessionTokenProvider,
|
55
55
|
TokenProvider,
|
56
56
|
)
|
57
|
-
from airbyte_cdk.sources.declarative.checks import
|
58
|
-
CheckDynamicStream,
|
59
|
-
CheckStream,
|
60
|
-
DynamicStreamCheckConfig,
|
61
|
-
)
|
57
|
+
from airbyte_cdk.sources.declarative.checks import CheckDynamicStream, CheckStream
|
62
58
|
from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
|
63
59
|
from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
|
64
60
|
from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
|
@@ -106,7 +102,6 @@ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_mi
|
|
106
102
|
)
|
107
103
|
from airbyte_cdk.sources.declarative.models import (
|
108
104
|
CustomStateMigration,
|
109
|
-
GzipDecoder,
|
110
105
|
)
|
111
106
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
112
107
|
AddedFieldDefinition as AddedFieldDefinitionModel,
|
@@ -223,10 +218,10 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
223
218
|
DynamicSchemaLoader as DynamicSchemaLoaderModel,
|
224
219
|
)
|
225
220
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
226
|
-
|
221
|
+
ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
|
227
222
|
)
|
228
223
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
229
|
-
|
224
|
+
FileUploader as FileUploaderModel,
|
230
225
|
)
|
231
226
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
232
227
|
FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel,
|
@@ -234,12 +229,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
234
229
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
235
230
|
FlattenFields as FlattenFieldsModel,
|
236
231
|
)
|
237
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
238
|
-
GroupByKeyMergeStrategy as GroupByKeyMergeStrategyModel,
|
239
|
-
)
|
240
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
241
|
-
GroupingPartitionRouter as GroupingPartitionRouterModel,
|
242
|
-
)
|
243
232
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
244
233
|
GzipDecoder as GzipDecoderModel,
|
245
234
|
)
|
@@ -327,18 +316,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
327
316
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
328
317
|
ParentStreamConfig as ParentStreamConfigModel,
|
329
318
|
)
|
330
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
331
|
-
PropertiesFromEndpoint as PropertiesFromEndpointModel,
|
332
|
-
)
|
333
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
334
|
-
PropertyChunking as PropertyChunkingModel,
|
335
|
-
)
|
336
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
337
|
-
PropertyLimitType as PropertyLimitTypeModel,
|
338
|
-
)
|
339
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
340
|
-
QueryProperties as QueryPropertiesModel,
|
341
|
-
)
|
342
319
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
343
320
|
Rate as RateModel,
|
344
321
|
)
|
@@ -410,7 +387,6 @@ from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
|
|
410
387
|
)
|
411
388
|
from airbyte_cdk.sources.declarative.partition_routers import (
|
412
389
|
CartesianProductStreamSlicer,
|
413
|
-
GroupingPartitionRouter,
|
414
390
|
ListPartitionRouter,
|
415
391
|
PartitionRouter,
|
416
392
|
SinglePartitionRouter,
|
@@ -447,17 +423,6 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies import (
|
|
447
423
|
PageIncrement,
|
448
424
|
StopConditionPaginationStrategyDecorator,
|
449
425
|
)
|
450
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties import (
|
451
|
-
PropertiesFromEndpoint,
|
452
|
-
PropertyChunking,
|
453
|
-
QueryProperties,
|
454
|
-
)
|
455
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import (
|
456
|
-
PropertyLimitType,
|
457
|
-
)
|
458
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties.strategies import (
|
459
|
-
GroupByKey,
|
460
|
-
)
|
461
426
|
from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType
|
462
427
|
from airbyte_cdk.sources.declarative.requesters.request_options import (
|
463
428
|
DatetimeBasedRequestOptionsProvider,
|
@@ -479,6 +444,7 @@ from airbyte_cdk.sources.declarative.retrievers import (
|
|
479
444
|
SimpleRetriever,
|
480
445
|
SimpleRetrieverTestReadDecorator,
|
481
446
|
)
|
447
|
+
from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
|
482
448
|
from airbyte_cdk.sources.declarative.schema import (
|
483
449
|
ComplexFieldType,
|
484
450
|
DefaultSchemaLoader,
|
@@ -592,7 +558,6 @@ class ModelToComponentFactory:
|
|
592
558
|
BasicHttpAuthenticatorModel: self.create_basic_http_authenticator,
|
593
559
|
BearerAuthenticatorModel: self.create_bearer_authenticator,
|
594
560
|
CheckStreamModel: self.create_check_stream,
|
595
|
-
DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config,
|
596
561
|
CheckDynamicStreamModel: self.create_check_dynamic_stream,
|
597
562
|
CompositeErrorHandlerModel: self.create_composite_error_handler,
|
598
563
|
ConcurrencyLevelModel: self.create_concurrency_level,
|
@@ -622,7 +587,6 @@ class ModelToComponentFactory:
|
|
622
587
|
ResponseToFileExtractorModel: self.create_response_to_file_extractor,
|
623
588
|
ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy,
|
624
589
|
SessionTokenAuthenticatorModel: self.create_session_token_authenticator,
|
625
|
-
GroupByKeyMergeStrategyModel: self.create_group_by_key,
|
626
590
|
HttpRequesterModel: self.create_http_requester,
|
627
591
|
HttpResponseFilterModel: self.create_http_response_filter,
|
628
592
|
InlineSchemaLoaderModel: self.create_inline_schema_loader,
|
@@ -652,9 +616,6 @@ class ModelToComponentFactory:
|
|
652
616
|
OffsetIncrementModel: self.create_offset_increment,
|
653
617
|
PageIncrementModel: self.create_page_increment,
|
654
618
|
ParentStreamConfigModel: self.create_parent_stream_config,
|
655
|
-
PropertiesFromEndpointModel: self.create_properties_from_endpoint,
|
656
|
-
PropertyChunkingModel: self.create_property_chunking,
|
657
|
-
QueryPropertiesModel: self.create_query_properties,
|
658
619
|
RecordFilterModel: self.create_record_filter,
|
659
620
|
RecordSelectorModel: self.create_record_selector,
|
660
621
|
RemoveFieldsModel: self.create_remove_fields,
|
@@ -675,12 +636,12 @@ class ModelToComponentFactory:
|
|
675
636
|
ComponentMappingDefinitionModel: self.create_components_mapping_definition,
|
676
637
|
ZipfileDecoderModel: self.create_zipfile_decoder,
|
677
638
|
HTTPAPIBudgetModel: self.create_http_api_budget,
|
639
|
+
FileUploaderModel: self.create_file_uploader,
|
678
640
|
FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
|
679
641
|
MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
|
680
642
|
UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
|
681
643
|
RateModel: self.create_rate,
|
682
644
|
HttpRequestRegexMatcherModel: self.create_http_request_matcher,
|
683
|
-
GroupingPartitionRouterModel: self.create_grouping_partition_router,
|
684
645
|
}
|
685
646
|
|
686
647
|
# Needed for the case where we need to perform a second parse on the fields of a custom component
|
@@ -974,36 +935,8 @@ class ModelToComponentFactory:
|
|
974
935
|
)
|
975
936
|
|
976
937
|
@staticmethod
|
977
|
-
def
|
978
|
-
model
|
979
|
-
) -> DynamicStreamCheckConfig:
|
980
|
-
return DynamicStreamCheckConfig(
|
981
|
-
dynamic_stream_name=model.dynamic_stream_name,
|
982
|
-
stream_count=model.stream_count or 0,
|
983
|
-
)
|
984
|
-
|
985
|
-
def create_check_stream(
|
986
|
-
self, model: CheckStreamModel, config: Config, **kwargs: Any
|
987
|
-
) -> CheckStream:
|
988
|
-
if model.dynamic_streams_check_configs is None and model.stream_names is None:
|
989
|
-
raise ValueError(
|
990
|
-
"Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream"
|
991
|
-
)
|
992
|
-
|
993
|
-
dynamic_streams_check_configs = (
|
994
|
-
[
|
995
|
-
self._create_component_from_model(model=dynamic_stream_check_config, config=config)
|
996
|
-
for dynamic_stream_check_config in model.dynamic_streams_check_configs
|
997
|
-
]
|
998
|
-
if model.dynamic_streams_check_configs
|
999
|
-
else []
|
1000
|
-
)
|
1001
|
-
|
1002
|
-
return CheckStream(
|
1003
|
-
stream_names=model.stream_names or [],
|
1004
|
-
dynamic_streams_check_configs=dynamic_streams_check_configs,
|
1005
|
-
parameters={},
|
1006
|
-
)
|
938
|
+
def create_check_stream(model: CheckStreamModel, config: Config, **kwargs: Any) -> CheckStream:
|
939
|
+
return CheckStream(stream_names=model.stream_names, parameters={})
|
1007
940
|
|
1008
941
|
@staticmethod
|
1009
942
|
def create_check_dynamic_stream(
|
@@ -1426,9 +1359,6 @@ class ModelToComponentFactory:
|
|
1426
1359
|
)
|
1427
1360
|
stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
|
1428
1361
|
|
1429
|
-
# Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state
|
1430
|
-
use_global_cursor = isinstance(partition_router, GroupingPartitionRouter)
|
1431
|
-
|
1432
1362
|
# Return the concurrent cursor and state converter
|
1433
1363
|
return ConcurrentPerPartitionCursor(
|
1434
1364
|
cursor_factory=cursor_factory,
|
@@ -1440,7 +1370,6 @@ class ModelToComponentFactory:
|
|
1440
1370
|
connector_state_manager=state_manager,
|
1441
1371
|
connector_state_converter=connector_state_converter,
|
1442
1372
|
cursor_field=cursor_field,
|
1443
|
-
use_global_cursor=use_global_cursor,
|
1444
1373
|
)
|
1445
1374
|
|
1446
1375
|
@staticmethod
|
@@ -1826,6 +1755,11 @@ class ModelToComponentFactory:
|
|
1826
1755
|
transformations.append(
|
1827
1756
|
self._create_component_from_model(model=transformation_model, config=config)
|
1828
1757
|
)
|
1758
|
+
file_uploader = None
|
1759
|
+
if model.file_uploader:
|
1760
|
+
file_uploader = self._create_component_from_model(
|
1761
|
+
model=model.file_uploader, config=config
|
1762
|
+
)
|
1829
1763
|
|
1830
1764
|
retriever = self._create_component_from_model(
|
1831
1765
|
model=model.retriever,
|
@@ -1837,6 +1771,7 @@ class ModelToComponentFactory:
|
|
1837
1771
|
stop_condition_on_cursor=stop_condition_on_cursor,
|
1838
1772
|
client_side_incremental_sync=client_side_incremental_sync,
|
1839
1773
|
transformations=transformations,
|
1774
|
+
file_uploader=file_uploader,
|
1840
1775
|
incremental_sync=model.incremental_sync,
|
1841
1776
|
)
|
1842
1777
|
cursor_field = model.incremental_sync.cursor_field if model.incremental_sync else None
|
@@ -2113,8 +2048,8 @@ class ModelToComponentFactory:
|
|
2113
2048
|
parameters=model.parameters or {},
|
2114
2049
|
)
|
2115
2050
|
|
2116
|
-
@staticmethod
|
2117
2051
|
def create_response_to_file_extractor(
|
2052
|
+
self,
|
2118
2053
|
model: ResponseToFileExtractorModel,
|
2119
2054
|
**kwargs: Any,
|
2120
2055
|
) -> ResponseToFileExtractor:
|
@@ -2128,17 +2063,11 @@ class ModelToComponentFactory:
|
|
2128
2063
|
factor=model.factor or 5, parameters=model.parameters or {}, config=config
|
2129
2064
|
)
|
2130
2065
|
|
2131
|
-
@staticmethod
|
2132
|
-
def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey:
|
2133
|
-
return GroupByKey(model.key, config=config, parameters=model.parameters or {})
|
2134
|
-
|
2135
2066
|
def create_http_requester(
|
2136
2067
|
self,
|
2137
2068
|
model: HttpRequesterModel,
|
2138
2069
|
config: Config,
|
2139
2070
|
decoder: Decoder = JsonDecoder(parameters={}),
|
2140
|
-
query_properties_key: Optional[str] = None,
|
2141
|
-
use_cache: Optional[bool] = None,
|
2142
2071
|
*,
|
2143
2072
|
name: str,
|
2144
2073
|
) -> HttpRequester:
|
@@ -2171,7 +2100,6 @@ class ModelToComponentFactory:
|
|
2171
2100
|
request_body_json=model.request_body_json,
|
2172
2101
|
request_headers=model.request_headers,
|
2173
2102
|
request_parameters=model.request_parameters,
|
2174
|
-
query_properties_key=query_properties_key,
|
2175
2103
|
config=config,
|
2176
2104
|
parameters=model.parameters or {},
|
2177
2105
|
)
|
@@ -2179,7 +2107,7 @@ class ModelToComponentFactory:
|
|
2179
2107
|
assert model.use_cache is not None # for mypy
|
2180
2108
|
assert model.http_method is not None # for mypy
|
2181
2109
|
|
2182
|
-
|
2110
|
+
use_cache = model.use_cache and not self._disable_cache
|
2183
2111
|
|
2184
2112
|
return HttpRequester(
|
2185
2113
|
name=name,
|
@@ -2194,7 +2122,7 @@ class ModelToComponentFactory:
|
|
2194
2122
|
disable_retries=self._disable_retries,
|
2195
2123
|
parameters=model.parameters or {},
|
2196
2124
|
message_repository=self._message_repository,
|
2197
|
-
use_cache=
|
2125
|
+
use_cache=use_cache,
|
2198
2126
|
decoder=decoder,
|
2199
2127
|
stream_response=decoder.is_stream_response() if decoder else False,
|
2200
2128
|
)
|
@@ -2298,11 +2226,10 @@ class ModelToComponentFactory:
|
|
2298
2226
|
retriever = self._create_component_from_model(
|
2299
2227
|
model=model.retriever,
|
2300
2228
|
config=config,
|
2301
|
-
name="
|
2229
|
+
name="",
|
2302
2230
|
primary_key=None,
|
2303
2231
|
stream_slicer=combined_slicers,
|
2304
2232
|
transformations=[],
|
2305
|
-
use_cache=True,
|
2306
2233
|
)
|
2307
2234
|
schema_type_identifier = self._create_component_from_model(
|
2308
2235
|
model.schema_type_identifier, config=config, parameters=model.parameters or {}
|
@@ -2640,79 +2567,6 @@ class ModelToComponentFactory:
|
|
2640
2567
|
lazy_read_pointer=model_lazy_read_pointer,
|
2641
2568
|
)
|
2642
2569
|
|
2643
|
-
def create_properties_from_endpoint(
|
2644
|
-
self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any
|
2645
|
-
) -> PropertiesFromEndpoint:
|
2646
|
-
retriever = self._create_component_from_model(
|
2647
|
-
model=model.retriever,
|
2648
|
-
config=config,
|
2649
|
-
name="dynamic_properties",
|
2650
|
-
primary_key=None,
|
2651
|
-
stream_slicer=None,
|
2652
|
-
transformations=[],
|
2653
|
-
use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different
|
2654
|
-
)
|
2655
|
-
return PropertiesFromEndpoint(
|
2656
|
-
property_field_path=model.property_field_path,
|
2657
|
-
retriever=retriever,
|
2658
|
-
config=config,
|
2659
|
-
parameters=model.parameters or {},
|
2660
|
-
)
|
2661
|
-
|
2662
|
-
def create_property_chunking(
|
2663
|
-
self, model: PropertyChunkingModel, config: Config, **kwargs: Any
|
2664
|
-
) -> PropertyChunking:
|
2665
|
-
record_merge_strategy = (
|
2666
|
-
self._create_component_from_model(
|
2667
|
-
model=model.record_merge_strategy, config=config, **kwargs
|
2668
|
-
)
|
2669
|
-
if model.record_merge_strategy
|
2670
|
-
else None
|
2671
|
-
)
|
2672
|
-
|
2673
|
-
property_limit_type: PropertyLimitType
|
2674
|
-
match model.property_limit_type:
|
2675
|
-
case PropertyLimitTypeModel.property_count:
|
2676
|
-
property_limit_type = PropertyLimitType.property_count
|
2677
|
-
case PropertyLimitTypeModel.characters:
|
2678
|
-
property_limit_type = PropertyLimitType.characters
|
2679
|
-
case _:
|
2680
|
-
raise ValueError(f"Invalid PropertyLimitType {property_limit_type}")
|
2681
|
-
|
2682
|
-
return PropertyChunking(
|
2683
|
-
property_limit_type=property_limit_type,
|
2684
|
-
property_limit=model.property_limit,
|
2685
|
-
record_merge_strategy=record_merge_strategy,
|
2686
|
-
config=config,
|
2687
|
-
parameters=model.parameters or {},
|
2688
|
-
)
|
2689
|
-
|
2690
|
-
def create_query_properties(
|
2691
|
-
self, model: QueryPropertiesModel, config: Config, **kwargs: Any
|
2692
|
-
) -> QueryProperties:
|
2693
|
-
if isinstance(model.property_list, list):
|
2694
|
-
property_list = model.property_list
|
2695
|
-
else:
|
2696
|
-
property_list = self._create_component_from_model(
|
2697
|
-
model=model.property_list, config=config, **kwargs
|
2698
|
-
)
|
2699
|
-
|
2700
|
-
property_chunking = (
|
2701
|
-
self._create_component_from_model(
|
2702
|
-
model=model.property_chunking, config=config, **kwargs
|
2703
|
-
)
|
2704
|
-
if model.property_chunking
|
2705
|
-
else None
|
2706
|
-
)
|
2707
|
-
|
2708
|
-
return QueryProperties(
|
2709
|
-
property_list=property_list,
|
2710
|
-
always_include_properties=model.always_include_properties,
|
2711
|
-
property_chunking=property_chunking,
|
2712
|
-
config=config,
|
2713
|
-
parameters=model.parameters or {},
|
2714
|
-
)
|
2715
|
-
|
2716
2570
|
@staticmethod
|
2717
2571
|
def create_record_filter(
|
2718
2572
|
model: RecordFilterModel, config: Config, **kwargs: Any
|
@@ -2759,6 +2613,7 @@ class ModelToComponentFactory:
|
|
2759
2613
|
transformations: List[RecordTransformation] | None = None,
|
2760
2614
|
decoder: Decoder | None = None,
|
2761
2615
|
client_side_incremental_sync: Dict[str, Any] | None = None,
|
2616
|
+
file_uploader: Optional[FileUploader] = None,
|
2762
2617
|
**kwargs: Any,
|
2763
2618
|
) -> RecordSelector:
|
2764
2619
|
extractor = self._create_component_from_model(
|
@@ -2796,6 +2651,7 @@ class ModelToComponentFactory:
|
|
2796
2651
|
config=config,
|
2797
2652
|
record_filter=record_filter,
|
2798
2653
|
transformations=transformations or [],
|
2654
|
+
file_uploader=file_uploader,
|
2799
2655
|
schema_normalization=schema_normalization,
|
2800
2656
|
parameters=model.parameters or {},
|
2801
2657
|
transform_before_filtering=transform_before_filtering,
|
@@ -2853,12 +2709,12 @@ class ModelToComponentFactory:
|
|
2853
2709
|
stop_condition_on_cursor: bool = False,
|
2854
2710
|
client_side_incremental_sync: Optional[Dict[str, Any]] = None,
|
2855
2711
|
transformations: List[RecordTransformation],
|
2712
|
+
file_uploader: Optional[FileUploader] = None,
|
2856
2713
|
incremental_sync: Optional[
|
2857
2714
|
Union[
|
2858
2715
|
IncrementingCountCursorModel, DatetimeBasedCursorModel, CustomIncrementalSyncModel
|
2859
2716
|
]
|
2860
2717
|
] = None,
|
2861
|
-
use_cache: Optional[bool] = None,
|
2862
2718
|
**kwargs: Any,
|
2863
2719
|
) -> SimpleRetriever:
|
2864
2720
|
decoder = (
|
@@ -2866,6 +2722,9 @@ class ModelToComponentFactory:
|
|
2866
2722
|
if model.decoder
|
2867
2723
|
else JsonDecoder(parameters={})
|
2868
2724
|
)
|
2725
|
+
requester = self._create_component_from_model(
|
2726
|
+
model=model.requester, decoder=decoder, config=config, name=name
|
2727
|
+
)
|
2869
2728
|
record_selector = self._create_component_from_model(
|
2870
2729
|
model=model.record_selector,
|
2871
2730
|
name=name,
|
@@ -2873,57 +2732,7 @@ class ModelToComponentFactory:
|
|
2873
2732
|
decoder=decoder,
|
2874
2733
|
transformations=transformations,
|
2875
2734
|
client_side_incremental_sync=client_side_incremental_sync,
|
2876
|
-
|
2877
|
-
|
2878
|
-
query_properties: Optional[QueryProperties] = None
|
2879
|
-
query_properties_key: Optional[str] = None
|
2880
|
-
if (
|
2881
|
-
hasattr(model.requester, "request_parameters")
|
2882
|
-
and model.requester.request_parameters
|
2883
|
-
and isinstance(model.requester.request_parameters, Mapping)
|
2884
|
-
):
|
2885
|
-
query_properties_definitions = []
|
2886
|
-
for key, request_parameter in model.requester.request_parameters.items():
|
2887
|
-
# When translating JSON schema into Pydantic models, enforcing types for arrays containing both
|
2888
|
-
# concrete string complex object definitions like QueryProperties would get resolved to Union[str, Any].
|
2889
|
-
# This adds the extra validation that we couldn't get for free in Pydantic model generation
|
2890
|
-
if (
|
2891
|
-
isinstance(request_parameter, Mapping)
|
2892
|
-
and request_parameter.get("type") == "QueryProperties"
|
2893
|
-
):
|
2894
|
-
query_properties_key = key
|
2895
|
-
query_properties_definitions.append(request_parameter)
|
2896
|
-
elif not isinstance(request_parameter, str):
|
2897
|
-
raise ValueError(
|
2898
|
-
f"Each element of request_parameters should be of type str or QueryProperties, but received {request_parameter.get('type')}"
|
2899
|
-
)
|
2900
|
-
|
2901
|
-
if len(query_properties_definitions) > 1:
|
2902
|
-
raise ValueError(
|
2903
|
-
f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages"
|
2904
|
-
)
|
2905
|
-
|
2906
|
-
if len(query_properties_definitions) == 1:
|
2907
|
-
query_properties = self.create_component(
|
2908
|
-
model_type=QueryPropertiesModel,
|
2909
|
-
component_definition=query_properties_definitions[0],
|
2910
|
-
config=config,
|
2911
|
-
)
|
2912
|
-
|
2913
|
-
# Removes QueryProperties components from the interpolated mappings because it will be resolved in
|
2914
|
-
# the provider from the slice directly instead of through jinja interpolation
|
2915
|
-
if isinstance(model.requester.request_parameters, Mapping):
|
2916
|
-
model.requester.request_parameters = self._remove_query_properties(
|
2917
|
-
model.requester.request_parameters
|
2918
|
-
)
|
2919
|
-
|
2920
|
-
requester = self._create_component_from_model(
|
2921
|
-
model=model.requester,
|
2922
|
-
decoder=decoder,
|
2923
|
-
name=name,
|
2924
|
-
query_properties_key=query_properties_key,
|
2925
|
-
use_cache=use_cache,
|
2926
|
-
config=config,
|
2735
|
+
file_uploader=file_uploader,
|
2927
2736
|
)
|
2928
2737
|
url_base = (
|
2929
2738
|
model.requester.url_base
|
@@ -3030,21 +2839,9 @@ class ModelToComponentFactory:
|
|
3030
2839
|
cursor=cursor,
|
3031
2840
|
config=config,
|
3032
2841
|
ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
|
3033
|
-
additional_query_properties=query_properties,
|
3034
2842
|
parameters=model.parameters or {},
|
3035
2843
|
)
|
3036
2844
|
|
3037
|
-
@staticmethod
|
3038
|
-
def _remove_query_properties(
|
3039
|
-
request_parameters: Mapping[str, Union[Any, str]],
|
3040
|
-
) -> Mapping[str, Union[Any, str]]:
|
3041
|
-
return {
|
3042
|
-
parameter_field: request_parameter
|
3043
|
-
for parameter_field, request_parameter in request_parameters.items()
|
3044
|
-
if not isinstance(request_parameter, Mapping)
|
3045
|
-
or not request_parameter.get("type") == "QueryProperties"
|
3046
|
-
}
|
3047
|
-
|
3048
2845
|
def create_state_delegating_stream(
|
3049
2846
|
self,
|
3050
2847
|
model: StateDelegatingStreamModel,
|
@@ -3290,11 +3087,8 @@ class ModelToComponentFactory:
|
|
3290
3087
|
stream_slices,
|
3291
3088
|
self._job_tracker,
|
3292
3089
|
self._message_repository,
|
3293
|
-
# FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
|
3294
3090
|
has_bulk_parent=False,
|
3295
|
-
#
|
3296
|
-
# `None` == default retry is set to 3 attempts, under the hood.
|
3297
|
-
job_max_retry=1 if self._emit_connector_builder_messages else None,
|
3091
|
+
# FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
|
3298
3092
|
),
|
3299
3093
|
stream_slicer=stream_slicer,
|
3300
3094
|
config=config,
|
@@ -3538,6 +3332,30 @@ class ModelToComponentFactory:
|
|
3538
3332
|
matchers=matchers,
|
3539
3333
|
)
|
3540
3334
|
|
3335
|
+
def create_file_uploader(
|
3336
|
+
self, model: FileUploaderModel, config: Config, **kwargs: Any
|
3337
|
+
) -> FileUploader:
|
3338
|
+
name = "File Uploader"
|
3339
|
+
requester = self._create_component_from_model(
|
3340
|
+
model=model.requester,
|
3341
|
+
config=config,
|
3342
|
+
name=name,
|
3343
|
+
**kwargs,
|
3344
|
+
)
|
3345
|
+
download_target_extractor = self._create_component_from_model(
|
3346
|
+
model=model.download_target_extractor,
|
3347
|
+
config=config,
|
3348
|
+
name=name,
|
3349
|
+
**kwargs,
|
3350
|
+
)
|
3351
|
+
return FileUploader(
|
3352
|
+
requester=requester,
|
3353
|
+
download_target_extractor=download_target_extractor,
|
3354
|
+
config=config,
|
3355
|
+
parameters=model.parameters or {},
|
3356
|
+
filename_extractor=model.filename_extractor if model.filename_extractor else None,
|
3357
|
+
)
|
3358
|
+
|
3541
3359
|
def create_moving_window_call_rate_policy(
|
3542
3360
|
self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
|
3543
3361
|
) -> MovingWindowCallRatePolicy:
|
@@ -3587,34 +3405,3 @@ class ModelToComponentFactory:
|
|
3587
3405
|
self._api_budget = self.create_component(
|
3588
3406
|
model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config
|
3589
3407
|
)
|
3590
|
-
|
3591
|
-
def create_grouping_partition_router(
|
3592
|
-
self, model: GroupingPartitionRouterModel, config: Config, **kwargs: Any
|
3593
|
-
) -> GroupingPartitionRouter:
|
3594
|
-
underlying_router = self._create_component_from_model(
|
3595
|
-
model=model.underlying_partition_router, config=config
|
3596
|
-
)
|
3597
|
-
if model.group_size < 1:
|
3598
|
-
raise ValueError(f"Group size must be greater than 0, got {model.group_size}")
|
3599
|
-
|
3600
|
-
# Request options in underlying partition routers are not supported for GroupingPartitionRouter
|
3601
|
-
# because they are specific to individual partitions and cannot be aggregated or handled
|
3602
|
-
# when grouping, potentially leading to incorrect API calls. Any request customization
|
3603
|
-
# should be managed at the stream level through the requester's configuration.
|
3604
|
-
if isinstance(underlying_router, SubstreamPartitionRouter):
|
3605
|
-
if any(
|
3606
|
-
parent_config.request_option
|
3607
|
-
for parent_config in underlying_router.parent_stream_configs
|
3608
|
-
):
|
3609
|
-
raise ValueError("Request options are not supported for GroupingPartitionRouter.")
|
3610
|
-
|
3611
|
-
if isinstance(underlying_router, ListPartitionRouter):
|
3612
|
-
if underlying_router.request_option:
|
3613
|
-
raise ValueError("Request options are not supported for GroupingPartitionRouter.")
|
3614
|
-
|
3615
|
-
return GroupingPartitionRouter(
|
3616
|
-
group_size=model.group_size,
|
3617
|
-
underlying_partition_router=underlying_router,
|
3618
|
-
deduplicate=model.deduplicate if model.deduplicate is not None else True,
|
3619
|
-
config=config,
|
3620
|
-
)
|
@@ -8,9 +8,6 @@ from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_route
|
|
8
8
|
from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_slicer import (
|
9
9
|
CartesianProductStreamSlicer,
|
10
10
|
)
|
11
|
-
from airbyte_cdk.sources.declarative.partition_routers.grouping_partition_router import (
|
12
|
-
GroupingPartitionRouter,
|
13
|
-
)
|
14
11
|
from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import (
|
15
12
|
ListPartitionRouter,
|
16
13
|
)
|
@@ -25,7 +22,6 @@ from airbyte_cdk.sources.declarative.partition_routers.substream_partition_route
|
|
25
22
|
__all__ = [
|
26
23
|
"AsyncJobPartitionRouter",
|
27
24
|
"CartesianProductStreamSlicer",
|
28
|
-
"GroupingPartitionRouter",
|
29
25
|
"ListPartitionRouter",
|
30
26
|
"SinglePartitionRouter",
|
31
27
|
"SubstreamPartitionRouter",
|
@@ -374,11 +374,7 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
374
374
|
# Ignore per-partition states or invalid formats.
|
375
375
|
if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
|
376
376
|
# If a global state is present under the key "state", use its first value.
|
377
|
-
if (
|
378
|
-
"state" in stream_state
|
379
|
-
and isinstance(stream_state["state"], dict)
|
380
|
-
and stream_state["state"] != {}
|
381
|
-
):
|
377
|
+
if "state" in stream_state and isinstance(stream_state["state"], dict):
|
382
378
|
substream_state = list(stream_state["state"].values())[0]
|
383
379
|
else:
|
384
380
|
return {}
|
airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
#
|
2
|
-
# Copyright (c)
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass, field
|
6
|
-
from typing import Any,
|
6
|
+
from typing import Any, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
|
9
9
|
from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
|
@@ -40,7 +40,6 @@ class InterpolatedRequestOptionsProvider(RequestOptionsProvider):
|
|
40
40
|
request_headers: Optional[RequestInput] = None
|
41
41
|
request_body_data: Optional[RequestInput] = None
|
42
42
|
request_body_json: Optional[NestedMapping] = None
|
43
|
-
query_properties_key: Optional[str] = None
|
44
43
|
|
45
44
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
46
45
|
if self.request_parameters is None:
|
@@ -84,28 +83,6 @@ class InterpolatedRequestOptionsProvider(RequestOptionsProvider):
|
|
84
83
|
valid_value_types=ValidRequestTypes,
|
85
84
|
)
|
86
85
|
if isinstance(interpolated_value, dict):
|
87
|
-
if self.query_properties_key:
|
88
|
-
if not stream_slice:
|
89
|
-
raise ValueError(
|
90
|
-
"stream_slice should not be None if query properties in requests is enabled. Please contact Airbyte Support"
|
91
|
-
)
|
92
|
-
elif (
|
93
|
-
"query_properties" not in stream_slice.extra_fields
|
94
|
-
or stream_slice.extra_fields.get("query_properties") is None
|
95
|
-
):
|
96
|
-
raise ValueError(
|
97
|
-
"QueryProperties component is defined but stream_partition does not contain query_properties. Please contact Airbyte Support"
|
98
|
-
)
|
99
|
-
elif not isinstance(stream_slice.extra_fields.get("query_properties"), List):
|
100
|
-
raise ValueError(
|
101
|
-
"QueryProperties component is defined but stream_slice.extra_fields.query_properties is not a List. Please contact Airbyte Support"
|
102
|
-
)
|
103
|
-
interpolated_value = {
|
104
|
-
**interpolated_value,
|
105
|
-
self.query_properties_key: ",".join(
|
106
|
-
stream_slice.extra_fields.get("query_properties") # type: ignore # Earlier type checks validate query_properties type
|
107
|
-
),
|
108
|
-
}
|
109
86
|
return interpolated_value
|
110
87
|
return {}
|
111
88
|
|