airbyte-cdk 6.45.0__py3-none-any.whl → 6.45.0.dev4101__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. airbyte_cdk/connector_builder/connector_builder_handler.py +6 -45
  2. airbyte_cdk/connector_builder/main.py +2 -5
  3. airbyte_cdk/models/__init__.py +1 -0
  4. airbyte_cdk/models/airbyte_protocol.py +1 -3
  5. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +1 -1
  6. airbyte_cdk/sources/declarative/async_job/job.py +0 -6
  7. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +18 -18
  8. airbyte_cdk/sources/declarative/async_job/job_tracker.py +6 -22
  9. airbyte_cdk/sources/declarative/checks/__init__.py +2 -5
  10. airbyte_cdk/sources/declarative/checks/check_stream.py +11 -113
  11. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +8 -0
  12. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +50 -210
  13. airbyte_cdk/sources/declarative/extractors/record_selector.py +6 -1
  14. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +1 -2
  15. airbyte_cdk/sources/declarative/interpolation/macros.py +4 -8
  16. airbyte_cdk/sources/declarative/manifest_declarative_source.py +2 -23
  17. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +43 -142
  18. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +4 -16
  19. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +50 -263
  20. airbyte_cdk/sources/declarative/partition_routers/__init__.py +0 -4
  21. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +1 -5
  22. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +2 -25
  23. airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
  24. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +30 -101
  25. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +9 -4
  26. airbyte_cdk/sources/declarative/transformations/add_fields.py +1 -3
  27. airbyte_cdk/sources/file_based/file_based_stream_reader.py +32 -14
  28. airbyte_cdk/sources/file_based/file_record_data.py +24 -0
  29. airbyte_cdk/sources/file_based/file_types/file_transfer.py +8 -15
  30. airbyte_cdk/sources/file_based/schema_helpers.py +11 -1
  31. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +0 -1
  32. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +16 -31
  33. airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +1 -3
  34. airbyte_cdk/sources/streams/concurrent/default_stream.py +3 -0
  35. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +0 -4
  36. airbyte_cdk/sources/types.py +11 -2
  37. airbyte_cdk/sources/utils/files_directory.py +15 -0
  38. airbyte_cdk/sources/utils/record_helper.py +8 -8
  39. {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4101.dist-info}/METADATA +2 -2
  40. {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4101.dist-info}/RECORD +44 -50
  41. airbyte_cdk/models/file_transfer_record_message.py +0 -13
  42. airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py +0 -150
  43. airbyte_cdk/sources/declarative/requesters/query_properties/__init__.py +0 -13
  44. airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +0 -40
  45. airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py +0 -69
  46. airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +0 -58
  47. airbyte_cdk/sources/declarative/requesters/query_properties/strategies/__init__.py +0 -10
  48. airbyte_cdk/sources/declarative/requesters/query_properties/strategies/group_by_key.py +0 -33
  49. airbyte_cdk/sources/declarative/requesters/query_properties/strategies/merge_strategy.py +0 -19
  50. {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4101.dist-info}/LICENSE.txt +0 -0
  51. {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4101.dist-info}/LICENSE_SHORT +0 -0
  52. {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4101.dist-info}/WHEEL +0 -0
  53. {airbyte_cdk-6.45.0.dist-info → airbyte_cdk-6.45.0.dev4101.dist-info}/entry_points.txt +0 -0
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
3
  #
4
4
 
5
5
  from __future__ import annotations
@@ -54,11 +54,7 @@ from airbyte_cdk.sources.declarative.auth.token_provider import (
54
54
  SessionTokenProvider,
55
55
  TokenProvider,
56
56
  )
57
- from airbyte_cdk.sources.declarative.checks import (
58
- CheckDynamicStream,
59
- CheckStream,
60
- DynamicStreamCheckConfig,
61
- )
57
+ from airbyte_cdk.sources.declarative.checks import CheckDynamicStream, CheckStream
62
58
  from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
63
59
  from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
64
60
  from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
@@ -106,7 +102,6 @@ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_mi
106
102
  )
107
103
  from airbyte_cdk.sources.declarative.models import (
108
104
  CustomStateMigration,
109
- GzipDecoder,
110
105
  )
111
106
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
112
107
  AddedFieldDefinition as AddedFieldDefinitionModel,
@@ -223,10 +218,10 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
223
218
  DynamicSchemaLoader as DynamicSchemaLoaderModel,
224
219
  )
225
220
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
226
- DynamicStreamCheckConfig as DynamicStreamCheckConfigModel,
221
+ ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
227
222
  )
228
223
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
229
- ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
224
+ FileUploader as FileUploaderModel,
230
225
  )
231
226
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
232
227
  FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel,
@@ -234,12 +229,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
234
229
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
235
230
  FlattenFields as FlattenFieldsModel,
236
231
  )
237
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
238
- GroupByKeyMergeStrategy as GroupByKeyMergeStrategyModel,
239
- )
240
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
241
- GroupingPartitionRouter as GroupingPartitionRouterModel,
242
- )
243
232
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
244
233
  GzipDecoder as GzipDecoderModel,
245
234
  )
@@ -327,18 +316,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
327
316
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
328
317
  ParentStreamConfig as ParentStreamConfigModel,
329
318
  )
330
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
331
- PropertiesFromEndpoint as PropertiesFromEndpointModel,
332
- )
333
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
334
- PropertyChunking as PropertyChunkingModel,
335
- )
336
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
337
- PropertyLimitType as PropertyLimitTypeModel,
338
- )
339
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
340
- QueryProperties as QueryPropertiesModel,
341
- )
342
319
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
343
320
  Rate as RateModel,
344
321
  )
@@ -410,7 +387,6 @@ from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
410
387
  )
411
388
  from airbyte_cdk.sources.declarative.partition_routers import (
412
389
  CartesianProductStreamSlicer,
413
- GroupingPartitionRouter,
414
390
  ListPartitionRouter,
415
391
  PartitionRouter,
416
392
  SinglePartitionRouter,
@@ -447,17 +423,6 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies import (
447
423
  PageIncrement,
448
424
  StopConditionPaginationStrategyDecorator,
449
425
  )
450
- from airbyte_cdk.sources.declarative.requesters.query_properties import (
451
- PropertiesFromEndpoint,
452
- PropertyChunking,
453
- QueryProperties,
454
- )
455
- from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import (
456
- PropertyLimitType,
457
- )
458
- from airbyte_cdk.sources.declarative.requesters.query_properties.strategies import (
459
- GroupByKey,
460
- )
461
426
  from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType
462
427
  from airbyte_cdk.sources.declarative.requesters.request_options import (
463
428
  DatetimeBasedRequestOptionsProvider,
@@ -479,6 +444,7 @@ from airbyte_cdk.sources.declarative.retrievers import (
479
444
  SimpleRetriever,
480
445
  SimpleRetrieverTestReadDecorator,
481
446
  )
447
+ from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
482
448
  from airbyte_cdk.sources.declarative.schema import (
483
449
  ComplexFieldType,
484
450
  DefaultSchemaLoader,
@@ -592,7 +558,6 @@ class ModelToComponentFactory:
592
558
  BasicHttpAuthenticatorModel: self.create_basic_http_authenticator,
593
559
  BearerAuthenticatorModel: self.create_bearer_authenticator,
594
560
  CheckStreamModel: self.create_check_stream,
595
- DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config,
596
561
  CheckDynamicStreamModel: self.create_check_dynamic_stream,
597
562
  CompositeErrorHandlerModel: self.create_composite_error_handler,
598
563
  ConcurrencyLevelModel: self.create_concurrency_level,
@@ -622,7 +587,6 @@ class ModelToComponentFactory:
622
587
  ResponseToFileExtractorModel: self.create_response_to_file_extractor,
623
588
  ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy,
624
589
  SessionTokenAuthenticatorModel: self.create_session_token_authenticator,
625
- GroupByKeyMergeStrategyModel: self.create_group_by_key,
626
590
  HttpRequesterModel: self.create_http_requester,
627
591
  HttpResponseFilterModel: self.create_http_response_filter,
628
592
  InlineSchemaLoaderModel: self.create_inline_schema_loader,
@@ -652,9 +616,6 @@ class ModelToComponentFactory:
652
616
  OffsetIncrementModel: self.create_offset_increment,
653
617
  PageIncrementModel: self.create_page_increment,
654
618
  ParentStreamConfigModel: self.create_parent_stream_config,
655
- PropertiesFromEndpointModel: self.create_properties_from_endpoint,
656
- PropertyChunkingModel: self.create_property_chunking,
657
- QueryPropertiesModel: self.create_query_properties,
658
619
  RecordFilterModel: self.create_record_filter,
659
620
  RecordSelectorModel: self.create_record_selector,
660
621
  RemoveFieldsModel: self.create_remove_fields,
@@ -675,12 +636,12 @@ class ModelToComponentFactory:
675
636
  ComponentMappingDefinitionModel: self.create_components_mapping_definition,
676
637
  ZipfileDecoderModel: self.create_zipfile_decoder,
677
638
  HTTPAPIBudgetModel: self.create_http_api_budget,
639
+ FileUploaderModel: self.create_file_uploader,
678
640
  FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
679
641
  MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
680
642
  UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
681
643
  RateModel: self.create_rate,
682
644
  HttpRequestRegexMatcherModel: self.create_http_request_matcher,
683
- GroupingPartitionRouterModel: self.create_grouping_partition_router,
684
645
  }
685
646
 
686
647
  # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -974,36 +935,8 @@ class ModelToComponentFactory:
974
935
  )
975
936
 
976
937
  @staticmethod
977
- def create_dynamic_stream_check_config(
978
- model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any
979
- ) -> DynamicStreamCheckConfig:
980
- return DynamicStreamCheckConfig(
981
- dynamic_stream_name=model.dynamic_stream_name,
982
- stream_count=model.stream_count or 0,
983
- )
984
-
985
- def create_check_stream(
986
- self, model: CheckStreamModel, config: Config, **kwargs: Any
987
- ) -> CheckStream:
988
- if model.dynamic_streams_check_configs is None and model.stream_names is None:
989
- raise ValueError(
990
- "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream"
991
- )
992
-
993
- dynamic_streams_check_configs = (
994
- [
995
- self._create_component_from_model(model=dynamic_stream_check_config, config=config)
996
- for dynamic_stream_check_config in model.dynamic_streams_check_configs
997
- ]
998
- if model.dynamic_streams_check_configs
999
- else []
1000
- )
1001
-
1002
- return CheckStream(
1003
- stream_names=model.stream_names or [],
1004
- dynamic_streams_check_configs=dynamic_streams_check_configs,
1005
- parameters={},
1006
- )
938
+ def create_check_stream(model: CheckStreamModel, config: Config, **kwargs: Any) -> CheckStream:
939
+ return CheckStream(stream_names=model.stream_names, parameters={})
1007
940
 
1008
941
  @staticmethod
1009
942
  def create_check_dynamic_stream(
@@ -1426,9 +1359,6 @@ class ModelToComponentFactory:
1426
1359
  )
1427
1360
  stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
1428
1361
 
1429
- # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state
1430
- use_global_cursor = isinstance(partition_router, GroupingPartitionRouter)
1431
-
1432
1362
  # Return the concurrent cursor and state converter
1433
1363
  return ConcurrentPerPartitionCursor(
1434
1364
  cursor_factory=cursor_factory,
@@ -1440,7 +1370,6 @@ class ModelToComponentFactory:
1440
1370
  connector_state_manager=state_manager,
1441
1371
  connector_state_converter=connector_state_converter,
1442
1372
  cursor_field=cursor_field,
1443
- use_global_cursor=use_global_cursor,
1444
1373
  )
1445
1374
 
1446
1375
  @staticmethod
@@ -1826,6 +1755,11 @@ class ModelToComponentFactory:
1826
1755
  transformations.append(
1827
1756
  self._create_component_from_model(model=transformation_model, config=config)
1828
1757
  )
1758
+ file_uploader = None
1759
+ if model.file_uploader:
1760
+ file_uploader = self._create_component_from_model(
1761
+ model=model.file_uploader, config=config
1762
+ )
1829
1763
 
1830
1764
  retriever = self._create_component_from_model(
1831
1765
  model=model.retriever,
@@ -1837,6 +1771,7 @@ class ModelToComponentFactory:
1837
1771
  stop_condition_on_cursor=stop_condition_on_cursor,
1838
1772
  client_side_incremental_sync=client_side_incremental_sync,
1839
1773
  transformations=transformations,
1774
+ file_uploader=file_uploader,
1840
1775
  incremental_sync=model.incremental_sync,
1841
1776
  )
1842
1777
  cursor_field = model.incremental_sync.cursor_field if model.incremental_sync else None
@@ -2113,8 +2048,8 @@ class ModelToComponentFactory:
2113
2048
  parameters=model.parameters or {},
2114
2049
  )
2115
2050
 
2116
- @staticmethod
2117
2051
  def create_response_to_file_extractor(
2052
+ self,
2118
2053
  model: ResponseToFileExtractorModel,
2119
2054
  **kwargs: Any,
2120
2055
  ) -> ResponseToFileExtractor:
@@ -2128,17 +2063,11 @@ class ModelToComponentFactory:
2128
2063
  factor=model.factor or 5, parameters=model.parameters or {}, config=config
2129
2064
  )
2130
2065
 
2131
- @staticmethod
2132
- def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey:
2133
- return GroupByKey(model.key, config=config, parameters=model.parameters or {})
2134
-
2135
2066
  def create_http_requester(
2136
2067
  self,
2137
2068
  model: HttpRequesterModel,
2138
2069
  config: Config,
2139
2070
  decoder: Decoder = JsonDecoder(parameters={}),
2140
- query_properties_key: Optional[str] = None,
2141
- use_cache: Optional[bool] = None,
2142
2071
  *,
2143
2072
  name: str,
2144
2073
  ) -> HttpRequester:
@@ -2171,7 +2100,6 @@ class ModelToComponentFactory:
2171
2100
  request_body_json=model.request_body_json,
2172
2101
  request_headers=model.request_headers,
2173
2102
  request_parameters=model.request_parameters,
2174
- query_properties_key=query_properties_key,
2175
2103
  config=config,
2176
2104
  parameters=model.parameters or {},
2177
2105
  )
@@ -2179,7 +2107,7 @@ class ModelToComponentFactory:
2179
2107
  assert model.use_cache is not None # for mypy
2180
2108
  assert model.http_method is not None # for mypy
2181
2109
 
2182
- should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache
2110
+ use_cache = model.use_cache and not self._disable_cache
2183
2111
 
2184
2112
  return HttpRequester(
2185
2113
  name=name,
@@ -2194,7 +2122,7 @@ class ModelToComponentFactory:
2194
2122
  disable_retries=self._disable_retries,
2195
2123
  parameters=model.parameters or {},
2196
2124
  message_repository=self._message_repository,
2197
- use_cache=should_use_cache,
2125
+ use_cache=use_cache,
2198
2126
  decoder=decoder,
2199
2127
  stream_response=decoder.is_stream_response() if decoder else False,
2200
2128
  )
@@ -2298,11 +2226,10 @@ class ModelToComponentFactory:
2298
2226
  retriever = self._create_component_from_model(
2299
2227
  model=model.retriever,
2300
2228
  config=config,
2301
- name="dynamic_properties",
2229
+ name="",
2302
2230
  primary_key=None,
2303
2231
  stream_slicer=combined_slicers,
2304
2232
  transformations=[],
2305
- use_cache=True,
2306
2233
  )
2307
2234
  schema_type_identifier = self._create_component_from_model(
2308
2235
  model.schema_type_identifier, config=config, parameters=model.parameters or {}
@@ -2640,79 +2567,6 @@ class ModelToComponentFactory:
2640
2567
  lazy_read_pointer=model_lazy_read_pointer,
2641
2568
  )
2642
2569
 
2643
- def create_properties_from_endpoint(
2644
- self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any
2645
- ) -> PropertiesFromEndpoint:
2646
- retriever = self._create_component_from_model(
2647
- model=model.retriever,
2648
- config=config,
2649
- name="dynamic_properties",
2650
- primary_key=None,
2651
- stream_slicer=None,
2652
- transformations=[],
2653
- use_cache=True, # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different
2654
- )
2655
- return PropertiesFromEndpoint(
2656
- property_field_path=model.property_field_path,
2657
- retriever=retriever,
2658
- config=config,
2659
- parameters=model.parameters or {},
2660
- )
2661
-
2662
- def create_property_chunking(
2663
- self, model: PropertyChunkingModel, config: Config, **kwargs: Any
2664
- ) -> PropertyChunking:
2665
- record_merge_strategy = (
2666
- self._create_component_from_model(
2667
- model=model.record_merge_strategy, config=config, **kwargs
2668
- )
2669
- if model.record_merge_strategy
2670
- else None
2671
- )
2672
-
2673
- property_limit_type: PropertyLimitType
2674
- match model.property_limit_type:
2675
- case PropertyLimitTypeModel.property_count:
2676
- property_limit_type = PropertyLimitType.property_count
2677
- case PropertyLimitTypeModel.characters:
2678
- property_limit_type = PropertyLimitType.characters
2679
- case _:
2680
- raise ValueError(f"Invalid PropertyLimitType {property_limit_type}")
2681
-
2682
- return PropertyChunking(
2683
- property_limit_type=property_limit_type,
2684
- property_limit=model.property_limit,
2685
- record_merge_strategy=record_merge_strategy,
2686
- config=config,
2687
- parameters=model.parameters or {},
2688
- )
2689
-
2690
- def create_query_properties(
2691
- self, model: QueryPropertiesModel, config: Config, **kwargs: Any
2692
- ) -> QueryProperties:
2693
- if isinstance(model.property_list, list):
2694
- property_list = model.property_list
2695
- else:
2696
- property_list = self._create_component_from_model(
2697
- model=model.property_list, config=config, **kwargs
2698
- )
2699
-
2700
- property_chunking = (
2701
- self._create_component_from_model(
2702
- model=model.property_chunking, config=config, **kwargs
2703
- )
2704
- if model.property_chunking
2705
- else None
2706
- )
2707
-
2708
- return QueryProperties(
2709
- property_list=property_list,
2710
- always_include_properties=model.always_include_properties,
2711
- property_chunking=property_chunking,
2712
- config=config,
2713
- parameters=model.parameters or {},
2714
- )
2715
-
2716
2570
  @staticmethod
2717
2571
  def create_record_filter(
2718
2572
  model: RecordFilterModel, config: Config, **kwargs: Any
@@ -2759,6 +2613,7 @@ class ModelToComponentFactory:
2759
2613
  transformations: List[RecordTransformation] | None = None,
2760
2614
  decoder: Decoder | None = None,
2761
2615
  client_side_incremental_sync: Dict[str, Any] | None = None,
2616
+ file_uploader: Optional[FileUploader] = None,
2762
2617
  **kwargs: Any,
2763
2618
  ) -> RecordSelector:
2764
2619
  extractor = self._create_component_from_model(
@@ -2796,6 +2651,7 @@ class ModelToComponentFactory:
2796
2651
  config=config,
2797
2652
  record_filter=record_filter,
2798
2653
  transformations=transformations or [],
2654
+ file_uploader=file_uploader,
2799
2655
  schema_normalization=schema_normalization,
2800
2656
  parameters=model.parameters or {},
2801
2657
  transform_before_filtering=transform_before_filtering,
@@ -2853,12 +2709,12 @@ class ModelToComponentFactory:
2853
2709
  stop_condition_on_cursor: bool = False,
2854
2710
  client_side_incremental_sync: Optional[Dict[str, Any]] = None,
2855
2711
  transformations: List[RecordTransformation],
2712
+ file_uploader: Optional[FileUploader] = None,
2856
2713
  incremental_sync: Optional[
2857
2714
  Union[
2858
2715
  IncrementingCountCursorModel, DatetimeBasedCursorModel, CustomIncrementalSyncModel
2859
2716
  ]
2860
2717
  ] = None,
2861
- use_cache: Optional[bool] = None,
2862
2718
  **kwargs: Any,
2863
2719
  ) -> SimpleRetriever:
2864
2720
  decoder = (
@@ -2866,6 +2722,9 @@ class ModelToComponentFactory:
2866
2722
  if model.decoder
2867
2723
  else JsonDecoder(parameters={})
2868
2724
  )
2725
+ requester = self._create_component_from_model(
2726
+ model=model.requester, decoder=decoder, config=config, name=name
2727
+ )
2869
2728
  record_selector = self._create_component_from_model(
2870
2729
  model=model.record_selector,
2871
2730
  name=name,
@@ -2873,57 +2732,7 @@ class ModelToComponentFactory:
2873
2732
  decoder=decoder,
2874
2733
  transformations=transformations,
2875
2734
  client_side_incremental_sync=client_side_incremental_sync,
2876
- )
2877
-
2878
- query_properties: Optional[QueryProperties] = None
2879
- query_properties_key: Optional[str] = None
2880
- if (
2881
- hasattr(model.requester, "request_parameters")
2882
- and model.requester.request_parameters
2883
- and isinstance(model.requester.request_parameters, Mapping)
2884
- ):
2885
- query_properties_definitions = []
2886
- for key, request_parameter in model.requester.request_parameters.items():
2887
- # When translating JSON schema into Pydantic models, enforcing types for arrays containing both
2888
- # concrete string complex object definitions like QueryProperties would get resolved to Union[str, Any].
2889
- # This adds the extra validation that we couldn't get for free in Pydantic model generation
2890
- if (
2891
- isinstance(request_parameter, Mapping)
2892
- and request_parameter.get("type") == "QueryProperties"
2893
- ):
2894
- query_properties_key = key
2895
- query_properties_definitions.append(request_parameter)
2896
- elif not isinstance(request_parameter, str):
2897
- raise ValueError(
2898
- f"Each element of request_parameters should be of type str or QueryProperties, but received {request_parameter.get('type')}"
2899
- )
2900
-
2901
- if len(query_properties_definitions) > 1:
2902
- raise ValueError(
2903
- f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages"
2904
- )
2905
-
2906
- if len(query_properties_definitions) == 1:
2907
- query_properties = self.create_component(
2908
- model_type=QueryPropertiesModel,
2909
- component_definition=query_properties_definitions[0],
2910
- config=config,
2911
- )
2912
-
2913
- # Removes QueryProperties components from the interpolated mappings because it will be resolved in
2914
- # the provider from the slice directly instead of through jinja interpolation
2915
- if isinstance(model.requester.request_parameters, Mapping):
2916
- model.requester.request_parameters = self._remove_query_properties(
2917
- model.requester.request_parameters
2918
- )
2919
-
2920
- requester = self._create_component_from_model(
2921
- model=model.requester,
2922
- decoder=decoder,
2923
- name=name,
2924
- query_properties_key=query_properties_key,
2925
- use_cache=use_cache,
2926
- config=config,
2735
+ file_uploader=file_uploader,
2927
2736
  )
2928
2737
  url_base = (
2929
2738
  model.requester.url_base
@@ -3030,21 +2839,9 @@ class ModelToComponentFactory:
3030
2839
  cursor=cursor,
3031
2840
  config=config,
3032
2841
  ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
3033
- additional_query_properties=query_properties,
3034
2842
  parameters=model.parameters or {},
3035
2843
  )
3036
2844
 
3037
- @staticmethod
3038
- def _remove_query_properties(
3039
- request_parameters: Mapping[str, Union[Any, str]],
3040
- ) -> Mapping[str, Union[Any, str]]:
3041
- return {
3042
- parameter_field: request_parameter
3043
- for parameter_field, request_parameter in request_parameters.items()
3044
- if not isinstance(request_parameter, Mapping)
3045
- or not request_parameter.get("type") == "QueryProperties"
3046
- }
3047
-
3048
2845
  def create_state_delegating_stream(
3049
2846
  self,
3050
2847
  model: StateDelegatingStreamModel,
@@ -3290,11 +3087,8 @@ class ModelToComponentFactory:
3290
3087
  stream_slices,
3291
3088
  self._job_tracker,
3292
3089
  self._message_repository,
3293
- # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
3294
3090
  has_bulk_parent=False,
3295
- # set the `job_max_retry` to 1 for the `Connector Builder`` use-case.
3296
- # `None` == default retry is set to 3 attempts, under the hood.
3297
- job_max_retry=1 if self._emit_connector_builder_messages else None,
3091
+ # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
3298
3092
  ),
3299
3093
  stream_slicer=stream_slicer,
3300
3094
  config=config,
@@ -3538,6 +3332,30 @@ class ModelToComponentFactory:
3538
3332
  matchers=matchers,
3539
3333
  )
3540
3334
 
3335
+ def create_file_uploader(
3336
+ self, model: FileUploaderModel, config: Config, **kwargs: Any
3337
+ ) -> FileUploader:
3338
+ name = "File Uploader"
3339
+ requester = self._create_component_from_model(
3340
+ model=model.requester,
3341
+ config=config,
3342
+ name=name,
3343
+ **kwargs,
3344
+ )
3345
+ download_target_extractor = self._create_component_from_model(
3346
+ model=model.download_target_extractor,
3347
+ config=config,
3348
+ name=name,
3349
+ **kwargs,
3350
+ )
3351
+ return FileUploader(
3352
+ requester=requester,
3353
+ download_target_extractor=download_target_extractor,
3354
+ config=config,
3355
+ parameters=model.parameters or {},
3356
+ filename_extractor=model.filename_extractor if model.filename_extractor else None,
3357
+ )
3358
+
3541
3359
  def create_moving_window_call_rate_policy(
3542
3360
  self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
3543
3361
  ) -> MovingWindowCallRatePolicy:
@@ -3587,34 +3405,3 @@ class ModelToComponentFactory:
3587
3405
  self._api_budget = self.create_component(
3588
3406
  model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config
3589
3407
  )
3590
-
3591
- def create_grouping_partition_router(
3592
- self, model: GroupingPartitionRouterModel, config: Config, **kwargs: Any
3593
- ) -> GroupingPartitionRouter:
3594
- underlying_router = self._create_component_from_model(
3595
- model=model.underlying_partition_router, config=config
3596
- )
3597
- if model.group_size < 1:
3598
- raise ValueError(f"Group size must be greater than 0, got {model.group_size}")
3599
-
3600
- # Request options in underlying partition routers are not supported for GroupingPartitionRouter
3601
- # because they are specific to individual partitions and cannot be aggregated or handled
3602
- # when grouping, potentially leading to incorrect API calls. Any request customization
3603
- # should be managed at the stream level through the requester's configuration.
3604
- if isinstance(underlying_router, SubstreamPartitionRouter):
3605
- if any(
3606
- parent_config.request_option
3607
- for parent_config in underlying_router.parent_stream_configs
3608
- ):
3609
- raise ValueError("Request options are not supported for GroupingPartitionRouter.")
3610
-
3611
- if isinstance(underlying_router, ListPartitionRouter):
3612
- if underlying_router.request_option:
3613
- raise ValueError("Request options are not supported for GroupingPartitionRouter.")
3614
-
3615
- return GroupingPartitionRouter(
3616
- group_size=model.group_size,
3617
- underlying_partition_router=underlying_router,
3618
- deduplicate=model.deduplicate if model.deduplicate is not None else True,
3619
- config=config,
3620
- )
@@ -8,9 +8,6 @@ from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_route
8
8
  from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_slicer import (
9
9
  CartesianProductStreamSlicer,
10
10
  )
11
- from airbyte_cdk.sources.declarative.partition_routers.grouping_partition_router import (
12
- GroupingPartitionRouter,
13
- )
14
11
  from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import (
15
12
  ListPartitionRouter,
16
13
  )
@@ -25,7 +22,6 @@ from airbyte_cdk.sources.declarative.partition_routers.substream_partition_route
25
22
  __all__ = [
26
23
  "AsyncJobPartitionRouter",
27
24
  "CartesianProductStreamSlicer",
28
- "GroupingPartitionRouter",
29
25
  "ListPartitionRouter",
30
26
  "SinglePartitionRouter",
31
27
  "SubstreamPartitionRouter",
@@ -374,11 +374,7 @@ class SubstreamPartitionRouter(PartitionRouter):
374
374
  # Ignore per-partition states or invalid formats.
375
375
  if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
376
376
  # If a global state is present under the key "state", use its first value.
377
- if (
378
- "state" in stream_state
379
- and isinstance(stream_state["state"], dict)
380
- and stream_state["state"] != {}
381
- ):
377
+ if "state" in stream_state and isinstance(stream_state["state"], dict):
382
378
  substream_state = list(stream_state["state"].values())[0]
383
379
  else:
384
380
  return {}
@@ -1,9 +1,9 @@
1
1
  #
2
- # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
3
  #
4
4
 
5
5
  from dataclasses import InitVar, dataclass, field
6
- from typing import Any, List, Mapping, MutableMapping, Optional, Union
6
+ from typing import Any, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
9
9
  from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
@@ -40,7 +40,6 @@ class InterpolatedRequestOptionsProvider(RequestOptionsProvider):
40
40
  request_headers: Optional[RequestInput] = None
41
41
  request_body_data: Optional[RequestInput] = None
42
42
  request_body_json: Optional[NestedMapping] = None
43
- query_properties_key: Optional[str] = None
44
43
 
45
44
  def __post_init__(self, parameters: Mapping[str, Any]) -> None:
46
45
  if self.request_parameters is None:
@@ -84,28 +83,6 @@ class InterpolatedRequestOptionsProvider(RequestOptionsProvider):
84
83
  valid_value_types=ValidRequestTypes,
85
84
  )
86
85
  if isinstance(interpolated_value, dict):
87
- if self.query_properties_key:
88
- if not stream_slice:
89
- raise ValueError(
90
- "stream_slice should not be None if query properties in requests is enabled. Please contact Airbyte Support"
91
- )
92
- elif (
93
- "query_properties" not in stream_slice.extra_fields
94
- or stream_slice.extra_fields.get("query_properties") is None
95
- ):
96
- raise ValueError(
97
- "QueryProperties component is defined but stream_partition does not contain query_properties. Please contact Airbyte Support"
98
- )
99
- elif not isinstance(stream_slice.extra_fields.get("query_properties"), List):
100
- raise ValueError(
101
- "QueryProperties component is defined but stream_slice.extra_fields.query_properties is not a List. Please contact Airbyte Support"
102
- )
103
- interpolated_value = {
104
- **interpolated_value,
105
- self.query_properties_key: ",".join(
106
- stream_slice.extra_fields.get("query_properties") # type: ignore # Earlier type checks validate query_properties type
107
- ),
108
- }
109
86
  return interpolated_value
110
87
  return {}
111
88