airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.5.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. airbyte_cdk/__init__.py +17 -2
  2. airbyte_cdk/config_observation.py +10 -3
  3. airbyte_cdk/connector.py +19 -9
  4. airbyte_cdk/connector_builder/connector_builder_handler.py +28 -8
  5. airbyte_cdk/connector_builder/main.py +26 -6
  6. airbyte_cdk/connector_builder/message_grouper.py +95 -25
  7. airbyte_cdk/destinations/destination.py +47 -14
  8. airbyte_cdk/destinations/vector_db_based/config.py +36 -14
  9. airbyte_cdk/destinations/vector_db_based/document_processor.py +49 -11
  10. airbyte_cdk/destinations/vector_db_based/embedder.py +52 -11
  11. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  12. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  13. airbyte_cdk/destinations/vector_db_based/writer.py +15 -4
  14. airbyte_cdk/entrypoint.py +82 -26
  15. airbyte_cdk/exception_handler.py +13 -3
  16. airbyte_cdk/logger.py +10 -2
  17. airbyte_cdk/models/airbyte_protocol.py +11 -5
  18. airbyte_cdk/models/airbyte_protocol_serializers.py +9 -3
  19. airbyte_cdk/models/well_known_types.py +1 -1
  20. airbyte_cdk/sources/abstract_source.py +63 -17
  21. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +47 -14
  22. airbyte_cdk/sources/concurrent_source/concurrent_source.py +25 -7
  23. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +27 -6
  24. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +9 -3
  25. airbyte_cdk/sources/connector_state_manager.py +32 -10
  26. airbyte_cdk/sources/declarative/async_job/job.py +3 -1
  27. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +68 -14
  28. airbyte_cdk/sources/declarative/async_job/job_tracker.py +24 -6
  29. airbyte_cdk/sources/declarative/async_job/repository.py +3 -1
  30. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  31. airbyte_cdk/sources/declarative/auth/jwt.py +27 -7
  32. airbyte_cdk/sources/declarative/auth/oauth.py +35 -11
  33. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +3 -1
  34. airbyte_cdk/sources/declarative/auth/token.py +25 -8
  35. airbyte_cdk/sources/declarative/checks/check_stream.py +12 -4
  36. airbyte_cdk/sources/declarative/checks/connection_checker.py +3 -1
  37. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +11 -3
  38. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +106 -50
  39. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +20 -6
  40. airbyte_cdk/sources/declarative/declarative_source.py +3 -1
  41. airbyte_cdk/sources/declarative/declarative_stream.py +27 -6
  42. airbyte_cdk/sources/declarative/decoders/decoder.py +3 -1
  43. airbyte_cdk/sources/declarative/decoders/json_decoder.py +3 -1
  44. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +3 -1
  45. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +6 -2
  46. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +6 -2
  47. airbyte_cdk/sources/declarative/extractors/record_filter.py +24 -7
  48. airbyte_cdk/sources/declarative/extractors/record_selector.py +10 -3
  49. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +15 -5
  50. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +96 -31
  51. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +22 -8
  52. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +46 -15
  53. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +19 -5
  54. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +3 -1
  55. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +20 -2
  56. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +5 -1
  57. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +10 -3
  58. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +6 -2
  59. airbyte_cdk/sources/declarative/interpolation/interpolation.py +7 -1
  60. airbyte_cdk/sources/declarative/interpolation/jinja.py +6 -2
  61. airbyte_cdk/sources/declarative/interpolation/macros.py +19 -4
  62. airbyte_cdk/sources/declarative/manifest_declarative_source.py +106 -24
  63. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +7 -2
  64. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +656 -678
  65. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +13 -4
  66. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +9 -2
  67. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +782 -232
  68. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +29 -7
  69. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +25 -7
  70. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +54 -15
  71. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +6 -2
  72. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +3 -1
  73. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +17 -5
  74. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +15 -5
  75. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +3 -1
  76. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +18 -8
  77. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +16 -7
  78. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +51 -14
  79. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +29 -8
  80. airbyte_cdk/sources/declarative/requesters/http_requester.py +58 -16
  81. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +49 -14
  82. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +3 -1
  83. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +3 -1
  84. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +17 -5
  85. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +24 -7
  86. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +9 -3
  87. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +3 -1
  88. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +6 -2
  89. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +19 -6
  90. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +3 -1
  91. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +21 -7
  92. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +18 -6
  93. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +27 -8
  94. airbyte_cdk/sources/declarative/requesters/requester.py +3 -1
  95. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +12 -5
  96. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +105 -24
  97. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +3 -1
  98. airbyte_cdk/sources/declarative/spec/spec.py +8 -2
  99. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +3 -1
  100. airbyte_cdk/sources/declarative/transformations/add_fields.py +12 -3
  101. airbyte_cdk/sources/declarative/transformations/remove_fields.py +6 -2
  102. airbyte_cdk/sources/declarative/types.py +8 -1
  103. airbyte_cdk/sources/declarative/yaml_declarative_source.py +3 -1
  104. airbyte_cdk/sources/embedded/base_integration.py +14 -4
  105. airbyte_cdk/sources/embedded/catalog.py +16 -4
  106. airbyte_cdk/sources/embedded/runner.py +19 -3
  107. airbyte_cdk/sources/embedded/tools.py +3 -1
  108. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +12 -4
  109. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +27 -7
  110. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +12 -6
  111. airbyte_cdk/sources/file_based/config/csv_format.py +21 -9
  112. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +6 -2
  113. airbyte_cdk/sources/file_based/config/unstructured_format.py +10 -3
  114. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  115. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  116. airbyte_cdk/sources/file_based/exceptions.py +13 -15
  117. airbyte_cdk/sources/file_based/file_based_source.py +82 -24
  118. airbyte_cdk/sources/file_based/file_based_stream_reader.py +16 -5
  119. airbyte_cdk/sources/file_based/file_types/avro_parser.py +58 -17
  120. airbyte_cdk/sources/file_based/file_types/csv_parser.py +89 -26
  121. airbyte_cdk/sources/file_based/file_types/excel_parser.py +25 -7
  122. airbyte_cdk/sources/file_based/file_types/file_transfer.py +8 -2
  123. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  124. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +20 -6
  125. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +57 -16
  126. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +64 -15
  127. airbyte_cdk/sources/file_based/schema_helpers.py +33 -10
  128. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  129. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  130. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +33 -10
  131. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +47 -11
  132. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +13 -22
  133. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +53 -17
  134. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +17 -5
  135. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  136. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +26 -9
  137. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +67 -21
  138. airbyte_cdk/sources/http_logger.py +5 -1
  139. airbyte_cdk/sources/message/repository.py +18 -4
  140. airbyte_cdk/sources/source.py +17 -7
  141. airbyte_cdk/sources/streams/availability_strategy.py +9 -3
  142. airbyte_cdk/sources/streams/call_rate.py +63 -19
  143. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +31 -7
  144. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +6 -2
  145. airbyte_cdk/sources/streams/concurrent/adapters.py +77 -22
  146. airbyte_cdk/sources/streams/concurrent/cursor.py +56 -20
  147. airbyte_cdk/sources/streams/concurrent/default_stream.py +9 -2
  148. airbyte_cdk/sources/streams/concurrent/helpers.py +6 -2
  149. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +9 -2
  150. airbyte_cdk/sources/streams/concurrent/partition_reader.py +4 -1
  151. airbyte_cdk/sources/streams/concurrent/partitions/record.py +10 -2
  152. airbyte_cdk/sources/streams/concurrent/partitions/types.py +6 -2
  153. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +25 -10
  154. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +32 -16
  155. airbyte_cdk/sources/streams/core.py +77 -22
  156. airbyte_cdk/sources/streams/http/availability_strategy.py +3 -1
  157. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +4 -1
  158. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +3 -1
  159. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +16 -5
  160. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +9 -3
  161. airbyte_cdk/sources/streams/http/exceptions.py +2 -2
  162. airbyte_cdk/sources/streams/http/http.py +133 -33
  163. airbyte_cdk/sources/streams/http/http_client.py +91 -29
  164. airbyte_cdk/sources/streams/http/rate_limiting.py +23 -7
  165. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +19 -6
  166. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +38 -11
  167. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  168. airbyte_cdk/sources/types.py +5 -1
  169. airbyte_cdk/sources/utils/record_helper.py +12 -3
  170. airbyte_cdk/sources/utils/schema_helpers.py +9 -3
  171. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  172. airbyte_cdk/sources/utils/transform.py +24 -9
  173. airbyte_cdk/sql/exceptions.py +19 -6
  174. airbyte_cdk/sql/secrets.py +3 -1
  175. airbyte_cdk/sql/shared/catalog_providers.py +13 -4
  176. airbyte_cdk/sql/shared/sql_processor.py +44 -14
  177. airbyte_cdk/test/catalog_builder.py +19 -8
  178. airbyte_cdk/test/entrypoint_wrapper.py +27 -8
  179. airbyte_cdk/test/mock_http/mocker.py +41 -11
  180. airbyte_cdk/test/mock_http/request.py +9 -3
  181. airbyte_cdk/test/mock_http/response.py +3 -1
  182. airbyte_cdk/test/mock_http/response_builder.py +29 -7
  183. airbyte_cdk/test/state_builder.py +10 -2
  184. airbyte_cdk/test/utils/data.py +6 -2
  185. airbyte_cdk/test/utils/http_mocking.py +3 -1
  186. airbyte_cdk/utils/airbyte_secrets_utils.py +3 -1
  187. airbyte_cdk/utils/analytics_message.py +10 -2
  188. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  189. airbyte_cdk/utils/mapping_helpers.py +3 -1
  190. airbyte_cdk/utils/message_utils.py +11 -4
  191. airbyte_cdk/utils/print_buffer.py +6 -1
  192. airbyte_cdk/utils/schema_inferrer.py +30 -9
  193. airbyte_cdk/utils/spec_schema_transformations.py +3 -1
  194. airbyte_cdk/utils/traced_exception.py +35 -9
  195. {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/METADATA +7 -6
  196. {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/RECORD +198 -198
  197. {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/LICENSE.txt +0 -0
  198. {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/WHEEL +0 -0
@@ -9,7 +9,21 @@ import importlib
9
9
  import inspect
10
10
  import re
11
11
  from functools import partial
12
- from typing import Any, Callable, Dict, List, Mapping, MutableMapping, Optional, Tuple, Type, Union, get_args, get_origin, get_type_hints
12
+ from typing import (
13
+ Any,
14
+ Callable,
15
+ Dict,
16
+ List,
17
+ Mapping,
18
+ MutableMapping,
19
+ Optional,
20
+ Tuple,
21
+ Type,
22
+ Union,
23
+ get_args,
24
+ get_origin,
25
+ get_type_hints,
26
+ )
13
27
 
14
28
  from airbyte_cdk.models import FailureType, Level
15
29
  from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
@@ -18,9 +32,14 @@ from airbyte_cdk.sources.declarative.async_job.job_tracker import JobTracker
18
32
  from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository
19
33
  from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus
20
34
  from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator, JwtAuthenticator
21
- from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
35
+ from airbyte_cdk.sources.declarative.auth.declarative_authenticator import (
36
+ DeclarativeAuthenticator,
37
+ NoAuth,
38
+ )
22
39
  from airbyte_cdk.sources.declarative.auth.jwt import JwtAlgorithm
23
- from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
40
+ from airbyte_cdk.sources.declarative.auth.oauth import (
41
+ DeclarativeSingleUseRefreshTokenOauth2Authenticator,
42
+ )
24
43
  from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator
25
44
  from airbyte_cdk.sources.declarative.auth.token import (
26
45
  ApiKeyAuthenticator,
@@ -28,7 +47,11 @@ from airbyte_cdk.sources.declarative.auth.token import (
28
47
  BearerAuthenticator,
29
48
  LegacySessionTokenAuthenticator,
30
49
  )
31
- from airbyte_cdk.sources.declarative.auth.token_provider import InterpolatedStringTokenProvider, SessionTokenProvider, TokenProvider
50
+ from airbyte_cdk.sources.declarative.auth.token_provider import (
51
+ InterpolatedStringTokenProvider,
52
+ SessionTokenProvider,
53
+ TokenProvider,
54
+ )
32
55
  from airbyte_cdk.sources.declarative.checks import CheckStream
33
56
  from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
34
57
  from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
@@ -41,9 +64,18 @@ from airbyte_cdk.sources.declarative.decoders import (
41
64
  PaginationDecoderDecorator,
42
65
  XmlDecoder,
43
66
  )
44
- from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector, ResponseToFileExtractor
45
- from airbyte_cdk.sources.declarative.extractors.record_filter import ClientSideIncrementalRecordFilterDecorator
46
- from airbyte_cdk.sources.declarative.extractors.record_selector import SCHEMA_TRANSFORMER_TYPE_MAPPING
67
+ from airbyte_cdk.sources.declarative.extractors import (
68
+ DpathExtractor,
69
+ RecordFilter,
70
+ RecordSelector,
71
+ ResponseToFileExtractor,
72
+ )
73
+ from airbyte_cdk.sources.declarative.extractors.record_filter import (
74
+ ClientSideIncrementalRecordFilterDecorator,
75
+ )
76
+ from airbyte_cdk.sources.declarative.extractors.record_selector import (
77
+ SCHEMA_TRANSFORMER_TYPE_MAPPING,
78
+ )
47
79
  from airbyte_cdk.sources.declarative.incremental import (
48
80
  ChildPartitionResumableFullRefreshCursor,
49
81
  CursorFactory,
@@ -56,88 +88,216 @@ from airbyte_cdk.sources.declarative.incremental import (
56
88
  )
57
89
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
58
90
  from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
59
- from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import LegacyToPerPartitionStateMigration
91
+ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import (
92
+ LegacyToPerPartitionStateMigration,
93
+ )
60
94
  from airbyte_cdk.sources.declarative.models import CustomStateMigration
61
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import AddedFieldDefinition as AddedFieldDefinitionModel
62
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import AddFields as AddFieldsModel
63
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import ApiKeyAuthenticator as ApiKeyAuthenticatorModel
64
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import AsyncJobStatusMap as AsyncJobStatusMapModel
65
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import AsyncRetriever as AsyncRetrieverModel
66
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import BasicHttpAuthenticator as BasicHttpAuthenticatorModel
67
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import BearerAuthenticator as BearerAuthenticatorModel
68
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CheckStream as CheckStreamModel
69
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CompositeErrorHandler as CompositeErrorHandlerModel
70
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import ConcurrencyLevel as ConcurrencyLevelModel
71
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import ConstantBackoffStrategy as ConstantBackoffStrategyModel
72
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CursorPagination as CursorPaginationModel
73
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomAuthenticator as CustomAuthenticatorModel
74
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomBackoffStrategy as CustomBackoffStrategyModel
75
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomErrorHandler as CustomErrorHandlerModel
76
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomIncrementalSync as CustomIncrementalSyncModel
77
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomPaginationStrategy as CustomPaginationStrategyModel
78
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomPartitionRouter as CustomPartitionRouterModel
79
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRecordExtractor as CustomRecordExtractorModel
80
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRecordFilter as CustomRecordFilterModel
81
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRequester as CustomRequesterModel
82
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRetriever as CustomRetrieverModel
83
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomSchemaLoader as CustomSchemaLoader
84
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomTransformation as CustomTransformationModel
85
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import DatetimeBasedCursor as DatetimeBasedCursorModel
86
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import DeclarativeStream as DeclarativeStreamModel
87
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import DefaultErrorHandler as DefaultErrorHandlerModel
88
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import DefaultPaginator as DefaultPaginatorModel
89
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import DpathExtractor as DpathExtractorModel
95
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
96
+ AddedFieldDefinition as AddedFieldDefinitionModel,
97
+ )
98
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
99
+ AddFields as AddFieldsModel,
100
+ )
101
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
102
+ ApiKeyAuthenticator as ApiKeyAuthenticatorModel,
103
+ )
104
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
105
+ AsyncJobStatusMap as AsyncJobStatusMapModel,
106
+ )
107
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
108
+ AsyncRetriever as AsyncRetrieverModel,
109
+ )
110
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
111
+ BasicHttpAuthenticator as BasicHttpAuthenticatorModel,
112
+ )
113
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
114
+ BearerAuthenticator as BearerAuthenticatorModel,
115
+ )
116
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
117
+ CheckStream as CheckStreamModel,
118
+ )
119
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
120
+ CompositeErrorHandler as CompositeErrorHandlerModel,
121
+ )
122
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
123
+ ConcurrencyLevel as ConcurrencyLevelModel,
124
+ )
125
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
126
+ ConstantBackoffStrategy as ConstantBackoffStrategyModel,
127
+ )
128
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
129
+ CursorPagination as CursorPaginationModel,
130
+ )
131
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
132
+ CustomAuthenticator as CustomAuthenticatorModel,
133
+ )
134
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
135
+ CustomBackoffStrategy as CustomBackoffStrategyModel,
136
+ )
137
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
138
+ CustomErrorHandler as CustomErrorHandlerModel,
139
+ )
140
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
141
+ CustomIncrementalSync as CustomIncrementalSyncModel,
142
+ )
143
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
144
+ CustomPaginationStrategy as CustomPaginationStrategyModel,
145
+ )
146
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
147
+ CustomPartitionRouter as CustomPartitionRouterModel,
148
+ )
149
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
150
+ CustomRecordExtractor as CustomRecordExtractorModel,
151
+ )
152
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
153
+ CustomRecordFilter as CustomRecordFilterModel,
154
+ )
155
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
156
+ CustomRequester as CustomRequesterModel,
157
+ )
158
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
159
+ CustomRetriever as CustomRetrieverModel,
160
+ )
161
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
162
+ CustomSchemaLoader as CustomSchemaLoader,
163
+ )
164
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
165
+ CustomTransformation as CustomTransformationModel,
166
+ )
167
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
168
+ DatetimeBasedCursor as DatetimeBasedCursorModel,
169
+ )
170
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
171
+ DeclarativeStream as DeclarativeStreamModel,
172
+ )
173
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
174
+ DefaultErrorHandler as DefaultErrorHandlerModel,
175
+ )
176
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
177
+ DefaultPaginator as DefaultPaginatorModel,
178
+ )
179
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
180
+ DpathExtractor as DpathExtractorModel,
181
+ )
90
182
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
91
183
  ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
92
184
  )
93
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import HttpRequester as HttpRequesterModel
94
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import HttpResponseFilter as HttpResponseFilterModel
95
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import InlineSchemaLoader as InlineSchemaLoaderModel
96
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import IterableDecoder as IterableDecoderModel
97
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import JsonDecoder as JsonDecoderModel
98
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import JsonFileSchemaLoader as JsonFileSchemaLoaderModel
99
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import JsonlDecoder as JsonlDecoderModel
100
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import JwtAuthenticator as JwtAuthenticatorModel
101
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import JwtHeaders as JwtHeadersModel
102
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import JwtPayload as JwtPayloadModel
103
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import KeysToLower as KeysToLowerModel
185
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
186
+ HttpRequester as HttpRequesterModel,
187
+ )
188
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
189
+ HttpResponseFilter as HttpResponseFilterModel,
190
+ )
191
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
192
+ InlineSchemaLoader as InlineSchemaLoaderModel,
193
+ )
194
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
195
+ IterableDecoder as IterableDecoderModel,
196
+ )
197
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
198
+ JsonDecoder as JsonDecoderModel,
199
+ )
200
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
201
+ JsonFileSchemaLoader as JsonFileSchemaLoaderModel,
202
+ )
203
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
204
+ JsonlDecoder as JsonlDecoderModel,
205
+ )
206
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
207
+ JwtAuthenticator as JwtAuthenticatorModel,
208
+ )
209
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
210
+ JwtHeaders as JwtHeadersModel,
211
+ )
212
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
213
+ JwtPayload as JwtPayloadModel,
214
+ )
215
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
216
+ KeysToLower as KeysToLowerModel,
217
+ )
104
218
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
105
219
  LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel,
106
220
  )
107
221
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
108
222
  LegacyToPerPartitionStateMigration as LegacyToPerPartitionStateMigrationModel,
109
223
  )
110
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import ListPartitionRouter as ListPartitionRouterModel
111
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import MinMaxDatetime as MinMaxDatetimeModel
112
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import NoAuth as NoAuthModel
113
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import NoPagination as NoPaginationModel
114
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import OAuthAuthenticator as OAuthAuthenticatorModel
115
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel
116
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import PageIncrement as PageIncrementModel
117
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig as ParentStreamConfigModel
118
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import RecordFilter as RecordFilterModel
119
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import RecordSelector as RecordSelectorModel
120
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import RemoveFields as RemoveFieldsModel
121
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestOption as RequestOptionModel
122
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestPath as RequestPathModel
123
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator as SelectiveAuthenticatorModel
124
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import SessionTokenAuthenticator as SessionTokenAuthenticatorModel
125
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import SimpleRetriever as SimpleRetrieverModel
224
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
225
+ ListPartitionRouter as ListPartitionRouterModel,
226
+ )
227
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
228
+ MinMaxDatetime as MinMaxDatetimeModel,
229
+ )
230
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
231
+ NoAuth as NoAuthModel,
232
+ )
233
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
234
+ NoPagination as NoPaginationModel,
235
+ )
236
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
237
+ OAuthAuthenticator as OAuthAuthenticatorModel,
238
+ )
239
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
240
+ OffsetIncrement as OffsetIncrementModel,
241
+ )
242
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
243
+ PageIncrement as PageIncrementModel,
244
+ )
245
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
246
+ ParentStreamConfig as ParentStreamConfigModel,
247
+ )
248
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
249
+ RecordFilter as RecordFilterModel,
250
+ )
251
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
252
+ RecordSelector as RecordSelectorModel,
253
+ )
254
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
255
+ RemoveFields as RemoveFieldsModel,
256
+ )
257
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
258
+ RequestOption as RequestOptionModel,
259
+ )
260
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
261
+ RequestPath as RequestPathModel,
262
+ )
263
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
264
+ SelectiveAuthenticator as SelectiveAuthenticatorModel,
265
+ )
266
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
267
+ SessionTokenAuthenticator as SessionTokenAuthenticatorModel,
268
+ )
269
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
270
+ SimpleRetriever as SimpleRetrieverModel,
271
+ )
126
272
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel
127
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import SubstreamPartitionRouter as SubstreamPartitionRouterModel
273
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
274
+ SubstreamPartitionRouter as SubstreamPartitionRouterModel,
275
+ )
128
276
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
129
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import WaitTimeFromHeader as WaitTimeFromHeaderModel
130
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import WaitUntilTimeFromHeader as WaitUntilTimeFromHeaderModel
131
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import XmlDecoder as XmlDecoderModel
277
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
278
+ WaitTimeFromHeader as WaitTimeFromHeaderModel,
279
+ )
280
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
281
+ WaitUntilTimeFromHeader as WaitUntilTimeFromHeaderModel,
282
+ )
283
+ from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
284
+ XmlDecoder as XmlDecoderModel,
285
+ )
132
286
  from airbyte_cdk.sources.declarative.partition_routers import (
133
287
  CartesianProductStreamSlicer,
134
288
  ListPartitionRouter,
135
289
  SinglePartitionRouter,
136
290
  SubstreamPartitionRouter,
137
291
  )
138
- from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
292
+ from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import (
293
+ ParentStreamConfig,
294
+ )
139
295
  from airbyte_cdk.sources.declarative.requesters import HttpRequester, RequestOption
140
- from airbyte_cdk.sources.declarative.requesters.error_handlers import CompositeErrorHandler, DefaultErrorHandler, HttpResponseFilter
296
+ from airbyte_cdk.sources.declarative.requesters.error_handlers import (
297
+ CompositeErrorHandler,
298
+ DefaultErrorHandler,
299
+ HttpResponseFilter,
300
+ )
141
301
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import (
142
302
  ConstantBackoffStrategy,
143
303
  ExponentialBackoffStrategy,
@@ -145,7 +305,11 @@ from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategie
145
305
  WaitUntilTimeFromHeaderBackoffStrategy,
146
306
  )
147
307
  from airbyte_cdk.sources.declarative.requesters.http_job_repository import AsyncHttpJobRepository
148
- from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator, NoPagination, PaginatorTestReadDecorator
308
+ from airbyte_cdk.sources.declarative.requesters.paginators import (
309
+ DefaultPaginator,
310
+ NoPagination,
311
+ PaginatorTestReadDecorator,
312
+ )
149
313
  from airbyte_cdk.sources.declarative.requesters.paginators.strategies import (
150
314
  CursorPaginationStrategy,
151
315
  CursorStopCondition,
@@ -162,14 +326,32 @@ from airbyte_cdk.sources.declarative.requesters.request_options import (
162
326
  )
163
327
  from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
164
328
  from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
165
- from airbyte_cdk.sources.declarative.retrievers import AsyncRetriever, SimpleRetriever, SimpleRetrieverTestReadDecorator
166
- from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader, InlineSchemaLoader, JsonFileSchemaLoader
329
+ from airbyte_cdk.sources.declarative.retrievers import (
330
+ AsyncRetriever,
331
+ SimpleRetriever,
332
+ SimpleRetrieverTestReadDecorator,
333
+ )
334
+ from airbyte_cdk.sources.declarative.schema import (
335
+ DefaultSchemaLoader,
336
+ InlineSchemaLoader,
337
+ JsonFileSchemaLoader,
338
+ )
167
339
  from airbyte_cdk.sources.declarative.spec import Spec
168
340
  from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
169
- from airbyte_cdk.sources.declarative.transformations import AddFields, RecordTransformation, RemoveFields
341
+ from airbyte_cdk.sources.declarative.transformations import (
342
+ AddFields,
343
+ RecordTransformation,
344
+ RemoveFields,
345
+ )
170
346
  from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
171
- from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import KeysToLowerTransformation
172
- from airbyte_cdk.sources.message import InMemoryMessageRepository, LogAppenderMessageRepositoryDecorator, MessageRepository
347
+ from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import (
348
+ KeysToLowerTransformation,
349
+ )
350
+ from airbyte_cdk.sources.message import (
351
+ InMemoryMessageRepository,
352
+ LogAppenderMessageRepositoryDecorator,
353
+ MessageRepository,
354
+ )
173
355
  from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField
174
356
  from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
175
357
  CustomFormatConcurrentStreamStateConverter,
@@ -185,7 +367,6 @@ ComponentDefinition = Mapping[str, Any]
185
367
 
186
368
 
187
369
  class ModelToComponentFactory:
188
-
189
370
  EPOCH_DATETIME_FORMAT = "%s"
190
371
 
191
372
  def __init__(
@@ -277,7 +458,11 @@ class ModelToComponentFactory:
277
458
  self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR}
278
459
 
279
460
  def create_component(
280
- self, model_type: Type[BaseModel], component_definition: ComponentDefinition, config: Config, **kwargs: Any
461
+ self,
462
+ model_type: Type[BaseModel],
463
+ component_definition: ComponentDefinition,
464
+ config: Config,
465
+ **kwargs: Any,
281
466
  ) -> Any:
282
467
  """
283
468
  Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and
@@ -292,26 +477,38 @@ class ModelToComponentFactory:
292
477
 
293
478
  component_type = component_definition.get("type")
294
479
  if component_definition.get("type") != model_type.__name__:
295
- raise ValueError(f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead")
480
+ raise ValueError(
481
+ f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead"
482
+ )
296
483
 
297
484
  declarative_component_model = model_type.parse_obj(component_definition)
298
485
 
299
486
  if not isinstance(declarative_component_model, model_type):
300
- raise ValueError(f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}")
487
+ raise ValueError(
488
+ f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}"
489
+ )
301
490
 
302
- return self._create_component_from_model(model=declarative_component_model, config=config, **kwargs)
491
+ return self._create_component_from_model(
492
+ model=declarative_component_model, config=config, **kwargs
493
+ )
303
494
 
304
495
  def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any:
305
496
  if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR:
306
- raise ValueError(f"{model.__class__} with attributes {model} is not a valid component type")
497
+ raise ValueError(
498
+ f"{model.__class__} with attributes {model} is not a valid component type"
499
+ )
307
500
  component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__)
308
501
  if not component_constructor:
309
502
  raise ValueError(f"Could not find constructor for {model.__class__}")
310
503
  return component_constructor(model=model, config=config, **kwargs)
311
504
 
312
505
  @staticmethod
313
- def create_added_field_definition(model: AddedFieldDefinitionModel, config: Config, **kwargs: Any) -> AddedFieldDefinition:
314
- interpolated_value = InterpolatedString.create(model.value, parameters=model.parameters or {})
506
+ def create_added_field_definition(
507
+ model: AddedFieldDefinitionModel, config: Config, **kwargs: Any
508
+ ) -> AddedFieldDefinition:
509
+ interpolated_value = InterpolatedString.create(
510
+ model.value, parameters=model.parameters or {}
511
+ )
315
512
  return AddedFieldDefinition(
316
513
  path=model.path,
317
514
  value=interpolated_value,
@@ -323,14 +520,18 @@ class ModelToComponentFactory:
323
520
  added_field_definitions = [
324
521
  self._create_component_from_model(
325
522
  model=added_field_definition_model,
326
- value_type=ModelToComponentFactory._json_schema_type_name_to_type(added_field_definition_model.value_type),
523
+ value_type=ModelToComponentFactory._json_schema_type_name_to_type(
524
+ added_field_definition_model.value_type
525
+ ),
327
526
  config=config,
328
527
  )
329
528
  for added_field_definition_model in model.fields
330
529
  ]
331
530
  return AddFields(fields=added_field_definitions, parameters=model.parameters or {})
332
531
 
333
- def create_keys_to_lower_transformation(self, model: KeysToLowerModel, config: Config, **kwargs: Any) -> KeysToLowerTransformation:
532
+ def create_keys_to_lower_transformation(
533
+ self, model: KeysToLowerModel, config: Config, **kwargs: Any
534
+ ) -> KeysToLowerTransformation:
334
535
  return KeysToLowerTransformation()
335
536
 
336
537
  @staticmethod
@@ -347,16 +548,25 @@ class ModelToComponentFactory:
347
548
 
348
549
  @staticmethod
349
550
  def create_api_key_authenticator(
350
- model: ApiKeyAuthenticatorModel, config: Config, token_provider: Optional[TokenProvider] = None, **kwargs: Any
551
+ model: ApiKeyAuthenticatorModel,
552
+ config: Config,
553
+ token_provider: Optional[TokenProvider] = None,
554
+ **kwargs: Any,
351
555
  ) -> ApiKeyAuthenticator:
352
556
  if model.inject_into is None and model.header is None:
353
- raise ValueError("Expected either inject_into or header to be set for ApiKeyAuthenticator")
557
+ raise ValueError(
558
+ "Expected either inject_into or header to be set for ApiKeyAuthenticator"
559
+ )
354
560
 
355
561
  if model.inject_into is not None and model.header is not None:
356
- raise ValueError("inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option")
562
+ raise ValueError(
563
+ "inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option"
564
+ )
357
565
 
358
566
  if token_provider is not None and model.api_token != "":
359
- raise ValueError("If token_provider is set, api_token is ignored and has to be set to empty string.")
567
+ raise ValueError(
568
+ "If token_provider is set, api_token is ignored and has to be set to empty string."
569
+ )
360
570
 
361
571
  request_option = (
362
572
  RequestOption(
@@ -375,7 +585,11 @@ class ModelToComponentFactory:
375
585
  token_provider=(
376
586
  token_provider
377
587
  if token_provider is not None
378
- else InterpolatedStringTokenProvider(api_token=model.api_token or "", config=config, parameters=model.parameters or {})
588
+ else InterpolatedStringTokenProvider(
589
+ api_token=model.api_token or "",
590
+ config=config,
591
+ parameters=model.parameters or {},
592
+ )
379
593
  ),
380
594
  request_option=request_option,
381
595
  config=config,
@@ -394,26 +608,44 @@ class ModelToComponentFactory:
394
608
  f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever. Got {type(retriever)}"
395
609
  )
396
610
  partition_router = retriever.partition_router
397
- if not isinstance(partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel)):
611
+ if not isinstance(
612
+ partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel)
613
+ ):
398
614
  raise ValueError(
399
615
  f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got {type(partition_router)}"
400
616
  )
401
617
  if not hasattr(partition_router, "parent_stream_configs"):
402
- raise ValueError("LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration.")
618
+ raise ValueError(
619
+ "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration."
620
+ )
403
621
 
404
- return LegacyToPerPartitionStateMigration(declarative_stream.retriever.partition_router, declarative_stream.incremental_sync, config, declarative_stream.parameters) # type: ignore # The retriever type was already checked
622
+ return LegacyToPerPartitionStateMigration(
623
+ declarative_stream.retriever.partition_router,
624
+ declarative_stream.incremental_sync,
625
+ config,
626
+ declarative_stream.parameters,
627
+ ) # type: ignore # The retriever type was already checked
405
628
 
406
629
  def create_session_token_authenticator(
407
630
  self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any
408
631
  ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]:
409
- decoder = self._create_component_from_model(model=model.decoder, config=config) if model.decoder else JsonDecoder(parameters={})
632
+ decoder = (
633
+ self._create_component_from_model(model=model.decoder, config=config)
634
+ if model.decoder
635
+ else JsonDecoder(parameters={})
636
+ )
410
637
  login_requester = self._create_component_from_model(
411
- model=model.login_requester, config=config, name=f"{name}_login_requester", decoder=decoder
638
+ model=model.login_requester,
639
+ config=config,
640
+ name=f"{name}_login_requester",
641
+ decoder=decoder,
412
642
  )
413
643
  token_provider = SessionTokenProvider(
414
644
  login_requester=login_requester,
415
645
  session_token_path=model.session_token_path,
416
- expiration_duration=parse_duration(model.expiration_duration) if model.expiration_duration else None,
646
+ expiration_duration=parse_duration(model.expiration_duration)
647
+ if model.expiration_duration
648
+ else None,
417
649
  parameters=model.parameters or {},
418
650
  message_repository=self._message_repository,
419
651
  decoder=decoder,
@@ -426,28 +658,46 @@ class ModelToComponentFactory:
426
658
  )
427
659
  else:
428
660
  return ModelToComponentFactory.create_api_key_authenticator(
429
- ApiKeyAuthenticatorModel(type="ApiKeyAuthenticator", api_token="", inject_into=model.request_authentication.inject_into), # type: ignore # $parameters and headers default to None
661
+ ApiKeyAuthenticatorModel(
662
+ type="ApiKeyAuthenticator",
663
+ api_token="",
664
+ inject_into=model.request_authentication.inject_into,
665
+ ), # type: ignore # $parameters and headers default to None
430
666
  config=config,
431
667
  token_provider=token_provider,
432
668
  )
433
669
 
434
670
  @staticmethod
435
- def create_basic_http_authenticator(model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any) -> BasicHttpAuthenticator:
671
+ def create_basic_http_authenticator(
672
+ model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any
673
+ ) -> BasicHttpAuthenticator:
436
674
  return BasicHttpAuthenticator(
437
- password=model.password or "", username=model.username, config=config, parameters=model.parameters or {}
675
+ password=model.password or "",
676
+ username=model.username,
677
+ config=config,
678
+ parameters=model.parameters or {},
438
679
  )
439
680
 
440
681
  @staticmethod
441
682
  def create_bearer_authenticator(
442
- model: BearerAuthenticatorModel, config: Config, token_provider: Optional[TokenProvider] = None, **kwargs: Any
683
+ model: BearerAuthenticatorModel,
684
+ config: Config,
685
+ token_provider: Optional[TokenProvider] = None,
686
+ **kwargs: Any,
443
687
  ) -> BearerAuthenticator:
444
688
  if token_provider is not None and model.api_token != "":
445
- raise ValueError("If token_provider is set, api_token is ignored and has to be set to empty string.")
689
+ raise ValueError(
690
+ "If token_provider is set, api_token is ignored and has to be set to empty string."
691
+ )
446
692
  return BearerAuthenticator(
447
693
  token_provider=(
448
694
  token_provider
449
695
  if token_provider is not None
450
- else InterpolatedStringTokenProvider(api_token=model.api_token or "", config=config, parameters=model.parameters or {})
696
+ else InterpolatedStringTokenProvider(
697
+ api_token=model.api_token or "",
698
+ config=config,
699
+ parameters=model.parameters or {},
700
+ )
451
701
  ),
452
702
  config=config,
453
703
  parameters=model.parameters or {},
@@ -457,14 +707,21 @@ class ModelToComponentFactory:
457
707
  def create_check_stream(model: CheckStreamModel, config: Config, **kwargs: Any) -> CheckStream:
458
708
  return CheckStream(stream_names=model.stream_names, parameters={})
459
709
 
460
- def create_composite_error_handler(self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any) -> CompositeErrorHandler:
710
+ def create_composite_error_handler(
711
+ self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any
712
+ ) -> CompositeErrorHandler:
461
713
  error_handlers = [
462
- self._create_component_from_model(model=error_handler_model, config=config) for error_handler_model in model.error_handlers
714
+ self._create_component_from_model(model=error_handler_model, config=config)
715
+ for error_handler_model in model.error_handlers
463
716
  ]
464
- return CompositeErrorHandler(error_handlers=error_handlers, parameters=model.parameters or {})
717
+ return CompositeErrorHandler(
718
+ error_handlers=error_handlers, parameters=model.parameters or {}
719
+ )
465
720
 
466
721
  @staticmethod
467
- def create_concurrency_level(model: ConcurrencyLevelModel, config: Config, **kwargs: Any) -> ConcurrencyLevel:
722
+ def create_concurrency_level(
723
+ model: ConcurrencyLevelModel, config: Config, **kwargs: Any
724
+ ) -> ConcurrencyLevel:
468
725
  return ConcurrencyLevel(
469
726
  default_concurrency=model.default_concurrency,
470
727
  max_concurrency=model.max_concurrency,
@@ -483,26 +740,32 @@ class ModelToComponentFactory:
483
740
  stream_state: MutableMapping[str, Any],
484
741
  **kwargs: Any,
485
742
  ) -> Tuple[ConcurrentCursor, DateTimeStreamStateConverter]:
486
-
487
743
  component_type = component_definition.get("type")
488
744
  if component_definition.get("type") != model_type.__name__:
489
- raise ValueError(f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead")
745
+ raise ValueError(
746
+ f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead"
747
+ )
490
748
 
491
749
  datetime_based_cursor_model = model_type.parse_obj(component_definition)
492
750
 
493
751
  if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel):
494
- raise ValueError(f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}")
752
+ raise ValueError(
753
+ f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}"
754
+ )
495
755
 
496
756
  interpolated_cursor_field = InterpolatedString.create(
497
- datetime_based_cursor_model.cursor_field, parameters=datetime_based_cursor_model.parameters or {}
757
+ datetime_based_cursor_model.cursor_field,
758
+ parameters=datetime_based_cursor_model.parameters or {},
498
759
  )
499
760
  cursor_field = CursorField(interpolated_cursor_field.eval(config=config))
500
761
 
501
762
  interpolated_partition_field_start = InterpolatedString.create(
502
- datetime_based_cursor_model.partition_field_start or "start_time", parameters=datetime_based_cursor_model.parameters or {}
763
+ datetime_based_cursor_model.partition_field_start or "start_time",
764
+ parameters=datetime_based_cursor_model.parameters or {},
503
765
  )
504
766
  interpolated_partition_field_end = InterpolatedString.create(
505
- datetime_based_cursor_model.partition_field_end or "end_time", parameters=datetime_based_cursor_model.parameters or {}
767
+ datetime_based_cursor_model.partition_field_end or "end_time",
768
+ parameters=datetime_based_cursor_model.parameters or {},
506
769
  )
507
770
 
508
771
  slice_boundary_fields = (
@@ -513,12 +776,17 @@ class ModelToComponentFactory:
513
776
  datetime_format = datetime_based_cursor_model.datetime_format
514
777
 
515
778
  cursor_granularity = (
516
- parse_duration(datetime_based_cursor_model.cursor_granularity) if datetime_based_cursor_model.cursor_granularity else None
779
+ parse_duration(datetime_based_cursor_model.cursor_granularity)
780
+ if datetime_based_cursor_model.cursor_granularity
781
+ else None
517
782
  )
518
783
 
519
784
  lookback_window = None
520
785
  interpolated_lookback_window = (
521
- InterpolatedString.create(datetime_based_cursor_model.lookback_window, parameters=datetime_based_cursor_model.parameters or {})
786
+ InterpolatedString.create(
787
+ datetime_based_cursor_model.lookback_window,
788
+ parameters=datetime_based_cursor_model.parameters or {},
789
+ )
522
790
  if datetime_based_cursor_model.lookback_window
523
791
  else None
524
792
  )
@@ -538,21 +806,30 @@ class ModelToComponentFactory:
538
806
 
539
807
  start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime]
540
808
  if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel):
541
- start_date_runtime_value = self.create_min_max_datetime(model=datetime_based_cursor_model.start_datetime, config=config)
809
+ start_date_runtime_value = self.create_min_max_datetime(
810
+ model=datetime_based_cursor_model.start_datetime, config=config
811
+ )
542
812
  else:
543
813
  start_date_runtime_value = datetime_based_cursor_model.start_datetime
544
814
 
545
815
  end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]]
546
816
  if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel):
547
- end_date_runtime_value = self.create_min_max_datetime(model=datetime_based_cursor_model.end_datetime, config=config)
817
+ end_date_runtime_value = self.create_min_max_datetime(
818
+ model=datetime_based_cursor_model.end_datetime, config=config
819
+ )
548
820
  else:
549
821
  end_date_runtime_value = datetime_based_cursor_model.end_datetime
550
822
 
551
823
  interpolated_start_date = MinMaxDatetime.create(
552
- interpolated_string_or_min_max_datetime=start_date_runtime_value, parameters=datetime_based_cursor_model.parameters
824
+ interpolated_string_or_min_max_datetime=start_date_runtime_value,
825
+ parameters=datetime_based_cursor_model.parameters,
553
826
  )
554
827
  interpolated_end_date = (
555
- None if not end_date_runtime_value else MinMaxDatetime.create(end_date_runtime_value, datetime_based_cursor_model.parameters)
828
+ None
829
+ if not end_date_runtime_value
830
+ else MinMaxDatetime.create(
831
+ end_date_runtime_value, datetime_based_cursor_model.parameters
832
+ )
556
833
  )
557
834
 
558
835
  # If datetime format is not specified then start/end datetime should inherit it from the stream slicer
@@ -563,10 +840,14 @@ class ModelToComponentFactory:
563
840
 
564
841
  start_date = interpolated_start_date.get_datetime(config=config)
565
842
  end_date_provider = (
566
- partial(interpolated_end_date.get_datetime, config) if interpolated_end_date else connector_state_converter.get_end_provider()
843
+ partial(interpolated_end_date.get_datetime, config)
844
+ if interpolated_end_date
845
+ else connector_state_converter.get_end_provider()
567
846
  )
568
847
 
569
- if (datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity) or (
848
+ if (
849
+ datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity
850
+ ) or (
570
851
  not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity
571
852
  ):
572
853
  raise ValueError(
@@ -577,7 +858,10 @@ class ModelToComponentFactory:
577
858
  # When step is not defined, default to a step size from the starting date to the present moment
578
859
  step_length = datetime.timedelta.max
579
860
  interpolated_step = (
580
- InterpolatedString.create(datetime_based_cursor_model.step, parameters=datetime_based_cursor_model.parameters or {})
861
+ InterpolatedString.create(
862
+ datetime_based_cursor_model.step,
863
+ parameters=datetime_based_cursor_model.parameters or {},
864
+ )
581
865
  if datetime_based_cursor_model.step
582
866
  else None
583
867
  )
@@ -606,7 +890,9 @@ class ModelToComponentFactory:
606
890
  )
607
891
 
608
892
  @staticmethod
609
- def create_constant_backoff_strategy(model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any) -> ConstantBackoffStrategy:
893
+ def create_constant_backoff_strategy(
894
+ model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any
895
+ ) -> ConstantBackoffStrategy:
610
896
  return ConstantBackoffStrategy(
611
897
  backoff_time_in_seconds=model.backoff_time_in_seconds,
612
898
  config=config,
@@ -624,7 +910,9 @@ class ModelToComponentFactory:
624
910
  decoder_to_use = decoder
625
911
  else:
626
912
  if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
627
- raise ValueError(f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead.")
913
+ raise ValueError(
914
+ f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
915
+ )
628
916
  decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
629
917
 
630
918
  return CursorPaginationStrategy(
@@ -660,18 +948,28 @@ class ModelToComponentFactory:
660
948
  # the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components
661
949
  for model_field, model_value in model_args.items():
662
950
  # If a custom component field doesn't have a type set, we try to use the type hints to infer the type
663
- if isinstance(model_value, dict) and "type" not in model_value and model_field in component_fields:
664
- derived_type = self._derive_component_type_from_type_hints(component_fields.get(model_field))
951
+ if (
952
+ isinstance(model_value, dict)
953
+ and "type" not in model_value
954
+ and model_field in component_fields
955
+ ):
956
+ derived_type = self._derive_component_type_from_type_hints(
957
+ component_fields.get(model_field)
958
+ )
665
959
  if derived_type:
666
960
  model_value["type"] = derived_type
667
961
 
668
962
  if self._is_component(model_value):
669
- model_args[model_field] = self._create_nested_component(model, model_field, model_value, config)
963
+ model_args[model_field] = self._create_nested_component(
964
+ model, model_field, model_value, config
965
+ )
670
966
  elif isinstance(model_value, list):
671
967
  vals = []
672
968
  for v in model_value:
673
969
  if isinstance(v, dict) and "type" not in v and model_field in component_fields:
674
- derived_type = self._derive_component_type_from_type_hints(component_fields.get(model_field))
970
+ derived_type = self._derive_component_type_from_type_hints(
971
+ component_fields.get(model_field)
972
+ )
675
973
  if derived_type:
676
974
  v["type"] = derived_type
677
975
  if self._is_component(v):
@@ -680,7 +978,11 @@ class ModelToComponentFactory:
680
978
  vals.append(v)
681
979
  model_args[model_field] = vals
682
980
 
683
- kwargs = {class_field: model_args[class_field] for class_field in component_fields.keys() if class_field in model_args}
981
+ kwargs = {
982
+ class_field: model_args[class_field]
983
+ for class_field in component_fields.keys()
984
+ if class_field in model_args
985
+ }
684
986
  return custom_component_class(**kwargs)
685
987
 
686
988
  @staticmethod
@@ -724,7 +1026,9 @@ class ModelToComponentFactory:
724
1026
  else:
725
1027
  return []
726
1028
 
727
- def _create_nested_component(self, model: Any, model_field: str, model_value: Any, config: Config) -> Any:
1029
+ def _create_nested_component(
1030
+ self, model: Any, model_field: str, model_value: Any, config: Config
1031
+ ) -> Any:
728
1032
  type_name = model_value.get("type", None)
729
1033
  if not type_name:
730
1034
  # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent
@@ -743,16 +1047,29 @@ class ModelToComponentFactory:
743
1047
  model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__)
744
1048
  constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs
745
1049
  model_parameters = model_value.get("$parameters", {})
746
- matching_parameters = {kwarg: model_parameters[kwarg] for kwarg in constructor_kwargs if kwarg in model_parameters}
747
- return self._create_component_from_model(model=parsed_model, config=config, **matching_parameters)
1050
+ matching_parameters = {
1051
+ kwarg: model_parameters[kwarg]
1052
+ for kwarg in constructor_kwargs
1053
+ if kwarg in model_parameters
1054
+ }
1055
+ return self._create_component_from_model(
1056
+ model=parsed_model, config=config, **matching_parameters
1057
+ )
748
1058
  except TypeError as error:
749
1059
  missing_parameters = self._extract_missing_parameters(error)
750
1060
  if missing_parameters:
751
1061
  raise ValueError(
752
1062
  f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide "
753
- + ", ".join((f"{type_name}.$parameters.{parameter}" for parameter in missing_parameters))
1063
+ + ", ".join(
1064
+ (
1065
+ f"{type_name}.$parameters.{parameter}"
1066
+ for parameter in missing_parameters
1067
+ )
1068
+ )
754
1069
  )
755
- raise TypeError(f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}")
1070
+ raise TypeError(
1071
+ f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}"
1072
+ )
756
1073
  else:
757
1074
  raise ValueError(
758
1075
  f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'"
@@ -762,18 +1079,26 @@ class ModelToComponentFactory:
762
1079
  def _is_component(model_value: Any) -> bool:
763
1080
  return isinstance(model_value, dict) and model_value.get("type") is not None
764
1081
 
765
- def create_datetime_based_cursor(self, model: DatetimeBasedCursorModel, config: Config, **kwargs: Any) -> DatetimeBasedCursor:
1082
+ def create_datetime_based_cursor(
1083
+ self, model: DatetimeBasedCursorModel, config: Config, **kwargs: Any
1084
+ ) -> DatetimeBasedCursor:
766
1085
  start_datetime: Union[str, MinMaxDatetime] = (
767
- model.start_datetime if isinstance(model.start_datetime, str) else self.create_min_max_datetime(model.start_datetime, config)
1086
+ model.start_datetime
1087
+ if isinstance(model.start_datetime, str)
1088
+ else self.create_min_max_datetime(model.start_datetime, config)
768
1089
  )
769
1090
  end_datetime: Union[str, MinMaxDatetime, None] = None
770
1091
  if model.is_data_feed and model.end_datetime:
771
1092
  raise ValueError("Data feed does not support end_datetime")
772
1093
  if model.is_data_feed and model.is_client_side_incremental:
773
- raise ValueError("`Client side incremental` cannot be applied with `data feed`. Choose only 1 from them.")
1094
+ raise ValueError(
1095
+ "`Client side incremental` cannot be applied with `data feed`. Choose only 1 from them."
1096
+ )
774
1097
  if model.end_datetime:
775
1098
  end_datetime = (
776
- model.end_datetime if isinstance(model.end_datetime, str) else self.create_min_max_datetime(model.end_datetime, config)
1099
+ model.end_datetime
1100
+ if isinstance(model.end_datetime, str)
1101
+ else self.create_min_max_datetime(model.end_datetime, config)
777
1102
  )
778
1103
 
779
1104
  end_time_option = (
@@ -797,7 +1122,9 @@ class ModelToComponentFactory:
797
1122
 
798
1123
  return DatetimeBasedCursor(
799
1124
  cursor_field=model.cursor_field,
800
- cursor_datetime_formats=model.cursor_datetime_formats if model.cursor_datetime_formats else [],
1125
+ cursor_datetime_formats=model.cursor_datetime_formats
1126
+ if model.cursor_datetime_formats
1127
+ else [],
801
1128
  cursor_granularity=model.cursor_granularity,
802
1129
  datetime_format=model.datetime_format,
803
1130
  end_datetime=end_datetime,
@@ -814,7 +1141,9 @@ class ModelToComponentFactory:
814
1141
  parameters=model.parameters or {},
815
1142
  )
816
1143
 
817
- def create_declarative_stream(self, model: DeclarativeStreamModel, config: Config, **kwargs: Any) -> DeclarativeStream:
1144
+ def create_declarative_stream(
1145
+ self, model: DeclarativeStreamModel, config: Config, **kwargs: Any
1146
+ ) -> DeclarativeStream:
818
1147
  # When constructing a declarative stream, we assemble the incremental_sync component and retriever's partition_router field
819
1148
  # components if they exist into a single CartesianProductStreamSlicer. This is then passed back as an argument when constructing the
820
1149
  # Retriever. This is done in the declarative stream not the retriever to support custom retrievers. The custom create methods in
@@ -823,7 +1152,9 @@ class ModelToComponentFactory:
823
1152
 
824
1153
  primary_key = model.primary_key.__root__ if model.primary_key else None
825
1154
  stop_condition_on_cursor = (
826
- model.incremental_sync and hasattr(model.incremental_sync, "is_data_feed") and model.incremental_sync.is_data_feed
1155
+ model.incremental_sync
1156
+ and hasattr(model.incremental_sync, "is_data_feed")
1157
+ and model.incremental_sync.is_data_feed
827
1158
  )
828
1159
  client_side_incremental_sync = None
829
1160
  if (
@@ -831,13 +1162,25 @@ class ModelToComponentFactory:
831
1162
  and hasattr(model.incremental_sync, "is_client_side_incremental")
832
1163
  and model.incremental_sync.is_client_side_incremental
833
1164
  ):
834
- supported_slicers = (DatetimeBasedCursor, GlobalSubstreamCursor, PerPartitionWithGlobalCursor)
1165
+ supported_slicers = (
1166
+ DatetimeBasedCursor,
1167
+ GlobalSubstreamCursor,
1168
+ PerPartitionWithGlobalCursor,
1169
+ )
835
1170
  if combined_slicers and not isinstance(combined_slicers, supported_slicers):
836
- raise ValueError("Unsupported Slicer is used. PerPartitionWithGlobalCursor should be used here instead")
1171
+ raise ValueError(
1172
+ "Unsupported Slicer is used. PerPartitionWithGlobalCursor should be used here instead"
1173
+ )
837
1174
  client_side_incremental_sync = {
838
- "date_time_based_cursor": self._create_component_from_model(model=model.incremental_sync, config=config),
1175
+ "date_time_based_cursor": self._create_component_from_model(
1176
+ model=model.incremental_sync, config=config
1177
+ ),
839
1178
  "substream_cursor": (
840
- combined_slicers if isinstance(combined_slicers, (PerPartitionWithGlobalCursor, GlobalSubstreamCursor)) else None
1179
+ combined_slicers
1180
+ if isinstance(
1181
+ combined_slicers, (PerPartitionWithGlobalCursor, GlobalSubstreamCursor)
1182
+ )
1183
+ else None
841
1184
  ),
842
1185
  }
843
1186
 
@@ -877,7 +1220,9 @@ class ModelToComponentFactory:
877
1220
  transformations = []
878
1221
  if model.transformations:
879
1222
  for transformation_model in model.transformations:
880
- transformations.append(self._create_component_from_model(model=transformation_model, config=config))
1223
+ transformations.append(
1224
+ self._create_component_from_model(model=transformation_model, config=config)
1225
+ )
881
1226
  retriever = self._create_component_from_model(
882
1227
  model=model.retriever,
883
1228
  config=config,
@@ -900,7 +1245,9 @@ class ModelToComponentFactory:
900
1245
  state_transformations = []
901
1246
 
902
1247
  if model.schema_loader:
903
- schema_loader = self._create_component_from_model(model=model.schema_loader, config=config)
1248
+ schema_loader = self._create_component_from_model(
1249
+ model=model.schema_loader, config=config
1250
+ )
904
1251
  else:
905
1252
  options = model.parameters or {}
906
1253
  if "name" not in options:
@@ -918,7 +1265,9 @@ class ModelToComponentFactory:
918
1265
  parameters=model.parameters or {},
919
1266
  )
920
1267
 
921
- def _merge_stream_slicers(self, model: DeclarativeStreamModel, config: Config) -> Optional[StreamSlicer]:
1268
+ def _merge_stream_slicers(
1269
+ self, model: DeclarativeStreamModel, config: Config
1270
+ ) -> Optional[StreamSlicer]:
922
1271
  stream_slicer = None
923
1272
  if (
924
1273
  hasattr(model.retriever, "partition_router")
@@ -929,50 +1278,85 @@ class ModelToComponentFactory:
929
1278
 
930
1279
  if isinstance(stream_slicer_model, list):
931
1280
  stream_slicer = CartesianProductStreamSlicer(
932
- [self._create_component_from_model(model=slicer, config=config) for slicer in stream_slicer_model], parameters={}
1281
+ [
1282
+ self._create_component_from_model(model=slicer, config=config)
1283
+ for slicer in stream_slicer_model
1284
+ ],
1285
+ parameters={},
933
1286
  )
934
1287
  else:
935
- stream_slicer = self._create_component_from_model(model=stream_slicer_model, config=config)
1288
+ stream_slicer = self._create_component_from_model(
1289
+ model=stream_slicer_model, config=config
1290
+ )
936
1291
 
937
1292
  if model.incremental_sync and stream_slicer:
938
1293
  incremental_sync_model = model.incremental_sync
939
- if hasattr(incremental_sync_model, "global_substream_cursor") and incremental_sync_model.global_substream_cursor:
940
- cursor_component = self._create_component_from_model(model=incremental_sync_model, config=config)
941
- return GlobalSubstreamCursor(stream_cursor=cursor_component, partition_router=stream_slicer)
1294
+ if (
1295
+ hasattr(incremental_sync_model, "global_substream_cursor")
1296
+ and incremental_sync_model.global_substream_cursor
1297
+ ):
1298
+ cursor_component = self._create_component_from_model(
1299
+ model=incremental_sync_model, config=config
1300
+ )
1301
+ return GlobalSubstreamCursor(
1302
+ stream_cursor=cursor_component, partition_router=stream_slicer
1303
+ )
942
1304
  else:
943
- cursor_component = self._create_component_from_model(model=incremental_sync_model, config=config)
1305
+ cursor_component = self._create_component_from_model(
1306
+ model=incremental_sync_model, config=config
1307
+ )
944
1308
  return PerPartitionWithGlobalCursor(
945
1309
  cursor_factory=CursorFactory(
946
- lambda: self._create_component_from_model(model=incremental_sync_model, config=config),
1310
+ lambda: self._create_component_from_model(
1311
+ model=incremental_sync_model, config=config
1312
+ ),
947
1313
  ),
948
1314
  partition_router=stream_slicer,
949
1315
  stream_cursor=cursor_component,
950
1316
  )
951
1317
  elif model.incremental_sync:
952
- return self._create_component_from_model(model=model.incremental_sync, config=config) if model.incremental_sync else None
1318
+ return (
1319
+ self._create_component_from_model(model=model.incremental_sync, config=config)
1320
+ if model.incremental_sync
1321
+ else None
1322
+ )
953
1323
  elif stream_slicer:
954
1324
  # For the Full-Refresh sub-streams, we use the nested `ChildPartitionResumableFullRefreshCursor`
955
1325
  return PerPartitionCursor(
956
- cursor_factory=CursorFactory(create_function=partial(ChildPartitionResumableFullRefreshCursor, {})),
1326
+ cursor_factory=CursorFactory(
1327
+ create_function=partial(ChildPartitionResumableFullRefreshCursor, {})
1328
+ ),
957
1329
  partition_router=stream_slicer,
958
1330
  )
959
- elif hasattr(model.retriever, "paginator") and model.retriever.paginator and not stream_slicer:
1331
+ elif (
1332
+ hasattr(model.retriever, "paginator")
1333
+ and model.retriever.paginator
1334
+ and not stream_slicer
1335
+ ):
960
1336
  # For the regular Full-Refresh streams, we use the high level `ResumableFullRefreshCursor`
961
1337
  return ResumableFullRefreshCursor(parameters={})
962
1338
  else:
963
1339
  return None
964
1340
 
965
- def create_default_error_handler(self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any) -> DefaultErrorHandler:
1341
+ def create_default_error_handler(
1342
+ self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any
1343
+ ) -> DefaultErrorHandler:
966
1344
  backoff_strategies = []
967
1345
  if model.backoff_strategies:
968
1346
  for backoff_strategy_model in model.backoff_strategies:
969
- backoff_strategies.append(self._create_component_from_model(model=backoff_strategy_model, config=config))
1347
+ backoff_strategies.append(
1348
+ self._create_component_from_model(model=backoff_strategy_model, config=config)
1349
+ )
970
1350
 
971
1351
  response_filters = []
972
1352
  if model.response_filters:
973
1353
  for response_filter_model in model.response_filters:
974
- response_filters.append(self._create_component_from_model(model=response_filter_model, config=config))
975
- response_filters.append(HttpResponseFilter(config=config, parameters=model.parameters or {}))
1354
+ response_filters.append(
1355
+ self._create_component_from_model(model=response_filter_model, config=config)
1356
+ )
1357
+ response_filters.append(
1358
+ HttpResponseFilter(config=config, parameters=model.parameters or {})
1359
+ )
976
1360
 
977
1361
  return DefaultErrorHandler(
978
1362
  backoff_strategies=backoff_strategies,
@@ -993,17 +1377,25 @@ class ModelToComponentFactory:
993
1377
  ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]:
994
1378
  if decoder:
995
1379
  if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
996
- raise ValueError(f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead.")
1380
+ raise ValueError(
1381
+ f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
1382
+ )
997
1383
  decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
998
1384
  else:
999
1385
  decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
1000
1386
  page_size_option = (
1001
- self._create_component_from_model(model=model.page_size_option, config=config) if model.page_size_option else None
1387
+ self._create_component_from_model(model=model.page_size_option, config=config)
1388
+ if model.page_size_option
1389
+ else None
1002
1390
  )
1003
1391
  page_token_option = (
1004
- self._create_component_from_model(model=model.page_token_option, config=config) if model.page_token_option else None
1392
+ self._create_component_from_model(model=model.page_token_option, config=config)
1393
+ if model.page_token_option
1394
+ else None
1395
+ )
1396
+ pagination_strategy = self._create_component_from_model(
1397
+ model=model.pagination_strategy, config=config, decoder=decoder_to_use
1005
1398
  )
1006
- pagination_strategy = self._create_component_from_model(model=model.pagination_strategy, config=config, decoder=decoder_to_use)
1007
1399
  if cursor_used_for_stop_condition:
1008
1400
  pagination_strategy = StopConditionPaginationStrategyDecorator(
1009
1401
  pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition)
@@ -1022,29 +1414,55 @@ class ModelToComponentFactory:
1022
1414
  return paginator
1023
1415
 
1024
1416
  def create_dpath_extractor(
1025
- self, model: DpathExtractorModel, config: Config, decoder: Optional[Decoder] = None, **kwargs: Any
1417
+ self,
1418
+ model: DpathExtractorModel,
1419
+ config: Config,
1420
+ decoder: Optional[Decoder] = None,
1421
+ **kwargs: Any,
1026
1422
  ) -> DpathExtractor:
1027
1423
  if decoder:
1028
1424
  decoder_to_use = decoder
1029
1425
  else:
1030
1426
  decoder_to_use = JsonDecoder(parameters={})
1031
1427
  model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path]
1032
- return DpathExtractor(decoder=decoder_to_use, field_path=model_field_path, config=config, parameters=model.parameters or {})
1428
+ return DpathExtractor(
1429
+ decoder=decoder_to_use,
1430
+ field_path=model_field_path,
1431
+ config=config,
1432
+ parameters=model.parameters or {},
1433
+ )
1033
1434
 
1034
1435
  @staticmethod
1035
- def create_exponential_backoff_strategy(model: ExponentialBackoffStrategyModel, config: Config) -> ExponentialBackoffStrategy:
1036
- return ExponentialBackoffStrategy(factor=model.factor or 5, parameters=model.parameters or {}, config=config)
1436
+ def create_exponential_backoff_strategy(
1437
+ model: ExponentialBackoffStrategyModel, config: Config
1438
+ ) -> ExponentialBackoffStrategy:
1439
+ return ExponentialBackoffStrategy(
1440
+ factor=model.factor or 5, parameters=model.parameters or {}, config=config
1441
+ )
1037
1442
 
1038
- def create_http_requester(self, model: HttpRequesterModel, decoder: Decoder, config: Config, *, name: str) -> HttpRequester:
1443
+ def create_http_requester(
1444
+ self, model: HttpRequesterModel, decoder: Decoder, config: Config, *, name: str
1445
+ ) -> HttpRequester:
1039
1446
  authenticator = (
1040
- self._create_component_from_model(model=model.authenticator, config=config, url_base=model.url_base, name=name, decoder=decoder)
1447
+ self._create_component_from_model(
1448
+ model=model.authenticator,
1449
+ config=config,
1450
+ url_base=model.url_base,
1451
+ name=name,
1452
+ decoder=decoder,
1453
+ )
1041
1454
  if model.authenticator
1042
1455
  else None
1043
1456
  )
1044
1457
  error_handler = (
1045
1458
  self._create_component_from_model(model=model.error_handler, config=config)
1046
1459
  if model.error_handler
1047
- else DefaultErrorHandler(backoff_strategies=[], response_filters=[], config=config, parameters=model.parameters or {})
1460
+ else DefaultErrorHandler(
1461
+ backoff_strategies=[],
1462
+ response_filters=[],
1463
+ config=config,
1464
+ parameters=model.parameters or {},
1465
+ )
1048
1466
  )
1049
1467
 
1050
1468
  request_options_provider = InterpolatedRequestOptionsProvider(
@@ -1079,7 +1497,9 @@ class ModelToComponentFactory:
1079
1497
  )
1080
1498
 
1081
1499
  @staticmethod
1082
- def create_http_response_filter(model: HttpResponseFilterModel, config: Config, **kwargs: Any) -> HttpResponseFilter:
1500
+ def create_http_response_filter(
1501
+ model: HttpResponseFilterModel, config: Config, **kwargs: Any
1502
+ ) -> HttpResponseFilter:
1083
1503
  if model.action:
1084
1504
  action = ResponseAction(model.action.value)
1085
1505
  else:
@@ -1103,7 +1523,9 @@ class ModelToComponentFactory:
1103
1523
  )
1104
1524
 
1105
1525
  @staticmethod
1106
- def create_inline_schema_loader(model: InlineSchemaLoaderModel, config: Config, **kwargs: Any) -> InlineSchemaLoader:
1526
+ def create_inline_schema_loader(
1527
+ model: InlineSchemaLoaderModel, config: Config, **kwargs: Any
1528
+ ) -> InlineSchemaLoader:
1107
1529
  return InlineSchemaLoader(schema=model.schema_ or {}, parameters={})
1108
1530
 
1109
1531
  @staticmethod
@@ -1111,11 +1533,15 @@ class ModelToComponentFactory:
1111
1533
  return JsonDecoder(parameters={})
1112
1534
 
1113
1535
  @staticmethod
1114
- def create_jsonl_decoder(model: JsonlDecoderModel, config: Config, **kwargs: Any) -> JsonlDecoder:
1536
+ def create_jsonl_decoder(
1537
+ model: JsonlDecoderModel, config: Config, **kwargs: Any
1538
+ ) -> JsonlDecoder:
1115
1539
  return JsonlDecoder(parameters={})
1116
1540
 
1117
1541
  @staticmethod
1118
- def create_iterable_decoder(model: IterableDecoderModel, config: Config, **kwargs: Any) -> IterableDecoder:
1542
+ def create_iterable_decoder(
1543
+ model: IterableDecoderModel, config: Config, **kwargs: Any
1544
+ ) -> IterableDecoder:
1119
1545
  return IterableDecoder(parameters={})
1120
1546
 
1121
1547
  @staticmethod
@@ -1123,11 +1549,17 @@ class ModelToComponentFactory:
1123
1549
  return XmlDecoder(parameters={})
1124
1550
 
1125
1551
  @staticmethod
1126
- def create_json_file_schema_loader(model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any) -> JsonFileSchemaLoader:
1127
- return JsonFileSchemaLoader(file_path=model.file_path or "", config=config, parameters=model.parameters or {})
1552
+ def create_json_file_schema_loader(
1553
+ model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any
1554
+ ) -> JsonFileSchemaLoader:
1555
+ return JsonFileSchemaLoader(
1556
+ file_path=model.file_path or "", config=config, parameters=model.parameters or {}
1557
+ )
1128
1558
 
1129
1559
  @staticmethod
1130
- def create_jwt_authenticator(model: JwtAuthenticatorModel, config: Config, **kwargs: Any) -> JwtAuthenticator:
1560
+ def create_jwt_authenticator(
1561
+ model: JwtAuthenticatorModel, config: Config, **kwargs: Any
1562
+ ) -> JwtAuthenticator:
1131
1563
  jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None)
1132
1564
  jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None)
1133
1565
  return JwtAuthenticator(
@@ -1149,7 +1581,9 @@ class ModelToComponentFactory:
1149
1581
  )
1150
1582
 
1151
1583
  @staticmethod
1152
- def create_list_partition_router(model: ListPartitionRouterModel, config: Config, **kwargs: Any) -> ListPartitionRouter:
1584
+ def create_list_partition_router(
1585
+ model: ListPartitionRouterModel, config: Config, **kwargs: Any
1586
+ ) -> ListPartitionRouter:
1153
1587
  request_option = (
1154
1588
  RequestOption(
1155
1589
  inject_into=RequestOptionType(model.request_option.inject_into.value),
@@ -1168,7 +1602,9 @@ class ModelToComponentFactory:
1168
1602
  )
1169
1603
 
1170
1604
  @staticmethod
1171
- def create_min_max_datetime(model: MinMaxDatetimeModel, config: Config, **kwargs: Any) -> MinMaxDatetime:
1605
+ def create_min_max_datetime(
1606
+ model: MinMaxDatetimeModel, config: Config, **kwargs: Any
1607
+ ) -> MinMaxDatetime:
1172
1608
  return MinMaxDatetime(
1173
1609
  datetime=model.datetime,
1174
1610
  datetime_format=model.datetime_format or "",
@@ -1182,29 +1618,43 @@ class ModelToComponentFactory:
1182
1618
  return NoAuth(parameters=model.parameters or {})
1183
1619
 
1184
1620
  @staticmethod
1185
- def create_no_pagination(model: NoPaginationModel, config: Config, **kwargs: Any) -> NoPagination:
1621
+ def create_no_pagination(
1622
+ model: NoPaginationModel, config: Config, **kwargs: Any
1623
+ ) -> NoPagination:
1186
1624
  return NoPagination(parameters={})
1187
1625
 
1188
- def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any) -> DeclarativeOauth2Authenticator:
1626
+ def create_oauth_authenticator(
1627
+ self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any
1628
+ ) -> DeclarativeOauth2Authenticator:
1189
1629
  if model.refresh_token_updater:
1190
1630
  # ignore type error because fixing it would have a lot of dependencies, revisit later
1191
1631
  return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore
1192
1632
  config,
1193
- InterpolatedString.create(model.token_refresh_endpoint, parameters=model.parameters or {}).eval(config),
1633
+ InterpolatedString.create(
1634
+ model.token_refresh_endpoint, parameters=model.parameters or {}
1635
+ ).eval(config),
1194
1636
  access_token_name=InterpolatedString.create(
1195
1637
  model.access_token_name or "access_token", parameters=model.parameters or {}
1196
1638
  ).eval(config),
1197
1639
  refresh_token_name=model.refresh_token_updater.refresh_token_name,
1198
- expires_in_name=InterpolatedString.create(model.expires_in_name or "expires_in", parameters=model.parameters or {}).eval(
1199
- config
1200
- ),
1201
- client_id=InterpolatedString.create(model.client_id, parameters=model.parameters or {}).eval(config),
1202
- client_secret=InterpolatedString.create(model.client_secret, parameters=model.parameters or {}).eval(config),
1640
+ expires_in_name=InterpolatedString.create(
1641
+ model.expires_in_name or "expires_in", parameters=model.parameters or {}
1642
+ ).eval(config),
1643
+ client_id=InterpolatedString.create(
1644
+ model.client_id, parameters=model.parameters or {}
1645
+ ).eval(config),
1646
+ client_secret=InterpolatedString.create(
1647
+ model.client_secret, parameters=model.parameters or {}
1648
+ ).eval(config),
1203
1649
  access_token_config_path=model.refresh_token_updater.access_token_config_path,
1204
1650
  refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path,
1205
1651
  token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path,
1206
- grant_type=InterpolatedString.create(model.grant_type or "refresh_token", parameters=model.parameters or {}).eval(config),
1207
- refresh_request_body=InterpolatedMapping(model.refresh_request_body or {}, parameters=model.parameters or {}).eval(config),
1652
+ grant_type=InterpolatedString.create(
1653
+ model.grant_type or "refresh_token", parameters=model.parameters or {}
1654
+ ).eval(config),
1655
+ refresh_request_body=InterpolatedMapping(
1656
+ model.refresh_request_body or {}, parameters=model.parameters or {}
1657
+ ).eval(config),
1208
1658
  scopes=model.scopes,
1209
1659
  token_expiry_date_format=model.token_expiry_date_format,
1210
1660
  message_repository=self._message_repository,
@@ -1232,7 +1682,9 @@ class ModelToComponentFactory:
1232
1682
  )
1233
1683
 
1234
1684
  @staticmethod
1235
- def create_offset_increment(model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any) -> OffsetIncrement:
1685
+ def create_offset_increment(
1686
+ model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any
1687
+ ) -> OffsetIncrement:
1236
1688
  if isinstance(decoder, PaginationDecoderDecorator):
1237
1689
  if not isinstance(decoder.decoder, (JsonDecoder, XmlDecoder)):
1238
1690
  raise ValueError(
@@ -1241,7 +1693,9 @@ class ModelToComponentFactory:
1241
1693
  decoder_to_use = decoder
1242
1694
  else:
1243
1695
  if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
1244
- raise ValueError(f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead.")
1696
+ raise ValueError(
1697
+ f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
1698
+ )
1245
1699
  decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
1246
1700
  return OffsetIncrement(
1247
1701
  page_size=model.page_size,
@@ -1252,7 +1706,9 @@ class ModelToComponentFactory:
1252
1706
  )
1253
1707
 
1254
1708
  @staticmethod
1255
- def create_page_increment(model: PageIncrementModel, config: Config, **kwargs: Any) -> PageIncrement:
1709
+ def create_page_increment(
1710
+ model: PageIncrementModel, config: Config, **kwargs: Any
1711
+ ) -> PageIncrement:
1256
1712
  return PageIncrement(
1257
1713
  page_size=model.page_size,
1258
1714
  config=config,
@@ -1261,9 +1717,15 @@ class ModelToComponentFactory:
1261
1717
  parameters=model.parameters or {},
1262
1718
  )
1263
1719
 
1264
- def create_parent_stream_config(self, model: ParentStreamConfigModel, config: Config, **kwargs: Any) -> ParentStreamConfig:
1720
+ def create_parent_stream_config(
1721
+ self, model: ParentStreamConfigModel, config: Config, **kwargs: Any
1722
+ ) -> ParentStreamConfig:
1265
1723
  declarative_stream = self._create_component_from_model(model.stream, config=config)
1266
- request_option = self._create_component_from_model(model.request_option, config=config) if model.request_option else None
1724
+ request_option = (
1725
+ self._create_component_from_model(model.request_option, config=config)
1726
+ if model.request_option
1727
+ else None
1728
+ )
1267
1729
  return ParentStreamConfig(
1268
1730
  parent_key=model.parent_key,
1269
1731
  request_option=request_option,
@@ -1276,15 +1738,21 @@ class ModelToComponentFactory:
1276
1738
  )
1277
1739
 
1278
1740
  @staticmethod
1279
- def create_record_filter(model: RecordFilterModel, config: Config, **kwargs: Any) -> RecordFilter:
1280
- return RecordFilter(condition=model.condition or "", config=config, parameters=model.parameters or {})
1741
+ def create_record_filter(
1742
+ model: RecordFilterModel, config: Config, **kwargs: Any
1743
+ ) -> RecordFilter:
1744
+ return RecordFilter(
1745
+ condition=model.condition or "", config=config, parameters=model.parameters or {}
1746
+ )
1281
1747
 
1282
1748
  @staticmethod
1283
1749
  def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath:
1284
1750
  return RequestPath(parameters={})
1285
1751
 
1286
1752
  @staticmethod
1287
- def create_request_option(model: RequestOptionModel, config: Config, **kwargs: Any) -> RequestOption:
1753
+ def create_request_option(
1754
+ model: RequestOptionModel, config: Config, **kwargs: Any
1755
+ ) -> RequestOption:
1288
1756
  inject_into = RequestOptionType(model.inject_into.value)
1289
1757
  return RequestOption(field_name=model.field_name, inject_into=inject_into, parameters={})
1290
1758
 
@@ -1299,16 +1767,26 @@ class ModelToComponentFactory:
1299
1767
  **kwargs: Any,
1300
1768
  ) -> RecordSelector:
1301
1769
  assert model.schema_normalization is not None # for mypy
1302
- extractor = self._create_component_from_model(model=model.extractor, decoder=decoder, config=config)
1303
- record_filter = self._create_component_from_model(model.record_filter, config=config) if model.record_filter else None
1770
+ extractor = self._create_component_from_model(
1771
+ model=model.extractor, decoder=decoder, config=config
1772
+ )
1773
+ record_filter = (
1774
+ self._create_component_from_model(model.record_filter, config=config)
1775
+ if model.record_filter
1776
+ else None
1777
+ )
1304
1778
  if client_side_incremental_sync:
1305
1779
  record_filter = ClientSideIncrementalRecordFilterDecorator(
1306
1780
  config=config,
1307
1781
  parameters=model.parameters,
1308
- condition=model.record_filter.condition if (model.record_filter and hasattr(model.record_filter, "condition")) else None,
1782
+ condition=model.record_filter.condition
1783
+ if (model.record_filter and hasattr(model.record_filter, "condition"))
1784
+ else None,
1309
1785
  **client_side_incremental_sync,
1310
1786
  )
1311
- schema_normalization = TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization])
1787
+ schema_normalization = TypeTransformer(
1788
+ SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]
1789
+ )
1312
1790
 
1313
1791
  return RecordSelector(
1314
1792
  extractor=extractor,
@@ -1320,11 +1798,20 @@ class ModelToComponentFactory:
1320
1798
  )
1321
1799
 
1322
1800
  @staticmethod
1323
- def create_remove_fields(model: RemoveFieldsModel, config: Config, **kwargs: Any) -> RemoveFields:
1324
- return RemoveFields(field_pointers=model.field_pointers, condition=model.condition or "", parameters={})
1325
-
1326
- def create_selective_authenticator(self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any) -> DeclarativeAuthenticator:
1327
- authenticators = {name: self._create_component_from_model(model=auth, config=config) for name, auth in model.authenticators.items()}
1801
+ def create_remove_fields(
1802
+ model: RemoveFieldsModel, config: Config, **kwargs: Any
1803
+ ) -> RemoveFields:
1804
+ return RemoveFields(
1805
+ field_pointers=model.field_pointers, condition=model.condition or "", parameters={}
1806
+ )
1807
+
1808
+ def create_selective_authenticator(
1809
+ self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any
1810
+ ) -> DeclarativeAuthenticator:
1811
+ authenticators = {
1812
+ name: self._create_component_from_model(model=auth, config=config)
1813
+ for name, auth in model.authenticators.items()
1814
+ }
1328
1815
  # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error
1329
1816
  return SelectiveAuthenticator( # type: ignore[abstract]
1330
1817
  config=config,
@@ -1363,8 +1850,14 @@ class ModelToComponentFactory:
1363
1850
  client_side_incremental_sync: Optional[Dict[str, Any]] = None,
1364
1851
  transformations: List[RecordTransformation],
1365
1852
  ) -> SimpleRetriever:
1366
- decoder = self._create_component_from_model(model=model.decoder, config=config) if model.decoder else JsonDecoder(parameters={})
1367
- requester = self._create_component_from_model(model=model.requester, decoder=decoder, config=config, name=name)
1853
+ decoder = (
1854
+ self._create_component_from_model(model=model.decoder, config=config)
1855
+ if model.decoder
1856
+ else JsonDecoder(parameters={})
1857
+ )
1858
+ requester = self._create_component_from_model(
1859
+ model=model.requester, decoder=decoder, config=config, name=name
1860
+ )
1368
1861
  record_selector = self._create_component_from_model(
1369
1862
  model=model.record_selector,
1370
1863
  config=config,
@@ -1372,12 +1865,19 @@ class ModelToComponentFactory:
1372
1865
  transformations=transformations,
1373
1866
  client_side_incremental_sync=client_side_incremental_sync,
1374
1867
  )
1375
- url_base = model.requester.url_base if hasattr(model.requester, "url_base") else requester.get_url_base()
1868
+ url_base = (
1869
+ model.requester.url_base
1870
+ if hasattr(model.requester, "url_base")
1871
+ else requester.get_url_base()
1872
+ )
1376
1873
 
1377
1874
  # Define cursor only if per partition or common incremental support is needed
1378
1875
  cursor = stream_slicer if isinstance(stream_slicer, DeclarativeCursor) else None
1379
1876
 
1380
- if not isinstance(stream_slicer, DatetimeBasedCursor) or type(stream_slicer) is not DatetimeBasedCursor:
1877
+ if (
1878
+ not isinstance(stream_slicer, DatetimeBasedCursor)
1879
+ or type(stream_slicer) is not DatetimeBasedCursor
1880
+ ):
1381
1881
  # Many of the custom component implementations of DatetimeBasedCursor override get_request_params() (or other methods).
1382
1882
  # Because we're decoupling RequestOptionsProvider from the Cursor, custom components will eventually need to reimplement
1383
1883
  # their own RequestOptionsProvider. However, right now the existing StreamSlicer/Cursor still can act as the SimpleRetriever's
@@ -1401,7 +1901,9 @@ class ModelToComponentFactory:
1401
1901
  else NoPagination(parameters={})
1402
1902
  )
1403
1903
 
1404
- ignore_stream_slicer_parameters_on_paginated_requests = model.ignore_stream_slicer_parameters_on_paginated_requests or False
1904
+ ignore_stream_slicer_parameters_on_paginated_requests = (
1905
+ model.ignore_stream_slicer_parameters_on_paginated_requests or False
1906
+ )
1405
1907
 
1406
1908
  if self._limit_slices_fetched or self._emit_connector_builder_messages:
1407
1909
  return SimpleRetrieverTestReadDecorator(
@@ -1468,14 +1970,19 @@ class ModelToComponentFactory:
1468
1970
  config: Config,
1469
1971
  *,
1470
1972
  name: str,
1471
- primary_key: Optional[Union[str, List[str], List[List[str]]]], # this seems to be needed to match create_simple_retriever
1973
+ primary_key: Optional[
1974
+ Union[str, List[str], List[List[str]]]
1975
+ ], # this seems to be needed to match create_simple_retriever
1472
1976
  stream_slicer: Optional[StreamSlicer],
1473
1977
  client_side_incremental_sync: Optional[Dict[str, Any]] = None,
1474
1978
  transformations: List[RecordTransformation],
1475
1979
  **kwargs: Any,
1476
1980
  ) -> AsyncRetriever:
1477
-
1478
- decoder = self._create_component_from_model(model=model.decoder, config=config) if model.decoder else JsonDecoder(parameters={})
1981
+ decoder = (
1982
+ self._create_component_from_model(model=model.decoder, config=config)
1983
+ if model.decoder
1984
+ else JsonDecoder(parameters={})
1985
+ )
1479
1986
  record_selector = self._create_component_from_model(
1480
1987
  model=model.record_selector,
1481
1988
  config=config,
@@ -1485,14 +1992,23 @@ class ModelToComponentFactory:
1485
1992
  )
1486
1993
  stream_slicer = stream_slicer or SinglePartitionRouter(parameters={})
1487
1994
  creation_requester = self._create_component_from_model(
1488
- model=model.creation_requester, decoder=decoder, config=config, name=f"job creation - {name}"
1995
+ model=model.creation_requester,
1996
+ decoder=decoder,
1997
+ config=config,
1998
+ name=f"job creation - {name}",
1489
1999
  )
1490
2000
  polling_requester = self._create_component_from_model(
1491
- model=model.polling_requester, decoder=decoder, config=config, name=f"job polling - {name}"
2001
+ model=model.polling_requester,
2002
+ decoder=decoder,
2003
+ config=config,
2004
+ name=f"job polling - {name}",
1492
2005
  )
1493
2006
  job_download_components_name = f"job download - {name}"
1494
2007
  download_requester = self._create_component_from_model(
1495
- model=model.download_requester, decoder=decoder, config=config, name=job_download_components_name
2008
+ model=model.download_requester,
2009
+ decoder=decoder,
2010
+ config=config,
2011
+ name=job_download_components_name,
1496
2012
  )
1497
2013
  download_retriever = SimpleRetriever(
1498
2014
  requester=download_requester,
@@ -1507,7 +2023,9 @@ class ModelToComponentFactory:
1507
2023
  primary_key=None,
1508
2024
  name=job_download_components_name,
1509
2025
  paginator=(
1510
- self._create_component_from_model(model=model.download_paginator, decoder=decoder, config=config, url_base="")
2026
+ self._create_component_from_model(
2027
+ model=model.download_paginator, decoder=decoder, config=config, url_base=""
2028
+ )
1511
2029
  if model.download_paginator
1512
2030
  else NoPagination(parameters={})
1513
2031
  ),
@@ -1515,17 +2033,31 @@ class ModelToComponentFactory:
1515
2033
  parameters={},
1516
2034
  )
1517
2035
  abort_requester = (
1518
- self._create_component_from_model(model=model.abort_requester, decoder=decoder, config=config, name=f"job abort - {name}")
2036
+ self._create_component_from_model(
2037
+ model=model.abort_requester,
2038
+ decoder=decoder,
2039
+ config=config,
2040
+ name=f"job abort - {name}",
2041
+ )
1519
2042
  if model.abort_requester
1520
2043
  else None
1521
2044
  )
1522
2045
  delete_requester = (
1523
- self._create_component_from_model(model=model.delete_requester, decoder=decoder, config=config, name=f"job delete - {name}")
2046
+ self._create_component_from_model(
2047
+ model=model.delete_requester,
2048
+ decoder=decoder,
2049
+ config=config,
2050
+ name=f"job delete - {name}",
2051
+ )
1524
2052
  if model.delete_requester
1525
2053
  else None
1526
2054
  )
1527
- status_extractor = self._create_component_from_model(model=model.status_extractor, decoder=decoder, config=config, name=name)
1528
- urls_extractor = self._create_component_from_model(model=model.urls_extractor, decoder=decoder, config=config, name=name)
2055
+ status_extractor = self._create_component_from_model(
2056
+ model=model.status_extractor, decoder=decoder, config=config, name=name
2057
+ )
2058
+ urls_extractor = self._create_component_from_model(
2059
+ model=model.urls_extractor, decoder=decoder, config=config, name=name
2060
+ )
1529
2061
  job_repository: AsyncJobRepository = AsyncHttpJobRepository(
1530
2062
  creation_requester=creation_requester,
1531
2063
  polling_requester=polling_requester,
@@ -1541,7 +2073,9 @@ class ModelToComponentFactory:
1541
2073
  job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator(
1542
2074
  job_repository,
1543
2075
  stream_slices,
1544
- JobTracker(1), # FIXME eventually make the number of concurrent jobs in the API configurable. Until then, we limit to 1
2076
+ JobTracker(
2077
+ 1
2078
+ ), # FIXME eventually make the number of concurrent jobs in the API configurable. Until then, we limit to 1
1545
2079
  self._message_repository,
1546
2080
  has_bulk_parent=False, # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
1547
2081
  ),
@@ -1567,14 +2101,22 @@ class ModelToComponentFactory:
1567
2101
  if model.parent_stream_configs:
1568
2102
  parent_stream_configs.extend(
1569
2103
  [
1570
- self._create_message_repository_substream_wrapper(model=parent_stream_config, config=config)
2104
+ self._create_message_repository_substream_wrapper(
2105
+ model=parent_stream_config, config=config
2106
+ )
1571
2107
  for parent_stream_config in model.parent_stream_configs
1572
2108
  ]
1573
2109
  )
1574
2110
 
1575
- return SubstreamPartitionRouter(parent_stream_configs=parent_stream_configs, parameters=model.parameters or {}, config=config)
2111
+ return SubstreamPartitionRouter(
2112
+ parent_stream_configs=parent_stream_configs,
2113
+ parameters=model.parameters or {},
2114
+ config=config,
2115
+ )
1576
2116
 
1577
- def _create_message_repository_substream_wrapper(self, model: ParentStreamConfigModel, config: Config) -> Any:
2117
+ def _create_message_repository_substream_wrapper(
2118
+ self, model: ParentStreamConfigModel, config: Config
2119
+ ) -> Any:
1578
2120
  substream_factory = ModelToComponentFactory(
1579
2121
  limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice,
1580
2122
  limit_slices_fetched=self._limit_slices_fetched,
@@ -1590,13 +2132,17 @@ class ModelToComponentFactory:
1590
2132
  return substream_factory._create_component_from_model(model=model, config=config)
1591
2133
 
1592
2134
  @staticmethod
1593
- def create_wait_time_from_header(model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any) -> WaitTimeFromHeaderBackoffStrategy:
2135
+ def create_wait_time_from_header(
2136
+ model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any
2137
+ ) -> WaitTimeFromHeaderBackoffStrategy:
1594
2138
  return WaitTimeFromHeaderBackoffStrategy(
1595
2139
  header=model.header,
1596
2140
  parameters=model.parameters or {},
1597
2141
  config=config,
1598
2142
  regex=model.regex,
1599
- max_waiting_time_in_seconds=model.max_waiting_time_in_seconds if model.max_waiting_time_in_seconds is not None else None,
2143
+ max_waiting_time_in_seconds=model.max_waiting_time_in_seconds
2144
+ if model.max_waiting_time_in_seconds is not None
2145
+ else None,
1600
2146
  )
1601
2147
 
1602
2148
  @staticmethod
@@ -1604,7 +2150,11 @@ class ModelToComponentFactory:
1604
2150
  model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any
1605
2151
  ) -> WaitUntilTimeFromHeaderBackoffStrategy:
1606
2152
  return WaitUntilTimeFromHeaderBackoffStrategy(
1607
- header=model.header, parameters=model.parameters or {}, config=config, min_wait=model.min_wait, regex=model.regex
2153
+ header=model.header,
2154
+ parameters=model.parameters or {},
2155
+ config=config,
2156
+ min_wait=model.min_wait,
2157
+ regex=model.regex,
1608
2158
  )
1609
2159
 
1610
2160
  def get_message_repository(self) -> MessageRepository: