airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4107__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (517) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +421 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +65 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +25 -3
  79. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  80. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  81. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  82. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +159 -74
  83. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  84. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  85. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  86. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  91. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  92. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  93. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  94. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  95. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  96. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  97. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
  98. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  99. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  100. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  101. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1695 -225
  102. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  103. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  104. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  105. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  106. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  107. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  108. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  109. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  122. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  123. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  124. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  125. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  126. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  127. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  134. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  135. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  136. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  140. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  141. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  142. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  143. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  144. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  145. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  146. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  147. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  148. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  149. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +228 -72
  150. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  151. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  152. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  153. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  154. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  155. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  156. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  157. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  158. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  159. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  160. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  163. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  164. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  165. airbyte_cdk/sources/declarative/types.py +19 -110
  166. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  167. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  168. airbyte_cdk/sources/embedded/catalog.py +16 -4
  169. airbyte_cdk/sources/embedded/runner.py +19 -3
  170. airbyte_cdk/sources/embedded/tools.py +5 -2
  171. airbyte_cdk/sources/file_based/README.md +152 -0
  172. airbyte_cdk/sources/file_based/__init__.py +24 -0
  173. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  174. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  175. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  176. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +58 -10
  177. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  178. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  179. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  180. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  181. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  182. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  184. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  185. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  186. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  187. airbyte_cdk/sources/file_based/exceptions.py +52 -15
  188. airbyte_cdk/sources/file_based/file_based_source.py +163 -33
  189. airbyte_cdk/sources/file_based/file_based_stream_reader.py +83 -5
  190. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  191. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  192. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  193. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  194. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  195. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  196. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  197. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  198. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +147 -41
  199. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  200. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  201. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  202. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  204. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  205. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  206. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  210. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  211. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  212. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +175 -45
  213. airbyte_cdk/sources/http_logger.py +8 -3
  214. airbyte_cdk/sources/message/__init__.py +7 -1
  215. airbyte_cdk/sources/message/repository.py +18 -4
  216. airbyte_cdk/sources/source.py +42 -38
  217. airbyte_cdk/sources/streams/__init__.py +2 -2
  218. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  219. airbyte_cdk/sources/streams/call_rate.py +64 -21
  220. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  221. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  222. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  223. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  224. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  225. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  226. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  227. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  228. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  229. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  230. airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
  231. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  232. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  233. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  234. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  236. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  237. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  238. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  239. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  240. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  241. airbyte_cdk/sources/streams/core.py +412 -87
  242. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  243. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  244. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  245. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  253. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  254. airbyte_cdk/sources/streams/http/http.py +369 -246
  255. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  256. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  257. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  259. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  260. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  261. airbyte_cdk/sources/types.py +154 -0
  262. airbyte_cdk/sources/utils/record_helper.py +36 -21
  263. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  264. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  265. airbyte_cdk/sources/utils/transform.py +54 -20
  266. airbyte_cdk/sql/_util/hashing.py +34 -0
  267. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  268. airbyte_cdk/sql/constants.py +32 -0
  269. airbyte_cdk/sql/exceptions.py +235 -0
  270. airbyte_cdk/sql/secrets.py +123 -0
  271. airbyte_cdk/sql/shared/__init__.py +15 -0
  272. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  273. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  274. airbyte_cdk/sql/types.py +160 -0
  275. airbyte_cdk/test/catalog_builder.py +70 -18
  276. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  277. airbyte_cdk/test/mock_http/__init__.py +1 -1
  278. airbyte_cdk/test/mock_http/matcher.py +6 -0
  279. airbyte_cdk/test/mock_http/mocker.py +57 -10
  280. airbyte_cdk/test/mock_http/request.py +19 -3
  281. airbyte_cdk/test/mock_http/response.py +3 -1
  282. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  283. airbyte_cdk/test/state_builder.py +18 -10
  284. airbyte_cdk/test/utils/__init__.py +1 -0
  285. airbyte_cdk/test/utils/data.py +24 -0
  286. airbyte_cdk/test/utils/http_mocking.py +16 -0
  287. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  288. airbyte_cdk/test/utils/reading.py +26 -0
  289. airbyte_cdk/utils/__init__.py +2 -1
  290. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  291. airbyte_cdk/utils/analytics_message.py +10 -2
  292. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  293. airbyte_cdk/utils/event_timing.py +10 -10
  294. airbyte_cdk/utils/mapping_helpers.py +3 -1
  295. airbyte_cdk/utils/message_utils.py +20 -11
  296. airbyte_cdk/utils/print_buffer.py +75 -0
  297. airbyte_cdk/utils/schema_inferrer.py +198 -28
  298. airbyte_cdk/utils/slice_hasher.py +30 -0
  299. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  300. airbyte_cdk/utils/stream_status_utils.py +8 -1
  301. airbyte_cdk/utils/traced_exception.py +61 -21
  302. airbyte_cdk-6.13.1.dev4107.dist-info/METADATA +109 -0
  303. airbyte_cdk-6.13.1.dev4107.dist-info/RECORD +349 -0
  304. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.13.1.dev4107.dist-info}/WHEEL +1 -2
  305. airbyte_cdk-6.13.1.dev4107.dist-info/entry_points.txt +3 -0
  306. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  307. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  308. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  309. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  311. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  312. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  313. airbyte_cdk/sources/deprecated/client.py +0 -99
  314. airbyte_cdk/sources/singer/__init__.py +0 -8
  315. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  316. airbyte_cdk/sources/singer/source.py +0 -186
  317. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  318. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  319. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  320. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  321. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  322. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  323. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  324. airbyte_cdk/sources/utils/schema_models.py +0 -84
  325. airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
  326. airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
  327. airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
  328. source_declarative_manifest/main.py +0 -29
  329. unit_tests/connector_builder/__init__.py +0 -3
  330. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  331. unit_tests/connector_builder/test_message_grouper.py +0 -713
  332. unit_tests/connector_builder/utils.py +0 -27
  333. unit_tests/destinations/test_destination.py +0 -243
  334. unit_tests/singer/test_singer_helpers.py +0 -56
  335. unit_tests/singer/test_singer_source.py +0 -112
  336. unit_tests/sources/__init__.py +0 -0
  337. unit_tests/sources/concurrent_source/__init__.py +0 -3
  338. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  339. unit_tests/sources/declarative/__init__.py +0 -3
  340. unit_tests/sources/declarative/auth/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  342. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  343. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  344. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  345. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  346. unit_tests/sources/declarative/checks/__init__.py +0 -3
  347. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  348. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  349. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  350. unit_tests/sources/declarative/external_component.py +0 -13
  351. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  352. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  353. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  355. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  356. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  357. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  359. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  360. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  361. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  362. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  363. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  364. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  365. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  366. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  367. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  368. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  369. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  370. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
  371. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  372. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  373. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  374. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  375. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  376. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  377. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  384. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  385. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  386. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  387. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  388. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  389. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  390. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  391. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  392. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  393. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  394. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  395. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  396. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  397. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  398. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  399. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  400. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  401. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  402. unit_tests/sources/declarative/schema/__init__.py +0 -6
  403. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  404. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  405. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  406. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  407. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  408. unit_tests/sources/declarative/states/__init__.py +0 -3
  409. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  411. unit_tests/sources/declarative/test_create_partial.py +0 -83
  412. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  413. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  414. unit_tests/sources/declarative/test_types.py +0 -39
  415. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  416. unit_tests/sources/file_based/__init__.py +0 -0
  417. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  419. unit_tests/sources/file_based/config/__init__.py +0 -0
  420. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  421. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  422. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  423. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  424. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  425. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  426. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  427. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  428. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  429. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  430. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  431. unit_tests/sources/file_based/helpers.py +0 -70
  432. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  433. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  434. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  435. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  436. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  437. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  438. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  439. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  440. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  441. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  442. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  443. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  444. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  445. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  446. unit_tests/sources/file_based/stream/__init__.py +0 -0
  447. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  449. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  450. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  451. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  452. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  453. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  454. unit_tests/sources/file_based/test_scenarios.py +0 -253
  455. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  456. unit_tests/sources/fixtures/__init__.py +0 -3
  457. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  458. unit_tests/sources/message/__init__.py +0 -0
  459. unit_tests/sources/message/test_repository.py +0 -153
  460. unit_tests/sources/streams/__init__.py +0 -0
  461. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  462. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  464. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  466. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  467. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  469. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  470. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  471. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  472. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  473. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  474. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  475. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  476. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  477. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  478. unit_tests/sources/streams/http/__init__.py +0 -0
  479. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  481. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  482. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  483. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  484. unit_tests/sources/streams/http/test_http.py +0 -635
  485. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  486. unit_tests/sources/streams/test_call_rate.py +0 -300
  487. unit_tests/sources/streams/test_stream_read.py +0 -405
  488. unit_tests/sources/streams/test_streams_core.py +0 -184
  489. unit_tests/sources/test_abstract_source.py +0 -1442
  490. unit_tests/sources/test_concurrent_source.py +0 -112
  491. unit_tests/sources/test_config.py +0 -92
  492. unit_tests/sources/test_connector_state_manager.py +0 -482
  493. unit_tests/sources/test_http_logger.py +0 -252
  494. unit_tests/sources/test_integration_source.py +0 -86
  495. unit_tests/sources/test_source.py +0 -684
  496. unit_tests/sources/test_source_read.py +0 -460
  497. unit_tests/test/__init__.py +0 -0
  498. unit_tests/test/mock_http/__init__.py +0 -0
  499. unit_tests/test/mock_http/test_matcher.py +0 -53
  500. unit_tests/test/mock_http/test_mocker.py +0 -214
  501. unit_tests/test/mock_http/test_request.py +0 -117
  502. unit_tests/test/mock_http/test_response_builder.py +0 -177
  503. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  504. unit_tests/utils/__init__.py +0 -0
  505. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  506. unit_tests/utils/test_mapping_helpers.py +0 -54
  507. unit_tests/utils/test_message_utils.py +0 -91
  508. unit_tests/utils/test_rate_limiting.py +0 -26
  509. unit_tests/utils/test_schema_inferrer.py +0 -202
  510. unit_tests/utils/test_secret_utils.py +0 -135
  511. unit_tests/utils/test_stream_status_utils.py +0 -61
  512. unit_tests/utils/test_traced_exception.py +0 -107
  513. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  514. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  515. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  516. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  517. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.13.1.dev4107.dist-info}/LICENSE.txt +0 -0
@@ -1,746 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
-
6
- from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError
7
- from airbyte_cdk.test.catalog_builder import CatalogBuilder
8
- from airbyte_protocol.models import SyncMode
9
- from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder
10
- from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder
11
-
12
- """
13
- User input schema rules:
14
- - `check`: Successful if the schema conforms to a record, otherwise ConfigValidationError.
15
- - `discover`: User-input schema is output if the schema is valid, otherwise ConfigValidationError.
16
- - `read`: If the schema is valid, record values are cast to types in the schema; if this is successful
17
- the records are emitted. otherwise an error is logged. If the schema is not valid, ConfigValidationError.
18
- """
19
-
20
-
21
- _base_user_input_schema_scenario = (
22
- TestScenarioBuilder()
23
- .set_source_builder(
24
- FileBasedSourceBuilder()
25
- .set_files(
26
- {
27
- "a.csv": {
28
- "contents": [
29
- ("col1", "col2"),
30
- ("val11", "val12"),
31
- ("val21", "val22"),
32
- ],
33
- "last_modified": "2023-06-05T03:54:07.000Z",
34
- }
35
- }
36
- )
37
- .set_file_type("csv")
38
- )
39
- .set_expected_catalog(
40
- {
41
- "streams": [
42
- {
43
- "default_cursor_field": ["_ab_source_file_last_modified"],
44
- "json_schema": {
45
- "type": "object",
46
- "properties": {
47
- "col1": {"type": "string"},
48
- "col2": {"type": "string"},
49
- "_ab_source_file_last_modified": {"type": "string"},
50
- "_ab_source_file_url": {"type": "string"},
51
- },
52
- },
53
- "name": "stream1",
54
- "source_defined_cursor": True,
55
- "supported_sync_modes": ["full_refresh", "incremental"],
56
- }
57
- ]
58
- }
59
- )
60
- .set_expected_records(
61
- [
62
- {
63
- "data": {
64
- "col1": "val11",
65
- "col2": "val12",
66
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
67
- "_ab_source_file_url": "a.csv",
68
- },
69
- "stream": "stream1",
70
- },
71
- {
72
- "data": {
73
- "col1": "val21",
74
- "col2": "val22",
75
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
76
- "_ab_source_file_url": "a.csv",
77
- },
78
- "stream": "stream1",
79
- },
80
- ]
81
- )
82
- )
83
-
84
-
85
- valid_single_stream_user_input_schema_scenario = (
86
- _base_user_input_schema_scenario.copy()
87
- .set_name("valid_single_stream_user_input_schema_scenario")
88
- .set_config(
89
- {
90
- "streams": [
91
- {
92
- "name": "stream1",
93
- "format": {"filetype": "csv"},
94
- "globs": ["*"],
95
- "validation_policy": "Emit Record",
96
- "input_schema": '{"col1": "string", "col2": "string"}',
97
- }
98
- ]
99
- }
100
- )
101
- .set_expected_check_status("SUCCEEDED")
102
- ).build()
103
-
104
-
105
- single_stream_user_input_schema_scenario_schema_is_invalid = (
106
- _base_user_input_schema_scenario.copy()
107
- .set_name("single_stream_user_input_schema_scenario_schema_is_invalid")
108
- .set_config(
109
- {
110
- "streams": [
111
- {
112
- "name": "stream1",
113
- "format": {"filetype": "csv"},
114
- "globs": ["*"],
115
- "validation_policy": "Emit Record",
116
- "input_schema": '{"col1": "x", "col2": "string"}',
117
- }
118
- ]
119
- }
120
- )
121
- .set_catalog(CatalogBuilder().with_stream("stream1", SyncMode.full_refresh).build())
122
- .set_expected_check_status("FAILED")
123
- .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value)
124
- .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value)
125
- .set_expected_read_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value)
126
- ).build()
127
-
128
-
129
- single_stream_user_input_schema_scenario_emit_nonconforming_records = (
130
- _base_user_input_schema_scenario.copy()
131
- .set_name("single_stream_user_input_schema_scenario_emit_nonconforming_records")
132
- .set_config(
133
- {
134
- "streams": [
135
- {
136
- "name": "stream1",
137
- "format": {"filetype": "csv"},
138
- "globs": ["*"],
139
- "validation_policy": "Emit Record",
140
- "input_schema": '{"col1": "integer", "col2": "string"}',
141
- }
142
- ]
143
- }
144
- )
145
- .set_expected_check_status("FAILED")
146
- .set_expected_check_error(None, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value)
147
- .set_expected_catalog(
148
- {
149
- "streams": [
150
- {
151
- "default_cursor_field": ["_ab_source_file_last_modified"],
152
- "json_schema": {
153
- "type": "object",
154
- "properties": {
155
- "col1": {"type": "integer"},
156
- "col2": {"type": "string"},
157
- "_ab_source_file_last_modified": {"type": "string"},
158
- "_ab_source_file_url": {"type": "string"},
159
- },
160
- },
161
- "name": "stream1",
162
- "source_defined_cursor": True,
163
- "supported_sync_modes": ["full_refresh", "incremental"],
164
- }
165
- ]
166
- }
167
- )
168
- ).build()
169
-
170
-
171
- single_stream_user_input_schema_scenario_skip_nonconforming_records = (
172
- _base_user_input_schema_scenario.copy()
173
- .set_name("single_stream_user_input_schema_scenario_skip_nonconforming_records")
174
- .set_config(
175
- {
176
- "streams": [
177
- {
178
- "name": "stream1",
179
- "format": {"filetype": "csv"},
180
- "globs": ["*"],
181
- "validation_policy": "Skip Record",
182
- "input_schema": '{"col1": "integer", "col2": "string"}',
183
- }
184
- ]
185
- }
186
- )
187
- .set_expected_check_status("FAILED")
188
- .set_expected_check_error(None, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value)
189
- .set_expected_catalog(
190
- {
191
- "streams": [
192
- {
193
- "default_cursor_field": ["_ab_source_file_last_modified"],
194
- "json_schema": {
195
- "type": "object",
196
- "properties": {
197
- "col1": {"type": "integer"},
198
- "col2": {"type": "string"},
199
- "_ab_source_file_last_modified": {"type": "string"},
200
- "_ab_source_file_url": {"type": "string"},
201
- },
202
- },
203
- "name": "stream1",
204
- "source_defined_cursor": True,
205
- "supported_sync_modes": ["full_refresh", "incremental"],
206
- }
207
- ]
208
- }
209
- )
210
- .set_expected_records([])
211
- .set_expected_logs(
212
- {
213
- "read": [
214
- {
215
- "level": "WARN",
216
- "message": "Records in file did not pass validation policy. stream=stream1 file=a.csv n_skipped=2 validation_policy=skip_record",
217
- },
218
- {
219
- "level": "WARN",
220
- "message": "Could not cast the value to the expected type.: col1: value=val11,expected_type=integer",
221
- },
222
- {
223
- "level": "WARN",
224
- "message": "Could not cast the value to the expected type.: col1: value=val21,expected_type=integer",
225
- },
226
- ]
227
- }
228
- )
229
- ).build()
230
-
231
-
232
- _base_multi_stream_user_input_schema_scenario = (
233
- TestScenarioBuilder()
234
- .set_source_builder(
235
- FileBasedSourceBuilder()
236
- .set_files(
237
- {
238
- "a.csv": {
239
- "contents": [
240
- ("col1", "col2"),
241
- ("val11a", 21),
242
- ("val12a", 22),
243
- ],
244
- "last_modified": "2023-06-05T03:54:07.000Z",
245
- },
246
- "b.csv": {
247
- "contents": [
248
- ("col1", "col2", "col3"),
249
- ("val11b", "val12b", "val13b"),
250
- ("val21b", "val22b", "val23b"),
251
- ],
252
- "last_modified": "2023-06-05T03:54:07.000Z",
253
- },
254
- "c.csv": {
255
- "contents": [
256
- ("col1",),
257
- ("val11c",),
258
- ("val21c",),
259
- ],
260
- "last_modified": "2023-06-05T03:54:07.000Z",
261
- },
262
- }
263
- )
264
- .set_file_type("csv")
265
- )
266
- .set_expected_catalog(
267
- {
268
- "streams": [
269
- {
270
- "default_cursor_field": ["_ab_source_file_last_modified"],
271
- "json_schema": {
272
- "type": "object",
273
- "properties": {
274
- "col1": {
275
- "type": "string",
276
- },
277
- "col2": {
278
- "type": "integer",
279
- },
280
- "_ab_source_file_last_modified": {"type": "string"},
281
- "_ab_source_file_url": {"type": "string"},
282
- },
283
- },
284
- "name": "stream1",
285
- "source_defined_cursor": True,
286
- "supported_sync_modes": ["full_refresh", "incremental"],
287
- },
288
- {
289
- "default_cursor_field": ["_ab_source_file_last_modified"],
290
- "json_schema": {
291
- "type": "object",
292
- "properties": {
293
- "col1": {
294
- "type": "string",
295
- },
296
- "col2": {
297
- "type": "string",
298
- },
299
- "col3": {
300
- "type": "string",
301
- },
302
- "_ab_source_file_last_modified": {"type": "string"},
303
- "_ab_source_file_url": {"type": "string"},
304
- },
305
- },
306
- "name": "stream2",
307
- "source_defined_cursor": True,
308
- "supported_sync_modes": ["full_refresh", "incremental"],
309
- },
310
- {
311
- "default_cursor_field": ["_ab_source_file_last_modified"],
312
- "json_schema": {
313
- "type": "object",
314
- "properties": {
315
- "col1": {
316
- "type": ["null", "string"],
317
- },
318
- "_ab_source_file_last_modified": {"type": "string"},
319
- "_ab_source_file_url": {"type": "string"},
320
- },
321
- },
322
- "name": "stream3",
323
- "source_defined_cursor": True,
324
- "supported_sync_modes": ["full_refresh", "incremental"],
325
- },
326
- ]
327
- }
328
- )
329
- .set_expected_records(
330
- [
331
- {
332
- "data": {
333
- "col1": "val11a",
334
- "col2": 21,
335
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
336
- "_ab_source_file_url": "a.csv",
337
- },
338
- "stream": "stream1",
339
- },
340
- {
341
- "data": {
342
- "col1": "val12a",
343
- "col2": 22,
344
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
345
- "_ab_source_file_url": "a.csv",
346
- },
347
- "stream": "stream1",
348
- },
349
- # The files in b.csv are emitted despite having an invalid schema
350
- {
351
- "data": {
352
- "col1": "val11b",
353
- "col2": "val12b",
354
- "col3": "val13b",
355
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
356
- "_ab_source_file_url": "b.csv",
357
- },
358
- "stream": "stream2",
359
- },
360
- {
361
- "data": {
362
- "col1": "val21b",
363
- "col2": "val22b",
364
- "col3": "val23b",
365
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
366
- "_ab_source_file_url": "b.csv",
367
- },
368
- "stream": "stream2",
369
- },
370
- {
371
- "data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"},
372
- "stream": "stream3",
373
- },
374
- {
375
- "data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"},
376
- "stream": "stream3",
377
- },
378
- ]
379
- )
380
- )
381
-
382
-
383
- valid_multi_stream_user_input_schema_scenario = (
384
- _base_multi_stream_user_input_schema_scenario.copy()
385
- .set_name("valid_multi_stream_user_input_schema_scenario")
386
- .set_config(
387
- {
388
- "streams": [
389
- {
390
- "name": "stream1",
391
- "format": {"filetype": "csv"},
392
- "globs": ["a.csv"],
393
- "validation_policy": "Emit Record",
394
- "input_schema": '{"col1": "string", "col2": "integer"}',
395
- },
396
- {
397
- "name": "stream2",
398
- "format": {"filetype": "csv"},
399
- "globs": ["b.csv"],
400
- "validation_policy": "Emit Record",
401
- "input_schema": '{"col1": "string", "col2": "string", "col3": "string"}',
402
- },
403
- {
404
- "name": "stream3",
405
- "format": {"filetype": "csv"},
406
- "globs": ["c.csv"],
407
- "validation_policy": "Emit Record",
408
- },
409
- ]
410
- }
411
- )
412
- .set_expected_check_status("SUCCEEDED")
413
- ).build()
414
-
415
-
416
- multi_stream_user_input_schema_scenario_schema_is_invalid = (
417
- _base_multi_stream_user_input_schema_scenario.copy()
418
- .set_name("multi_stream_user_input_schema_scenario_schema_is_invalid")
419
- .set_config(
420
- {
421
- "streams": [
422
- {
423
- "name": "stream1",
424
- "format": {"filetype": "csv"},
425
- "globs": ["a.csv"],
426
- "validation_policy": "Emit Record",
427
- "input_schema": '{"col1": "string", "col2": "integer"}',
428
- },
429
- {
430
- "name": "stream2",
431
- "format": {"filetype": "csv"},
432
- "globs": ["b.csv"],
433
- "validation_policy": "Emit Record",
434
- "input_schema": '{"col1": "x", "col2": "string", "col3": "string"}', # this stream's schema is invalid
435
- },
436
- {
437
- "name": "stream3",
438
- "format": {"filetype": "csv"},
439
- "globs": ["c.csv"],
440
- "validation_policy": "Emit Record",
441
- },
442
- ]
443
- }
444
- )
445
- .set_catalog(
446
- CatalogBuilder()
447
- .with_stream("stream1", SyncMode.full_refresh)
448
- .with_stream("stream2", SyncMode.full_refresh)
449
- .with_stream("stream3", SyncMode.full_refresh)
450
- .build()
451
- )
452
- .set_expected_check_status("FAILED")
453
- .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value)
454
- .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value)
455
- .set_expected_read_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value)
456
- ).build()
457
-
458
-
459
- multi_stream_user_input_schema_scenario_emit_nonconforming_records = (
460
- _base_multi_stream_user_input_schema_scenario.copy()
461
- .set_name("multi_stream_user_input_schema_scenario_emit_nonconforming_records")
462
- .set_config(
463
- {
464
- "streams": [
465
- {
466
- "name": "stream1",
467
- "format": {"filetype": "csv"},
468
- "globs": ["a.csv"],
469
- "validation_policy": "Emit Record",
470
- "input_schema": '{"col1": "string", "col2": "integer"}',
471
- },
472
- {
473
- "name": "stream2",
474
- "format": {"filetype": "csv"},
475
- "globs": ["b.csv"],
476
- "validation_policy": "Emit Record",
477
- "input_schema": '{"col1": "string", "col2": "integer", "col3": "string"}', # this stream's records do not conform to the schema
478
- },
479
- {
480
- "name": "stream3",
481
- "format": {"filetype": "csv"},
482
- "globs": ["c.csv"],
483
- "validation_policy": "Emit Record",
484
- },
485
- ]
486
- }
487
- )
488
- .set_expected_catalog(
489
- {
490
- "streams": [
491
- {
492
- "default_cursor_field": ["_ab_source_file_last_modified"],
493
- "json_schema": {
494
- "type": "object",
495
- "properties": {
496
- "col1": {"type": "string"},
497
- "col2": {"type": "integer"},
498
- "_ab_source_file_last_modified": {"type": "string"},
499
- "_ab_source_file_url": {"type": "string"},
500
- },
501
- },
502
- "name": "stream1",
503
- "source_defined_cursor": True,
504
- "supported_sync_modes": ["full_refresh", "incremental"],
505
- },
506
- {
507
- "default_cursor_field": ["_ab_source_file_last_modified"],
508
- "json_schema": {
509
- "type": "object",
510
- "properties": {
511
- "col1": {"type": "string"},
512
- "col2": {"type": "integer"},
513
- "col3": {"type": "string"},
514
- "_ab_source_file_last_modified": {"type": "string"},
515
- "_ab_source_file_url": {"type": "string"},
516
- },
517
- },
518
- "name": "stream2",
519
- "source_defined_cursor": True,
520
- "supported_sync_modes": ["full_refresh", "incremental"],
521
- },
522
- {
523
- "default_cursor_field": ["_ab_source_file_last_modified"],
524
- "json_schema": {
525
- "type": "object",
526
- "properties": {
527
- "col1": {
528
- "type": ["null", "string"],
529
- },
530
- "_ab_source_file_last_modified": {"type": "string"},
531
- "_ab_source_file_url": {"type": "string"},
532
- },
533
- },
534
- "name": "stream3",
535
- "source_defined_cursor": True,
536
- "supported_sync_modes": ["full_refresh", "incremental"],
537
- },
538
- ]
539
- }
540
- )
541
- .set_expected_check_status("FAILED")
542
- .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value)
543
- .set_expected_records(
544
- [
545
- {
546
- "data": {
547
- "col1": "val11a",
548
- "col2": 21,
549
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
550
- "_ab_source_file_url": "a.csv",
551
- },
552
- "stream": "stream1",
553
- },
554
- {
555
- "data": {
556
- "col1": "val12a",
557
- "col2": 22,
558
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
559
- "_ab_source_file_url": "a.csv",
560
- },
561
- "stream": "stream1",
562
- },
563
- {
564
- "data": {
565
- "col1": "val11b",
566
- "col2": "val12b",
567
- "col3": "val13b",
568
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
569
- "_ab_source_file_url": "b.csv",
570
- },
571
- "stream": "stream2",
572
- },
573
- {
574
- "data": {
575
- "col1": "val21b",
576
- "col2": "val22b",
577
- "col3": "val23b",
578
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
579
- "_ab_source_file_url": "b.csv",
580
- },
581
- "stream": "stream2",
582
- },
583
- {
584
- "data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"},
585
- "stream": "stream3",
586
- },
587
- {
588
- "data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"},
589
- "stream": "stream3",
590
- },
591
- ]
592
- )
593
- .set_expected_logs(
594
- {
595
- "read": [
596
- {
597
- "level": "WARN",
598
- "message": "Could not cast the value to the expected type.: col2: value=val12b,expected_type=integer",
599
- },
600
- {
601
- "level": "WARN",
602
- "message": "Could not cast the value to the expected type.: col2: value=val22b,expected_type=integer",
603
- },
604
- ]
605
- }
606
- )
607
- ).build()
608
-
609
-
610
- multi_stream_user_input_schema_scenario_skip_nonconforming_records = (
611
- _base_multi_stream_user_input_schema_scenario.copy()
612
- .set_name("multi_stream_user_input_schema_scenario_skip_nonconforming_records")
613
- .set_config(
614
- {
615
- "streams": [
616
- {
617
- "name": "stream1",
618
- "format": {"filetype": "csv"},
619
- "globs": ["a.csv"],
620
- "validation_policy": "Emit Record",
621
- "input_schema": '{"col1": "string", "col2": "integer"}',
622
- },
623
- {
624
- "name": "stream2",
625
- "format": {"filetype": "csv"},
626
- "globs": ["b.csv"],
627
- "validation_policy": "Skip Record",
628
- "input_schema": '{"col1": "string", "col2": "integer", "col3": "string"}', # this stream's records do not conform to the schema
629
- },
630
- {
631
- "name": "stream3",
632
- "format": {"filetype": "csv"},
633
- "globs": ["c.csv"],
634
- "validation_policy": "Emit Record",
635
- },
636
- ]
637
- }
638
- )
639
- .set_expected_catalog(
640
- {
641
- "streams": [
642
- {
643
- "default_cursor_field": ["_ab_source_file_last_modified"],
644
- "json_schema": {
645
- "type": "object",
646
- "properties": {
647
- "col1": {"type": "string"},
648
- "col2": {"type": "integer"},
649
- "_ab_source_file_last_modified": {"type": "string"},
650
- "_ab_source_file_url": {"type": "string"},
651
- },
652
- },
653
- "name": "stream1",
654
- "source_defined_cursor": True,
655
- "supported_sync_modes": ["full_refresh", "incremental"],
656
- },
657
- {
658
- "default_cursor_field": ["_ab_source_file_last_modified"],
659
- "json_schema": {
660
- "type": "object",
661
- "properties": {
662
- "col1": {"type": "string"},
663
- "col2": {"type": "integer"},
664
- "col3": {"type": "string"},
665
- "_ab_source_file_last_modified": {"type": "string"},
666
- "_ab_source_file_url": {"type": "string"},
667
- },
668
- },
669
- "name": "stream2",
670
- "source_defined_cursor": True,
671
- "supported_sync_modes": ["full_refresh", "incremental"],
672
- },
673
- {
674
- "default_cursor_field": ["_ab_source_file_last_modified"],
675
- "json_schema": {
676
- "type": "object",
677
- "properties": {
678
- "col1": {
679
- "type": ["null", "string"],
680
- },
681
- "_ab_source_file_last_modified": {"type": "string"},
682
- "_ab_source_file_url": {"type": "string"},
683
- },
684
- },
685
- "name": "stream3",
686
- "source_defined_cursor": True,
687
- "supported_sync_modes": ["full_refresh", "incremental"],
688
- },
689
- ]
690
- }
691
- )
692
- .set_expected_check_status("FAILED")
693
- .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value)
694
- .set_expected_records(
695
- [
696
- {
697
- "data": {
698
- "col1": "val11a",
699
- "col2": 21,
700
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
701
- "_ab_source_file_url": "a.csv",
702
- },
703
- "stream": "stream1",
704
- },
705
- {
706
- "data": {
707
- "col1": "val12a",
708
- "col2": 22,
709
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
710
- "_ab_source_file_url": "a.csv",
711
- },
712
- "stream": "stream1",
713
- },
714
- # {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
715
- # "_ab_source_file_url": "b.csv"}, "stream": "stream2"},
716
- # {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
717
- # "_ab_source_file_url": "b.csv"}, "stream": "stream2"},
718
- {
719
- "data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"},
720
- "stream": "stream3",
721
- },
722
- {
723
- "data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"},
724
- "stream": "stream3",
725
- },
726
- ]
727
- )
728
- .set_expected_logs(
729
- {
730
- "read": [
731
- {
732
- "level": "WARN",
733
- "message": "Records in file did not pass validation policy. stream=stream2 file=b.csv n_skipped=2 validation_policy=skip_record",
734
- },
735
- {
736
- "level": "WARN",
737
- "message": "Could not cast the value to the expected type.: col2: value=val12b,expected_type=integer",
738
- },
739
- {
740
- "level": "WARN",
741
- "message": "Could not cast the value to the expected type.: col2: value=val22b,expected_type=integer",
742
- },
743
- ]
744
- }
745
- )
746
- ).build()