airbyte-cdk 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (368) hide show
  1. airbyte_cdk/__init__.py +358 -0
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +104 -0
  7. airbyte_cdk/connector.py +123 -0
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/__init__.py +3 -0
  10. airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
  11. airbyte_cdk/connector_builder/main.py +107 -0
  12. airbyte_cdk/connector_builder/models.py +73 -0
  13. airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
  14. airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
  15. airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
  16. airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
  17. airbyte_cdk/connector_builder/test_reader/types.py +83 -0
  18. airbyte_cdk/destinations/__init__.py +8 -0
  19. airbyte_cdk/destinations/destination.py +154 -0
  20. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  21. airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
  22. airbyte_cdk/destinations/vector_db_based/config.py +298 -0
  23. airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
  24. airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
  25. airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
  26. airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
  27. airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
  28. airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
  29. airbyte_cdk/entrypoint.py +414 -0
  30. airbyte_cdk/exception_handler.py +56 -0
  31. airbyte_cdk/logger.py +109 -0
  32. airbyte_cdk/models/__init__.py +72 -0
  33. airbyte_cdk/models/airbyte_protocol.py +88 -0
  34. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  35. airbyte_cdk/models/well_known_types.py +5 -0
  36. airbyte_cdk/py.typed +0 -0
  37. airbyte_cdk/sources/__init__.py +26 -0
  38. airbyte_cdk/sources/abstract_source.py +326 -0
  39. airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
  40. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
  41. airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
  42. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
  43. airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
  44. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  45. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
  46. airbyte_cdk/sources/config.py +27 -0
  47. airbyte_cdk/sources/connector_state_manager.py +161 -0
  48. airbyte_cdk/sources/declarative/__init__.py +3 -0
  49. airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
  50. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  51. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
  52. airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
  53. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  54. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  55. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  56. airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
  57. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
  58. airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
  59. airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
  60. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
  61. airbyte_cdk/sources/declarative/auth/token.py +267 -0
  62. airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
  63. airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
  64. airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
  65. airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
  66. airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
  67. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  68. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  69. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
  70. airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
  71. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
  72. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
  73. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
  74. airbyte_cdk/sources/declarative/declarative_source.py +36 -0
  75. airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
  76. airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
  77. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
  78. airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
  79. airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
  80. airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
  81. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  82. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  83. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  84. airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
  85. airbyte_cdk/sources/declarative/exceptions.py +9 -0
  86. airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
  87. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
  88. airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
  89. airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
  90. airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
  91. airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
  92. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
  93. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  94. airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
  95. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
  96. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
  97. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  98. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
  99. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
  100. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  101. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  102. airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
  103. airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
  104. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
  105. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
  106. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
  107. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
  108. airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
  109. airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
  110. airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
  111. airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
  112. airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
  113. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  114. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  115. airbyte_cdk/sources/declarative/models/__init__.py +2 -0
  116. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
  117. airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
  118. airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
  119. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
  120. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
  121. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
  122. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
  123. airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
  124. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  125. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  126. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
  127. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  128. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
  129. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
  130. airbyte_cdk/sources/declarative/requesters/README.md +56 -0
  131. airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
  132. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
  133. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
  134. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
  135. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
  136. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
  137. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
  138. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
  139. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
  140. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
  141. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
  142. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  143. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
  144. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
  145. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
  146. airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
  147. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
  148. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
  149. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
  150. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
  151. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
  152. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
  153. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
  154. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
  155. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
  156. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
  157. airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
  158. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
  159. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
  160. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  161. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
  162. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
  163. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
  164. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
  165. airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
  166. airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
  167. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  168. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  169. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  170. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  171. airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
  172. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
  173. airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
  174. airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
  175. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
  176. airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
  177. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
  178. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
  179. airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
  180. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
  181. airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
  182. airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
  183. airbyte_cdk/sources/declarative/spec/spec.py +48 -0
  184. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
  185. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
  186. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
  187. airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
  188. airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
  189. airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
  190. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  191. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  192. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  193. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  194. airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
  195. airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
  196. airbyte_cdk/sources/declarative/types.py +25 -0
  197. airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
  198. airbyte_cdk/sources/file_based/README.md +152 -0
  199. airbyte_cdk/sources/file_based/__init__.py +24 -0
  200. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
  201. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
  202. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
  203. airbyte_cdk/sources/file_based/config/__init__.py +0 -0
  204. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
  205. airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
  206. airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
  207. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  208. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
  209. airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
  210. airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
  211. airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
  212. airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
  213. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
  214. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
  215. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
  216. airbyte_cdk/sources/file_based/exceptions.py +159 -0
  217. airbyte_cdk/sources/file_based/file_based_source.py +466 -0
  218. airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
  219. airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
  220. airbyte_cdk/sources/file_based/file_record_data.py +22 -0
  221. airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
  222. airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
  223. airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
  224. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  225. airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
  226. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
  227. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
  228. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
  229. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
  230. airbyte_cdk/sources/file_based/remote_file.py +18 -0
  231. airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
  232. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
  233. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
  234. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
  235. airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
  236. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
  237. airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
  238. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
  239. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
  240. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
  241. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
  242. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
  243. airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
  244. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
  245. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
  246. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
  247. airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
  248. airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
  249. airbyte_cdk/sources/file_based/types.py +10 -0
  250. airbyte_cdk/sources/http_config.py +10 -0
  251. airbyte_cdk/sources/http_logger.py +55 -0
  252. airbyte_cdk/sources/message/__init__.py +19 -0
  253. airbyte_cdk/sources/message/repository.py +137 -0
  254. airbyte_cdk/sources/source.py +95 -0
  255. airbyte_cdk/sources/specs/transfer_modes.py +26 -0
  256. airbyte_cdk/sources/streams/__init__.py +8 -0
  257. airbyte_cdk/sources/streams/availability_strategy.py +84 -0
  258. airbyte_cdk/sources/streams/call_rate.py +704 -0
  259. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  260. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  261. airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
  262. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  263. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  264. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  265. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  266. airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
  267. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
  268. airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
  269. airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
  270. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
  271. airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
  272. airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
  273. airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
  274. airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
  275. airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
  276. airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
  277. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
  278. airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
  279. airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
  280. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
  281. airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
  282. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  283. airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
  284. airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
  285. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
  286. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
  287. airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
  288. airbyte_cdk/sources/streams/core.py +703 -0
  289. airbyte_cdk/sources/streams/http/__init__.py +10 -0
  290. airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
  291. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  292. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  293. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  294. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  295. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  296. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  297. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  298. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  299. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  300. airbyte_cdk/sources/streams/http/exceptions.py +61 -0
  301. airbyte_cdk/sources/streams/http/http.py +673 -0
  302. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  303. airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
  304. airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
  305. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
  306. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
  307. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
  308. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
  309. airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
  310. airbyte_cdk/sources/streams/utils/__init__.py +3 -0
  311. airbyte_cdk/sources/types.py +169 -0
  312. airbyte_cdk/sources/utils/__init__.py +7 -0
  313. airbyte_cdk/sources/utils/casing.py +12 -0
  314. airbyte_cdk/sources/utils/files_directory.py +15 -0
  315. airbyte_cdk/sources/utils/record_helper.py +53 -0
  316. airbyte_cdk/sources/utils/schema_helpers.py +230 -0
  317. airbyte_cdk/sources/utils/slice_logger.py +57 -0
  318. airbyte_cdk/sources/utils/transform.py +277 -0
  319. airbyte_cdk/sources/utils/types.py +7 -0
  320. airbyte_cdk/sql/__init__.py +0 -0
  321. airbyte_cdk/sql/_util/__init__.py +0 -0
  322. airbyte_cdk/sql/_util/hashing.py +34 -0
  323. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  324. airbyte_cdk/sql/constants.py +32 -0
  325. airbyte_cdk/sql/exceptions.py +235 -0
  326. airbyte_cdk/sql/secrets.py +123 -0
  327. airbyte_cdk/sql/shared/__init__.py +15 -0
  328. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  329. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  330. airbyte_cdk/sql/types.py +160 -0
  331. airbyte_cdk/test/__init__.py +7 -0
  332. airbyte_cdk/test/catalog_builder.py +81 -0
  333. airbyte_cdk/test/entrypoint_wrapper.py +250 -0
  334. airbyte_cdk/test/mock_http/__init__.py +6 -0
  335. airbyte_cdk/test/mock_http/matcher.py +41 -0
  336. airbyte_cdk/test/mock_http/mocker.py +185 -0
  337. airbyte_cdk/test/mock_http/request.py +103 -0
  338. airbyte_cdk/test/mock_http/response.py +28 -0
  339. airbyte_cdk/test/mock_http/response_builder.py +237 -0
  340. airbyte_cdk/test/state_builder.py +33 -0
  341. airbyte_cdk/test/utils/__init__.py +1 -0
  342. airbyte_cdk/test/utils/data.py +24 -0
  343. airbyte_cdk/test/utils/http_mocking.py +16 -0
  344. airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
  345. airbyte_cdk/test/utils/reading.py +26 -0
  346. airbyte_cdk/utils/__init__.py +10 -0
  347. airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
  348. airbyte_cdk/utils/analytics_message.py +25 -0
  349. airbyte_cdk/utils/constants.py +5 -0
  350. airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
  351. airbyte_cdk/utils/datetime_helpers.py +499 -0
  352. airbyte_cdk/utils/event_timing.py +85 -0
  353. airbyte_cdk/utils/is_cloud_environment.py +18 -0
  354. airbyte_cdk/utils/mapping_helpers.py +162 -0
  355. airbyte_cdk/utils/message_utils.py +26 -0
  356. airbyte_cdk/utils/oneof_option_config.py +33 -0
  357. airbyte_cdk/utils/print_buffer.py +75 -0
  358. airbyte_cdk/utils/schema_inferrer.py +270 -0
  359. airbyte_cdk/utils/slice_hasher.py +37 -0
  360. airbyte_cdk/utils/spec_schema_transformations.py +26 -0
  361. airbyte_cdk/utils/stream_status_utils.py +43 -0
  362. airbyte_cdk/utils/traced_exception.py +145 -0
  363. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
  364. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
  365. airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
  366. airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
  367. airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
  368. airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,36 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ import logging
6
+ from abc import abstractmethod
7
+ from typing import Any, Mapping, Tuple
8
+
9
+ from airbyte_cdk.sources.abstract_source import AbstractSource
10
+ from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
11
+
12
+
13
+ class DeclarativeSource(AbstractSource):
14
+ """
15
+ Base class for declarative Source. Concrete sources need to define the connection_checker to use
16
+ """
17
+
18
+ @property
19
+ @abstractmethod
20
+ def connection_checker(self) -> ConnectionChecker:
21
+ """Returns the ConnectionChecker to use for the `check` operation"""
22
+
23
+ def check_connection(
24
+ self, logger: logging.Logger, config: Mapping[str, Any]
25
+ ) -> Tuple[bool, Any]:
26
+ """
27
+ :param logger: The source logger
28
+ :param config: The user-provided configuration as specified by the source's spec.
29
+ This usually contains information required to check connection e.g. tokens, secrets and keys etc.
30
+ :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful
31
+ and we can connect to the underlying data source using the provided configuration.
32
+ Otherwise, the input config cannot be used to connect to the underlying data source,
33
+ and the "error" object should describe what went wrong.
34
+ The error object will be cast to string to display the problem to the user.
35
+ """
36
+ return self.connection_checker.check_connection(self, logger, config)
@@ -0,0 +1,241 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+ import logging
5
+ from dataclasses import InitVar, dataclass, field
6
+ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
7
+
8
+ from airbyte_cdk.models import SyncMode
9
+ from airbyte_cdk.sources.declarative.incremental import (
10
+ GlobalSubstreamCursor,
11
+ PerPartitionCursor,
12
+ PerPartitionWithGlobalCursor,
13
+ )
14
+ from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
15
+ from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
16
+ from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever
17
+ from airbyte_cdk.sources.declarative.retrievers.async_retriever import AsyncRetriever
18
+ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
19
+ from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
20
+ from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
21
+ from airbyte_cdk.sources.streams.checkpoint import (
22
+ CheckpointMode,
23
+ CheckpointReader,
24
+ Cursor,
25
+ CursorBasedCheckpointReader,
26
+ )
27
+ from airbyte_cdk.sources.streams.core import Stream
28
+ from airbyte_cdk.sources.types import Config, StreamSlice
29
+
30
+
31
+ @dataclass
32
+ class DeclarativeStream(Stream):
33
+ """
34
+ DeclarativeStream is a Stream that delegates most of its logic to its schema_load and retriever
35
+
36
+ Attributes:
37
+ name (str): stream name
38
+ primary_key (Optional[Union[str, List[str], List[List[str]]]]): the primary key of the stream
39
+ schema_loader (SchemaLoader): The schema loader
40
+ retriever (Retriever): The retriever
41
+ config (Config): The user-provided configuration as specified by the source's spec
42
+ stream_cursor_field (Optional[Union[InterpolatedString, str]]): The cursor field
43
+ stream. Transformations are applied in the order in which they are defined.
44
+ """
45
+
46
+ retriever: Retriever
47
+ config: Config
48
+ parameters: InitVar[Mapping[str, Any]]
49
+ name: str
50
+ primary_key: Optional[Union[str, List[str], List[List[str]]]]
51
+ state_migrations: List[StateMigration] = field(repr=True, default_factory=list)
52
+ schema_loader: Optional[SchemaLoader] = None
53
+ _name: str = field(init=False, repr=False, default="")
54
+ _primary_key: str = field(init=False, repr=False, default="")
55
+ stream_cursor_field: Optional[Union[InterpolatedString, str]] = None
56
+
57
+ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
58
+ self._stream_cursor_field = (
59
+ InterpolatedString.create(self.stream_cursor_field, parameters=parameters)
60
+ if isinstance(self.stream_cursor_field, str)
61
+ else self.stream_cursor_field
62
+ )
63
+ self._schema_loader = (
64
+ self.schema_loader
65
+ if self.schema_loader
66
+ else DefaultSchemaLoader(config=self.config, parameters=parameters)
67
+ )
68
+
69
+ @property # type: ignore
70
+ def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
71
+ return self._primary_key
72
+
73
+ @primary_key.setter
74
+ def primary_key(self, value: str) -> None:
75
+ if not isinstance(value, property):
76
+ self._primary_key = value
77
+
78
+ @property
79
+ def exit_on_rate_limit(self) -> bool:
80
+ if isinstance(self.retriever, AsyncRetriever):
81
+ return self.retriever.exit_on_rate_limit
82
+
83
+ return self.retriever.requester.exit_on_rate_limit # type: ignore # abstract Retriever class has not requester attribute
84
+
85
+ @exit_on_rate_limit.setter
86
+ def exit_on_rate_limit(self, value: bool) -> None:
87
+ if isinstance(self.retriever, AsyncRetriever):
88
+ self.retriever.exit_on_rate_limit = value
89
+ else:
90
+ self.retriever.requester.exit_on_rate_limit = value # type: ignore[attr-defined]
91
+
92
+ @property # type: ignore
93
+ def name(self) -> str:
94
+ """
95
+ :return: Stream name. By default this is the implementing class name, but it can be overridden as needed.
96
+ """
97
+ return self._name
98
+
99
+ @name.setter
100
+ def name(self, value: str) -> None:
101
+ if not isinstance(value, property):
102
+ self._name = value
103
+
104
+ @property
105
+ def state(self) -> MutableMapping[str, Any]:
106
+ return self.retriever.state # type: ignore
107
+
108
+ @state.setter
109
+ def state(self, value: MutableMapping[str, Any]) -> None:
110
+ """State setter, accept state serialized by state getter."""
111
+ state: Mapping[str, Any] = value
112
+ if self.state_migrations:
113
+ for migration in self.state_migrations:
114
+ if migration.should_migrate(state):
115
+ state = migration.migrate(state)
116
+ self.retriever.state = state
117
+
118
+ def get_updated_state(
119
+ self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
120
+ ) -> MutableMapping[str, Any]:
121
+ return self.state
122
+
123
+ @property
124
+ def cursor_field(self) -> Union[str, List[str]]:
125
+ """
126
+ Override to return the default cursor field used by this stream e.g: an API entity might always use created_at as the cursor field.
127
+ :return: The name of the field used as a cursor. If the cursor is nested, return an array consisting of the path to the cursor.
128
+ """
129
+ cursor = self._stream_cursor_field.eval(self.config) # type: ignore # _stream_cursor_field is always cast to interpolated string
130
+ return cursor if cursor else []
131
+
132
+ @property
133
+ def is_resumable(self) -> bool:
134
+ # Declarative sources always implement state getter/setter, but whether it supports checkpointing is based on
135
+ # if the retriever has a cursor defined.
136
+ return self.retriever.cursor is not None if hasattr(self.retriever, "cursor") else False
137
+
138
+ def read_records(
139
+ self,
140
+ sync_mode: SyncMode,
141
+ cursor_field: Optional[List[str]] = None,
142
+ stream_slice: Optional[Mapping[str, Any]] = None,
143
+ stream_state: Optional[Mapping[str, Any]] = None,
144
+ ) -> Iterable[Mapping[str, Any]]:
145
+ """
146
+ :param: stream_state We knowingly avoid using stream_state as we want cursors to manage their own state.
147
+ """
148
+ if stream_slice is None or (
149
+ not isinstance(stream_slice, StreamSlice) and stream_slice == {}
150
+ ):
151
+ # As the parameter is Optional, many would just call `read_records(sync_mode)` during testing without specifying the field
152
+ # As part of the declarative model without custom components, this should never happen as the CDK would wire up a
153
+ # SinglePartitionRouter that would create this StreamSlice properly
154
+ # As part of the declarative model with custom components, a user that would return a `None` slice would now have the default
155
+ # empty slice which seems to make sense.
156
+ stream_slice = StreamSlice(partition={}, cursor_slice={})
157
+ if not isinstance(stream_slice, StreamSlice):
158
+ raise ValueError(
159
+ f"DeclarativeStream does not support stream_slices that are not StreamSlice. Got {stream_slice}"
160
+ )
161
+ yield from self.retriever.read_records(self.get_json_schema(), stream_slice) # type: ignore # records are of the correct type
162
+
163
+ def get_json_schema(self) -> Mapping[str, Any]: # type: ignore
164
+ """
165
+ :return: A dict of the JSON schema representing this stream.
166
+
167
+ The default implementation of this method looks for a JSONSchema file with the same name as this stream's "name" property.
168
+ Override as needed.
169
+ """
170
+ return self._schema_loader.get_json_schema()
171
+
172
+ def stream_slices(
173
+ self,
174
+ *,
175
+ sync_mode: SyncMode,
176
+ cursor_field: Optional[List[str]] = None,
177
+ stream_state: Optional[Mapping[str, Any]] = None,
178
+ ) -> Iterable[Optional[StreamSlice]]:
179
+ """
180
+ Override to define the slices for this stream. See the stream slicing section of the docs for more information.
181
+
182
+ :param sync_mode:
183
+ :param cursor_field:
184
+ :param stream_state: we knowingly avoid using stream_state as we want cursors to manage their own state
185
+ :return:
186
+ """
187
+ return self.retriever.stream_slices()
188
+
189
+ @property
190
+ def state_checkpoint_interval(self) -> Optional[int]:
191
+ """
192
+ We explicitly disable checkpointing here. There are a couple reasons for that and not all are documented here but:
193
+ * In the case where records are not ordered, the granularity of what is ordered is the slice. Therefore, we will only update the
194
+ cursor value once at the end of every slice.
195
+ * Updating the state once every record would generate issues for data feed stop conditions or semi-incremental syncs where the
196
+ important state is the one at the beginning of the slice
197
+ """
198
+ return None
199
+
200
+ def get_cursor(self) -> Optional[Cursor]:
201
+ if self.retriever and isinstance(self.retriever, SimpleRetriever):
202
+ return self.retriever.cursor
203
+ return None
204
+
205
+ def _get_checkpoint_reader(
206
+ self,
207
+ logger: logging.Logger,
208
+ cursor_field: Optional[List[str]],
209
+ sync_mode: SyncMode,
210
+ stream_state: MutableMapping[str, Any],
211
+ ) -> CheckpointReader:
212
+ """
213
+ This method is overridden to prevent issues with stream slice classification for incremental streams that have parent streams.
214
+
215
+ The classification logic, when used with `itertools.tee`, creates a copy of the stream slices. When `stream_slices` is called
216
+ the second time, the parent records generated during the classification phase are lost. This occurs because `itertools.tee`
217
+ only buffers the results, meaning the logic in `simple_retriever` that observes and updates the cursor isn't executed again.
218
+
219
+ By overriding this method, we ensure that the stream slices are processed correctly and parent records are not lost,
220
+ allowing the cursor to function as expected.
221
+ """
222
+ mappings_or_slices = self.stream_slices(
223
+ cursor_field=cursor_field,
224
+ sync_mode=sync_mode, # todo: change this interface to no longer rely on sync_mode for behavior
225
+ stream_state=stream_state,
226
+ )
227
+
228
+ cursor = self.get_cursor()
229
+ checkpoint_mode = self._checkpoint_mode
230
+
231
+ if isinstance(
232
+ cursor, (GlobalSubstreamCursor, PerPartitionCursor, PerPartitionWithGlobalCursor)
233
+ ):
234
+ self.has_multiple_slices = True
235
+ return CursorBasedCheckpointReader(
236
+ stream_slices=mappings_or_slices,
237
+ cursor=cursor,
238
+ read_state_from_cursor=checkpoint_mode == CheckpointMode.RESUMABLE_FULL_REFRESH,
239
+ )
240
+
241
+ return super()._get_checkpoint_reader(logger, cursor_field, sync_mode, stream_state)
@@ -0,0 +1,33 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import (
6
+ CompositeRawDecoder,
7
+ GzipParser,
8
+ JsonParser,
9
+ Parser,
10
+ )
11
+ from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
12
+ from airbyte_cdk.sources.declarative.decoders.json_decoder import (
13
+ IterableDecoder,
14
+ JsonDecoder,
15
+ )
16
+ from airbyte_cdk.sources.declarative.decoders.noop_decoder import NoopDecoder
17
+ from airbyte_cdk.sources.declarative.decoders.pagination_decoder_decorator import (
18
+ PaginationDecoderDecorator,
19
+ )
20
+ from airbyte_cdk.sources.declarative.decoders.xml_decoder import XmlDecoder
21
+ from airbyte_cdk.sources.declarative.decoders.zipfile_decoder import ZipfileDecoder
22
+
23
+ __all__ = [
24
+ "Decoder",
25
+ "CompositeRawDecoder",
26
+ "JsonDecoder",
27
+ "JsonParser",
28
+ "IterableDecoder",
29
+ "NoopDecoder",
30
+ "PaginationDecoderDecorator",
31
+ "XmlDecoder",
32
+ "ZipfileDecoder",
33
+ ]
@@ -0,0 +1,218 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ import csv
6
+ import gzip
7
+ import io
8
+ import json
9
+ import logging
10
+ from dataclasses import dataclass
11
+ from io import BufferedIOBase, TextIOWrapper
12
+ from typing import Any, Optional
13
+
14
+ import orjson
15
+ import requests
16
+
17
+ from airbyte_cdk.models import FailureType
18
+ from airbyte_cdk.sources.declarative.decoders.decoder import DECODER_OUTPUT_TYPE, Decoder
19
+ from airbyte_cdk.sources.declarative.decoders.decoder_parser import (
20
+ PARSER_OUTPUT_TYPE,
21
+ PARSERS_BY_HEADER_TYPE,
22
+ PARSERS_TYPE,
23
+ Parser,
24
+ )
25
+ from airbyte_cdk.utils import AirbyteTracedException
26
+
27
+ logger = logging.getLogger("airbyte")
28
+
29
+
30
+ @dataclass
31
+ class GzipParser(Parser):
32
+ inner_parser: Parser
33
+
34
+ def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
35
+ """
36
+ Decompress gzipped bytes and pass decompressed data to the inner parser.
37
+
38
+ IMPORTANT:
39
+ - If the data is not gzipped, reset the pointer and pass the data to the inner parser as is.
40
+
41
+ Note:
42
+ - The data is not decoded by default.
43
+ """
44
+
45
+ with gzip.GzipFile(fileobj=data, mode="rb") as gzipobj:
46
+ yield from self.inner_parser.parse(gzipobj)
47
+
48
+
49
+ @dataclass
50
+ class JsonParser(Parser):
51
+ encoding: str = "utf-8"
52
+
53
+ def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
54
+ """
55
+ Attempts to deserialize data using orjson library. As an extra layer of safety we fallback on the json library to deserialize the data.
56
+ """
57
+ raw_data = data.read()
58
+ body_json = self._parse_orjson(raw_data) or self._parse_json(raw_data)
59
+
60
+ if body_json is None:
61
+ raise AirbyteTracedException(
62
+ message="Response JSON data failed to be parsed. See logs for more information.",
63
+ internal_message=f"Response JSON data failed to be parsed.",
64
+ failure_type=FailureType.system_error,
65
+ )
66
+
67
+ if isinstance(body_json, list):
68
+ yield from body_json
69
+ else:
70
+ yield from [body_json]
71
+
72
+ def _parse_orjson(self, raw_data: bytes) -> Optional[Any]:
73
+ try:
74
+ return orjson.loads(raw_data.decode(self.encoding))
75
+ except Exception as exc:
76
+ logger.debug(
77
+ f"Failed to parse JSON data using orjson library. Falling back to json library. {exc}"
78
+ )
79
+ return None
80
+
81
+ def _parse_json(self, raw_data: bytes) -> Optional[Any]:
82
+ try:
83
+ return json.loads(raw_data.decode(self.encoding))
84
+ except Exception as exc:
85
+ logger.error(f"Failed to parse JSON data using json library. {exc}")
86
+ return None
87
+
88
+
89
+ @dataclass
90
+ class JsonLineParser(Parser):
91
+ encoding: Optional[str] = "utf-8"
92
+
93
+ def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
94
+ for line in data:
95
+ try:
96
+ yield json.loads(line.decode(encoding=self.encoding or "utf-8"))
97
+ except json.JSONDecodeError as e:
98
+ logger.warning(f"Cannot decode/parse line {line!r} as JSON, error: {e}")
99
+
100
+
101
+ @dataclass
102
+ class CsvParser(Parser):
103
+ # TODO: migrate implementation to re-use file-base classes
104
+ encoding: Optional[str] = "utf-8"
105
+ delimiter: Optional[str] = ","
106
+
107
+ def _get_delimiter(self) -> Optional[str]:
108
+ """
109
+ Get delimiter from the configuration. Check for the escape character and decode it.
110
+ """
111
+ if self.delimiter is not None:
112
+ if self.delimiter.startswith("\\"):
113
+ self.delimiter = self.delimiter.encode("utf-8").decode("unicode_escape")
114
+
115
+ return self.delimiter
116
+
117
+ def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
118
+ """
119
+ Parse CSV data from decompressed bytes.
120
+ """
121
+ text_data = TextIOWrapper(data, encoding=self.encoding) # type: ignore
122
+ reader = csv.DictReader(text_data, delimiter=self._get_delimiter() or ",")
123
+ for row in reader:
124
+ yield row
125
+
126
+
127
+ class CompositeRawDecoder(Decoder):
128
+ """
129
+ Decoder strategy to transform a requests.Response into a PARSER_OUTPUT_TYPE
130
+ passed response.raw to parser(s).
131
+
132
+ Note: response.raw is not decoded/decompressed by default. Parsers should be instantiated recursively.
133
+
134
+ Example:
135
+ composite_raw_decoder = CompositeRawDecoder(
136
+ parser=GzipParser(
137
+ inner_parser=JsonLineParser(encoding="iso-8859-1")
138
+ )
139
+ )
140
+ """
141
+
142
+ def __init__(
143
+ self,
144
+ parser: Parser,
145
+ stream_response: bool = True,
146
+ parsers_by_header: PARSERS_BY_HEADER_TYPE = None,
147
+ ) -> None:
148
+ # since we moved from using `dataclass` to `__init__` method,
149
+ # we need to keep using the `parser` to be able to resolve the depenencies
150
+ # between the parsers correctly.
151
+ self.parser = parser
152
+
153
+ self._parsers_by_header = parsers_by_header if parsers_by_header else {}
154
+ self._stream_response = stream_response
155
+
156
+ @classmethod
157
+ def by_headers(
158
+ cls,
159
+ parsers: PARSERS_TYPE,
160
+ stream_response: bool,
161
+ fallback_parser: Parser,
162
+ ) -> "CompositeRawDecoder":
163
+ """
164
+ Create a CompositeRawDecoder instance based on header values.
165
+
166
+ Args:
167
+ parsers (PARSERS_TYPE): A list of tuples where each tuple contains headers, header values, and a parser.
168
+ stream_response (bool): A flag indicating whether the response should be streamed.
169
+ fallback_parser (Parser): A parser to use if no matching header is found.
170
+
171
+ Returns:
172
+ CompositeRawDecoder: An instance of CompositeRawDecoder configured with the provided parsers.
173
+ """
174
+ parsers_by_header = {}
175
+ for headers, header_values, parser in parsers:
176
+ for header in headers:
177
+ parsers_by_header[header] = {header_value: parser for header_value in header_values}
178
+ return cls(fallback_parser, stream_response, parsers_by_header)
179
+
180
+ def is_stream_response(self) -> bool:
181
+ return self._stream_response
182
+
183
+ def decode(self, response: requests.Response) -> DECODER_OUTPUT_TYPE:
184
+ parser = self._select_parser(response)
185
+ if self.is_stream_response():
186
+ # urllib mentions that some interfaces don't play nice with auto_close
187
+ # More info here: https://urllib3.readthedocs.io/en/stable/user-guide.html#using-io-wrappers-with-response-content
188
+ # We have indeed observed some issues with CSV parsing.
189
+ # Hence, we will manage the closing of the file ourselves until we find a better solution.
190
+ response.raw.auto_close = False
191
+ yield from parser.parse(
192
+ data=response.raw, # type: ignore[arg-type]
193
+ )
194
+ response.raw.close()
195
+ else:
196
+ yield from parser.parse(data=io.BytesIO(response.content))
197
+
198
+ def _select_parser(self, response: requests.Response) -> Parser:
199
+ """
200
+ Selects the appropriate parser based on the response headers.
201
+
202
+ This method iterates through the `_parsers_by_header` dictionary to find a matching parser
203
+ based on the headers in the response. If a matching header and header value are found,
204
+ the corresponding parser is returned. If no match is found, the default parser is returned.
205
+
206
+ Args:
207
+ response (requests.Response): The HTTP response object containing headers to check.
208
+
209
+ Returns:
210
+ Parser: The parser corresponding to the matched header value, or the default parser if no match is found.
211
+ """
212
+ for header, parser_by_header_value in self._parsers_by_header.items():
213
+ if (
214
+ header in response.headers
215
+ and response.headers[header] in parser_by_header_value.keys()
216
+ ):
217
+ return parser_by_header_value[response.headers[header]]
218
+ return self.parser
@@ -0,0 +1,32 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from abc import abstractmethod
6
+ from dataclasses import dataclass
7
+ from typing import Any, Generator, MutableMapping
8
+
9
+ import requests
10
+
11
+ DECODER_OUTPUT_TYPE = Generator[MutableMapping[str, Any], None, None]
12
+
13
+
14
+ @dataclass
15
+ class Decoder:
16
+ """
17
+ Decoder strategy to transform a requests.Response into a Mapping[str, Any]
18
+ """
19
+
20
+ @abstractmethod
21
+ def is_stream_response(self) -> bool:
22
+ """
23
+ Set to True if you'd like to use stream=True option in http requester
24
+ """
25
+
26
+ @abstractmethod
27
+ def decode(self, response: requests.Response) -> DECODER_OUTPUT_TYPE:
28
+ """
29
+ Decodes a requests.Response into a Mapping[str, Any] or an array
30
+ :param response: the response to decode
31
+ :return: Generator of Mapping describing the response
32
+ """
@@ -0,0 +1,30 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+
6
+ import logging
7
+ from abc import ABC, abstractmethod
8
+ from dataclasses import dataclass
9
+ from io import BufferedIOBase
10
+ from typing import Any, Dict, Generator, List, MutableMapping, Optional, Set, Tuple
11
+
12
+ logger = logging.getLogger("airbyte")
13
+
14
+
15
+ PARSER_OUTPUT_TYPE = Generator[MutableMapping[str, Any], None, None]
16
+
17
+
18
+ @dataclass
19
+ class Parser(ABC):
20
+ @abstractmethod
21
+ def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
22
+ """
23
+ Parse data and yield dictionaries.
24
+ """
25
+ pass
26
+
27
+
28
+ # reusable parser types
29
+ PARSERS_TYPE = List[Tuple[Set[str], Set[str], Parser]]
30
+ PARSERS_BY_HEADER_TYPE = Optional[Dict[str, Dict[str, Parser]]]
@@ -0,0 +1,65 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+ import codecs
5
+ import logging
6
+ from dataclasses import InitVar, dataclass
7
+ from gzip import decompress
8
+ from typing import Any, Generator, List, Mapping, MutableMapping, Optional
9
+
10
+ import orjson
11
+ import requests
12
+
13
+ from airbyte_cdk.sources.declarative.decoders import CompositeRawDecoder, JsonParser
14
+ from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
15
+
16
+ logger = logging.getLogger("airbyte")
17
+
18
+
19
+ class JsonDecoder(Decoder):
20
+ """
21
+ Decoder strategy that returns the json-encoded content of a response, if any.
22
+
23
+ Usually, we would try to instantiate the equivalent `CompositeRawDecoder(parser=JsonParser(), stream_response=False)` but there were specific historical behaviors related to the JsonDecoder that we didn't know if we could remove like the fallback on {} in case of errors.
24
+ """
25
+
26
+ def __init__(self, parameters: Mapping[str, Any]):
27
+ self._decoder = CompositeRawDecoder(parser=JsonParser(), stream_response=False)
28
+
29
+ def is_stream_response(self) -> bool:
30
+ return self._decoder.is_stream_response()
31
+
32
+ def decode(
33
+ self, response: requests.Response
34
+ ) -> Generator[MutableMapping[str, Any], None, None]:
35
+ """
36
+ Given the response is an empty string or an emtpy list, the function will return a generator with an empty mapping.
37
+ """
38
+ has_yielded = False
39
+ try:
40
+ for element in self._decoder.decode(response):
41
+ yield element
42
+ has_yielded = True
43
+ except Exception:
44
+ yield {}
45
+
46
+ if not has_yielded:
47
+ yield {}
48
+
49
+
50
+ @dataclass
51
+ class IterableDecoder(Decoder):
52
+ """
53
+ Decoder strategy that returns the string content of the response, if any.
54
+ """
55
+
56
+ parameters: InitVar[Mapping[str, Any]]
57
+
58
+ def is_stream_response(self) -> bool:
59
+ return True
60
+
61
+ def decode(
62
+ self, response: requests.Response
63
+ ) -> Generator[MutableMapping[str, Any], None, None]:
64
+ for line in response.iter_lines():
65
+ yield {"record": line.decode()}
@@ -0,0 +1,21 @@
1
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
+
3
+ import logging
4
+ from typing import Any, Generator, Mapping
5
+
6
+ import requests
7
+
8
+ from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
9
+
10
+ logger = logging.getLogger("airbyte")
11
+
12
+
13
+ class NoopDecoder(Decoder):
14
+ def is_stream_response(self) -> bool:
15
+ return False
16
+
17
+ def decode( # type: ignore[override] # Signature doesn't match base class
18
+ self,
19
+ response: requests.Response,
20
+ ) -> Generator[Mapping[str, Any], None, None]:
21
+ yield from [{}]