airbyte-cdk 0.61.2__py3-none-any.whl → 0.62.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. airbyte_cdk/sources/abstract_source.py +14 -33
  2. airbyte_cdk/sources/connector_state_manager.py +16 -4
  3. airbyte_cdk/sources/file_based/file_based_source.py +87 -35
  4. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +3 -0
  5. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +15 -13
  6. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -0
  7. airbyte_cdk/sources/file_based/stream/concurrent/{cursor.py → cursor/abstract_concurrent_file_based_cursor.py} +22 -44
  8. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +279 -0
  9. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py +56 -0
  10. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +11 -2
  11. airbyte_cdk/test/mock_http/mocker.py +3 -1
  12. airbyte_cdk/test/mock_http/response.py +9 -1
  13. airbyte_cdk/utils/traced_exception.py +1 -16
  14. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/METADATA +1 -1
  15. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/RECORD +33 -26
  16. unit_tests/sources/file_based/helpers.py +5 -0
  17. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +2860 -0
  18. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +11 -0
  19. unit_tests/sources/file_based/scenarios/scenario_builder.py +6 -2
  20. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  21. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +365 -0
  22. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +462 -0
  23. unit_tests/sources/file_based/test_file_based_scenarios.py +45 -0
  24. unit_tests/sources/file_based/test_scenarios.py +16 -8
  25. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +13 -2
  26. unit_tests/sources/test_abstract_source.py +36 -170
  27. unit_tests/sources/test_connector_state_manager.py +20 -13
  28. unit_tests/sources/test_integration_source.py +8 -25
  29. unit_tests/sources/test_source_read.py +1 -1
  30. unit_tests/test/mock_http/test_mocker.py +3 -1
  31. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/LICENSE.txt +0 -0
  32. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/WHEEL +0 -0
  33. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/top_level.txt +0 -0
@@ -13,7 +13,6 @@ import pytest
13
13
  from airbyte_cdk.models import (
14
14
  AirbyteCatalog,
15
15
  AirbyteConnectionStatus,
16
- AirbyteErrorTraceMessage,
17
16
  AirbyteLogMessage,
18
17
  AirbyteMessage,
19
18
  AirbyteRecordMessage,
@@ -28,7 +27,6 @@ from airbyte_cdk.models import (
28
27
  ConfiguredAirbyteCatalog,
29
28
  ConfiguredAirbyteStream,
30
29
  DestinationSyncMode,
31
- FailureType,
32
30
  Level,
33
31
  Status,
34
32
  StreamDescriptor,
@@ -42,7 +40,6 @@ from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
42
40
  from airbyte_cdk.sources.message import MessageRepository
43
41
  from airbyte_cdk.sources.streams import IncrementalMixin, Stream
44
42
  from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
45
- from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets
46
43
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
47
44
  from pytest import fixture
48
45
 
@@ -57,14 +54,12 @@ class MockSource(AbstractSource):
57
54
  per_stream: bool = True,
58
55
  message_repository: MessageRepository = None,
59
56
  exception_on_missing_stream: bool = True,
60
- stop_sync_on_stream_failure: bool = False,
61
57
  ):
62
58
  self._streams = streams
63
59
  self.check_lambda = check_lambda
64
60
  self.per_stream = per_stream
65
61
  self.exception_on_missing_stream = exception_on_missing_stream
66
62
  self._message_repository = message_repository
67
- self._stop_sync_on_stream_failure = stop_sync_on_stream_failure
68
63
 
69
64
  def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
70
65
  if self.check_lambda:
@@ -89,12 +84,6 @@ class MockSource(AbstractSource):
89
84
  return self._message_repository
90
85
 
91
86
 
92
- class MockSourceWithStopSyncFalseOverride(MockSource):
93
- @property
94
- def stop_sync_on_stream_failure(self) -> bool:
95
- return False
96
-
97
-
98
87
  class StreamNoStateMethod(Stream):
99
88
  name = "managers"
100
89
  primary_key = None
@@ -126,11 +115,8 @@ class StreamRaisesException(Stream):
126
115
  name = "lamentations"
127
116
  primary_key = None
128
117
 
129
- def __init__(self, exception_to_raise):
130
- self._exception_to_raise = exception_to_raise
131
-
132
118
  def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
133
- raise self._exception_to_raise
119
+ raise AirbyteTracedException(message="I was born only to crash like Icarus")
134
120
 
135
121
 
136
122
  MESSAGE_FROM_REPOSITORY = Mock()
@@ -305,7 +291,7 @@ def test_read_stream_emits_repository_message_on_error(mocker, message_repositor
305
291
 
306
292
  source = MockSource(streams=[stream], message_repository=message_repository)
307
293
 
308
- with pytest.raises(AirbyteTracedException):
294
+ with pytest.raises(RuntimeError):
309
295
  messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])))
310
296
  assert MESSAGE_FROM_REPOSITORY in messages
311
297
 
@@ -320,14 +306,14 @@ def test_read_stream_with_error_gets_display_message(mocker):
320
306
  catalog = ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])
321
307
 
322
308
  # without get_error_display_message
323
- with pytest.raises(AirbyteTracedException):
309
+ with pytest.raises(RuntimeError, match="oh no!"):
324
310
  list(source.read(logger, {}, catalog))
325
311
 
326
312
  mocker.patch.object(MockStream, "get_error_display_message", return_value="my message")
327
313
 
328
- with pytest.raises(AirbyteTracedException) as exc:
314
+ with pytest.raises(AirbyteTracedException, match="oh no!") as exc:
329
315
  list(source.read(logger, {}, catalog))
330
- assert "oh no!" in exc.value.message
316
+ assert exc.value.message == "my message"
331
317
 
332
318
 
333
319
  GLOBAL_EMITTED_AT = 1
@@ -372,22 +358,6 @@ def _as_state(state_data: Dict[str, Any], stream_name: str = "", per_stream_stat
372
358
  return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=state_data))
373
359
 
374
360
 
375
- def _as_error_trace(stream: str, error_message: str, internal_message: Optional[str], failure_type: Optional[FailureType], stack_trace: Optional[str]) -> AirbyteMessage:
376
- trace_message = AirbyteTraceMessage(
377
- emitted_at=datetime.datetime.now().timestamp() * 1000.0,
378
- type=TraceType.ERROR,
379
- error=AirbyteErrorTraceMessage(
380
- stream_descriptor=StreamDescriptor(name=stream),
381
- message=error_message,
382
- internal_message=internal_message,
383
- failure_type=failure_type,
384
- stack_trace=stack_trace,
385
- ),
386
- )
387
-
388
- return AirbyteMessage(type=MessageType.TRACE, trace=trace_message)
389
-
390
-
391
361
  def _configured_stream(stream: Stream, sync_mode: SyncMode):
392
362
  return ConfiguredAirbyteStream(
393
363
  stream=stream.as_airbyte_stream(),
@@ -1184,7 +1154,13 @@ class TestIncrementalRead:
1184
1154
  def test_checkpoint_state_from_stream_instance():
1185
1155
  teams_stream = MockStreamOverridesStateMethod()
1186
1156
  managers_stream = StreamNoStateMethod()
1187
- state_manager = ConnectorStateManager({"teams": teams_stream, "managers": managers_stream}, [])
1157
+ state_manager = ConnectorStateManager(
1158
+ {
1159
+ "teams": AirbyteStream(name="teams", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]),
1160
+ "managers": AirbyteStream(name="managers", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental])
1161
+ },
1162
+ [],
1163
+ )
1188
1164
 
1189
1165
  # The stream_state passed to checkpoint_state() should be ignored since stream implements state function
1190
1166
  teams_stream.state = {"updated_at": "2022-09-11"}
@@ -1198,124 +1174,21 @@ def test_checkpoint_state_from_stream_instance():
1198
1174
  )
1199
1175
 
1200
1176
 
1201
- @pytest.mark.parametrize(
1202
- "exception_to_raise,expected_error_message,expected_internal_message",
1203
- [
1204
- pytest.param(AirbyteTracedException(message="I was born only to crash like Icarus"), "I was born only to crash like Icarus", None, id="test_raises_traced_exception"),
1205
- pytest.param(Exception("Generic connector error message"), "Something went wrong in the connector. See the logs for more details.", "Generic connector error message", id="test_raises_generic_exception"),
1206
- ]
1207
- )
1208
- def test_continue_sync_with_failed_streams(mocker, exception_to_raise, expected_error_message, expected_internal_message):
1209
- """
1210
- Tests that running a sync for a connector with multiple streams will continue syncing when one stream fails
1211
- with an error. This source does not override the default behavior defined in the AbstractSource class.
1212
- """
1213
- stream_output = [{"k1": "v1"}, {"k2": "v2"}]
1214
- s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
1215
- s2 = StreamRaisesException(exception_to_raise=exception_to_raise)
1216
- s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
1217
-
1218
- mocker.patch.object(MockStream, "get_json_schema", return_value={})
1219
- mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
1220
-
1221
- src = MockSource(streams=[s1, s2, s3])
1222
- catalog = ConfiguredAirbyteCatalog(
1223
- streams=[
1224
- _configured_stream(s1, SyncMode.full_refresh),
1225
- _configured_stream(s2, SyncMode.full_refresh),
1226
- _configured_stream(s3, SyncMode.full_refresh),
1227
- ]
1228
- )
1229
-
1230
- expected = _fix_emitted_at(
1231
- [
1232
- _as_stream_status("s1", AirbyteStreamStatus.STARTED),
1233
- _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
1234
- *_as_records("s1", stream_output),
1235
- _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
1236
- _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
1237
- _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
1238
- _as_error_trace("lamentations", expected_error_message, expected_internal_message, FailureType.system_error, None),
1239
- _as_stream_status("s3", AirbyteStreamStatus.STARTED),
1240
- _as_stream_status("s3", AirbyteStreamStatus.RUNNING),
1241
- *_as_records("s3", stream_output),
1242
- _as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
1243
- ]
1244
- )
1245
-
1246
- with pytest.raises(AirbyteTracedException) as exc:
1247
- messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
1248
- messages = _fix_emitted_at(messages)
1249
-
1250
- assert expected == messages
1251
-
1252
- assert "lamentations" in exc.value.message
1253
-
1254
-
1255
- def test_continue_sync_source_override_false(mocker):
1256
- """
1257
- Tests that running a sync for a connector explicitly overriding the default AbstractSource.stop_sync_on_stream_failure
1258
- property to be False which will continue syncing stream even if one encountered an exception.
1259
- """
1260
- update_secrets(["API_KEY_VALUE"])
1261
-
1262
- stream_output = [{"k1": "v1"}, {"k2": "v2"}]
1263
- s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
1264
- s2 = StreamRaisesException(exception_to_raise=AirbyteTracedException(message="I was born only to crash like Icarus"))
1265
- s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
1266
-
1267
- mocker.patch.object(MockStream, "get_json_schema", return_value={})
1268
- mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
1269
-
1270
- src = MockSourceWithStopSyncFalseOverride(streams=[s1, s2, s3])
1271
- catalog = ConfiguredAirbyteCatalog(
1272
- streams=[
1273
- _configured_stream(s1, SyncMode.full_refresh),
1274
- _configured_stream(s2, SyncMode.full_refresh),
1275
- _configured_stream(s3, SyncMode.full_refresh),
1276
- ]
1277
- )
1278
-
1279
- expected = _fix_emitted_at(
1280
- [
1281
- _as_stream_status("s1", AirbyteStreamStatus.STARTED),
1282
- _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
1283
- *_as_records("s1", stream_output),
1284
- _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
1285
- _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
1286
- _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
1287
- _as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None),
1288
- _as_stream_status("s3", AirbyteStreamStatus.STARTED),
1289
- _as_stream_status("s3", AirbyteStreamStatus.RUNNING),
1290
- *_as_records("s3", stream_output),
1291
- _as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
1292
- ]
1293
- )
1294
-
1295
- with pytest.raises(AirbyteTracedException) as exc:
1296
- messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
1297
- messages = _fix_emitted_at(messages)
1298
-
1299
- assert expected == messages
1300
-
1301
- assert "lamentations" in exc.value.message
1302
-
1303
-
1304
- def test_sync_error_trace_messages_obfuscate_secrets(mocker):
1177
+ def test_continue_sync_with_failed_streams(mocker):
1305
1178
  """
1306
- Tests that exceptions emitted as trace messages by a source have secrets properly sanitized
1179
+ Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure enabled continues
1180
+ syncing even when one stream fails with an error.
1307
1181
  """
1308
- update_secrets(["API_KEY_VALUE"])
1309
-
1310
1182
  stream_output = [{"k1": "v1"}, {"k2": "v2"}]
1311
1183
  s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
1312
- s2 = StreamRaisesException(exception_to_raise=AirbyteTracedException(message="My api_key value API_KEY_VALUE flew too close to the sun."))
1184
+ s2 = StreamRaisesException()
1313
1185
  s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
1314
1186
 
1315
1187
  mocker.patch.object(MockStream, "get_json_schema", return_value={})
1316
1188
  mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
1317
1189
 
1318
1190
  src = MockSource(streams=[s1, s2, s3])
1191
+ mocker.patch.object(MockSource, "continue_sync_on_stream_failure", return_value=True)
1319
1192
  catalog = ConfiguredAirbyteCatalog(
1320
1193
  streams=[
1321
1194
  _configured_stream(s1, SyncMode.full_refresh),
@@ -1332,7 +1205,6 @@ def test_sync_error_trace_messages_obfuscate_secrets(mocker):
1332
1205
  _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
1333
1206
  _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
1334
1207
  _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
1335
- _as_error_trace("lamentations", "My api_key value **** flew too close to the sun.", None, FailureType.system_error, None),
1336
1208
  _as_stream_status("s3", AirbyteStreamStatus.STARTED),
1337
1209
  _as_stream_status("s3", AirbyteStreamStatus.RUNNING),
1338
1210
  *_as_records("s3", stream_output),
@@ -1340,30 +1212,32 @@ def test_sync_error_trace_messages_obfuscate_secrets(mocker):
1340
1212
  ]
1341
1213
  )
1342
1214
 
1215
+ messages = []
1343
1216
  with pytest.raises(AirbyteTracedException) as exc:
1344
- messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
1345
- messages = _fix_emitted_at(messages)
1346
-
1347
- assert expected == messages
1217
+ # We can't use list comprehension or list() here because we are still raising a final exception for the
1218
+ # failed streams and that disrupts parsing the generator into the messages emitted before
1219
+ for message in src.read(logger, {}, catalog):
1220
+ messages.append(message)
1348
1221
 
1222
+ messages = _fix_emitted_at(messages)
1223
+ assert expected == messages
1349
1224
  assert "lamentations" in exc.value.message
1350
1225
 
1351
1226
 
1352
- def test_continue_sync_with_failed_streams_with_override_false(mocker):
1227
+ def test_stop_sync_with_failed_streams(mocker):
1353
1228
  """
1354
- Tests that running a sync for a connector with multiple streams and stop_sync_on_stream_failure enabled stops
1355
- the sync when one stream fails with an error.
1229
+ Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure disabled stops
1230
+ syncing once a stream fails with an error.
1356
1231
  """
1357
1232
  stream_output = [{"k1": "v1"}, {"k2": "v2"}]
1358
1233
  s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
1359
- s2 = StreamRaisesException(AirbyteTracedException(message="I was born only to crash like Icarus"))
1234
+ s2 = StreamRaisesException()
1360
1235
  s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
1361
1236
 
1362
1237
  mocker.patch.object(MockStream, "get_json_schema", return_value={})
1363
1238
  mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
1364
1239
 
1365
1240
  src = MockSource(streams=[s1, s2, s3])
1366
- mocker.patch.object(MockSource, "stop_sync_on_stream_failure", return_value=True)
1367
1241
  catalog = ConfiguredAirbyteCatalog(
1368
1242
  streams=[
1369
1243
  _configured_stream(s1, SyncMode.full_refresh),
@@ -1380,23 +1254,15 @@ def test_continue_sync_with_failed_streams_with_override_false(mocker):
1380
1254
  _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
1381
1255
  _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
1382
1256
  _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
1383
- _as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None),
1384
1257
  ]
1385
1258
  )
1386
1259
 
1387
- with pytest.raises(AirbyteTracedException) as exc:
1388
- messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
1389
- messages = _fix_emitted_at(messages)
1390
-
1391
- assert expected == messages
1392
-
1393
- assert "lamentations" in exc.value.message
1394
-
1260
+ messages = []
1261
+ with pytest.raises(AirbyteTracedException):
1262
+ # We can't use list comprehension or list() here because we are still raising a final exception for the
1263
+ # failed streams and that disrupts parsing the generator into the messages emitted before
1264
+ for message in src.read(logger, {}, catalog):
1265
+ messages.append(message)
1395
1266
 
1396
- def _remove_stack_trace(message: AirbyteMessage) -> AirbyteMessage:
1397
- """
1398
- Helper method that removes the stack trace from Airbyte trace messages to make asserting against expected records easier
1399
- """
1400
- if message.trace and message.trace.error and message.trace.error.stack_trace:
1401
- message.trace.error.stack_trace = None
1402
- return message
1267
+ messages = _fix_emitted_at(messages)
1268
+ assert expected == messages
@@ -3,21 +3,21 @@
3
3
  #
4
4
 
5
5
  from contextlib import nullcontext as does_not_raise
6
- from typing import Any, Iterable, List, Mapping
6
+ from typing import List
7
7
 
8
8
  import pytest
9
- from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor
9
+ from airbyte_cdk.models import (
10
+ AirbyteMessage,
11
+ AirbyteStateBlob,
12
+ AirbyteStateMessage,
13
+ AirbyteStateType,
14
+ AirbyteStream,
15
+ AirbyteStreamState,
16
+ StreamDescriptor,
17
+ SyncMode,
18
+ )
10
19
  from airbyte_cdk.models import Type as MessageType
11
20
  from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager, HashableStreamDescriptor
12
- from airbyte_cdk.sources.streams import Stream
13
-
14
-
15
- class StreamWithNamespace(Stream):
16
- primary_key = None
17
- namespace = "public"
18
-
19
- def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
20
- return {}
21
21
 
22
22
 
23
23
  @pytest.mark.parametrize(
@@ -156,7 +156,11 @@ class StreamWithNamespace(Stream):
156
156
  ),
157
157
  )
158
158
  def test_initialize_state_manager(input_stream_state, expected_stream_state, expected_error):
159
- stream_to_instance_map = {"actors": StreamWithNamespace()}
159
+ stream_to_instance_map = {
160
+ "actors": AirbyteStream(
161
+ name="actors", namespace="public", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental],
162
+ )
163
+ }
160
164
 
161
165
  if isinstance(input_stream_state, List):
162
166
  input_stream_state = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_stream_state)]
@@ -264,7 +268,10 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp
264
268
  ],
265
269
  )
266
270
  def test_get_stream_state(input_state, stream_name, namespace, expected_state):
267
- stream_to_instance_map = {"users": StreamWithNamespace()}
271
+ stream_to_instance_map = {stream_name: AirbyteStream(
272
+ name=stream_name, namespace=namespace, json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]
273
+ )
274
+ }
268
275
  state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_state)]
269
276
  state_manager = ConnectorStateManager(stream_to_instance_map, state_messages)
270
277
 
@@ -2,9 +2,7 @@
2
2
  # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
3
  #
4
4
 
5
- import json
6
5
  import os
7
- from typing import Any, List, Mapping
8
6
  from unittest import mock
9
7
  from unittest.mock import patch
10
8
 
@@ -24,9 +22,9 @@ from unit_tests.sources.fixtures.source_test_fixture import (
24
22
  "deployment_mode, url_base, expected_records, expected_error",
25
23
  [
26
24
  pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
27
- pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], "system_error", id="test_cloud_read_with_unsecured_url"),
28
- pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], "config_error", id="test_cloud_read_with_private_endpoint"),
29
- pytest.param("CLOUD", "https://localhost:80/api/v1/", [], "config_error", id="test_cloud_read_with_localhost"),
25
+ pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
26
+ pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
27
+ pytest.param("CLOUD", "https://localhost:80/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_localhost"),
30
28
  pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
31
29
  pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
32
30
  ],
@@ -39,10 +37,8 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
39
37
  with mock.patch.object(HttpTestStream, "url_base", url_base):
40
38
  args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"]
41
39
  if expected_error:
42
- with pytest.raises(AirbyteTracedException):
40
+ with pytest.raises(expected_error):
43
41
  launch(source, args)
44
- messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
45
- assert contains_error_trace_message(messages, expected_error)
46
42
  else:
47
43
  launch(source, args)
48
44
 
@@ -51,14 +47,14 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
51
47
  "deployment_mode, token_refresh_url, expected_records, expected_error",
52
48
  [
53
49
  pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
54
- pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], "system_error", id="test_cloud_read_with_unsecured_url"),
55
- pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], "config_error", id="test_cloud_read_with_private_endpoint"),
50
+ pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
51
+ pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
56
52
  pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
57
53
  pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
58
54
  ],
59
55
  )
60
56
  @patch.object(requests.Session, "send", fixture_mock_send)
61
- def test_external_oauth_request_source(capsys, deployment_mode, token_refresh_url, expected_records, expected_error):
57
+ def test_external_oauth_request_source(deployment_mode, token_refresh_url, expected_records, expected_error):
62
58
  oauth_authenticator = SourceFixtureOauthAuthenticator(
63
59
  client_id="nora", client_secret="hae_sung", refresh_token="arthur", token_refresh_endpoint=token_refresh_url
64
60
  )
@@ -67,20 +63,7 @@ def test_external_oauth_request_source(capsys, deployment_mode, token_refresh_ur
67
63
  with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict
68
64
  args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"]
69
65
  if expected_error:
70
- with pytest.raises(AirbyteTracedException):
66
+ with pytest.raises(expected_error):
71
67
  launch(source, args)
72
- messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
73
- assert contains_error_trace_message(messages, expected_error)
74
68
  else:
75
69
  launch(source, args)
76
-
77
-
78
- def contains_error_trace_message(messages: List[Mapping[str, Any]], expected_error: str) -> bool:
79
- for message in messages:
80
- if message.get("type") != "TRACE":
81
- continue
82
- elif message.get("trace").get("type") != "ERROR":
83
- continue
84
- elif message.get("trace").get("error").get("failure_type") == expected_error:
85
- return True
86
- return False
@@ -343,7 +343,7 @@ def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_e
343
343
  source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger)
344
344
  config = {}
345
345
  catalog = _create_configured_catalog(source._streams)
346
- messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, AirbyteTracedException)
346
+ messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, RuntimeError)
347
347
  messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, RuntimeError)
348
348
 
349
349
  expected_messages = [
@@ -15,6 +15,7 @@ _ANOTHER_RESPONSE_BODY = "another body"
15
15
  _A_RESPONSE = HttpResponse("any response")
16
16
  _SOME_QUERY_PARAMS = {"q1": "query value"}
17
17
  _SOME_HEADERS = {"h1": "header value"}
18
+ _OTHER_HEADERS = {"h2": "another header value"}
18
19
  _SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2}
19
20
  _SOME_REQUEST_BODY_STR = "some_request_body"
20
21
 
@@ -24,13 +25,14 @@ class HttpMockerTest(TestCase):
24
25
  def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker):
25
26
  http_mocker.get(
26
27
  HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
27
- HttpResponse(_A_RESPONSE_BODY, 474),
28
+ HttpResponse(_A_RESPONSE_BODY, 474, _OTHER_HEADERS),
28
29
  )
29
30
 
30
31
  response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
31
32
 
32
33
  assert response.text == _A_RESPONSE_BODY
33
34
  assert response.status_code == 474
35
+ assert response.headers == _OTHER_HEADERS
34
36
 
35
37
  @HttpMocker()
36
38
  def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker):