airbyte-cdk 0.61.2__py3-none-any.whl → 0.62.1__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. airbyte_cdk/sources/abstract_source.py +14 -33
  2. airbyte_cdk/sources/connector_state_manager.py +16 -4
  3. airbyte_cdk/sources/file_based/file_based_source.py +87 -35
  4. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +3 -0
  5. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +15 -13
  6. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -0
  7. airbyte_cdk/sources/file_based/stream/concurrent/{cursor.py → cursor/abstract_concurrent_file_based_cursor.py} +22 -44
  8. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +279 -0
  9. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py +56 -0
  10. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +11 -2
  11. airbyte_cdk/test/mock_http/mocker.py +3 -1
  12. airbyte_cdk/test/mock_http/response.py +9 -1
  13. airbyte_cdk/utils/traced_exception.py +1 -16
  14. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/METADATA +1 -1
  15. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/RECORD +33 -26
  16. unit_tests/sources/file_based/helpers.py +5 -0
  17. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +2860 -0
  18. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +11 -0
  19. unit_tests/sources/file_based/scenarios/scenario_builder.py +6 -2
  20. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  21. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +365 -0
  22. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +462 -0
  23. unit_tests/sources/file_based/test_file_based_scenarios.py +45 -0
  24. unit_tests/sources/file_based/test_scenarios.py +16 -8
  25. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +13 -2
  26. unit_tests/sources/test_abstract_source.py +36 -170
  27. unit_tests/sources/test_connector_state_manager.py +20 -13
  28. unit_tests/sources/test_integration_source.py +8 -25
  29. unit_tests/sources/test_source_read.py +1 -1
  30. unit_tests/test/mock_http/test_mocker.py +3 -1
  31. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/LICENSE.txt +0 -0
  32. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/WHEEL +0 -0
  33. {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/top_level.txt +0 -0
@@ -13,7 +13,6 @@ import pytest
13
13
  from airbyte_cdk.models import (
14
14
  AirbyteCatalog,
15
15
  AirbyteConnectionStatus,
16
- AirbyteErrorTraceMessage,
17
16
  AirbyteLogMessage,
18
17
  AirbyteMessage,
19
18
  AirbyteRecordMessage,
@@ -28,7 +27,6 @@ from airbyte_cdk.models import (
28
27
  ConfiguredAirbyteCatalog,
29
28
  ConfiguredAirbyteStream,
30
29
  DestinationSyncMode,
31
- FailureType,
32
30
  Level,
33
31
  Status,
34
32
  StreamDescriptor,
@@ -42,7 +40,6 @@ from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
42
40
  from airbyte_cdk.sources.message import MessageRepository
43
41
  from airbyte_cdk.sources.streams import IncrementalMixin, Stream
44
42
  from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
45
- from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets
46
43
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
47
44
  from pytest import fixture
48
45
 
@@ -57,14 +54,12 @@ class MockSource(AbstractSource):
57
54
  per_stream: bool = True,
58
55
  message_repository: MessageRepository = None,
59
56
  exception_on_missing_stream: bool = True,
60
- stop_sync_on_stream_failure: bool = False,
61
57
  ):
62
58
  self._streams = streams
63
59
  self.check_lambda = check_lambda
64
60
  self.per_stream = per_stream
65
61
  self.exception_on_missing_stream = exception_on_missing_stream
66
62
  self._message_repository = message_repository
67
- self._stop_sync_on_stream_failure = stop_sync_on_stream_failure
68
63
 
69
64
  def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
70
65
  if self.check_lambda:
@@ -89,12 +84,6 @@ class MockSource(AbstractSource):
89
84
  return self._message_repository
90
85
 
91
86
 
92
- class MockSourceWithStopSyncFalseOverride(MockSource):
93
- @property
94
- def stop_sync_on_stream_failure(self) -> bool:
95
- return False
96
-
97
-
98
87
  class StreamNoStateMethod(Stream):
99
88
  name = "managers"
100
89
  primary_key = None
@@ -126,11 +115,8 @@ class StreamRaisesException(Stream):
126
115
  name = "lamentations"
127
116
  primary_key = None
128
117
 
129
- def __init__(self, exception_to_raise):
130
- self._exception_to_raise = exception_to_raise
131
-
132
118
  def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
133
- raise self._exception_to_raise
119
+ raise AirbyteTracedException(message="I was born only to crash like Icarus")
134
120
 
135
121
 
136
122
  MESSAGE_FROM_REPOSITORY = Mock()
@@ -305,7 +291,7 @@ def test_read_stream_emits_repository_message_on_error(mocker, message_repositor
305
291
 
306
292
  source = MockSource(streams=[stream], message_repository=message_repository)
307
293
 
308
- with pytest.raises(AirbyteTracedException):
294
+ with pytest.raises(RuntimeError):
309
295
  messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])))
310
296
  assert MESSAGE_FROM_REPOSITORY in messages
311
297
 
@@ -320,14 +306,14 @@ def test_read_stream_with_error_gets_display_message(mocker):
320
306
  catalog = ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])
321
307
 
322
308
  # without get_error_display_message
323
- with pytest.raises(AirbyteTracedException):
309
+ with pytest.raises(RuntimeError, match="oh no!"):
324
310
  list(source.read(logger, {}, catalog))
325
311
 
326
312
  mocker.patch.object(MockStream, "get_error_display_message", return_value="my message")
327
313
 
328
- with pytest.raises(AirbyteTracedException) as exc:
314
+ with pytest.raises(AirbyteTracedException, match="oh no!") as exc:
329
315
  list(source.read(logger, {}, catalog))
330
- assert "oh no!" in exc.value.message
316
+ assert exc.value.message == "my message"
331
317
 
332
318
 
333
319
  GLOBAL_EMITTED_AT = 1
@@ -372,22 +358,6 @@ def _as_state(state_data: Dict[str, Any], stream_name: str = "", per_stream_stat
372
358
  return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=state_data))
373
359
 
374
360
 
375
- def _as_error_trace(stream: str, error_message: str, internal_message: Optional[str], failure_type: Optional[FailureType], stack_trace: Optional[str]) -> AirbyteMessage:
376
- trace_message = AirbyteTraceMessage(
377
- emitted_at=datetime.datetime.now().timestamp() * 1000.0,
378
- type=TraceType.ERROR,
379
- error=AirbyteErrorTraceMessage(
380
- stream_descriptor=StreamDescriptor(name=stream),
381
- message=error_message,
382
- internal_message=internal_message,
383
- failure_type=failure_type,
384
- stack_trace=stack_trace,
385
- ),
386
- )
387
-
388
- return AirbyteMessage(type=MessageType.TRACE, trace=trace_message)
389
-
390
-
391
361
  def _configured_stream(stream: Stream, sync_mode: SyncMode):
392
362
  return ConfiguredAirbyteStream(
393
363
  stream=stream.as_airbyte_stream(),
@@ -1184,7 +1154,13 @@ class TestIncrementalRead:
1184
1154
  def test_checkpoint_state_from_stream_instance():
1185
1155
  teams_stream = MockStreamOverridesStateMethod()
1186
1156
  managers_stream = StreamNoStateMethod()
1187
- state_manager = ConnectorStateManager({"teams": teams_stream, "managers": managers_stream}, [])
1157
+ state_manager = ConnectorStateManager(
1158
+ {
1159
+ "teams": AirbyteStream(name="teams", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]),
1160
+ "managers": AirbyteStream(name="managers", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental])
1161
+ },
1162
+ [],
1163
+ )
1188
1164
 
1189
1165
  # The stream_state passed to checkpoint_state() should be ignored since stream implements state function
1190
1166
  teams_stream.state = {"updated_at": "2022-09-11"}
@@ -1198,124 +1174,21 @@ def test_checkpoint_state_from_stream_instance():
1198
1174
  )
1199
1175
 
1200
1176
 
1201
- @pytest.mark.parametrize(
1202
- "exception_to_raise,expected_error_message,expected_internal_message",
1203
- [
1204
- pytest.param(AirbyteTracedException(message="I was born only to crash like Icarus"), "I was born only to crash like Icarus", None, id="test_raises_traced_exception"),
1205
- pytest.param(Exception("Generic connector error message"), "Something went wrong in the connector. See the logs for more details.", "Generic connector error message", id="test_raises_generic_exception"),
1206
- ]
1207
- )
1208
- def test_continue_sync_with_failed_streams(mocker, exception_to_raise, expected_error_message, expected_internal_message):
1209
- """
1210
- Tests that running a sync for a connector with multiple streams will continue syncing when one stream fails
1211
- with an error. This source does not override the default behavior defined in the AbstractSource class.
1212
- """
1213
- stream_output = [{"k1": "v1"}, {"k2": "v2"}]
1214
- s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
1215
- s2 = StreamRaisesException(exception_to_raise=exception_to_raise)
1216
- s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
1217
-
1218
- mocker.patch.object(MockStream, "get_json_schema", return_value={})
1219
- mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
1220
-
1221
- src = MockSource(streams=[s1, s2, s3])
1222
- catalog = ConfiguredAirbyteCatalog(
1223
- streams=[
1224
- _configured_stream(s1, SyncMode.full_refresh),
1225
- _configured_stream(s2, SyncMode.full_refresh),
1226
- _configured_stream(s3, SyncMode.full_refresh),
1227
- ]
1228
- )
1229
-
1230
- expected = _fix_emitted_at(
1231
- [
1232
- _as_stream_status("s1", AirbyteStreamStatus.STARTED),
1233
- _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
1234
- *_as_records("s1", stream_output),
1235
- _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
1236
- _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
1237
- _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
1238
- _as_error_trace("lamentations", expected_error_message, expected_internal_message, FailureType.system_error, None),
1239
- _as_stream_status("s3", AirbyteStreamStatus.STARTED),
1240
- _as_stream_status("s3", AirbyteStreamStatus.RUNNING),
1241
- *_as_records("s3", stream_output),
1242
- _as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
1243
- ]
1244
- )
1245
-
1246
- with pytest.raises(AirbyteTracedException) as exc:
1247
- messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
1248
- messages = _fix_emitted_at(messages)
1249
-
1250
- assert expected == messages
1251
-
1252
- assert "lamentations" in exc.value.message
1253
-
1254
-
1255
- def test_continue_sync_source_override_false(mocker):
1256
- """
1257
- Tests that running a sync for a connector explicitly overriding the default AbstractSource.stop_sync_on_stream_failure
1258
- property to be False which will continue syncing stream even if one encountered an exception.
1259
- """
1260
- update_secrets(["API_KEY_VALUE"])
1261
-
1262
- stream_output = [{"k1": "v1"}, {"k2": "v2"}]
1263
- s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
1264
- s2 = StreamRaisesException(exception_to_raise=AirbyteTracedException(message="I was born only to crash like Icarus"))
1265
- s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
1266
-
1267
- mocker.patch.object(MockStream, "get_json_schema", return_value={})
1268
- mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
1269
-
1270
- src = MockSourceWithStopSyncFalseOverride(streams=[s1, s2, s3])
1271
- catalog = ConfiguredAirbyteCatalog(
1272
- streams=[
1273
- _configured_stream(s1, SyncMode.full_refresh),
1274
- _configured_stream(s2, SyncMode.full_refresh),
1275
- _configured_stream(s3, SyncMode.full_refresh),
1276
- ]
1277
- )
1278
-
1279
- expected = _fix_emitted_at(
1280
- [
1281
- _as_stream_status("s1", AirbyteStreamStatus.STARTED),
1282
- _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
1283
- *_as_records("s1", stream_output),
1284
- _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
1285
- _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
1286
- _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
1287
- _as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None),
1288
- _as_stream_status("s3", AirbyteStreamStatus.STARTED),
1289
- _as_stream_status("s3", AirbyteStreamStatus.RUNNING),
1290
- *_as_records("s3", stream_output),
1291
- _as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
1292
- ]
1293
- )
1294
-
1295
- with pytest.raises(AirbyteTracedException) as exc:
1296
- messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
1297
- messages = _fix_emitted_at(messages)
1298
-
1299
- assert expected == messages
1300
-
1301
- assert "lamentations" in exc.value.message
1302
-
1303
-
1304
- def test_sync_error_trace_messages_obfuscate_secrets(mocker):
1177
+ def test_continue_sync_with_failed_streams(mocker):
1305
1178
  """
1306
- Tests that exceptions emitted as trace messages by a source have secrets properly sanitized
1179
+ Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure enabled continues
1180
+ syncing even when one stream fails with an error.
1307
1181
  """
1308
- update_secrets(["API_KEY_VALUE"])
1309
-
1310
1182
  stream_output = [{"k1": "v1"}, {"k2": "v2"}]
1311
1183
  s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
1312
- s2 = StreamRaisesException(exception_to_raise=AirbyteTracedException(message="My api_key value API_KEY_VALUE flew too close to the sun."))
1184
+ s2 = StreamRaisesException()
1313
1185
  s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
1314
1186
 
1315
1187
  mocker.patch.object(MockStream, "get_json_schema", return_value={})
1316
1188
  mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
1317
1189
 
1318
1190
  src = MockSource(streams=[s1, s2, s3])
1191
+ mocker.patch.object(MockSource, "continue_sync_on_stream_failure", return_value=True)
1319
1192
  catalog = ConfiguredAirbyteCatalog(
1320
1193
  streams=[
1321
1194
  _configured_stream(s1, SyncMode.full_refresh),
@@ -1332,7 +1205,6 @@ def test_sync_error_trace_messages_obfuscate_secrets(mocker):
1332
1205
  _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
1333
1206
  _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
1334
1207
  _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
1335
- _as_error_trace("lamentations", "My api_key value **** flew too close to the sun.", None, FailureType.system_error, None),
1336
1208
  _as_stream_status("s3", AirbyteStreamStatus.STARTED),
1337
1209
  _as_stream_status("s3", AirbyteStreamStatus.RUNNING),
1338
1210
  *_as_records("s3", stream_output),
@@ -1340,30 +1212,32 @@ def test_sync_error_trace_messages_obfuscate_secrets(mocker):
1340
1212
  ]
1341
1213
  )
1342
1214
 
1215
+ messages = []
1343
1216
  with pytest.raises(AirbyteTracedException) as exc:
1344
- messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
1345
- messages = _fix_emitted_at(messages)
1346
-
1347
- assert expected == messages
1217
+ # We can't use list comprehension or list() here because we are still raising a final exception for the
1218
+ # failed streams and that disrupts parsing the generator into the messages emitted before
1219
+ for message in src.read(logger, {}, catalog):
1220
+ messages.append(message)
1348
1221
 
1222
+ messages = _fix_emitted_at(messages)
1223
+ assert expected == messages
1349
1224
  assert "lamentations" in exc.value.message
1350
1225
 
1351
1226
 
1352
- def test_continue_sync_with_failed_streams_with_override_false(mocker):
1227
+ def test_stop_sync_with_failed_streams(mocker):
1353
1228
  """
1354
- Tests that running a sync for a connector with multiple streams and stop_sync_on_stream_failure enabled stops
1355
- the sync when one stream fails with an error.
1229
+ Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure disabled stops
1230
+ syncing once a stream fails with an error.
1356
1231
  """
1357
1232
  stream_output = [{"k1": "v1"}, {"k2": "v2"}]
1358
1233
  s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
1359
- s2 = StreamRaisesException(AirbyteTracedException(message="I was born only to crash like Icarus"))
1234
+ s2 = StreamRaisesException()
1360
1235
  s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
1361
1236
 
1362
1237
  mocker.patch.object(MockStream, "get_json_schema", return_value={})
1363
1238
  mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
1364
1239
 
1365
1240
  src = MockSource(streams=[s1, s2, s3])
1366
- mocker.patch.object(MockSource, "stop_sync_on_stream_failure", return_value=True)
1367
1241
  catalog = ConfiguredAirbyteCatalog(
1368
1242
  streams=[
1369
1243
  _configured_stream(s1, SyncMode.full_refresh),
@@ -1380,23 +1254,15 @@ def test_continue_sync_with_failed_streams_with_override_false(mocker):
1380
1254
  _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
1381
1255
  _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
1382
1256
  _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
1383
- _as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None),
1384
1257
  ]
1385
1258
  )
1386
1259
 
1387
- with pytest.raises(AirbyteTracedException) as exc:
1388
- messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
1389
- messages = _fix_emitted_at(messages)
1390
-
1391
- assert expected == messages
1392
-
1393
- assert "lamentations" in exc.value.message
1394
-
1260
+ messages = []
1261
+ with pytest.raises(AirbyteTracedException):
1262
+ # We can't use list comprehension or list() here because we are still raising a final exception for the
1263
+ # failed streams and that disrupts parsing the generator into the messages emitted before
1264
+ for message in src.read(logger, {}, catalog):
1265
+ messages.append(message)
1395
1266
 
1396
- def _remove_stack_trace(message: AirbyteMessage) -> AirbyteMessage:
1397
- """
1398
- Helper method that removes the stack trace from Airbyte trace messages to make asserting against expected records easier
1399
- """
1400
- if message.trace and message.trace.error and message.trace.error.stack_trace:
1401
- message.trace.error.stack_trace = None
1402
- return message
1267
+ messages = _fix_emitted_at(messages)
1268
+ assert expected == messages
@@ -3,21 +3,21 @@
3
3
  #
4
4
 
5
5
  from contextlib import nullcontext as does_not_raise
6
- from typing import Any, Iterable, List, Mapping
6
+ from typing import List
7
7
 
8
8
  import pytest
9
- from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor
9
+ from airbyte_cdk.models import (
10
+ AirbyteMessage,
11
+ AirbyteStateBlob,
12
+ AirbyteStateMessage,
13
+ AirbyteStateType,
14
+ AirbyteStream,
15
+ AirbyteStreamState,
16
+ StreamDescriptor,
17
+ SyncMode,
18
+ )
10
19
  from airbyte_cdk.models import Type as MessageType
11
20
  from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager, HashableStreamDescriptor
12
- from airbyte_cdk.sources.streams import Stream
13
-
14
-
15
- class StreamWithNamespace(Stream):
16
- primary_key = None
17
- namespace = "public"
18
-
19
- def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
20
- return {}
21
21
 
22
22
 
23
23
  @pytest.mark.parametrize(
@@ -156,7 +156,11 @@ class StreamWithNamespace(Stream):
156
156
  ),
157
157
  )
158
158
  def test_initialize_state_manager(input_stream_state, expected_stream_state, expected_error):
159
- stream_to_instance_map = {"actors": StreamWithNamespace()}
159
+ stream_to_instance_map = {
160
+ "actors": AirbyteStream(
161
+ name="actors", namespace="public", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental],
162
+ )
163
+ }
160
164
 
161
165
  if isinstance(input_stream_state, List):
162
166
  input_stream_state = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_stream_state)]
@@ -264,7 +268,10 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp
264
268
  ],
265
269
  )
266
270
  def test_get_stream_state(input_state, stream_name, namespace, expected_state):
267
- stream_to_instance_map = {"users": StreamWithNamespace()}
271
+ stream_to_instance_map = {stream_name: AirbyteStream(
272
+ name=stream_name, namespace=namespace, json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]
273
+ )
274
+ }
268
275
  state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_state)]
269
276
  state_manager = ConnectorStateManager(stream_to_instance_map, state_messages)
270
277
 
@@ -2,9 +2,7 @@
2
2
  # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
3
  #
4
4
 
5
- import json
6
5
  import os
7
- from typing import Any, List, Mapping
8
6
  from unittest import mock
9
7
  from unittest.mock import patch
10
8
 
@@ -24,9 +22,9 @@ from unit_tests.sources.fixtures.source_test_fixture import (
24
22
  "deployment_mode, url_base, expected_records, expected_error",
25
23
  [
26
24
  pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
27
- pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], "system_error", id="test_cloud_read_with_unsecured_url"),
28
- pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], "config_error", id="test_cloud_read_with_private_endpoint"),
29
- pytest.param("CLOUD", "https://localhost:80/api/v1/", [], "config_error", id="test_cloud_read_with_localhost"),
25
+ pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
26
+ pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
27
+ pytest.param("CLOUD", "https://localhost:80/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_localhost"),
30
28
  pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
31
29
  pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
32
30
  ],
@@ -39,10 +37,8 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
39
37
  with mock.patch.object(HttpTestStream, "url_base", url_base):
40
38
  args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"]
41
39
  if expected_error:
42
- with pytest.raises(AirbyteTracedException):
40
+ with pytest.raises(expected_error):
43
41
  launch(source, args)
44
- messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
45
- assert contains_error_trace_message(messages, expected_error)
46
42
  else:
47
43
  launch(source, args)
48
44
 
@@ -51,14 +47,14 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
51
47
  "deployment_mode, token_refresh_url, expected_records, expected_error",
52
48
  [
53
49
  pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
54
- pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], "system_error", id="test_cloud_read_with_unsecured_url"),
55
- pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], "config_error", id="test_cloud_read_with_private_endpoint"),
50
+ pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
51
+ pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
56
52
  pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
57
53
  pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
58
54
  ],
59
55
  )
60
56
  @patch.object(requests.Session, "send", fixture_mock_send)
61
- def test_external_oauth_request_source(capsys, deployment_mode, token_refresh_url, expected_records, expected_error):
57
+ def test_external_oauth_request_source(deployment_mode, token_refresh_url, expected_records, expected_error):
62
58
  oauth_authenticator = SourceFixtureOauthAuthenticator(
63
59
  client_id="nora", client_secret="hae_sung", refresh_token="arthur", token_refresh_endpoint=token_refresh_url
64
60
  )
@@ -67,20 +63,7 @@ def test_external_oauth_request_source(capsys, deployment_mode, token_refresh_ur
67
63
  with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict
68
64
  args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"]
69
65
  if expected_error:
70
- with pytest.raises(AirbyteTracedException):
66
+ with pytest.raises(expected_error):
71
67
  launch(source, args)
72
- messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
73
- assert contains_error_trace_message(messages, expected_error)
74
68
  else:
75
69
  launch(source, args)
76
-
77
-
78
- def contains_error_trace_message(messages: List[Mapping[str, Any]], expected_error: str) -> bool:
79
- for message in messages:
80
- if message.get("type") != "TRACE":
81
- continue
82
- elif message.get("trace").get("type") != "ERROR":
83
- continue
84
- elif message.get("trace").get("error").get("failure_type") == expected_error:
85
- return True
86
- return False
@@ -343,7 +343,7 @@ def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_e
343
343
  source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger)
344
344
  config = {}
345
345
  catalog = _create_configured_catalog(source._streams)
346
- messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, AirbyteTracedException)
346
+ messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, RuntimeError)
347
347
  messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, RuntimeError)
348
348
 
349
349
  expected_messages = [
@@ -15,6 +15,7 @@ _ANOTHER_RESPONSE_BODY = "another body"
15
15
  _A_RESPONSE = HttpResponse("any response")
16
16
  _SOME_QUERY_PARAMS = {"q1": "query value"}
17
17
  _SOME_HEADERS = {"h1": "header value"}
18
+ _OTHER_HEADERS = {"h2": "another header value"}
18
19
  _SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2}
19
20
  _SOME_REQUEST_BODY_STR = "some_request_body"
20
21
 
@@ -24,13 +25,14 @@ class HttpMockerTest(TestCase):
24
25
  def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker):
25
26
  http_mocker.get(
26
27
  HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
27
- HttpResponse(_A_RESPONSE_BODY, 474),
28
+ HttpResponse(_A_RESPONSE_BODY, 474, _OTHER_HEADERS),
28
29
  )
29
30
 
30
31
  response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
31
32
 
32
33
  assert response.text == _A_RESPONSE_BODY
33
34
  assert response.status_code == 474
35
+ assert response.headers == _OTHER_HEADERS
34
36
 
35
37
  @HttpMocker()
36
38
  def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker):