airbyte-cdk 0.61.2__py3-none-any.whl → 0.62.1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/sources/abstract_source.py +14 -33
- airbyte_cdk/sources/connector_state_manager.py +16 -4
- airbyte_cdk/sources/file_based/file_based_source.py +87 -35
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +3 -0
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +15 -13
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -0
- airbyte_cdk/sources/file_based/stream/concurrent/{cursor.py → cursor/abstract_concurrent_file_based_cursor.py} +22 -44
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +279 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py +56 -0
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +11 -2
- airbyte_cdk/test/mock_http/mocker.py +3 -1
- airbyte_cdk/test/mock_http/response.py +9 -1
- airbyte_cdk/utils/traced_exception.py +1 -16
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/METADATA +1 -1
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/RECORD +33 -26
- unit_tests/sources/file_based/helpers.py +5 -0
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +2860 -0
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +11 -0
- unit_tests/sources/file_based/scenarios/scenario_builder.py +6 -2
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +365 -0
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +462 -0
- unit_tests/sources/file_based/test_file_based_scenarios.py +45 -0
- unit_tests/sources/file_based/test_scenarios.py +16 -8
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +13 -2
- unit_tests/sources/test_abstract_source.py +36 -170
- unit_tests/sources/test_connector_state_manager.py +20 -13
- unit_tests/sources/test_integration_source.py +8 -25
- unit_tests/sources/test_source_read.py +1 -1
- unit_tests/test/mock_http/test_mocker.py +3 -1
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/top_level.txt +0 -0
@@ -13,7 +13,6 @@ import pytest
|
|
13
13
|
from airbyte_cdk.models import (
|
14
14
|
AirbyteCatalog,
|
15
15
|
AirbyteConnectionStatus,
|
16
|
-
AirbyteErrorTraceMessage,
|
17
16
|
AirbyteLogMessage,
|
18
17
|
AirbyteMessage,
|
19
18
|
AirbyteRecordMessage,
|
@@ -28,7 +27,6 @@ from airbyte_cdk.models import (
|
|
28
27
|
ConfiguredAirbyteCatalog,
|
29
28
|
ConfiguredAirbyteStream,
|
30
29
|
DestinationSyncMode,
|
31
|
-
FailureType,
|
32
30
|
Level,
|
33
31
|
Status,
|
34
32
|
StreamDescriptor,
|
@@ -42,7 +40,6 @@ from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
|
42
40
|
from airbyte_cdk.sources.message import MessageRepository
|
43
41
|
from airbyte_cdk.sources.streams import IncrementalMixin, Stream
|
44
42
|
from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
|
45
|
-
from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets
|
46
43
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
47
44
|
from pytest import fixture
|
48
45
|
|
@@ -57,14 +54,12 @@ class MockSource(AbstractSource):
|
|
57
54
|
per_stream: bool = True,
|
58
55
|
message_repository: MessageRepository = None,
|
59
56
|
exception_on_missing_stream: bool = True,
|
60
|
-
stop_sync_on_stream_failure: bool = False,
|
61
57
|
):
|
62
58
|
self._streams = streams
|
63
59
|
self.check_lambda = check_lambda
|
64
60
|
self.per_stream = per_stream
|
65
61
|
self.exception_on_missing_stream = exception_on_missing_stream
|
66
62
|
self._message_repository = message_repository
|
67
|
-
self._stop_sync_on_stream_failure = stop_sync_on_stream_failure
|
68
63
|
|
69
64
|
def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
|
70
65
|
if self.check_lambda:
|
@@ -89,12 +84,6 @@ class MockSource(AbstractSource):
|
|
89
84
|
return self._message_repository
|
90
85
|
|
91
86
|
|
92
|
-
class MockSourceWithStopSyncFalseOverride(MockSource):
|
93
|
-
@property
|
94
|
-
def stop_sync_on_stream_failure(self) -> bool:
|
95
|
-
return False
|
96
|
-
|
97
|
-
|
98
87
|
class StreamNoStateMethod(Stream):
|
99
88
|
name = "managers"
|
100
89
|
primary_key = None
|
@@ -126,11 +115,8 @@ class StreamRaisesException(Stream):
|
|
126
115
|
name = "lamentations"
|
127
116
|
primary_key = None
|
128
117
|
|
129
|
-
def __init__(self, exception_to_raise):
|
130
|
-
self._exception_to_raise = exception_to_raise
|
131
|
-
|
132
118
|
def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
|
133
|
-
raise
|
119
|
+
raise AirbyteTracedException(message="I was born only to crash like Icarus")
|
134
120
|
|
135
121
|
|
136
122
|
MESSAGE_FROM_REPOSITORY = Mock()
|
@@ -305,7 +291,7 @@ def test_read_stream_emits_repository_message_on_error(mocker, message_repositor
|
|
305
291
|
|
306
292
|
source = MockSource(streams=[stream], message_repository=message_repository)
|
307
293
|
|
308
|
-
with pytest.raises(
|
294
|
+
with pytest.raises(RuntimeError):
|
309
295
|
messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])))
|
310
296
|
assert MESSAGE_FROM_REPOSITORY in messages
|
311
297
|
|
@@ -320,14 +306,14 @@ def test_read_stream_with_error_gets_display_message(mocker):
|
|
320
306
|
catalog = ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])
|
321
307
|
|
322
308
|
# without get_error_display_message
|
323
|
-
with pytest.raises(
|
309
|
+
with pytest.raises(RuntimeError, match="oh no!"):
|
324
310
|
list(source.read(logger, {}, catalog))
|
325
311
|
|
326
312
|
mocker.patch.object(MockStream, "get_error_display_message", return_value="my message")
|
327
313
|
|
328
|
-
with pytest.raises(AirbyteTracedException) as exc:
|
314
|
+
with pytest.raises(AirbyteTracedException, match="oh no!") as exc:
|
329
315
|
list(source.read(logger, {}, catalog))
|
330
|
-
assert
|
316
|
+
assert exc.value.message == "my message"
|
331
317
|
|
332
318
|
|
333
319
|
GLOBAL_EMITTED_AT = 1
|
@@ -372,22 +358,6 @@ def _as_state(state_data: Dict[str, Any], stream_name: str = "", per_stream_stat
|
|
372
358
|
return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=state_data))
|
373
359
|
|
374
360
|
|
375
|
-
def _as_error_trace(stream: str, error_message: str, internal_message: Optional[str], failure_type: Optional[FailureType], stack_trace: Optional[str]) -> AirbyteMessage:
|
376
|
-
trace_message = AirbyteTraceMessage(
|
377
|
-
emitted_at=datetime.datetime.now().timestamp() * 1000.0,
|
378
|
-
type=TraceType.ERROR,
|
379
|
-
error=AirbyteErrorTraceMessage(
|
380
|
-
stream_descriptor=StreamDescriptor(name=stream),
|
381
|
-
message=error_message,
|
382
|
-
internal_message=internal_message,
|
383
|
-
failure_type=failure_type,
|
384
|
-
stack_trace=stack_trace,
|
385
|
-
),
|
386
|
-
)
|
387
|
-
|
388
|
-
return AirbyteMessage(type=MessageType.TRACE, trace=trace_message)
|
389
|
-
|
390
|
-
|
391
361
|
def _configured_stream(stream: Stream, sync_mode: SyncMode):
|
392
362
|
return ConfiguredAirbyteStream(
|
393
363
|
stream=stream.as_airbyte_stream(),
|
@@ -1184,7 +1154,13 @@ class TestIncrementalRead:
|
|
1184
1154
|
def test_checkpoint_state_from_stream_instance():
|
1185
1155
|
teams_stream = MockStreamOverridesStateMethod()
|
1186
1156
|
managers_stream = StreamNoStateMethod()
|
1187
|
-
state_manager = ConnectorStateManager(
|
1157
|
+
state_manager = ConnectorStateManager(
|
1158
|
+
{
|
1159
|
+
"teams": AirbyteStream(name="teams", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]),
|
1160
|
+
"managers": AirbyteStream(name="managers", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental])
|
1161
|
+
},
|
1162
|
+
[],
|
1163
|
+
)
|
1188
1164
|
|
1189
1165
|
# The stream_state passed to checkpoint_state() should be ignored since stream implements state function
|
1190
1166
|
teams_stream.state = {"updated_at": "2022-09-11"}
|
@@ -1198,124 +1174,21 @@ def test_checkpoint_state_from_stream_instance():
|
|
1198
1174
|
)
|
1199
1175
|
|
1200
1176
|
|
1201
|
-
|
1202
|
-
"exception_to_raise,expected_error_message,expected_internal_message",
|
1203
|
-
[
|
1204
|
-
pytest.param(AirbyteTracedException(message="I was born only to crash like Icarus"), "I was born only to crash like Icarus", None, id="test_raises_traced_exception"),
|
1205
|
-
pytest.param(Exception("Generic connector error message"), "Something went wrong in the connector. See the logs for more details.", "Generic connector error message", id="test_raises_generic_exception"),
|
1206
|
-
]
|
1207
|
-
)
|
1208
|
-
def test_continue_sync_with_failed_streams(mocker, exception_to_raise, expected_error_message, expected_internal_message):
|
1209
|
-
"""
|
1210
|
-
Tests that running a sync for a connector with multiple streams will continue syncing when one stream fails
|
1211
|
-
with an error. This source does not override the default behavior defined in the AbstractSource class.
|
1212
|
-
"""
|
1213
|
-
stream_output = [{"k1": "v1"}, {"k2": "v2"}]
|
1214
|
-
s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
|
1215
|
-
s2 = StreamRaisesException(exception_to_raise=exception_to_raise)
|
1216
|
-
s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
|
1217
|
-
|
1218
|
-
mocker.patch.object(MockStream, "get_json_schema", return_value={})
|
1219
|
-
mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
|
1220
|
-
|
1221
|
-
src = MockSource(streams=[s1, s2, s3])
|
1222
|
-
catalog = ConfiguredAirbyteCatalog(
|
1223
|
-
streams=[
|
1224
|
-
_configured_stream(s1, SyncMode.full_refresh),
|
1225
|
-
_configured_stream(s2, SyncMode.full_refresh),
|
1226
|
-
_configured_stream(s3, SyncMode.full_refresh),
|
1227
|
-
]
|
1228
|
-
)
|
1229
|
-
|
1230
|
-
expected = _fix_emitted_at(
|
1231
|
-
[
|
1232
|
-
_as_stream_status("s1", AirbyteStreamStatus.STARTED),
|
1233
|
-
_as_stream_status("s1", AirbyteStreamStatus.RUNNING),
|
1234
|
-
*_as_records("s1", stream_output),
|
1235
|
-
_as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
|
1236
|
-
_as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
|
1237
|
-
_as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
|
1238
|
-
_as_error_trace("lamentations", expected_error_message, expected_internal_message, FailureType.system_error, None),
|
1239
|
-
_as_stream_status("s3", AirbyteStreamStatus.STARTED),
|
1240
|
-
_as_stream_status("s3", AirbyteStreamStatus.RUNNING),
|
1241
|
-
*_as_records("s3", stream_output),
|
1242
|
-
_as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
|
1243
|
-
]
|
1244
|
-
)
|
1245
|
-
|
1246
|
-
with pytest.raises(AirbyteTracedException) as exc:
|
1247
|
-
messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
|
1248
|
-
messages = _fix_emitted_at(messages)
|
1249
|
-
|
1250
|
-
assert expected == messages
|
1251
|
-
|
1252
|
-
assert "lamentations" in exc.value.message
|
1253
|
-
|
1254
|
-
|
1255
|
-
def test_continue_sync_source_override_false(mocker):
|
1256
|
-
"""
|
1257
|
-
Tests that running a sync for a connector explicitly overriding the default AbstractSource.stop_sync_on_stream_failure
|
1258
|
-
property to be False which will continue syncing stream even if one encountered an exception.
|
1259
|
-
"""
|
1260
|
-
update_secrets(["API_KEY_VALUE"])
|
1261
|
-
|
1262
|
-
stream_output = [{"k1": "v1"}, {"k2": "v2"}]
|
1263
|
-
s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
|
1264
|
-
s2 = StreamRaisesException(exception_to_raise=AirbyteTracedException(message="I was born only to crash like Icarus"))
|
1265
|
-
s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
|
1266
|
-
|
1267
|
-
mocker.patch.object(MockStream, "get_json_schema", return_value={})
|
1268
|
-
mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
|
1269
|
-
|
1270
|
-
src = MockSourceWithStopSyncFalseOverride(streams=[s1, s2, s3])
|
1271
|
-
catalog = ConfiguredAirbyteCatalog(
|
1272
|
-
streams=[
|
1273
|
-
_configured_stream(s1, SyncMode.full_refresh),
|
1274
|
-
_configured_stream(s2, SyncMode.full_refresh),
|
1275
|
-
_configured_stream(s3, SyncMode.full_refresh),
|
1276
|
-
]
|
1277
|
-
)
|
1278
|
-
|
1279
|
-
expected = _fix_emitted_at(
|
1280
|
-
[
|
1281
|
-
_as_stream_status("s1", AirbyteStreamStatus.STARTED),
|
1282
|
-
_as_stream_status("s1", AirbyteStreamStatus.RUNNING),
|
1283
|
-
*_as_records("s1", stream_output),
|
1284
|
-
_as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
|
1285
|
-
_as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
|
1286
|
-
_as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
|
1287
|
-
_as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None),
|
1288
|
-
_as_stream_status("s3", AirbyteStreamStatus.STARTED),
|
1289
|
-
_as_stream_status("s3", AirbyteStreamStatus.RUNNING),
|
1290
|
-
*_as_records("s3", stream_output),
|
1291
|
-
_as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
|
1292
|
-
]
|
1293
|
-
)
|
1294
|
-
|
1295
|
-
with pytest.raises(AirbyteTracedException) as exc:
|
1296
|
-
messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
|
1297
|
-
messages = _fix_emitted_at(messages)
|
1298
|
-
|
1299
|
-
assert expected == messages
|
1300
|
-
|
1301
|
-
assert "lamentations" in exc.value.message
|
1302
|
-
|
1303
|
-
|
1304
|
-
def test_sync_error_trace_messages_obfuscate_secrets(mocker):
|
1177
|
+
def test_continue_sync_with_failed_streams(mocker):
|
1305
1178
|
"""
|
1306
|
-
Tests that
|
1179
|
+
Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure enabled continues
|
1180
|
+
syncing even when one stream fails with an error.
|
1307
1181
|
"""
|
1308
|
-
update_secrets(["API_KEY_VALUE"])
|
1309
|
-
|
1310
1182
|
stream_output = [{"k1": "v1"}, {"k2": "v2"}]
|
1311
1183
|
s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
|
1312
|
-
s2 = StreamRaisesException(
|
1184
|
+
s2 = StreamRaisesException()
|
1313
1185
|
s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
|
1314
1186
|
|
1315
1187
|
mocker.patch.object(MockStream, "get_json_schema", return_value={})
|
1316
1188
|
mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
|
1317
1189
|
|
1318
1190
|
src = MockSource(streams=[s1, s2, s3])
|
1191
|
+
mocker.patch.object(MockSource, "continue_sync_on_stream_failure", return_value=True)
|
1319
1192
|
catalog = ConfiguredAirbyteCatalog(
|
1320
1193
|
streams=[
|
1321
1194
|
_configured_stream(s1, SyncMode.full_refresh),
|
@@ -1332,7 +1205,6 @@ def test_sync_error_trace_messages_obfuscate_secrets(mocker):
|
|
1332
1205
|
_as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
|
1333
1206
|
_as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
|
1334
1207
|
_as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
|
1335
|
-
_as_error_trace("lamentations", "My api_key value **** flew too close to the sun.", None, FailureType.system_error, None),
|
1336
1208
|
_as_stream_status("s3", AirbyteStreamStatus.STARTED),
|
1337
1209
|
_as_stream_status("s3", AirbyteStreamStatus.RUNNING),
|
1338
1210
|
*_as_records("s3", stream_output),
|
@@ -1340,30 +1212,32 @@ def test_sync_error_trace_messages_obfuscate_secrets(mocker):
|
|
1340
1212
|
]
|
1341
1213
|
)
|
1342
1214
|
|
1215
|
+
messages = []
|
1343
1216
|
with pytest.raises(AirbyteTracedException) as exc:
|
1344
|
-
|
1345
|
-
messages
|
1346
|
-
|
1347
|
-
|
1217
|
+
# We can't use list comprehension or list() here because we are still raising a final exception for the
|
1218
|
+
# failed streams and that disrupts parsing the generator into the messages emitted before
|
1219
|
+
for message in src.read(logger, {}, catalog):
|
1220
|
+
messages.append(message)
|
1348
1221
|
|
1222
|
+
messages = _fix_emitted_at(messages)
|
1223
|
+
assert expected == messages
|
1349
1224
|
assert "lamentations" in exc.value.message
|
1350
1225
|
|
1351
1226
|
|
1352
|
-
def
|
1227
|
+
def test_stop_sync_with_failed_streams(mocker):
|
1353
1228
|
"""
|
1354
|
-
Tests that running a sync for a connector with multiple streams and
|
1355
|
-
|
1229
|
+
Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure disabled stops
|
1230
|
+
syncing once a stream fails with an error.
|
1356
1231
|
"""
|
1357
1232
|
stream_output = [{"k1": "v1"}, {"k2": "v2"}]
|
1358
1233
|
s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
|
1359
|
-
s2 = StreamRaisesException(
|
1234
|
+
s2 = StreamRaisesException()
|
1360
1235
|
s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
|
1361
1236
|
|
1362
1237
|
mocker.patch.object(MockStream, "get_json_schema", return_value={})
|
1363
1238
|
mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
|
1364
1239
|
|
1365
1240
|
src = MockSource(streams=[s1, s2, s3])
|
1366
|
-
mocker.patch.object(MockSource, "stop_sync_on_stream_failure", return_value=True)
|
1367
1241
|
catalog = ConfiguredAirbyteCatalog(
|
1368
1242
|
streams=[
|
1369
1243
|
_configured_stream(s1, SyncMode.full_refresh),
|
@@ -1380,23 +1254,15 @@ def test_continue_sync_with_failed_streams_with_override_false(mocker):
|
|
1380
1254
|
_as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
|
1381
1255
|
_as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
|
1382
1256
|
_as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
|
1383
|
-
_as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None),
|
1384
1257
|
]
|
1385
1258
|
)
|
1386
1259
|
|
1387
|
-
|
1388
|
-
|
1389
|
-
|
1390
|
-
|
1391
|
-
|
1392
|
-
|
1393
|
-
assert "lamentations" in exc.value.message
|
1394
|
-
|
1260
|
+
messages = []
|
1261
|
+
with pytest.raises(AirbyteTracedException):
|
1262
|
+
# We can't use list comprehension or list() here because we are still raising a final exception for the
|
1263
|
+
# failed streams and that disrupts parsing the generator into the messages emitted before
|
1264
|
+
for message in src.read(logger, {}, catalog):
|
1265
|
+
messages.append(message)
|
1395
1266
|
|
1396
|
-
|
1397
|
-
|
1398
|
-
Helper method that removes the stack trace from Airbyte trace messages to make asserting against expected records easier
|
1399
|
-
"""
|
1400
|
-
if message.trace and message.trace.error and message.trace.error.stack_trace:
|
1401
|
-
message.trace.error.stack_trace = None
|
1402
|
-
return message
|
1267
|
+
messages = _fix_emitted_at(messages)
|
1268
|
+
assert expected == messages
|
@@ -3,21 +3,21 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from contextlib import nullcontext as does_not_raise
|
6
|
-
from typing import
|
6
|
+
from typing import List
|
7
7
|
|
8
8
|
import pytest
|
9
|
-
from airbyte_cdk.models import
|
9
|
+
from airbyte_cdk.models import (
|
10
|
+
AirbyteMessage,
|
11
|
+
AirbyteStateBlob,
|
12
|
+
AirbyteStateMessage,
|
13
|
+
AirbyteStateType,
|
14
|
+
AirbyteStream,
|
15
|
+
AirbyteStreamState,
|
16
|
+
StreamDescriptor,
|
17
|
+
SyncMode,
|
18
|
+
)
|
10
19
|
from airbyte_cdk.models import Type as MessageType
|
11
20
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager, HashableStreamDescriptor
|
12
|
-
from airbyte_cdk.sources.streams import Stream
|
13
|
-
|
14
|
-
|
15
|
-
class StreamWithNamespace(Stream):
|
16
|
-
primary_key = None
|
17
|
-
namespace = "public"
|
18
|
-
|
19
|
-
def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
|
20
|
-
return {}
|
21
21
|
|
22
22
|
|
23
23
|
@pytest.mark.parametrize(
|
@@ -156,7 +156,11 @@ class StreamWithNamespace(Stream):
|
|
156
156
|
),
|
157
157
|
)
|
158
158
|
def test_initialize_state_manager(input_stream_state, expected_stream_state, expected_error):
|
159
|
-
stream_to_instance_map = {
|
159
|
+
stream_to_instance_map = {
|
160
|
+
"actors": AirbyteStream(
|
161
|
+
name="actors", namespace="public", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental],
|
162
|
+
)
|
163
|
+
}
|
160
164
|
|
161
165
|
if isinstance(input_stream_state, List):
|
162
166
|
input_stream_state = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_stream_state)]
|
@@ -264,7 +268,10 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp
|
|
264
268
|
],
|
265
269
|
)
|
266
270
|
def test_get_stream_state(input_state, stream_name, namespace, expected_state):
|
267
|
-
stream_to_instance_map = {
|
271
|
+
stream_to_instance_map = {stream_name: AirbyteStream(
|
272
|
+
name=stream_name, namespace=namespace, json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]
|
273
|
+
)
|
274
|
+
}
|
268
275
|
state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_state)]
|
269
276
|
state_manager = ConnectorStateManager(stream_to_instance_map, state_messages)
|
270
277
|
|
@@ -2,9 +2,7 @@
|
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
-
import json
|
6
5
|
import os
|
7
|
-
from typing import Any, List, Mapping
|
8
6
|
from unittest import mock
|
9
7
|
from unittest.mock import patch
|
10
8
|
|
@@ -24,9 +22,9 @@ from unit_tests.sources.fixtures.source_test_fixture import (
|
|
24
22
|
"deployment_mode, url_base, expected_records, expected_error",
|
25
23
|
[
|
26
24
|
pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
|
27
|
-
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [],
|
28
|
-
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [],
|
29
|
-
pytest.param("CLOUD", "https://localhost:80/api/v1/", [],
|
25
|
+
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
|
26
|
+
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
|
27
|
+
pytest.param("CLOUD", "https://localhost:80/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_localhost"),
|
30
28
|
pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
|
31
29
|
pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
|
32
30
|
],
|
@@ -39,10 +37,8 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
|
|
39
37
|
with mock.patch.object(HttpTestStream, "url_base", url_base):
|
40
38
|
args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"]
|
41
39
|
if expected_error:
|
42
|
-
with pytest.raises(
|
40
|
+
with pytest.raises(expected_error):
|
43
41
|
launch(source, args)
|
44
|
-
messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
|
45
|
-
assert contains_error_trace_message(messages, expected_error)
|
46
42
|
else:
|
47
43
|
launch(source, args)
|
48
44
|
|
@@ -51,14 +47,14 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
|
|
51
47
|
"deployment_mode, token_refresh_url, expected_records, expected_error",
|
52
48
|
[
|
53
49
|
pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
|
54
|
-
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [],
|
55
|
-
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [],
|
50
|
+
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
|
51
|
+
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
|
56
52
|
pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
|
57
53
|
pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
|
58
54
|
],
|
59
55
|
)
|
60
56
|
@patch.object(requests.Session, "send", fixture_mock_send)
|
61
|
-
def test_external_oauth_request_source(
|
57
|
+
def test_external_oauth_request_source(deployment_mode, token_refresh_url, expected_records, expected_error):
|
62
58
|
oauth_authenticator = SourceFixtureOauthAuthenticator(
|
63
59
|
client_id="nora", client_secret="hae_sung", refresh_token="arthur", token_refresh_endpoint=token_refresh_url
|
64
60
|
)
|
@@ -67,20 +63,7 @@ def test_external_oauth_request_source(capsys, deployment_mode, token_refresh_ur
|
|
67
63
|
with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict
|
68
64
|
args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"]
|
69
65
|
if expected_error:
|
70
|
-
with pytest.raises(
|
66
|
+
with pytest.raises(expected_error):
|
71
67
|
launch(source, args)
|
72
|
-
messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
|
73
|
-
assert contains_error_trace_message(messages, expected_error)
|
74
68
|
else:
|
75
69
|
launch(source, args)
|
76
|
-
|
77
|
-
|
78
|
-
def contains_error_trace_message(messages: List[Mapping[str, Any]], expected_error: str) -> bool:
|
79
|
-
for message in messages:
|
80
|
-
if message.get("type") != "TRACE":
|
81
|
-
continue
|
82
|
-
elif message.get("trace").get("type") != "ERROR":
|
83
|
-
continue
|
84
|
-
elif message.get("trace").get("error").get("failure_type") == expected_error:
|
85
|
-
return True
|
86
|
-
return False
|
@@ -343,7 +343,7 @@ def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_e
|
|
343
343
|
source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger)
|
344
344
|
config = {}
|
345
345
|
catalog = _create_configured_catalog(source._streams)
|
346
|
-
messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state,
|
346
|
+
messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, RuntimeError)
|
347
347
|
messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, RuntimeError)
|
348
348
|
|
349
349
|
expected_messages = [
|
@@ -15,6 +15,7 @@ _ANOTHER_RESPONSE_BODY = "another body"
|
|
15
15
|
_A_RESPONSE = HttpResponse("any response")
|
16
16
|
_SOME_QUERY_PARAMS = {"q1": "query value"}
|
17
17
|
_SOME_HEADERS = {"h1": "header value"}
|
18
|
+
_OTHER_HEADERS = {"h2": "another header value"}
|
18
19
|
_SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2}
|
19
20
|
_SOME_REQUEST_BODY_STR = "some_request_body"
|
20
21
|
|
@@ -24,13 +25,14 @@ class HttpMockerTest(TestCase):
|
|
24
25
|
def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker):
|
25
26
|
http_mocker.get(
|
26
27
|
HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
|
27
|
-
HttpResponse(_A_RESPONSE_BODY, 474),
|
28
|
+
HttpResponse(_A_RESPONSE_BODY, 474, _OTHER_HEADERS),
|
28
29
|
)
|
29
30
|
|
30
31
|
response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
|
31
32
|
|
32
33
|
assert response.text == _A_RESPONSE_BODY
|
33
34
|
assert response.status_code == 474
|
35
|
+
assert response.headers == _OTHER_HEADERS
|
34
36
|
|
35
37
|
@HttpMocker()
|
36
38
|
def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker):
|
File without changes
|
File without changes
|
File without changes
|