airbyte-cdk 0.61.2__py3-none-any.whl → 0.62.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/abstract_source.py +14 -33
- airbyte_cdk/sources/connector_state_manager.py +16 -4
- airbyte_cdk/sources/file_based/file_based_source.py +87 -35
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +3 -0
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +15 -13
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -0
- airbyte_cdk/sources/file_based/stream/concurrent/{cursor.py → cursor/abstract_concurrent_file_based_cursor.py} +22 -44
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +279 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py +56 -0
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +11 -2
- airbyte_cdk/test/mock_http/mocker.py +3 -1
- airbyte_cdk/test/mock_http/response.py +9 -1
- airbyte_cdk/utils/traced_exception.py +1 -16
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/METADATA +1 -1
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/RECORD +33 -26
- unit_tests/sources/file_based/helpers.py +5 -0
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +2860 -0
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +11 -0
- unit_tests/sources/file_based/scenarios/scenario_builder.py +6 -2
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +365 -0
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +462 -0
- unit_tests/sources/file_based/test_file_based_scenarios.py +45 -0
- unit_tests/sources/file_based/test_scenarios.py +16 -8
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +13 -2
- unit_tests/sources/test_abstract_source.py +36 -170
- unit_tests/sources/test_connector_state_manager.py +20 -13
- unit_tests/sources/test_integration_source.py +8 -25
- unit_tests/sources/test_source_read.py +1 -1
- unit_tests/test/mock_http/test_mocker.py +3 -1
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/top_level.txt +0 -0
@@ -13,7 +13,6 @@ import pytest
|
|
13
13
|
from airbyte_cdk.models import (
|
14
14
|
AirbyteCatalog,
|
15
15
|
AirbyteConnectionStatus,
|
16
|
-
AirbyteErrorTraceMessage,
|
17
16
|
AirbyteLogMessage,
|
18
17
|
AirbyteMessage,
|
19
18
|
AirbyteRecordMessage,
|
@@ -28,7 +27,6 @@ from airbyte_cdk.models import (
|
|
28
27
|
ConfiguredAirbyteCatalog,
|
29
28
|
ConfiguredAirbyteStream,
|
30
29
|
DestinationSyncMode,
|
31
|
-
FailureType,
|
32
30
|
Level,
|
33
31
|
Status,
|
34
32
|
StreamDescriptor,
|
@@ -42,7 +40,6 @@ from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
|
42
40
|
from airbyte_cdk.sources.message import MessageRepository
|
43
41
|
from airbyte_cdk.sources.streams import IncrementalMixin, Stream
|
44
42
|
from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
|
45
|
-
from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets
|
46
43
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
47
44
|
from pytest import fixture
|
48
45
|
|
@@ -57,14 +54,12 @@ class MockSource(AbstractSource):
|
|
57
54
|
per_stream: bool = True,
|
58
55
|
message_repository: MessageRepository = None,
|
59
56
|
exception_on_missing_stream: bool = True,
|
60
|
-
stop_sync_on_stream_failure: bool = False,
|
61
57
|
):
|
62
58
|
self._streams = streams
|
63
59
|
self.check_lambda = check_lambda
|
64
60
|
self.per_stream = per_stream
|
65
61
|
self.exception_on_missing_stream = exception_on_missing_stream
|
66
62
|
self._message_repository = message_repository
|
67
|
-
self._stop_sync_on_stream_failure = stop_sync_on_stream_failure
|
68
63
|
|
69
64
|
def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
|
70
65
|
if self.check_lambda:
|
@@ -89,12 +84,6 @@ class MockSource(AbstractSource):
|
|
89
84
|
return self._message_repository
|
90
85
|
|
91
86
|
|
92
|
-
class MockSourceWithStopSyncFalseOverride(MockSource):
|
93
|
-
@property
|
94
|
-
def stop_sync_on_stream_failure(self) -> bool:
|
95
|
-
return False
|
96
|
-
|
97
|
-
|
98
87
|
class StreamNoStateMethod(Stream):
|
99
88
|
name = "managers"
|
100
89
|
primary_key = None
|
@@ -126,11 +115,8 @@ class StreamRaisesException(Stream):
|
|
126
115
|
name = "lamentations"
|
127
116
|
primary_key = None
|
128
117
|
|
129
|
-
def __init__(self, exception_to_raise):
|
130
|
-
self._exception_to_raise = exception_to_raise
|
131
|
-
|
132
118
|
def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
|
133
|
-
raise
|
119
|
+
raise AirbyteTracedException(message="I was born only to crash like Icarus")
|
134
120
|
|
135
121
|
|
136
122
|
MESSAGE_FROM_REPOSITORY = Mock()
|
@@ -305,7 +291,7 @@ def test_read_stream_emits_repository_message_on_error(mocker, message_repositor
|
|
305
291
|
|
306
292
|
source = MockSource(streams=[stream], message_repository=message_repository)
|
307
293
|
|
308
|
-
with pytest.raises(
|
294
|
+
with pytest.raises(RuntimeError):
|
309
295
|
messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])))
|
310
296
|
assert MESSAGE_FROM_REPOSITORY in messages
|
311
297
|
|
@@ -320,14 +306,14 @@ def test_read_stream_with_error_gets_display_message(mocker):
|
|
320
306
|
catalog = ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])
|
321
307
|
|
322
308
|
# without get_error_display_message
|
323
|
-
with pytest.raises(
|
309
|
+
with pytest.raises(RuntimeError, match="oh no!"):
|
324
310
|
list(source.read(logger, {}, catalog))
|
325
311
|
|
326
312
|
mocker.patch.object(MockStream, "get_error_display_message", return_value="my message")
|
327
313
|
|
328
|
-
with pytest.raises(AirbyteTracedException) as exc:
|
314
|
+
with pytest.raises(AirbyteTracedException, match="oh no!") as exc:
|
329
315
|
list(source.read(logger, {}, catalog))
|
330
|
-
assert
|
316
|
+
assert exc.value.message == "my message"
|
331
317
|
|
332
318
|
|
333
319
|
GLOBAL_EMITTED_AT = 1
|
@@ -372,22 +358,6 @@ def _as_state(state_data: Dict[str, Any], stream_name: str = "", per_stream_stat
|
|
372
358
|
return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=state_data))
|
373
359
|
|
374
360
|
|
375
|
-
def _as_error_trace(stream: str, error_message: str, internal_message: Optional[str], failure_type: Optional[FailureType], stack_trace: Optional[str]) -> AirbyteMessage:
|
376
|
-
trace_message = AirbyteTraceMessage(
|
377
|
-
emitted_at=datetime.datetime.now().timestamp() * 1000.0,
|
378
|
-
type=TraceType.ERROR,
|
379
|
-
error=AirbyteErrorTraceMessage(
|
380
|
-
stream_descriptor=StreamDescriptor(name=stream),
|
381
|
-
message=error_message,
|
382
|
-
internal_message=internal_message,
|
383
|
-
failure_type=failure_type,
|
384
|
-
stack_trace=stack_trace,
|
385
|
-
),
|
386
|
-
)
|
387
|
-
|
388
|
-
return AirbyteMessage(type=MessageType.TRACE, trace=trace_message)
|
389
|
-
|
390
|
-
|
391
361
|
def _configured_stream(stream: Stream, sync_mode: SyncMode):
|
392
362
|
return ConfiguredAirbyteStream(
|
393
363
|
stream=stream.as_airbyte_stream(),
|
@@ -1184,7 +1154,13 @@ class TestIncrementalRead:
|
|
1184
1154
|
def test_checkpoint_state_from_stream_instance():
|
1185
1155
|
teams_stream = MockStreamOverridesStateMethod()
|
1186
1156
|
managers_stream = StreamNoStateMethod()
|
1187
|
-
state_manager = ConnectorStateManager(
|
1157
|
+
state_manager = ConnectorStateManager(
|
1158
|
+
{
|
1159
|
+
"teams": AirbyteStream(name="teams", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]),
|
1160
|
+
"managers": AirbyteStream(name="managers", namespace="", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental])
|
1161
|
+
},
|
1162
|
+
[],
|
1163
|
+
)
|
1188
1164
|
|
1189
1165
|
# The stream_state passed to checkpoint_state() should be ignored since stream implements state function
|
1190
1166
|
teams_stream.state = {"updated_at": "2022-09-11"}
|
@@ -1198,124 +1174,21 @@ def test_checkpoint_state_from_stream_instance():
|
|
1198
1174
|
)
|
1199
1175
|
|
1200
1176
|
|
1201
|
-
|
1202
|
-
"exception_to_raise,expected_error_message,expected_internal_message",
|
1203
|
-
[
|
1204
|
-
pytest.param(AirbyteTracedException(message="I was born only to crash like Icarus"), "I was born only to crash like Icarus", None, id="test_raises_traced_exception"),
|
1205
|
-
pytest.param(Exception("Generic connector error message"), "Something went wrong in the connector. See the logs for more details.", "Generic connector error message", id="test_raises_generic_exception"),
|
1206
|
-
]
|
1207
|
-
)
|
1208
|
-
def test_continue_sync_with_failed_streams(mocker, exception_to_raise, expected_error_message, expected_internal_message):
|
1209
|
-
"""
|
1210
|
-
Tests that running a sync for a connector with multiple streams will continue syncing when one stream fails
|
1211
|
-
with an error. This source does not override the default behavior defined in the AbstractSource class.
|
1212
|
-
"""
|
1213
|
-
stream_output = [{"k1": "v1"}, {"k2": "v2"}]
|
1214
|
-
s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
|
1215
|
-
s2 = StreamRaisesException(exception_to_raise=exception_to_raise)
|
1216
|
-
s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
|
1217
|
-
|
1218
|
-
mocker.patch.object(MockStream, "get_json_schema", return_value={})
|
1219
|
-
mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
|
1220
|
-
|
1221
|
-
src = MockSource(streams=[s1, s2, s3])
|
1222
|
-
catalog = ConfiguredAirbyteCatalog(
|
1223
|
-
streams=[
|
1224
|
-
_configured_stream(s1, SyncMode.full_refresh),
|
1225
|
-
_configured_stream(s2, SyncMode.full_refresh),
|
1226
|
-
_configured_stream(s3, SyncMode.full_refresh),
|
1227
|
-
]
|
1228
|
-
)
|
1229
|
-
|
1230
|
-
expected = _fix_emitted_at(
|
1231
|
-
[
|
1232
|
-
_as_stream_status("s1", AirbyteStreamStatus.STARTED),
|
1233
|
-
_as_stream_status("s1", AirbyteStreamStatus.RUNNING),
|
1234
|
-
*_as_records("s1", stream_output),
|
1235
|
-
_as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
|
1236
|
-
_as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
|
1237
|
-
_as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
|
1238
|
-
_as_error_trace("lamentations", expected_error_message, expected_internal_message, FailureType.system_error, None),
|
1239
|
-
_as_stream_status("s3", AirbyteStreamStatus.STARTED),
|
1240
|
-
_as_stream_status("s3", AirbyteStreamStatus.RUNNING),
|
1241
|
-
*_as_records("s3", stream_output),
|
1242
|
-
_as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
|
1243
|
-
]
|
1244
|
-
)
|
1245
|
-
|
1246
|
-
with pytest.raises(AirbyteTracedException) as exc:
|
1247
|
-
messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
|
1248
|
-
messages = _fix_emitted_at(messages)
|
1249
|
-
|
1250
|
-
assert expected == messages
|
1251
|
-
|
1252
|
-
assert "lamentations" in exc.value.message
|
1253
|
-
|
1254
|
-
|
1255
|
-
def test_continue_sync_source_override_false(mocker):
|
1256
|
-
"""
|
1257
|
-
Tests that running a sync for a connector explicitly overriding the default AbstractSource.stop_sync_on_stream_failure
|
1258
|
-
property to be False which will continue syncing stream even if one encountered an exception.
|
1259
|
-
"""
|
1260
|
-
update_secrets(["API_KEY_VALUE"])
|
1261
|
-
|
1262
|
-
stream_output = [{"k1": "v1"}, {"k2": "v2"}]
|
1263
|
-
s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
|
1264
|
-
s2 = StreamRaisesException(exception_to_raise=AirbyteTracedException(message="I was born only to crash like Icarus"))
|
1265
|
-
s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
|
1266
|
-
|
1267
|
-
mocker.patch.object(MockStream, "get_json_schema", return_value={})
|
1268
|
-
mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
|
1269
|
-
|
1270
|
-
src = MockSourceWithStopSyncFalseOverride(streams=[s1, s2, s3])
|
1271
|
-
catalog = ConfiguredAirbyteCatalog(
|
1272
|
-
streams=[
|
1273
|
-
_configured_stream(s1, SyncMode.full_refresh),
|
1274
|
-
_configured_stream(s2, SyncMode.full_refresh),
|
1275
|
-
_configured_stream(s3, SyncMode.full_refresh),
|
1276
|
-
]
|
1277
|
-
)
|
1278
|
-
|
1279
|
-
expected = _fix_emitted_at(
|
1280
|
-
[
|
1281
|
-
_as_stream_status("s1", AirbyteStreamStatus.STARTED),
|
1282
|
-
_as_stream_status("s1", AirbyteStreamStatus.RUNNING),
|
1283
|
-
*_as_records("s1", stream_output),
|
1284
|
-
_as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
|
1285
|
-
_as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
|
1286
|
-
_as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
|
1287
|
-
_as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None),
|
1288
|
-
_as_stream_status("s3", AirbyteStreamStatus.STARTED),
|
1289
|
-
_as_stream_status("s3", AirbyteStreamStatus.RUNNING),
|
1290
|
-
*_as_records("s3", stream_output),
|
1291
|
-
_as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
|
1292
|
-
]
|
1293
|
-
)
|
1294
|
-
|
1295
|
-
with pytest.raises(AirbyteTracedException) as exc:
|
1296
|
-
messages = [_remove_stack_trace(message) for message in src.read(logger, {}, catalog)]
|
1297
|
-
messages = _fix_emitted_at(messages)
|
1298
|
-
|
1299
|
-
assert expected == messages
|
1300
|
-
|
1301
|
-
assert "lamentations" in exc.value.message
|
1302
|
-
|
1303
|
-
|
1304
|
-
def test_sync_error_trace_messages_obfuscate_secrets(mocker):
|
1177
|
+
def test_continue_sync_with_failed_streams(mocker):
|
1305
1178
|
"""
|
1306
|
-
Tests that
|
1179
|
+
Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure enabled continues
|
1180
|
+
syncing even when one stream fails with an error.
|
1307
1181
|
"""
|
1308
|
-
update_secrets(["API_KEY_VALUE"])
|
1309
|
-
|
1310
1182
|
stream_output = [{"k1": "v1"}, {"k2": "v2"}]
|
1311
1183
|
s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
|
1312
|
-
s2 = StreamRaisesException(
|
1184
|
+
s2 = StreamRaisesException()
|
1313
1185
|
s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
|
1314
1186
|
|
1315
1187
|
mocker.patch.object(MockStream, "get_json_schema", return_value={})
|
1316
1188
|
mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
|
1317
1189
|
|
1318
1190
|
src = MockSource(streams=[s1, s2, s3])
|
1191
|
+
mocker.patch.object(MockSource, "continue_sync_on_stream_failure", return_value=True)
|
1319
1192
|
catalog = ConfiguredAirbyteCatalog(
|
1320
1193
|
streams=[
|
1321
1194
|
_configured_stream(s1, SyncMode.full_refresh),
|
@@ -1332,7 +1205,6 @@ def test_sync_error_trace_messages_obfuscate_secrets(mocker):
|
|
1332
1205
|
_as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
|
1333
1206
|
_as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
|
1334
1207
|
_as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
|
1335
|
-
_as_error_trace("lamentations", "My api_key value **** flew too close to the sun.", None, FailureType.system_error, None),
|
1336
1208
|
_as_stream_status("s3", AirbyteStreamStatus.STARTED),
|
1337
1209
|
_as_stream_status("s3", AirbyteStreamStatus.RUNNING),
|
1338
1210
|
*_as_records("s3", stream_output),
|
@@ -1340,30 +1212,32 @@ def test_sync_error_trace_messages_obfuscate_secrets(mocker):
|
|
1340
1212
|
]
|
1341
1213
|
)
|
1342
1214
|
|
1215
|
+
messages = []
|
1343
1216
|
with pytest.raises(AirbyteTracedException) as exc:
|
1344
|
-
|
1345
|
-
messages
|
1346
|
-
|
1347
|
-
|
1217
|
+
# We can't use list comprehension or list() here because we are still raising a final exception for the
|
1218
|
+
# failed streams and that disrupts parsing the generator into the messages emitted before
|
1219
|
+
for message in src.read(logger, {}, catalog):
|
1220
|
+
messages.append(message)
|
1348
1221
|
|
1222
|
+
messages = _fix_emitted_at(messages)
|
1223
|
+
assert expected == messages
|
1349
1224
|
assert "lamentations" in exc.value.message
|
1350
1225
|
|
1351
1226
|
|
1352
|
-
def
|
1227
|
+
def test_stop_sync_with_failed_streams(mocker):
|
1353
1228
|
"""
|
1354
|
-
Tests that running a sync for a connector with multiple streams and
|
1355
|
-
|
1229
|
+
Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure disabled stops
|
1230
|
+
syncing once a stream fails with an error.
|
1356
1231
|
"""
|
1357
1232
|
stream_output = [{"k1": "v1"}, {"k2": "v2"}]
|
1358
1233
|
s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
|
1359
|
-
s2 = StreamRaisesException(
|
1234
|
+
s2 = StreamRaisesException()
|
1360
1235
|
s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
|
1361
1236
|
|
1362
1237
|
mocker.patch.object(MockStream, "get_json_schema", return_value={})
|
1363
1238
|
mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
|
1364
1239
|
|
1365
1240
|
src = MockSource(streams=[s1, s2, s3])
|
1366
|
-
mocker.patch.object(MockSource, "stop_sync_on_stream_failure", return_value=True)
|
1367
1241
|
catalog = ConfiguredAirbyteCatalog(
|
1368
1242
|
streams=[
|
1369
1243
|
_configured_stream(s1, SyncMode.full_refresh),
|
@@ -1380,23 +1254,15 @@ def test_continue_sync_with_failed_streams_with_override_false(mocker):
|
|
1380
1254
|
_as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
|
1381
1255
|
_as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
|
1382
1256
|
_as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
|
1383
|
-
_as_error_trace("lamentations", "I was born only to crash like Icarus", None, FailureType.system_error, None),
|
1384
1257
|
]
|
1385
1258
|
)
|
1386
1259
|
|
1387
|
-
|
1388
|
-
|
1389
|
-
|
1390
|
-
|
1391
|
-
|
1392
|
-
|
1393
|
-
assert "lamentations" in exc.value.message
|
1394
|
-
|
1260
|
+
messages = []
|
1261
|
+
with pytest.raises(AirbyteTracedException):
|
1262
|
+
# We can't use list comprehension or list() here because we are still raising a final exception for the
|
1263
|
+
# failed streams and that disrupts parsing the generator into the messages emitted before
|
1264
|
+
for message in src.read(logger, {}, catalog):
|
1265
|
+
messages.append(message)
|
1395
1266
|
|
1396
|
-
|
1397
|
-
|
1398
|
-
Helper method that removes the stack trace from Airbyte trace messages to make asserting against expected records easier
|
1399
|
-
"""
|
1400
|
-
if message.trace and message.trace.error and message.trace.error.stack_trace:
|
1401
|
-
message.trace.error.stack_trace = None
|
1402
|
-
return message
|
1267
|
+
messages = _fix_emitted_at(messages)
|
1268
|
+
assert expected == messages
|
@@ -3,21 +3,21 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from contextlib import nullcontext as does_not_raise
|
6
|
-
from typing import
|
6
|
+
from typing import List
|
7
7
|
|
8
8
|
import pytest
|
9
|
-
from airbyte_cdk.models import
|
9
|
+
from airbyte_cdk.models import (
|
10
|
+
AirbyteMessage,
|
11
|
+
AirbyteStateBlob,
|
12
|
+
AirbyteStateMessage,
|
13
|
+
AirbyteStateType,
|
14
|
+
AirbyteStream,
|
15
|
+
AirbyteStreamState,
|
16
|
+
StreamDescriptor,
|
17
|
+
SyncMode,
|
18
|
+
)
|
10
19
|
from airbyte_cdk.models import Type as MessageType
|
11
20
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager, HashableStreamDescriptor
|
12
|
-
from airbyte_cdk.sources.streams import Stream
|
13
|
-
|
14
|
-
|
15
|
-
class StreamWithNamespace(Stream):
|
16
|
-
primary_key = None
|
17
|
-
namespace = "public"
|
18
|
-
|
19
|
-
def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
|
20
|
-
return {}
|
21
21
|
|
22
22
|
|
23
23
|
@pytest.mark.parametrize(
|
@@ -156,7 +156,11 @@ class StreamWithNamespace(Stream):
|
|
156
156
|
),
|
157
157
|
)
|
158
158
|
def test_initialize_state_manager(input_stream_state, expected_stream_state, expected_error):
|
159
|
-
stream_to_instance_map = {
|
159
|
+
stream_to_instance_map = {
|
160
|
+
"actors": AirbyteStream(
|
161
|
+
name="actors", namespace="public", json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental],
|
162
|
+
)
|
163
|
+
}
|
160
164
|
|
161
165
|
if isinstance(input_stream_state, List):
|
162
166
|
input_stream_state = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_stream_state)]
|
@@ -264,7 +268,10 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp
|
|
264
268
|
],
|
265
269
|
)
|
266
270
|
def test_get_stream_state(input_state, stream_name, namespace, expected_state):
|
267
|
-
stream_to_instance_map = {
|
271
|
+
stream_to_instance_map = {stream_name: AirbyteStream(
|
272
|
+
name=stream_name, namespace=namespace, json_schema={}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]
|
273
|
+
)
|
274
|
+
}
|
268
275
|
state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_state)]
|
269
276
|
state_manager = ConnectorStateManager(stream_to_instance_map, state_messages)
|
270
277
|
|
@@ -2,9 +2,7 @@
|
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
-
import json
|
6
5
|
import os
|
7
|
-
from typing import Any, List, Mapping
|
8
6
|
from unittest import mock
|
9
7
|
from unittest.mock import patch
|
10
8
|
|
@@ -24,9 +22,9 @@ from unit_tests.sources.fixtures.source_test_fixture import (
|
|
24
22
|
"deployment_mode, url_base, expected_records, expected_error",
|
25
23
|
[
|
26
24
|
pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
|
27
|
-
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [],
|
28
|
-
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [],
|
29
|
-
pytest.param("CLOUD", "https://localhost:80/api/v1/", [],
|
25
|
+
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
|
26
|
+
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
|
27
|
+
pytest.param("CLOUD", "https://localhost:80/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_localhost"),
|
30
28
|
pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
|
31
29
|
pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
|
32
30
|
],
|
@@ -39,10 +37,8 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
|
|
39
37
|
with mock.patch.object(HttpTestStream, "url_base", url_base):
|
40
38
|
args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"]
|
41
39
|
if expected_error:
|
42
|
-
with pytest.raises(
|
40
|
+
with pytest.raises(expected_error):
|
43
41
|
launch(source, args)
|
44
|
-
messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
|
45
|
-
assert contains_error_trace_message(messages, expected_error)
|
46
42
|
else:
|
47
43
|
launch(source, args)
|
48
44
|
|
@@ -51,14 +47,14 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec
|
|
51
47
|
"deployment_mode, token_refresh_url, expected_records, expected_error",
|
52
48
|
[
|
53
49
|
pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"),
|
54
|
-
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [],
|
55
|
-
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [],
|
50
|
+
pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"),
|
51
|
+
pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"),
|
56
52
|
pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"),
|
57
53
|
pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"),
|
58
54
|
],
|
59
55
|
)
|
60
56
|
@patch.object(requests.Session, "send", fixture_mock_send)
|
61
|
-
def test_external_oauth_request_source(
|
57
|
+
def test_external_oauth_request_source(deployment_mode, token_refresh_url, expected_records, expected_error):
|
62
58
|
oauth_authenticator = SourceFixtureOauthAuthenticator(
|
63
59
|
client_id="nora", client_secret="hae_sung", refresh_token="arthur", token_refresh_endpoint=token_refresh_url
|
64
60
|
)
|
@@ -67,20 +63,7 @@ def test_external_oauth_request_source(capsys, deployment_mode, token_refresh_ur
|
|
67
63
|
with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict
|
68
64
|
args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"]
|
69
65
|
if expected_error:
|
70
|
-
with pytest.raises(
|
66
|
+
with pytest.raises(expected_error):
|
71
67
|
launch(source, args)
|
72
|
-
messages = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
|
73
|
-
assert contains_error_trace_message(messages, expected_error)
|
74
68
|
else:
|
75
69
|
launch(source, args)
|
76
|
-
|
77
|
-
|
78
|
-
def contains_error_trace_message(messages: List[Mapping[str, Any]], expected_error: str) -> bool:
|
79
|
-
for message in messages:
|
80
|
-
if message.get("type") != "TRACE":
|
81
|
-
continue
|
82
|
-
elif message.get("trace").get("type") != "ERROR":
|
83
|
-
continue
|
84
|
-
elif message.get("trace").get("error").get("failure_type") == expected_error:
|
85
|
-
return True
|
86
|
-
return False
|
@@ -343,7 +343,7 @@ def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_e
|
|
343
343
|
source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger)
|
344
344
|
config = {}
|
345
345
|
catalog = _create_configured_catalog(source._streams)
|
346
|
-
messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state,
|
346
|
+
messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, RuntimeError)
|
347
347
|
messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, RuntimeError)
|
348
348
|
|
349
349
|
expected_messages = [
|
@@ -15,6 +15,7 @@ _ANOTHER_RESPONSE_BODY = "another body"
|
|
15
15
|
_A_RESPONSE = HttpResponse("any response")
|
16
16
|
_SOME_QUERY_PARAMS = {"q1": "query value"}
|
17
17
|
_SOME_HEADERS = {"h1": "header value"}
|
18
|
+
_OTHER_HEADERS = {"h2": "another header value"}
|
18
19
|
_SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2}
|
19
20
|
_SOME_REQUEST_BODY_STR = "some_request_body"
|
20
21
|
|
@@ -24,13 +25,14 @@ class HttpMockerTest(TestCase):
|
|
24
25
|
def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker):
|
25
26
|
http_mocker.get(
|
26
27
|
HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
|
27
|
-
HttpResponse(_A_RESPONSE_BODY, 474),
|
28
|
+
HttpResponse(_A_RESPONSE_BODY, 474, _OTHER_HEADERS),
|
28
29
|
)
|
29
30
|
|
30
31
|
response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
|
31
32
|
|
32
33
|
assert response.text == _A_RESPONSE_BODY
|
33
34
|
assert response.status_code == 474
|
35
|
+
assert response.headers == _OTHER_HEADERS
|
34
36
|
|
35
37
|
@HttpMocker()
|
36
38
|
def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker):
|
File without changes
|
File without changes
|
File without changes
|