airbyte-cdk 0.67.0__py3-none-any.whl → 0.67.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/abstract_source.py +30 -69
- airbyte_cdk/sources/connector_state_manager.py +12 -26
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +552 -524
- airbyte_cdk/sources/file_based/config/csv_format.py +2 -0
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +32 -14
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +3 -19
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -3
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +3 -19
- airbyte_cdk/sources/streams/concurrent/cursor.py +1 -3
- airbyte_cdk/sources/streams/core.py +36 -34
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/METADATA +3 -2
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/RECORD +31 -31
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +2 -1
- unit_tests/sources/file_based/config/test_csv_format.py +6 -1
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +51 -6
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +139 -199
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +91 -133
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +2 -13
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +2 -2
- unit_tests/sources/file_based/test_scenarios.py +2 -2
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +9 -9
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +5 -5
- unit_tests/sources/streams/concurrent/test_adapters.py +2 -13
- unit_tests/sources/streams/test_stream_read.py +221 -11
- unit_tests/sources/test_abstract_source.py +142 -130
- unit_tests/sources/test_connector_state_manager.py +3 -124
- unit_tests/sources/test_source.py +18 -14
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/WHEEL +0 -0
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/top_level.txt +0 -0
|
@@ -191,12 +191,57 @@ def test_value_dictionary() -> None:
|
|
|
191
191
|
assert py_value == {"indices": [0, 1, 2, 0, 1], "values": ["apple", "banana", "cherry"]}
|
|
192
192
|
|
|
193
193
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
194
|
+
@pytest.mark.parametrize(
|
|
195
|
+
"parquet_type, parquet_format",
|
|
196
|
+
[
|
|
197
|
+
pytest.param(pa.bool_(), _default_parquet_format, id="test_parquet_bool"),
|
|
198
|
+
pytest.param(pa.int8(), _default_parquet_format, id="test_parquet_int8"),
|
|
199
|
+
pytest.param(pa.int16(), _default_parquet_format, id="test_parquet_int16"),
|
|
200
|
+
pytest.param(pa.int32(), _default_parquet_format, id="test_parquet_int32"),
|
|
201
|
+
pytest.param(pa.int64(), _default_parquet_format, id="test_parquet_int64"),
|
|
202
|
+
pytest.param(pa.uint8(), _default_parquet_format, id="test_parquet_uint8"),
|
|
203
|
+
pytest.param(pa.uint16(), _default_parquet_format, id="test_parquet_uint16"),
|
|
204
|
+
pytest.param(pa.uint32(), _default_parquet_format, id="test_parquet_uint32"),
|
|
205
|
+
pytest.param(pa.uint64(), _default_parquet_format, id="test_parquet_uint64"),
|
|
206
|
+
pytest.param(pa.float16(), _default_parquet_format, id="test_parquet_float16"),
|
|
207
|
+
pytest.param(pa.float32(), _default_parquet_format, id="test_parquet_float32"),
|
|
208
|
+
pytest.param(pa.float64(), _default_parquet_format, id="test_parquet_float64"),
|
|
209
|
+
pytest.param(pa.time32("s"), _default_parquet_format, id="test_parquet_time32s"),
|
|
210
|
+
pytest.param(pa.time32("ms"), _default_parquet_format, id="test_parquet_time32ms"),
|
|
211
|
+
pytest.param(pa.time64("us"), _default_parquet_format, id="test_parquet_time64us"),
|
|
212
|
+
pytest.param(pa.time64("ns"), _default_parquet_format, id="test_parquet_time64ns"),
|
|
213
|
+
pytest.param(pa.timestamp("s"), _default_parquet_format, id="test_parquet_timestamps_s"),
|
|
214
|
+
pytest.param(pa.timestamp("ms"), _default_parquet_format, id="test_parquet_timestamp_ms"),
|
|
215
|
+
pytest.param(pa.timestamp("s", "utc"), _default_parquet_format, id="test_parquet_timestamps_s_with_tz"),
|
|
216
|
+
pytest.param(pa.timestamp("ms", "est"), _default_parquet_format, id="test_parquet_timestamps_ms_with_tz"),
|
|
217
|
+
pytest.param(pa.date32(), _default_parquet_format, id="test_parquet_date32"),
|
|
218
|
+
pytest.param(pa.date64(), _default_parquet_format, id="test_parquet_date64"),
|
|
219
|
+
pytest.param(pa.duration("s"), _default_parquet_format, id="test_duration_s"),
|
|
220
|
+
pytest.param(pa.duration("ms"), _default_parquet_format, id="test_duration_ms"),
|
|
221
|
+
pytest.param(pa.duration("us"), _default_parquet_format, id="test_duration_us"),
|
|
222
|
+
pytest.param(pa.duration("ns"), _default_parquet_format, id="test_duration_ns"),
|
|
223
|
+
pytest.param(pa.month_day_nano_interval(), _default_parquet_format, id="test_parquet_month_day_nano_interval"),
|
|
224
|
+
pytest.param(pa.binary(), _default_parquet_format, id="test_binary"),
|
|
225
|
+
pytest.param(pa.binary(2), _default_parquet_format, id="test_fixed_size_binary"),
|
|
226
|
+
pytest.param(pa.string(), _default_parquet_format, id="test_parquet_string"),
|
|
227
|
+
pytest.param(pa.utf8(), _default_parquet_format, id="test_utf8"),
|
|
228
|
+
pytest.param(pa.large_binary(), _default_parquet_format, id="test_large_binary"),
|
|
229
|
+
pytest.param(pa.large_string(), _default_parquet_format, id="test_large_string"),
|
|
230
|
+
pytest.param(pa.large_utf8(), _default_parquet_format, id="test_large_utf8"),
|
|
231
|
+
pytest.param(pa.dictionary(pa.int32(), pa.string()), _default_parquet_format, id="test_dictionary"),
|
|
232
|
+
pytest.param(pa.struct([pa.field("field", pa.int32())]), _default_parquet_format, id="test_struct"),
|
|
233
|
+
pytest.param(pa.list_(pa.int32()), _default_parquet_format, id="test_list"),
|
|
234
|
+
pytest.param(pa.large_list(pa.int32()), _default_parquet_format, id="test_large_list"),
|
|
235
|
+
pytest.param(pa.decimal128(2), _default_parquet_format, id="test_decimal128"),
|
|
236
|
+
pytest.param(pa.decimal256(2), _default_parquet_format, id="test_decimal256"),
|
|
237
|
+
pytest.param(pa.decimal128(2), _decimal_as_float_parquet_format, id="test_decimal128_as_float"),
|
|
238
|
+
pytest.param(pa.decimal256(2), _decimal_as_float_parquet_format, id="test_decimal256_as_float"),
|
|
239
|
+
pytest.param(pa.map_(pa.int32(), pa.int32()), _default_parquet_format, id="test_map"),
|
|
240
|
+
pytest.param(pa.null(), _default_parquet_format, id="test_null"),
|
|
241
|
+
])
|
|
242
|
+
def test_null_value_does_not_throw(parquet_type, parquet_format) -> None:
|
|
243
|
+
pyarrow_value = pa.scalar(None, type=parquet_type)
|
|
244
|
+
assert ParquetParser._to_output_value(pyarrow_value, parquet_format) is None
|
|
200
245
|
|
|
201
246
|
|
|
202
247
|
@pytest.mark.parametrize(
|
|
@@ -74,10 +74,8 @@ single_csv_input_state_is_earlier_scenario_concurrent = (
|
|
|
74
74
|
"stream": "stream1",
|
|
75
75
|
},
|
|
76
76
|
{
|
|
77
|
-
"
|
|
78
|
-
|
|
79
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
|
|
80
|
-
}
|
|
77
|
+
"history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z", "a.csv": "2023-06-05T03:54:07.000000Z"},
|
|
78
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
|
|
81
79
|
},
|
|
82
80
|
]
|
|
83
81
|
)
|
|
@@ -156,10 +154,8 @@ single_csv_file_is_skipped_if_same_modified_at_as_in_history_concurrent = (
|
|
|
156
154
|
.set_expected_records(
|
|
157
155
|
[
|
|
158
156
|
{
|
|
159
|
-
"
|
|
160
|
-
|
|
161
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
|
|
162
|
-
}
|
|
157
|
+
"history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
|
|
158
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
|
|
163
159
|
}
|
|
164
160
|
]
|
|
165
161
|
)
|
|
@@ -256,10 +252,8 @@ single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history_concurre
|
|
|
256
252
|
"stream": "stream1",
|
|
257
253
|
},
|
|
258
254
|
{
|
|
259
|
-
"
|
|
260
|
-
|
|
261
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
|
|
262
|
-
}
|
|
255
|
+
"history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
|
|
256
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
|
|
263
257
|
},
|
|
264
258
|
]
|
|
265
259
|
)
|
|
@@ -368,10 +362,8 @@ single_csv_no_input_state_scenario_concurrent = (
|
|
|
368
362
|
"stream": "stream1",
|
|
369
363
|
},
|
|
370
364
|
{
|
|
371
|
-
"
|
|
372
|
-
|
|
373
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
|
|
374
|
-
}
|
|
365
|
+
"history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
|
|
366
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
|
|
375
367
|
},
|
|
376
368
|
]
|
|
377
369
|
)
|
|
@@ -491,10 +483,8 @@ multi_csv_same_timestamp_scenario_concurrent = (
|
|
|
491
483
|
"stream": "stream1",
|
|
492
484
|
},
|
|
493
485
|
{
|
|
494
|
-
"
|
|
495
|
-
|
|
496
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
|
|
497
|
-
}
|
|
486
|
+
"history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
|
|
487
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
|
|
498
488
|
},
|
|
499
489
|
]
|
|
500
490
|
)
|
|
@@ -583,13 +573,11 @@ single_csv_input_state_is_later_scenario_concurrent = (
|
|
|
583
573
|
"stream": "stream1",
|
|
584
574
|
},
|
|
585
575
|
{
|
|
586
|
-
"
|
|
587
|
-
"
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
"_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv",
|
|
592
|
-
}
|
|
576
|
+
"history": {
|
|
577
|
+
"recent_file.csv": "2023-07-15T23:59:59.000000Z",
|
|
578
|
+
"a.csv": "2023-06-05T03:54:07.000000Z",
|
|
579
|
+
},
|
|
580
|
+
"_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv",
|
|
593
581
|
},
|
|
594
582
|
]
|
|
595
583
|
)
|
|
@@ -697,12 +685,10 @@ multi_csv_different_timestamps_scenario_concurrent = (
|
|
|
697
685
|
"stream": "stream1",
|
|
698
686
|
},
|
|
699
687
|
{
|
|
700
|
-
"
|
|
701
|
-
"
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
"_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv",
|
|
705
|
-
}
|
|
688
|
+
"history": {
|
|
689
|
+
"a.csv": "2023-06-04T03:54:07.000000Z",
|
|
690
|
+
},
|
|
691
|
+
"_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv",
|
|
706
692
|
},
|
|
707
693
|
{
|
|
708
694
|
"data": {
|
|
@@ -725,10 +711,8 @@ multi_csv_different_timestamps_scenario_concurrent = (
|
|
|
725
711
|
"stream": "stream1",
|
|
726
712
|
},
|
|
727
713
|
{
|
|
728
|
-
"
|
|
729
|
-
|
|
730
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
|
|
731
|
-
}
|
|
714
|
+
"history": {"a.csv": "2023-06-04T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
|
|
715
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
|
|
732
716
|
},
|
|
733
717
|
]
|
|
734
718
|
)
|
|
@@ -856,10 +840,8 @@ multi_csv_per_timestamp_scenario_concurrent = (
|
|
|
856
840
|
"stream": "stream1",
|
|
857
841
|
},
|
|
858
842
|
{
|
|
859
|
-
"
|
|
860
|
-
|
|
861
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
|
|
862
|
-
}
|
|
843
|
+
"history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
|
|
844
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
|
|
863
845
|
},
|
|
864
846
|
{
|
|
865
847
|
"data": {
|
|
@@ -882,14 +864,12 @@ multi_csv_per_timestamp_scenario_concurrent = (
|
|
|
882
864
|
"stream": "stream1",
|
|
883
865
|
},
|
|
884
866
|
{
|
|
885
|
-
"
|
|
886
|
-
"
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
|
|
892
|
-
}
|
|
867
|
+
"history": {
|
|
868
|
+
"a.csv": "2023-06-05T03:54:07.000000Z",
|
|
869
|
+
"b.csv": "2023-06-05T03:54:07.000000Z",
|
|
870
|
+
"c.csv": "2023-06-06T03:54:07.000000Z",
|
|
871
|
+
},
|
|
872
|
+
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
|
|
893
873
|
},
|
|
894
874
|
]
|
|
895
875
|
)
|
|
@@ -1001,10 +981,8 @@ multi_csv_skip_file_if_already_in_history_concurrent = (
|
|
|
1001
981
|
"stream": "stream1",
|
|
1002
982
|
},
|
|
1003
983
|
{
|
|
1004
|
-
"
|
|
1005
|
-
|
|
1006
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
|
|
1007
|
-
}
|
|
984
|
+
"history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
|
|
985
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
|
|
1008
986
|
},
|
|
1009
987
|
{
|
|
1010
988
|
"data": {
|
|
@@ -1027,14 +1005,12 @@ multi_csv_skip_file_if_already_in_history_concurrent = (
|
|
|
1027
1005
|
"stream": "stream1",
|
|
1028
1006
|
},
|
|
1029
1007
|
{
|
|
1030
|
-
"
|
|
1031
|
-
"
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
|
|
1037
|
-
}
|
|
1008
|
+
"history": {
|
|
1009
|
+
"a.csv": "2023-06-05T03:54:07.000000Z",
|
|
1010
|
+
"b.csv": "2023-06-05T03:54:07.000000Z",
|
|
1011
|
+
"c.csv": "2023-06-06T03:54:07.000000Z",
|
|
1012
|
+
},
|
|
1013
|
+
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
|
|
1038
1014
|
},
|
|
1039
1015
|
]
|
|
1040
1016
|
)
|
|
@@ -1153,14 +1129,12 @@ multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_newer
|
|
|
1153
1129
|
# {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped
|
|
1154
1130
|
# {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped
|
|
1155
1131
|
{
|
|
1156
|
-
"
|
|
1157
|
-
"
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
|
|
1163
|
-
}
|
|
1132
|
+
"history": {
|
|
1133
|
+
"a.csv": "2023-06-05T03:54:07.000000Z",
|
|
1134
|
+
"b.csv": "2023-06-05T03:54:07.000000Z",
|
|
1135
|
+
"c.csv": "2023-06-06T03:54:07.000000Z",
|
|
1136
|
+
},
|
|
1137
|
+
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
|
|
1164
1138
|
},
|
|
1165
1139
|
]
|
|
1166
1140
|
)
|
|
@@ -1282,14 +1256,12 @@ multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_older
|
|
|
1282
1256
|
# {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped
|
|
1283
1257
|
# {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped
|
|
1284
1258
|
{
|
|
1285
|
-
"
|
|
1286
|
-
"
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
|
|
1292
|
-
}
|
|
1259
|
+
"history": {
|
|
1260
|
+
"a.csv": "2023-06-05T03:54:07.000000Z",
|
|
1261
|
+
"b.csv": "2023-06-05T03:54:07.000000Z",
|
|
1262
|
+
"c.csv": "2023-06-06T03:54:07.000000Z",
|
|
1263
|
+
},
|
|
1264
|
+
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
|
|
1293
1265
|
},
|
|
1294
1266
|
]
|
|
1295
1267
|
)
|
|
@@ -1405,14 +1377,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
|
|
|
1405
1377
|
"stream": "stream1",
|
|
1406
1378
|
},
|
|
1407
1379
|
{
|
|
1408
|
-
"
|
|
1409
|
-
"
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
|
|
1415
|
-
}
|
|
1380
|
+
"history": {
|
|
1381
|
+
"very_old_file.csv": "2023-06-02T03:54:07.000000Z",
|
|
1382
|
+
"old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1383
|
+
"a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1384
|
+
},
|
|
1385
|
+
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
|
|
1416
1386
|
},
|
|
1417
1387
|
{
|
|
1418
1388
|
"data": {
|
|
@@ -1435,14 +1405,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
|
|
|
1435
1405
|
"stream": "stream1",
|
|
1436
1406
|
},
|
|
1437
1407
|
{
|
|
1438
|
-
"
|
|
1439
|
-
"
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
"_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
|
|
1445
|
-
}
|
|
1408
|
+
"history": {
|
|
1409
|
+
"old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1410
|
+
"a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1411
|
+
"b.csv": "2023-06-07T03:54:07.000000Z",
|
|
1412
|
+
},
|
|
1413
|
+
"_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
|
|
1446
1414
|
},
|
|
1447
1415
|
{
|
|
1448
1416
|
"data": {
|
|
@@ -1465,14 +1433,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
|
|
|
1465
1433
|
"stream": "stream1",
|
|
1466
1434
|
},
|
|
1467
1435
|
{
|
|
1468
|
-
"
|
|
1469
|
-
"
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
"_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
|
|
1475
|
-
}
|
|
1436
|
+
"history": {
|
|
1437
|
+
"old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1438
|
+
"b.csv": "2023-06-07T03:54:07.000000Z",
|
|
1439
|
+
"c.csv": "2023-06-10T03:54:07.000000Z",
|
|
1440
|
+
},
|
|
1441
|
+
"_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
|
|
1476
1442
|
},
|
|
1477
1443
|
]
|
|
1478
1444
|
)
|
|
@@ -1592,14 +1558,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
|
|
|
1592
1558
|
"stream": "stream1",
|
|
1593
1559
|
},
|
|
1594
1560
|
{
|
|
1595
|
-
"
|
|
1596
|
-
"
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
|
|
1602
|
-
}
|
|
1561
|
+
"history": {
|
|
1562
|
+
"very_old_file.csv": "2023-06-02T03:54:07.000000Z",
|
|
1563
|
+
"old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1564
|
+
"a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1565
|
+
},
|
|
1566
|
+
"_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
|
|
1603
1567
|
},
|
|
1604
1568
|
{
|
|
1605
1569
|
"data": {
|
|
@@ -1622,14 +1586,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
|
|
|
1622
1586
|
"stream": "stream1",
|
|
1623
1587
|
},
|
|
1624
1588
|
{
|
|
1625
|
-
"
|
|
1626
|
-
"
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
"_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
|
|
1632
|
-
}
|
|
1589
|
+
"history": {
|
|
1590
|
+
"old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1591
|
+
"a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1592
|
+
"b.csv": "2023-06-07T03:54:07.000000Z",
|
|
1593
|
+
},
|
|
1594
|
+
"_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
|
|
1633
1595
|
},
|
|
1634
1596
|
{
|
|
1635
1597
|
"data": {
|
|
@@ -1652,14 +1614,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
|
|
|
1652
1614
|
"stream": "stream1",
|
|
1653
1615
|
},
|
|
1654
1616
|
{
|
|
1655
|
-
"
|
|
1656
|
-
"
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
"_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
|
|
1662
|
-
}
|
|
1617
|
+
"history": {
|
|
1618
|
+
"old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
|
|
1619
|
+
"b.csv": "2023-06-07T03:54:07.000000Z",
|
|
1620
|
+
"c.csv": "2023-06-10T03:54:07.000000Z",
|
|
1621
|
+
},
|
|
1622
|
+
"_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
|
|
1663
1623
|
},
|
|
1664
1624
|
]
|
|
1665
1625
|
)
|
|
@@ -1848,14 +1808,12 @@ multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor
|
|
|
1848
1808
|
"stream": "stream1",
|
|
1849
1809
|
},
|
|
1850
1810
|
{
|
|
1851
|
-
"
|
|
1852
|
-
"
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
|
|
1858
|
-
}
|
|
1811
|
+
"history": {
|
|
1812
|
+
"b.csv": "2023-06-05T03:54:07.000000Z",
|
|
1813
|
+
"c.csv": "2023-06-05T03:54:07.000000Z",
|
|
1814
|
+
"d.csv": "2023-06-05T03:54:07.000000Z",
|
|
1815
|
+
},
|
|
1816
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
|
|
1859
1817
|
},
|
|
1860
1818
|
]
|
|
1861
1819
|
)
|
|
@@ -2032,14 +1990,12 @@ multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor
|
|
|
2032
1990
|
"stream": "stream1",
|
|
2033
1991
|
},
|
|
2034
1992
|
{
|
|
2035
|
-
"
|
|
2036
|
-
"
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
|
|
2042
|
-
}
|
|
1993
|
+
"history": {
|
|
1994
|
+
"b.csv": "2023-06-05T03:54:07.000000Z",
|
|
1995
|
+
"c.csv": "2023-06-05T03:54:07.000000Z",
|
|
1996
|
+
"d.csv": "2023-06-05T03:54:07.000000Z",
|
|
1997
|
+
},
|
|
1998
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
|
|
2043
1999
|
},
|
|
2044
2000
|
]
|
|
2045
2001
|
)
|
|
@@ -2138,14 +2094,12 @@ multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_
|
|
|
2138
2094
|
.set_expected_records(
|
|
2139
2095
|
[
|
|
2140
2096
|
{
|
|
2141
|
-
"
|
|
2142
|
-
"
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
|
|
2148
|
-
}
|
|
2097
|
+
"history": {
|
|
2098
|
+
"b.csv": "2023-06-05T03:54:07.000000Z",
|
|
2099
|
+
"c.csv": "2023-06-05T03:54:07.000000Z",
|
|
2100
|
+
"d.csv": "2023-06-05T03:54:07.000000Z",
|
|
2101
|
+
},
|
|
2102
|
+
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
|
|
2149
2103
|
}
|
|
2150
2104
|
]
|
|
2151
2105
|
)
|
|
@@ -2256,14 +2210,12 @@ multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_
|
|
|
2256
2210
|
.set_expected_records(
|
|
2257
2211
|
[
|
|
2258
2212
|
{
|
|
2259
|
-
"stream1": {
|
|
2260
2213
|
"history": {
|
|
2261
2214
|
"b.csv": "2023-06-05T03:54:07.000000Z",
|
|
2262
2215
|
"c.csv": "2023-06-05T03:54:07.000000Z",
|
|
2263
2216
|
"d.csv": "2023-06-05T03:54:07.000000Z",
|
|
2264
2217
|
},
|
|
2265
2218
|
"_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
|
|
2266
|
-
}
|
|
2267
2219
|
}
|
|
2268
2220
|
]
|
|
2269
2221
|
)
|
|
@@ -2397,14 +2349,12 @@ multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_time
|
|
|
2397
2349
|
"stream": "stream1",
|
|
2398
2350
|
},
|
|
2399
2351
|
{
|
|
2400
|
-
"
|
|
2401
|
-
"
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
|
|
2407
|
-
}
|
|
2352
|
+
"history": {
|
|
2353
|
+
"c.csv": "2023-06-07T03:54:07.000000Z",
|
|
2354
|
+
"d.csv": "2023-06-08T03:54:07.000000Z",
|
|
2355
|
+
"e.csv": "2023-06-08T03:54:07.000000Z",
|
|
2356
|
+
},
|
|
2357
|
+
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
|
|
2408
2358
|
},
|
|
2409
2359
|
]
|
|
2410
2360
|
)
|
|
@@ -2537,14 +2487,12 @@ multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_time
|
|
|
2537
2487
|
"stream": "stream1",
|
|
2538
2488
|
},
|
|
2539
2489
|
{
|
|
2540
|
-
"
|
|
2541
|
-
"
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
|
|
2547
|
-
}
|
|
2490
|
+
"history": {
|
|
2491
|
+
"c.csv": "2023-06-07T03:54:07.000000Z",
|
|
2492
|
+
"d.csv": "2023-06-08T03:54:07.000000Z",
|
|
2493
|
+
"e.csv": "2023-06-08T03:54:07.000000Z",
|
|
2494
|
+
},
|
|
2495
|
+
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
|
|
2548
2496
|
},
|
|
2549
2497
|
]
|
|
2550
2498
|
)
|
|
@@ -2675,14 +2623,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
|
|
|
2675
2623
|
"stream": "stream1",
|
|
2676
2624
|
},
|
|
2677
2625
|
{
|
|
2678
|
-
"
|
|
2679
|
-
"
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
|
|
2685
|
-
}
|
|
2626
|
+
"history": {
|
|
2627
|
+
"a.csv": "2023-06-05T03:54:07.000000Z",
|
|
2628
|
+
"c.csv": "2023-06-07T03:54:07.000000Z",
|
|
2629
|
+
"d.csv": "2023-06-08T03:54:07.000000Z",
|
|
2630
|
+
},
|
|
2631
|
+
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
|
|
2686
2632
|
},
|
|
2687
2633
|
{
|
|
2688
2634
|
"data": {
|
|
@@ -2705,14 +2651,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
|
|
|
2705
2651
|
"stream": "stream1",
|
|
2706
2652
|
},
|
|
2707
2653
|
{
|
|
2708
|
-
"
|
|
2709
|
-
"
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
|
|
2715
|
-
}
|
|
2654
|
+
"history": {
|
|
2655
|
+
"b.csv": "2023-06-06T03:54:07.000000Z",
|
|
2656
|
+
"c.csv": "2023-06-07T03:54:07.000000Z",
|
|
2657
|
+
"d.csv": "2023-06-08T03:54:07.000000Z",
|
|
2658
|
+
},
|
|
2659
|
+
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
|
|
2716
2660
|
},
|
|
2717
2661
|
]
|
|
2718
2662
|
)
|
|
@@ -2843,14 +2787,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
|
|
|
2843
2787
|
"stream": "stream1",
|
|
2844
2788
|
},
|
|
2845
2789
|
{
|
|
2846
|
-
"
|
|
2847
|
-
"
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
|
|
2853
|
-
}
|
|
2790
|
+
"history": {
|
|
2791
|
+
"a.csv": "2023-06-05T03:54:07.000000Z",
|
|
2792
|
+
"c.csv": "2023-06-07T03:54:07.000000Z",
|
|
2793
|
+
"d.csv": "2023-06-08T03:54:07.000000Z",
|
|
2794
|
+
},
|
|
2795
|
+
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
|
|
2854
2796
|
},
|
|
2855
2797
|
{
|
|
2856
2798
|
"data": {
|
|
@@ -2873,14 +2815,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
|
|
|
2873
2815
|
"stream": "stream1",
|
|
2874
2816
|
},
|
|
2875
2817
|
{
|
|
2876
|
-
"
|
|
2877
|
-
"
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
|
|
2883
|
-
}
|
|
2818
|
+
"history": {
|
|
2819
|
+
"b.csv": "2023-06-06T03:54:07.000000Z",
|
|
2820
|
+
"c.csv": "2023-06-07T03:54:07.000000Z",
|
|
2821
|
+
"d.csv": "2023-06-08T03:54:07.000000Z",
|
|
2822
|
+
},
|
|
2823
|
+
"_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
|
|
2884
2824
|
},
|
|
2885
2825
|
]
|
|
2886
2826
|
)
|