airbyte-cdk 0.67.0__py3-none-any.whl → 0.67.2__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (31) hide show
  1. airbyte_cdk/sources/abstract_source.py +30 -69
  2. airbyte_cdk/sources/connector_state_manager.py +12 -26
  3. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +552 -524
  4. airbyte_cdk/sources/file_based/config/csv_format.py +2 -0
  5. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +32 -14
  6. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +3 -19
  7. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -3
  8. airbyte_cdk/sources/streams/__init__.py +2 -2
  9. airbyte_cdk/sources/streams/concurrent/adapters.py +3 -19
  10. airbyte_cdk/sources/streams/concurrent/cursor.py +1 -3
  11. airbyte_cdk/sources/streams/core.py +36 -34
  12. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/METADATA +3 -2
  13. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/RECORD +31 -31
  14. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +2 -1
  15. unit_tests/sources/file_based/config/test_csv_format.py +6 -1
  16. unit_tests/sources/file_based/file_types/test_parquet_parser.py +51 -6
  17. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +139 -199
  18. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +91 -133
  19. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +2 -13
  20. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +2 -2
  21. unit_tests/sources/file_based/test_scenarios.py +2 -2
  22. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +9 -9
  23. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +5 -5
  24. unit_tests/sources/streams/concurrent/test_adapters.py +2 -13
  25. unit_tests/sources/streams/test_stream_read.py +221 -11
  26. unit_tests/sources/test_abstract_source.py +142 -130
  27. unit_tests/sources/test_connector_state_manager.py +3 -124
  28. unit_tests/sources/test_source.py +18 -14
  29. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/LICENSE.txt +0 -0
  30. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/WHEEL +0 -0
  31. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/top_level.txt +0 -0
unit_tests/sources/file_based/file_types/test_parquet_parser.py CHANGED
@@ -191,12 +191,57 @@ def test_value_dictionary() -> None:
191
191
  assert py_value == {"indices": [0, 1, 2, 0, 1], "values": ["apple", "banana", "cherry"]}
192
192
 
193
193
 
194
- def test_value_none_binary() -> None:
195
- none_binary_scalar = pa.scalar(None, type=pa.binary())
196
- try:
197
- ParquetParser._to_output_value(none_binary_scalar, _default_parquet_format)
198
- except AttributeError:
199
- assert False, "`None` type binary should be handled properly"
194
+ @pytest.mark.parametrize(
195
+ "parquet_type, parquet_format",
196
+ [
197
+ pytest.param(pa.bool_(), _default_parquet_format, id="test_parquet_bool"),
198
+ pytest.param(pa.int8(), _default_parquet_format, id="test_parquet_int8"),
199
+ pytest.param(pa.int16(), _default_parquet_format, id="test_parquet_int16"),
200
+ pytest.param(pa.int32(), _default_parquet_format, id="test_parquet_int32"),
201
+ pytest.param(pa.int64(), _default_parquet_format, id="test_parquet_int64"),
202
+ pytest.param(pa.uint8(), _default_parquet_format, id="test_parquet_uint8"),
203
+ pytest.param(pa.uint16(), _default_parquet_format, id="test_parquet_uint16"),
204
+ pytest.param(pa.uint32(), _default_parquet_format, id="test_parquet_uint32"),
205
+ pytest.param(pa.uint64(), _default_parquet_format, id="test_parquet_uint64"),
206
+ pytest.param(pa.float16(), _default_parquet_format, id="test_parquet_float16"),
207
+ pytest.param(pa.float32(), _default_parquet_format, id="test_parquet_float32"),
208
+ pytest.param(pa.float64(), _default_parquet_format, id="test_parquet_float64"),
209
+ pytest.param(pa.time32("s"), _default_parquet_format, id="test_parquet_time32s"),
210
+ pytest.param(pa.time32("ms"), _default_parquet_format, id="test_parquet_time32ms"),
211
+ pytest.param(pa.time64("us"), _default_parquet_format, id="test_parquet_time64us"),
212
+ pytest.param(pa.time64("ns"), _default_parquet_format, id="test_parquet_time64ns"),
213
+ pytest.param(pa.timestamp("s"), _default_parquet_format, id="test_parquet_timestamps_s"),
214
+ pytest.param(pa.timestamp("ms"), _default_parquet_format, id="test_parquet_timestamp_ms"),
215
+ pytest.param(pa.timestamp("s", "utc"), _default_parquet_format, id="test_parquet_timestamps_s_with_tz"),
216
+ pytest.param(pa.timestamp("ms", "est"), _default_parquet_format, id="test_parquet_timestamps_ms_with_tz"),
217
+ pytest.param(pa.date32(), _default_parquet_format, id="test_parquet_date32"),
218
+ pytest.param(pa.date64(), _default_parquet_format, id="test_parquet_date64"),
219
+ pytest.param(pa.duration("s"), _default_parquet_format, id="test_duration_s"),
220
+ pytest.param(pa.duration("ms"), _default_parquet_format, id="test_duration_ms"),
221
+ pytest.param(pa.duration("us"), _default_parquet_format, id="test_duration_us"),
222
+ pytest.param(pa.duration("ns"), _default_parquet_format, id="test_duration_ns"),
223
+ pytest.param(pa.month_day_nano_interval(), _default_parquet_format, id="test_parquet_month_day_nano_interval"),
224
+ pytest.param(pa.binary(), _default_parquet_format, id="test_binary"),
225
+ pytest.param(pa.binary(2), _default_parquet_format, id="test_fixed_size_binary"),
226
+ pytest.param(pa.string(), _default_parquet_format, id="test_parquet_string"),
227
+ pytest.param(pa.utf8(), _default_parquet_format, id="test_utf8"),
228
+ pytest.param(pa.large_binary(), _default_parquet_format, id="test_large_binary"),
229
+ pytest.param(pa.large_string(), _default_parquet_format, id="test_large_string"),
230
+ pytest.param(pa.large_utf8(), _default_parquet_format, id="test_large_utf8"),
231
+ pytest.param(pa.dictionary(pa.int32(), pa.string()), _default_parquet_format, id="test_dictionary"),
232
+ pytest.param(pa.struct([pa.field("field", pa.int32())]), _default_parquet_format, id="test_struct"),
233
+ pytest.param(pa.list_(pa.int32()), _default_parquet_format, id="test_list"),
234
+ pytest.param(pa.large_list(pa.int32()), _default_parquet_format, id="test_large_list"),
235
+ pytest.param(pa.decimal128(2), _default_parquet_format, id="test_decimal128"),
236
+ pytest.param(pa.decimal256(2), _default_parquet_format, id="test_decimal256"),
237
+ pytest.param(pa.decimal128(2), _decimal_as_float_parquet_format, id="test_decimal128_as_float"),
238
+ pytest.param(pa.decimal256(2), _decimal_as_float_parquet_format, id="test_decimal256_as_float"),
239
+ pytest.param(pa.map_(pa.int32(), pa.int32()), _default_parquet_format, id="test_map"),
240
+ pytest.param(pa.null(), _default_parquet_format, id="test_null"),
241
+ ])
242
+ def test_null_value_does_not_throw(parquet_type, parquet_format) -> None:
243
+ pyarrow_value = pa.scalar(None, type=parquet_type)
244
+ assert ParquetParser._to_output_value(pyarrow_value, parquet_format) is None
200
245
 
201
246
 
202
247
  @pytest.mark.parametrize(
unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py CHANGED
@@ -74,10 +74,8 @@ single_csv_input_state_is_earlier_scenario_concurrent = (
74
74
  "stream": "stream1",
75
75
  },
76
76
  {
77
- "stream1": {
78
- "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z", "a.csv": "2023-06-05T03:54:07.000000Z"},
79
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
80
- }
77
+ "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z", "a.csv": "2023-06-05T03:54:07.000000Z"},
78
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
81
79
  },
82
80
  ]
83
81
  )
@@ -156,10 +154,8 @@ single_csv_file_is_skipped_if_same_modified_at_as_in_history_concurrent = (
156
154
  .set_expected_records(
157
155
  [
158
156
  {
159
- "stream1": {
160
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
161
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
162
- }
157
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
158
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
163
159
  }
164
160
  ]
165
161
  )
@@ -256,10 +252,8 @@ single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history_concurre
256
252
  "stream": "stream1",
257
253
  },
258
254
  {
259
- "stream1": {
260
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
261
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
262
- }
255
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
256
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
263
257
  },
264
258
  ]
265
259
  )
@@ -368,10 +362,8 @@ single_csv_no_input_state_scenario_concurrent = (
368
362
  "stream": "stream1",
369
363
  },
370
364
  {
371
- "stream1": {
372
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
373
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
374
- }
365
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
366
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
375
367
  },
376
368
  ]
377
369
  )
@@ -491,10 +483,8 @@ multi_csv_same_timestamp_scenario_concurrent = (
491
483
  "stream": "stream1",
492
484
  },
493
485
  {
494
- "stream1": {
495
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
496
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
497
- }
486
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
487
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
498
488
  },
499
489
  ]
500
490
  )
@@ -583,13 +573,11 @@ single_csv_input_state_is_later_scenario_concurrent = (
583
573
  "stream": "stream1",
584
574
  },
585
575
  {
586
- "stream1": {
587
- "history": {
588
- "recent_file.csv": "2023-07-15T23:59:59.000000Z",
589
- "a.csv": "2023-06-05T03:54:07.000000Z",
590
- },
591
- "_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv",
592
- }
576
+ "history": {
577
+ "recent_file.csv": "2023-07-15T23:59:59.000000Z",
578
+ "a.csv": "2023-06-05T03:54:07.000000Z",
579
+ },
580
+ "_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv",
593
581
  },
594
582
  ]
595
583
  )
@@ -697,12 +685,10 @@ multi_csv_different_timestamps_scenario_concurrent = (
697
685
  "stream": "stream1",
698
686
  },
699
687
  {
700
- "stream1": {
701
- "history": {
702
- "a.csv": "2023-06-04T03:54:07.000000Z",
703
- },
704
- "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv",
705
- }
688
+ "history": {
689
+ "a.csv": "2023-06-04T03:54:07.000000Z",
690
+ },
691
+ "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv",
706
692
  },
707
693
  {
708
694
  "data": {
@@ -725,10 +711,8 @@ multi_csv_different_timestamps_scenario_concurrent = (
725
711
  "stream": "stream1",
726
712
  },
727
713
  {
728
- "stream1": {
729
- "history": {"a.csv": "2023-06-04T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
730
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
731
- }
714
+ "history": {"a.csv": "2023-06-04T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
715
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
732
716
  },
733
717
  ]
734
718
  )
@@ -856,10 +840,8 @@ multi_csv_per_timestamp_scenario_concurrent = (
856
840
  "stream": "stream1",
857
841
  },
858
842
  {
859
- "stream1": {
860
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
861
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
862
- }
843
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
844
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
863
845
  },
864
846
  {
865
847
  "data": {
@@ -882,14 +864,12 @@ multi_csv_per_timestamp_scenario_concurrent = (
882
864
  "stream": "stream1",
883
865
  },
884
866
  {
885
- "stream1": {
886
- "history": {
887
- "a.csv": "2023-06-05T03:54:07.000000Z",
888
- "b.csv": "2023-06-05T03:54:07.000000Z",
889
- "c.csv": "2023-06-06T03:54:07.000000Z",
890
- },
891
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
892
- }
867
+ "history": {
868
+ "a.csv": "2023-06-05T03:54:07.000000Z",
869
+ "b.csv": "2023-06-05T03:54:07.000000Z",
870
+ "c.csv": "2023-06-06T03:54:07.000000Z",
871
+ },
872
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
893
873
  },
894
874
  ]
895
875
  )
@@ -1001,10 +981,8 @@ multi_csv_skip_file_if_already_in_history_concurrent = (
1001
981
  "stream": "stream1",
1002
982
  },
1003
983
  {
1004
- "stream1": {
1005
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
1006
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
1007
- }
984
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
985
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
1008
986
  },
1009
987
  {
1010
988
  "data": {
@@ -1027,14 +1005,12 @@ multi_csv_skip_file_if_already_in_history_concurrent = (
1027
1005
  "stream": "stream1",
1028
1006
  },
1029
1007
  {
1030
- "stream1": {
1031
- "history": {
1032
- "a.csv": "2023-06-05T03:54:07.000000Z",
1033
- "b.csv": "2023-06-05T03:54:07.000000Z",
1034
- "c.csv": "2023-06-06T03:54:07.000000Z",
1035
- },
1036
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1037
- }
1008
+ "history": {
1009
+ "a.csv": "2023-06-05T03:54:07.000000Z",
1010
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1011
+ "c.csv": "2023-06-06T03:54:07.000000Z",
1012
+ },
1013
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1038
1014
  },
1039
1015
  ]
1040
1016
  )
@@ -1153,14 +1129,12 @@ multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_newer
1153
1129
  # {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped
1154
1130
  # {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped
1155
1131
  {
1156
- "stream1": {
1157
- "history": {
1158
- "a.csv": "2023-06-05T03:54:07.000000Z",
1159
- "b.csv": "2023-06-05T03:54:07.000000Z",
1160
- "c.csv": "2023-06-06T03:54:07.000000Z",
1161
- },
1162
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1163
- }
1132
+ "history": {
1133
+ "a.csv": "2023-06-05T03:54:07.000000Z",
1134
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1135
+ "c.csv": "2023-06-06T03:54:07.000000Z",
1136
+ },
1137
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1164
1138
  },
1165
1139
  ]
1166
1140
  )
@@ -1282,14 +1256,12 @@ multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_older
1282
1256
  # {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped
1283
1257
  # {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped
1284
1258
  {
1285
- "stream1": {
1286
- "history": {
1287
- "a.csv": "2023-06-05T03:54:07.000000Z",
1288
- "b.csv": "2023-06-05T03:54:07.000000Z",
1289
- "c.csv": "2023-06-06T03:54:07.000000Z",
1290
- },
1291
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1292
- }
1259
+ "history": {
1260
+ "a.csv": "2023-06-05T03:54:07.000000Z",
1261
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1262
+ "c.csv": "2023-06-06T03:54:07.000000Z",
1263
+ },
1264
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1293
1265
  },
1294
1266
  ]
1295
1267
  )
@@ -1405,14 +1377,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
1405
1377
  "stream": "stream1",
1406
1378
  },
1407
1379
  {
1408
- "stream1": {
1409
- "history": {
1410
- "very_old_file.csv": "2023-06-02T03:54:07.000000Z",
1411
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1412
- "a.csv": "2023-06-06T03:54:07.000000Z",
1413
- },
1414
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
1415
- }
1380
+ "history": {
1381
+ "very_old_file.csv": "2023-06-02T03:54:07.000000Z",
1382
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1383
+ "a.csv": "2023-06-06T03:54:07.000000Z",
1384
+ },
1385
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
1416
1386
  },
1417
1387
  {
1418
1388
  "data": {
@@ -1435,14 +1405,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
1435
1405
  "stream": "stream1",
1436
1406
  },
1437
1407
  {
1438
- "stream1": {
1439
- "history": {
1440
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1441
- "a.csv": "2023-06-06T03:54:07.000000Z",
1442
- "b.csv": "2023-06-07T03:54:07.000000Z",
1443
- },
1444
- "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
1445
- }
1408
+ "history": {
1409
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1410
+ "a.csv": "2023-06-06T03:54:07.000000Z",
1411
+ "b.csv": "2023-06-07T03:54:07.000000Z",
1412
+ },
1413
+ "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
1446
1414
  },
1447
1415
  {
1448
1416
  "data": {
@@ -1465,14 +1433,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
1465
1433
  "stream": "stream1",
1466
1434
  },
1467
1435
  {
1468
- "stream1": {
1469
- "history": {
1470
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1471
- "b.csv": "2023-06-07T03:54:07.000000Z",
1472
- "c.csv": "2023-06-10T03:54:07.000000Z",
1473
- },
1474
- "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
1475
- }
1436
+ "history": {
1437
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1438
+ "b.csv": "2023-06-07T03:54:07.000000Z",
1439
+ "c.csv": "2023-06-10T03:54:07.000000Z",
1440
+ },
1441
+ "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
1476
1442
  },
1477
1443
  ]
1478
1444
  )
@@ -1592,14 +1558,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
1592
1558
  "stream": "stream1",
1593
1559
  },
1594
1560
  {
1595
- "stream1": {
1596
- "history": {
1597
- "very_old_file.csv": "2023-06-02T03:54:07.000000Z",
1598
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1599
- "a.csv": "2023-06-06T03:54:07.000000Z",
1600
- },
1601
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
1602
- }
1561
+ "history": {
1562
+ "very_old_file.csv": "2023-06-02T03:54:07.000000Z",
1563
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1564
+ "a.csv": "2023-06-06T03:54:07.000000Z",
1565
+ },
1566
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
1603
1567
  },
1604
1568
  {
1605
1569
  "data": {
@@ -1622,14 +1586,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
1622
1586
  "stream": "stream1",
1623
1587
  },
1624
1588
  {
1625
- "stream1": {
1626
- "history": {
1627
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1628
- "a.csv": "2023-06-06T03:54:07.000000Z",
1629
- "b.csv": "2023-06-07T03:54:07.000000Z",
1630
- },
1631
- "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
1632
- }
1589
+ "history": {
1590
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1591
+ "a.csv": "2023-06-06T03:54:07.000000Z",
1592
+ "b.csv": "2023-06-07T03:54:07.000000Z",
1593
+ },
1594
+ "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
1633
1595
  },
1634
1596
  {
1635
1597
  "data": {
@@ -1652,14 +1614,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
1652
1614
  "stream": "stream1",
1653
1615
  },
1654
1616
  {
1655
- "stream1": {
1656
- "history": {
1657
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1658
- "b.csv": "2023-06-07T03:54:07.000000Z",
1659
- "c.csv": "2023-06-10T03:54:07.000000Z",
1660
- },
1661
- "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
1662
- }
1617
+ "history": {
1618
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1619
+ "b.csv": "2023-06-07T03:54:07.000000Z",
1620
+ "c.csv": "2023-06-10T03:54:07.000000Z",
1621
+ },
1622
+ "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
1663
1623
  },
1664
1624
  ]
1665
1625
  )
@@ -1848,14 +1808,12 @@ multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor
1848
1808
  "stream": "stream1",
1849
1809
  },
1850
1810
  {
1851
- "stream1": {
1852
- "history": {
1853
- "b.csv": "2023-06-05T03:54:07.000000Z",
1854
- "c.csv": "2023-06-05T03:54:07.000000Z",
1855
- "d.csv": "2023-06-05T03:54:07.000000Z",
1856
- },
1857
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
1858
- }
1811
+ "history": {
1812
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1813
+ "c.csv": "2023-06-05T03:54:07.000000Z",
1814
+ "d.csv": "2023-06-05T03:54:07.000000Z",
1815
+ },
1816
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
1859
1817
  },
1860
1818
  ]
1861
1819
  )
@@ -2032,14 +1990,12 @@ multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor
2032
1990
  "stream": "stream1",
2033
1991
  },
2034
1992
  {
2035
- "stream1": {
2036
- "history": {
2037
- "b.csv": "2023-06-05T03:54:07.000000Z",
2038
- "c.csv": "2023-06-05T03:54:07.000000Z",
2039
- "d.csv": "2023-06-05T03:54:07.000000Z",
2040
- },
2041
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2042
- }
1993
+ "history": {
1994
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1995
+ "c.csv": "2023-06-05T03:54:07.000000Z",
1996
+ "d.csv": "2023-06-05T03:54:07.000000Z",
1997
+ },
1998
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2043
1999
  },
2044
2000
  ]
2045
2001
  )
@@ -2138,14 +2094,12 @@ multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_
2138
2094
  .set_expected_records(
2139
2095
  [
2140
2096
  {
2141
- "stream1": {
2142
- "history": {
2143
- "b.csv": "2023-06-05T03:54:07.000000Z",
2144
- "c.csv": "2023-06-05T03:54:07.000000Z",
2145
- "d.csv": "2023-06-05T03:54:07.000000Z",
2146
- },
2147
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2148
- }
2097
+ "history": {
2098
+ "b.csv": "2023-06-05T03:54:07.000000Z",
2099
+ "c.csv": "2023-06-05T03:54:07.000000Z",
2100
+ "d.csv": "2023-06-05T03:54:07.000000Z",
2101
+ },
2102
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2149
2103
  }
2150
2104
  ]
2151
2105
  )
@@ -2256,14 +2210,12 @@ multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_
2256
2210
  .set_expected_records(
2257
2211
  [
2258
2212
  {
2259
- "stream1": {
2260
2213
  "history": {
2261
2214
  "b.csv": "2023-06-05T03:54:07.000000Z",
2262
2215
  "c.csv": "2023-06-05T03:54:07.000000Z",
2263
2216
  "d.csv": "2023-06-05T03:54:07.000000Z",
2264
2217
  },
2265
2218
  "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2266
- }
2267
2219
  }
2268
2220
  ]
2269
2221
  )
@@ -2397,14 +2349,12 @@ multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_time
2397
2349
  "stream": "stream1",
2398
2350
  },
2399
2351
  {
2400
- "stream1": {
2401
- "history": {
2402
- "c.csv": "2023-06-07T03:54:07.000000Z",
2403
- "d.csv": "2023-06-08T03:54:07.000000Z",
2404
- "e.csv": "2023-06-08T03:54:07.000000Z",
2405
- },
2406
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
2407
- }
2352
+ "history": {
2353
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2354
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2355
+ "e.csv": "2023-06-08T03:54:07.000000Z",
2356
+ },
2357
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
2408
2358
  },
2409
2359
  ]
2410
2360
  )
@@ -2537,14 +2487,12 @@ multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_time
2537
2487
  "stream": "stream1",
2538
2488
  },
2539
2489
  {
2540
- "stream1": {
2541
- "history": {
2542
- "c.csv": "2023-06-07T03:54:07.000000Z",
2543
- "d.csv": "2023-06-08T03:54:07.000000Z",
2544
- "e.csv": "2023-06-08T03:54:07.000000Z",
2545
- },
2546
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
2547
- }
2490
+ "history": {
2491
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2492
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2493
+ "e.csv": "2023-06-08T03:54:07.000000Z",
2494
+ },
2495
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
2548
2496
  },
2549
2497
  ]
2550
2498
  )
@@ -2675,14 +2623,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
2675
2623
  "stream": "stream1",
2676
2624
  },
2677
2625
  {
2678
- "stream1": {
2679
- "history": {
2680
- "a.csv": "2023-06-05T03:54:07.000000Z",
2681
- "c.csv": "2023-06-07T03:54:07.000000Z",
2682
- "d.csv": "2023-06-08T03:54:07.000000Z",
2683
- },
2684
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2685
- }
2626
+ "history": {
2627
+ "a.csv": "2023-06-05T03:54:07.000000Z",
2628
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2629
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2630
+ },
2631
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2686
2632
  },
2687
2633
  {
2688
2634
  "data": {
@@ -2705,14 +2651,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
2705
2651
  "stream": "stream1",
2706
2652
  },
2707
2653
  {
2708
- "stream1": {
2709
- "history": {
2710
- "b.csv": "2023-06-06T03:54:07.000000Z",
2711
- "c.csv": "2023-06-07T03:54:07.000000Z",
2712
- "d.csv": "2023-06-08T03:54:07.000000Z",
2713
- },
2714
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2715
- }
2654
+ "history": {
2655
+ "b.csv": "2023-06-06T03:54:07.000000Z",
2656
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2657
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2658
+ },
2659
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2716
2660
  },
2717
2661
  ]
2718
2662
  )
@@ -2843,14 +2787,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
2843
2787
  "stream": "stream1",
2844
2788
  },
2845
2789
  {
2846
- "stream1": {
2847
- "history": {
2848
- "a.csv": "2023-06-05T03:54:07.000000Z",
2849
- "c.csv": "2023-06-07T03:54:07.000000Z",
2850
- "d.csv": "2023-06-08T03:54:07.000000Z",
2851
- },
2852
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2853
- }
2790
+ "history": {
2791
+ "a.csv": "2023-06-05T03:54:07.000000Z",
2792
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2793
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2794
+ },
2795
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2854
2796
  },
2855
2797
  {
2856
2798
  "data": {
@@ -2873,14 +2815,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
2873
2815
  "stream": "stream1",
2874
2816
  },
2875
2817
  {
2876
- "stream1": {
2877
- "history": {
2878
- "b.csv": "2023-06-06T03:54:07.000000Z",
2879
- "c.csv": "2023-06-07T03:54:07.000000Z",
2880
- "d.csv": "2023-06-08T03:54:07.000000Z",
2881
- },
2882
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2883
- }
2818
+ "history": {
2819
+ "b.csv": "2023-06-06T03:54:07.000000Z",
2820
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2821
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2822
+ },
2823
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2884
2824
  },
2885
2825
  ]
2886
2826
  )