airbyte-cdk 0.67.0__py3-none-any.whl → 0.67.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. airbyte_cdk/sources/abstract_source.py +30 -69
  2. airbyte_cdk/sources/connector_state_manager.py +12 -26
  3. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +552 -524
  4. airbyte_cdk/sources/file_based/config/csv_format.py +2 -0
  5. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +32 -14
  6. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +3 -19
  7. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -3
  8. airbyte_cdk/sources/streams/__init__.py +2 -2
  9. airbyte_cdk/sources/streams/concurrent/adapters.py +3 -19
  10. airbyte_cdk/sources/streams/concurrent/cursor.py +1 -3
  11. airbyte_cdk/sources/streams/core.py +36 -34
  12. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/METADATA +3 -2
  13. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/RECORD +31 -31
  14. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +2 -1
  15. unit_tests/sources/file_based/config/test_csv_format.py +6 -1
  16. unit_tests/sources/file_based/file_types/test_parquet_parser.py +51 -6
  17. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +139 -199
  18. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +91 -133
  19. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +2 -13
  20. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +2 -2
  21. unit_tests/sources/file_based/test_scenarios.py +2 -2
  22. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +9 -9
  23. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +5 -5
  24. unit_tests/sources/streams/concurrent/test_adapters.py +2 -13
  25. unit_tests/sources/streams/test_stream_read.py +221 -11
  26. unit_tests/sources/test_abstract_source.py +142 -130
  27. unit_tests/sources/test_connector_state_manager.py +3 -124
  28. unit_tests/sources/test_source.py +18 -14
  29. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/LICENSE.txt +0 -0
  30. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/WHEEL +0 -0
  31. {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/top_level.txt +0 -0
@@ -191,12 +191,57 @@ def test_value_dictionary() -> None:
191
191
  assert py_value == {"indices": [0, 1, 2, 0, 1], "values": ["apple", "banana", "cherry"]}
192
192
 
193
193
 
194
- def test_value_none_binary() -> None:
195
- none_binary_scalar = pa.scalar(None, type=pa.binary())
196
- try:
197
- ParquetParser._to_output_value(none_binary_scalar, _default_parquet_format)
198
- except AttributeError:
199
- assert False, "`None` type binary should be handled properly"
194
+ @pytest.mark.parametrize(
195
+ "parquet_type, parquet_format",
196
+ [
197
+ pytest.param(pa.bool_(), _default_parquet_format, id="test_parquet_bool"),
198
+ pytest.param(pa.int8(), _default_parquet_format, id="test_parquet_int8"),
199
+ pytest.param(pa.int16(), _default_parquet_format, id="test_parquet_int16"),
200
+ pytest.param(pa.int32(), _default_parquet_format, id="test_parquet_int32"),
201
+ pytest.param(pa.int64(), _default_parquet_format, id="test_parquet_int64"),
202
+ pytest.param(pa.uint8(), _default_parquet_format, id="test_parquet_uint8"),
203
+ pytest.param(pa.uint16(), _default_parquet_format, id="test_parquet_uint16"),
204
+ pytest.param(pa.uint32(), _default_parquet_format, id="test_parquet_uint32"),
205
+ pytest.param(pa.uint64(), _default_parquet_format, id="test_parquet_uint64"),
206
+ pytest.param(pa.float16(), _default_parquet_format, id="test_parquet_float16"),
207
+ pytest.param(pa.float32(), _default_parquet_format, id="test_parquet_float32"),
208
+ pytest.param(pa.float64(), _default_parquet_format, id="test_parquet_float64"),
209
+ pytest.param(pa.time32("s"), _default_parquet_format, id="test_parquet_time32s"),
210
+ pytest.param(pa.time32("ms"), _default_parquet_format, id="test_parquet_time32ms"),
211
+ pytest.param(pa.time64("us"), _default_parquet_format, id="test_parquet_time64us"),
212
+ pytest.param(pa.time64("ns"), _default_parquet_format, id="test_parquet_time64ns"),
213
+ pytest.param(pa.timestamp("s"), _default_parquet_format, id="test_parquet_timestamps_s"),
214
+ pytest.param(pa.timestamp("ms"), _default_parquet_format, id="test_parquet_timestamp_ms"),
215
+ pytest.param(pa.timestamp("s", "utc"), _default_parquet_format, id="test_parquet_timestamps_s_with_tz"),
216
+ pytest.param(pa.timestamp("ms", "est"), _default_parquet_format, id="test_parquet_timestamps_ms_with_tz"),
217
+ pytest.param(pa.date32(), _default_parquet_format, id="test_parquet_date32"),
218
+ pytest.param(pa.date64(), _default_parquet_format, id="test_parquet_date64"),
219
+ pytest.param(pa.duration("s"), _default_parquet_format, id="test_duration_s"),
220
+ pytest.param(pa.duration("ms"), _default_parquet_format, id="test_duration_ms"),
221
+ pytest.param(pa.duration("us"), _default_parquet_format, id="test_duration_us"),
222
+ pytest.param(pa.duration("ns"), _default_parquet_format, id="test_duration_ns"),
223
+ pytest.param(pa.month_day_nano_interval(), _default_parquet_format, id="test_parquet_month_day_nano_interval"),
224
+ pytest.param(pa.binary(), _default_parquet_format, id="test_binary"),
225
+ pytest.param(pa.binary(2), _default_parquet_format, id="test_fixed_size_binary"),
226
+ pytest.param(pa.string(), _default_parquet_format, id="test_parquet_string"),
227
+ pytest.param(pa.utf8(), _default_parquet_format, id="test_utf8"),
228
+ pytest.param(pa.large_binary(), _default_parquet_format, id="test_large_binary"),
229
+ pytest.param(pa.large_string(), _default_parquet_format, id="test_large_string"),
230
+ pytest.param(pa.large_utf8(), _default_parquet_format, id="test_large_utf8"),
231
+ pytest.param(pa.dictionary(pa.int32(), pa.string()), _default_parquet_format, id="test_dictionary"),
232
+ pytest.param(pa.struct([pa.field("field", pa.int32())]), _default_parquet_format, id="test_struct"),
233
+ pytest.param(pa.list_(pa.int32()), _default_parquet_format, id="test_list"),
234
+ pytest.param(pa.large_list(pa.int32()), _default_parquet_format, id="test_large_list"),
235
+ pytest.param(pa.decimal128(2), _default_parquet_format, id="test_decimal128"),
236
+ pytest.param(pa.decimal256(2), _default_parquet_format, id="test_decimal256"),
237
+ pytest.param(pa.decimal128(2), _decimal_as_float_parquet_format, id="test_decimal128_as_float"),
238
+ pytest.param(pa.decimal256(2), _decimal_as_float_parquet_format, id="test_decimal256_as_float"),
239
+ pytest.param(pa.map_(pa.int32(), pa.int32()), _default_parquet_format, id="test_map"),
240
+ pytest.param(pa.null(), _default_parquet_format, id="test_null"),
241
+ ])
242
+ def test_null_value_does_not_throw(parquet_type, parquet_format) -> None:
243
+ pyarrow_value = pa.scalar(None, type=parquet_type)
244
+ assert ParquetParser._to_output_value(pyarrow_value, parquet_format) is None
200
245
 
201
246
 
202
247
  @pytest.mark.parametrize(
@@ -74,10 +74,8 @@ single_csv_input_state_is_earlier_scenario_concurrent = (
74
74
  "stream": "stream1",
75
75
  },
76
76
  {
77
- "stream1": {
78
- "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z", "a.csv": "2023-06-05T03:54:07.000000Z"},
79
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
80
- }
77
+ "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z", "a.csv": "2023-06-05T03:54:07.000000Z"},
78
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
81
79
  },
82
80
  ]
83
81
  )
@@ -156,10 +154,8 @@ single_csv_file_is_skipped_if_same_modified_at_as_in_history_concurrent = (
156
154
  .set_expected_records(
157
155
  [
158
156
  {
159
- "stream1": {
160
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
161
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
162
- }
157
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
158
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
163
159
  }
164
160
  ]
165
161
  )
@@ -256,10 +252,8 @@ single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history_concurre
256
252
  "stream": "stream1",
257
253
  },
258
254
  {
259
- "stream1": {
260
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
261
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
262
- }
255
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
256
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
263
257
  },
264
258
  ]
265
259
  )
@@ -368,10 +362,8 @@ single_csv_no_input_state_scenario_concurrent = (
368
362
  "stream": "stream1",
369
363
  },
370
364
  {
371
- "stream1": {
372
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
373
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
374
- }
365
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z"},
366
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv",
375
367
  },
376
368
  ]
377
369
  )
@@ -491,10 +483,8 @@ multi_csv_same_timestamp_scenario_concurrent = (
491
483
  "stream": "stream1",
492
484
  },
493
485
  {
494
- "stream1": {
495
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
496
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
497
- }
486
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
487
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
498
488
  },
499
489
  ]
500
490
  )
@@ -583,13 +573,11 @@ single_csv_input_state_is_later_scenario_concurrent = (
583
573
  "stream": "stream1",
584
574
  },
585
575
  {
586
- "stream1": {
587
- "history": {
588
- "recent_file.csv": "2023-07-15T23:59:59.000000Z",
589
- "a.csv": "2023-06-05T03:54:07.000000Z",
590
- },
591
- "_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv",
592
- }
576
+ "history": {
577
+ "recent_file.csv": "2023-07-15T23:59:59.000000Z",
578
+ "a.csv": "2023-06-05T03:54:07.000000Z",
579
+ },
580
+ "_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv",
593
581
  },
594
582
  ]
595
583
  )
@@ -697,12 +685,10 @@ multi_csv_different_timestamps_scenario_concurrent = (
697
685
  "stream": "stream1",
698
686
  },
699
687
  {
700
- "stream1": {
701
- "history": {
702
- "a.csv": "2023-06-04T03:54:07.000000Z",
703
- },
704
- "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv",
705
- }
688
+ "history": {
689
+ "a.csv": "2023-06-04T03:54:07.000000Z",
690
+ },
691
+ "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv",
706
692
  },
707
693
  {
708
694
  "data": {
@@ -725,10 +711,8 @@ multi_csv_different_timestamps_scenario_concurrent = (
725
711
  "stream": "stream1",
726
712
  },
727
713
  {
728
- "stream1": {
729
- "history": {"a.csv": "2023-06-04T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
730
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
731
- }
714
+ "history": {"a.csv": "2023-06-04T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
715
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
732
716
  },
733
717
  ]
734
718
  )
@@ -856,10 +840,8 @@ multi_csv_per_timestamp_scenario_concurrent = (
856
840
  "stream": "stream1",
857
841
  },
858
842
  {
859
- "stream1": {
860
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
861
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
862
- }
843
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
844
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
863
845
  },
864
846
  {
865
847
  "data": {
@@ -882,14 +864,12 @@ multi_csv_per_timestamp_scenario_concurrent = (
882
864
  "stream": "stream1",
883
865
  },
884
866
  {
885
- "stream1": {
886
- "history": {
887
- "a.csv": "2023-06-05T03:54:07.000000Z",
888
- "b.csv": "2023-06-05T03:54:07.000000Z",
889
- "c.csv": "2023-06-06T03:54:07.000000Z",
890
- },
891
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
892
- }
867
+ "history": {
868
+ "a.csv": "2023-06-05T03:54:07.000000Z",
869
+ "b.csv": "2023-06-05T03:54:07.000000Z",
870
+ "c.csv": "2023-06-06T03:54:07.000000Z",
871
+ },
872
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
893
873
  },
894
874
  ]
895
875
  )
@@ -1001,10 +981,8 @@ multi_csv_skip_file_if_already_in_history_concurrent = (
1001
981
  "stream": "stream1",
1002
982
  },
1003
983
  {
1004
- "stream1": {
1005
- "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
1006
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
1007
- }
984
+ "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"},
985
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv",
1008
986
  },
1009
987
  {
1010
988
  "data": {
@@ -1027,14 +1005,12 @@ multi_csv_skip_file_if_already_in_history_concurrent = (
1027
1005
  "stream": "stream1",
1028
1006
  },
1029
1007
  {
1030
- "stream1": {
1031
- "history": {
1032
- "a.csv": "2023-06-05T03:54:07.000000Z",
1033
- "b.csv": "2023-06-05T03:54:07.000000Z",
1034
- "c.csv": "2023-06-06T03:54:07.000000Z",
1035
- },
1036
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1037
- }
1008
+ "history": {
1009
+ "a.csv": "2023-06-05T03:54:07.000000Z",
1010
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1011
+ "c.csv": "2023-06-06T03:54:07.000000Z",
1012
+ },
1013
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1038
1014
  },
1039
1015
  ]
1040
1016
  )
@@ -1153,14 +1129,12 @@ multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_newer
1153
1129
  # {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped
1154
1130
  # {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped
1155
1131
  {
1156
- "stream1": {
1157
- "history": {
1158
- "a.csv": "2023-06-05T03:54:07.000000Z",
1159
- "b.csv": "2023-06-05T03:54:07.000000Z",
1160
- "c.csv": "2023-06-06T03:54:07.000000Z",
1161
- },
1162
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1163
- }
1132
+ "history": {
1133
+ "a.csv": "2023-06-05T03:54:07.000000Z",
1134
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1135
+ "c.csv": "2023-06-06T03:54:07.000000Z",
1136
+ },
1137
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1164
1138
  },
1165
1139
  ]
1166
1140
  )
@@ -1282,14 +1256,12 @@ multi_csv_include_missing_files_within_history_range_concurrent_cursor_is_older
1282
1256
  # {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped
1283
1257
  # {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped
1284
1258
  {
1285
- "stream1": {
1286
- "history": {
1287
- "a.csv": "2023-06-05T03:54:07.000000Z",
1288
- "b.csv": "2023-06-05T03:54:07.000000Z",
1289
- "c.csv": "2023-06-06T03:54:07.000000Z",
1290
- },
1291
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1292
- }
1259
+ "history": {
1260
+ "a.csv": "2023-06-05T03:54:07.000000Z",
1261
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1262
+ "c.csv": "2023-06-06T03:54:07.000000Z",
1263
+ },
1264
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv",
1293
1265
  },
1294
1266
  ]
1295
1267
  )
@@ -1405,14 +1377,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
1405
1377
  "stream": "stream1",
1406
1378
  },
1407
1379
  {
1408
- "stream1": {
1409
- "history": {
1410
- "very_old_file.csv": "2023-06-02T03:54:07.000000Z",
1411
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1412
- "a.csv": "2023-06-06T03:54:07.000000Z",
1413
- },
1414
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
1415
- }
1380
+ "history": {
1381
+ "very_old_file.csv": "2023-06-02T03:54:07.000000Z",
1382
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1383
+ "a.csv": "2023-06-06T03:54:07.000000Z",
1384
+ },
1385
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
1416
1386
  },
1417
1387
  {
1418
1388
  "data": {
@@ -1435,14 +1405,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
1435
1405
  "stream": "stream1",
1436
1406
  },
1437
1407
  {
1438
- "stream1": {
1439
- "history": {
1440
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1441
- "a.csv": "2023-06-06T03:54:07.000000Z",
1442
- "b.csv": "2023-06-07T03:54:07.000000Z",
1443
- },
1444
- "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
1445
- }
1408
+ "history": {
1409
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1410
+ "a.csv": "2023-06-06T03:54:07.000000Z",
1411
+ "b.csv": "2023-06-07T03:54:07.000000Z",
1412
+ },
1413
+ "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
1446
1414
  },
1447
1415
  {
1448
1416
  "data": {
@@ -1465,14 +1433,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_newe
1465
1433
  "stream": "stream1",
1466
1434
  },
1467
1435
  {
1468
- "stream1": {
1469
- "history": {
1470
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1471
- "b.csv": "2023-06-07T03:54:07.000000Z",
1472
- "c.csv": "2023-06-10T03:54:07.000000Z",
1473
- },
1474
- "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
1475
- }
1436
+ "history": {
1437
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1438
+ "b.csv": "2023-06-07T03:54:07.000000Z",
1439
+ "c.csv": "2023-06-10T03:54:07.000000Z",
1440
+ },
1441
+ "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
1476
1442
  },
1477
1443
  ]
1478
1444
  )
@@ -1592,14 +1558,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
1592
1558
  "stream": "stream1",
1593
1559
  },
1594
1560
  {
1595
- "stream1": {
1596
- "history": {
1597
- "very_old_file.csv": "2023-06-02T03:54:07.000000Z",
1598
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1599
- "a.csv": "2023-06-06T03:54:07.000000Z",
1600
- },
1601
- "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
1602
- }
1561
+ "history": {
1562
+ "very_old_file.csv": "2023-06-02T03:54:07.000000Z",
1563
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1564
+ "a.csv": "2023-06-06T03:54:07.000000Z",
1565
+ },
1566
+ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv",
1603
1567
  },
1604
1568
  {
1605
1569
  "data": {
@@ -1622,14 +1586,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
1622
1586
  "stream": "stream1",
1623
1587
  },
1624
1588
  {
1625
- "stream1": {
1626
- "history": {
1627
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1628
- "a.csv": "2023-06-06T03:54:07.000000Z",
1629
- "b.csv": "2023-06-07T03:54:07.000000Z",
1630
- },
1631
- "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
1632
- }
1589
+ "history": {
1590
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1591
+ "a.csv": "2023-06-06T03:54:07.000000Z",
1592
+ "b.csv": "2023-06-07T03:54:07.000000Z",
1593
+ },
1594
+ "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv",
1633
1595
  },
1634
1596
  {
1635
1597
  "data": {
@@ -1652,14 +1614,12 @@ multi_csv_remove_old_files_if_history_is_full_scenario_concurrent_cursor_is_olde
1652
1614
  "stream": "stream1",
1653
1615
  },
1654
1616
  {
1655
- "stream1": {
1656
- "history": {
1657
- "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1658
- "b.csv": "2023-06-07T03:54:07.000000Z",
1659
- "c.csv": "2023-06-10T03:54:07.000000Z",
1660
- },
1661
- "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
1662
- }
1617
+ "history": {
1618
+ "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z",
1619
+ "b.csv": "2023-06-07T03:54:07.000000Z",
1620
+ "c.csv": "2023-06-10T03:54:07.000000Z",
1621
+ },
1622
+ "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv",
1663
1623
  },
1664
1624
  ]
1665
1625
  )
@@ -1848,14 +1808,12 @@ multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor
1848
1808
  "stream": "stream1",
1849
1809
  },
1850
1810
  {
1851
- "stream1": {
1852
- "history": {
1853
- "b.csv": "2023-06-05T03:54:07.000000Z",
1854
- "c.csv": "2023-06-05T03:54:07.000000Z",
1855
- "d.csv": "2023-06-05T03:54:07.000000Z",
1856
- },
1857
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
1858
- }
1811
+ "history": {
1812
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1813
+ "c.csv": "2023-06-05T03:54:07.000000Z",
1814
+ "d.csv": "2023-06-05T03:54:07.000000Z",
1815
+ },
1816
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
1859
1817
  },
1860
1818
  ]
1861
1819
  )
@@ -2032,14 +1990,12 @@ multi_csv_same_timestamp_more_files_than_history_size_scenario_concurrent_cursor
2032
1990
  "stream": "stream1",
2033
1991
  },
2034
1992
  {
2035
- "stream1": {
2036
- "history": {
2037
- "b.csv": "2023-06-05T03:54:07.000000Z",
2038
- "c.csv": "2023-06-05T03:54:07.000000Z",
2039
- "d.csv": "2023-06-05T03:54:07.000000Z",
2040
- },
2041
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2042
- }
1993
+ "history": {
1994
+ "b.csv": "2023-06-05T03:54:07.000000Z",
1995
+ "c.csv": "2023-06-05T03:54:07.000000Z",
1996
+ "d.csv": "2023-06-05T03:54:07.000000Z",
1997
+ },
1998
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2043
1999
  },
2044
2000
  ]
2045
2001
  )
@@ -2138,14 +2094,12 @@ multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_
2138
2094
  .set_expected_records(
2139
2095
  [
2140
2096
  {
2141
- "stream1": {
2142
- "history": {
2143
- "b.csv": "2023-06-05T03:54:07.000000Z",
2144
- "c.csv": "2023-06-05T03:54:07.000000Z",
2145
- "d.csv": "2023-06-05T03:54:07.000000Z",
2146
- },
2147
- "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2148
- }
2097
+ "history": {
2098
+ "b.csv": "2023-06-05T03:54:07.000000Z",
2099
+ "c.csv": "2023-06-05T03:54:07.000000Z",
2100
+ "d.csv": "2023-06-05T03:54:07.000000Z",
2101
+ },
2102
+ "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2149
2103
  }
2150
2104
  ]
2151
2105
  )
@@ -2256,14 +2210,12 @@ multi_csv_sync_recent_files_if_history_is_incomplete_scenario_concurrent_cursor_
2256
2210
  .set_expected_records(
2257
2211
  [
2258
2212
  {
2259
- "stream1": {
2260
2213
  "history": {
2261
2214
  "b.csv": "2023-06-05T03:54:07.000000Z",
2262
2215
  "c.csv": "2023-06-05T03:54:07.000000Z",
2263
2216
  "d.csv": "2023-06-05T03:54:07.000000Z",
2264
2217
  },
2265
2218
  "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv",
2266
- }
2267
2219
  }
2268
2220
  ]
2269
2221
  )
@@ -2397,14 +2349,12 @@ multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_time
2397
2349
  "stream": "stream1",
2398
2350
  },
2399
2351
  {
2400
- "stream1": {
2401
- "history": {
2402
- "c.csv": "2023-06-07T03:54:07.000000Z",
2403
- "d.csv": "2023-06-08T03:54:07.000000Z",
2404
- "e.csv": "2023-06-08T03:54:07.000000Z",
2405
- },
2406
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
2407
- }
2352
+ "history": {
2353
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2354
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2355
+ "e.csv": "2023-06-08T03:54:07.000000Z",
2356
+ },
2357
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
2408
2358
  },
2409
2359
  ]
2410
2360
  )
@@ -2537,14 +2487,12 @@ multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_time
2537
2487
  "stream": "stream1",
2538
2488
  },
2539
2489
  {
2540
- "stream1": {
2541
- "history": {
2542
- "c.csv": "2023-06-07T03:54:07.000000Z",
2543
- "d.csv": "2023-06-08T03:54:07.000000Z",
2544
- "e.csv": "2023-06-08T03:54:07.000000Z",
2545
- },
2546
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
2547
- }
2490
+ "history": {
2491
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2492
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2493
+ "e.csv": "2023-06-08T03:54:07.000000Z",
2494
+ },
2495
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv",
2548
2496
  },
2549
2497
  ]
2550
2498
  )
@@ -2675,14 +2623,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
2675
2623
  "stream": "stream1",
2676
2624
  },
2677
2625
  {
2678
- "stream1": {
2679
- "history": {
2680
- "a.csv": "2023-06-05T03:54:07.000000Z",
2681
- "c.csv": "2023-06-07T03:54:07.000000Z",
2682
- "d.csv": "2023-06-08T03:54:07.000000Z",
2683
- },
2684
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2685
- }
2626
+ "history": {
2627
+ "a.csv": "2023-06-05T03:54:07.000000Z",
2628
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2629
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2630
+ },
2631
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2686
2632
  },
2687
2633
  {
2688
2634
  "data": {
@@ -2705,14 +2651,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
2705
2651
  "stream": "stream1",
2706
2652
  },
2707
2653
  {
2708
- "stream1": {
2709
- "history": {
2710
- "b.csv": "2023-06-06T03:54:07.000000Z",
2711
- "c.csv": "2023-06-07T03:54:07.000000Z",
2712
- "d.csv": "2023-06-08T03:54:07.000000Z",
2713
- },
2714
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2715
- }
2654
+ "history": {
2655
+ "b.csv": "2023-06-06T03:54:07.000000Z",
2656
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2657
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2658
+ },
2659
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2716
2660
  },
2717
2661
  ]
2718
2662
  )
@@ -2843,14 +2787,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
2843
2787
  "stream": "stream1",
2844
2788
  },
2845
2789
  {
2846
- "stream1": {
2847
- "history": {
2848
- "a.csv": "2023-06-05T03:54:07.000000Z",
2849
- "c.csv": "2023-06-07T03:54:07.000000Z",
2850
- "d.csv": "2023-06-08T03:54:07.000000Z",
2851
- },
2852
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2853
- }
2790
+ "history": {
2791
+ "a.csv": "2023-06-05T03:54:07.000000Z",
2792
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2793
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2794
+ },
2795
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2854
2796
  },
2855
2797
  {
2856
2798
  "data": {
@@ -2873,14 +2815,12 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
2873
2815
  "stream": "stream1",
2874
2816
  },
2875
2817
  {
2876
- "stream1": {
2877
- "history": {
2878
- "b.csv": "2023-06-06T03:54:07.000000Z",
2879
- "c.csv": "2023-06-07T03:54:07.000000Z",
2880
- "d.csv": "2023-06-08T03:54:07.000000Z",
2881
- },
2882
- "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2883
- }
2818
+ "history": {
2819
+ "b.csv": "2023-06-06T03:54:07.000000Z",
2820
+ "c.csv": "2023-06-07T03:54:07.000000Z",
2821
+ "d.csv": "2023-06-08T03:54:07.000000Z",
2822
+ },
2823
+ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv",
2884
2824
  },
2885
2825
  ]
2886
2826
  )