airbyte-cdk 0.51.15__py3-none-any.whl → 0.51.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +494 -522
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +1 -1
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +2 -37
- airbyte_cdk/sources/file_based/file_based_source.py +1 -1
- airbyte_cdk/sources/file_based/file_types/__init__.py +11 -6
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +1 -1
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +1 -1
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +2 -2
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +5 -5
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +7 -5
- airbyte_cdk/utils/datetime_format_inferrer.py +8 -4
- {airbyte_cdk-0.51.15.dist-info → airbyte_cdk-0.51.17.dist-info}/METADATA +1 -1
- {airbyte_cdk-0.51.15.dist-info → airbyte_cdk-0.51.17.dist-info}/RECORD +29 -29
- unit_tests/sources/file_based/file_types/test_avro_parser.py +6 -6
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +5 -6
- unit_tests/sources/file_based/scenarios/check_scenarios.py +8 -8
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +19 -42
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +15 -15
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +13 -12
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +5 -9
- unit_tests/sources/file_based/scenarios/scenario_builder.py +1 -1
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +16 -16
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +9 -9
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +2 -1
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +6 -3
- unit_tests/utils/test_datetime_format_inferrer.py +1 -0
- {airbyte_cdk-0.51.15.dist-info → airbyte_cdk-0.51.17.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-0.51.15.dist-info → airbyte_cdk-0.51.17.dist-info}/WHEEL +0 -0
- {airbyte_cdk-0.51.15.dist-info → airbyte_cdk-0.51.17.dist-info}/top_level.txt +0 -0
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
3
|
#
|
|
4
4
|
|
|
5
|
+
from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat
|
|
5
6
|
from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError
|
|
6
7
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
|
7
8
|
from unit_tests.sources.file_based.helpers import EmptySchemaParser, LowInferenceLimitDiscoveryPolicy
|
|
@@ -15,7 +16,7 @@ single_csv_scenario = (
|
|
|
15
16
|
"streams": [
|
|
16
17
|
{
|
|
17
18
|
"name": "stream1",
|
|
18
|
-
"
|
|
19
|
+
"format": {"filetype": "csv"},
|
|
19
20
|
"globs": ["*"],
|
|
20
21
|
"validation_policy": "Emit Record",
|
|
21
22
|
}
|
|
@@ -64,11 +65,6 @@ single_csv_scenario = (
|
|
|
64
65
|
"type": "object",
|
|
65
66
|
"properties": {
|
|
66
67
|
"name": {"title": "Name", "description": "The name of the stream.", "type": "string"},
|
|
67
|
-
"file_type": {
|
|
68
|
-
"title": "File Type",
|
|
69
|
-
"description": "The data file type that is being extracted for a stream.",
|
|
70
|
-
"type": "string",
|
|
71
|
-
},
|
|
72
68
|
"globs": {
|
|
73
69
|
"title": "Globs",
|
|
74
70
|
"description": 'The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look <a href="https://en.wikipedia.org/wiki/Glob_(programming)">here</a>.',
|
|
@@ -278,7 +274,7 @@ single_csv_scenario = (
|
|
|
278
274
|
"type": "boolean",
|
|
279
275
|
},
|
|
280
276
|
},
|
|
281
|
-
"required": ["name", "
|
|
277
|
+
"required": ["name", "format"],
|
|
282
278
|
},
|
|
283
279
|
},
|
|
284
280
|
},
|
|
@@ -339,7 +335,7 @@ multi_csv_scenario = (
|
|
|
339
335
|
"streams": [
|
|
340
336
|
{
|
|
341
337
|
"name": "stream1",
|
|
342
|
-
"
|
|
338
|
+
"format": {"filetype": "csv"},
|
|
343
339
|
"globs": ["*"],
|
|
344
340
|
"validation_policy": "Emit Record",
|
|
345
341
|
}
|
|
@@ -441,7 +437,7 @@ multi_csv_stream_n_file_exceeds_limit_for_inference = (
|
|
|
441
437
|
"streams": [
|
|
442
438
|
{
|
|
443
439
|
"name": "stream1",
|
|
444
|
-
"
|
|
440
|
+
"format": {"filetype": "csv"},
|
|
445
441
|
"globs": ["*"],
|
|
446
442
|
"validation_policy": "Emit Record",
|
|
447
443
|
}
|
|
@@ -535,13 +531,13 @@ multi_csv_stream_n_file_exceeds_limit_for_inference = (
|
|
|
535
531
|
|
|
536
532
|
invalid_csv_scenario = (
|
|
537
533
|
TestScenarioBuilder()
|
|
538
|
-
.set_name("invalid_csv_scenario")
|
|
534
|
+
.set_name("invalid_csv_scenario") # too many values for the number of headers
|
|
539
535
|
.set_config(
|
|
540
536
|
{
|
|
541
537
|
"streams": [
|
|
542
538
|
{
|
|
543
539
|
"name": "stream1",
|
|
544
|
-
"
|
|
540
|
+
"format": {"filetype": "csv"},
|
|
545
541
|
"globs": ["*"],
|
|
546
542
|
"validation_policy": "Emit Record",
|
|
547
543
|
}
|
|
@@ -604,7 +600,7 @@ csv_single_stream_scenario = (
|
|
|
604
600
|
"streams": [
|
|
605
601
|
{
|
|
606
602
|
"name": "stream1",
|
|
607
|
-
"
|
|
603
|
+
"format": {"filetype": "csv"},
|
|
608
604
|
"globs": ["*.csv"],
|
|
609
605
|
"validation_policy": "Emit Record",
|
|
610
606
|
}
|
|
@@ -684,13 +680,13 @@ csv_multi_stream_scenario = (
|
|
|
684
680
|
"streams": [
|
|
685
681
|
{
|
|
686
682
|
"name": "stream1",
|
|
687
|
-
"
|
|
683
|
+
"format": {"filetype": "csv"},
|
|
688
684
|
"globs": ["*.csv"],
|
|
689
685
|
"validation_policy": "Emit Record",
|
|
690
686
|
},
|
|
691
687
|
{
|
|
692
688
|
"name": "stream2",
|
|
693
|
-
"
|
|
689
|
+
"format": {"filetype": "csv"},
|
|
694
690
|
"globs": ["b.csv"],
|
|
695
691
|
"validation_policy": "Emit Record",
|
|
696
692
|
},
|
|
@@ -802,7 +798,6 @@ csv_custom_format_scenario = (
|
|
|
802
798
|
"streams": [
|
|
803
799
|
{
|
|
804
800
|
"name": "stream1",
|
|
805
|
-
"file_type": "csv",
|
|
806
801
|
"globs": ["*"],
|
|
807
802
|
"validation_policy": "Emit Record",
|
|
808
803
|
"format": {
|
|
@@ -908,14 +903,12 @@ multi_stream_custom_format = (
|
|
|
908
903
|
"streams": [
|
|
909
904
|
{
|
|
910
905
|
"name": "stream1",
|
|
911
|
-
"file_type": "csv",
|
|
912
906
|
"globs": ["*.csv"],
|
|
913
907
|
"validation_policy": "Emit Record",
|
|
914
908
|
"format": {"filetype": "csv", "delimiter": "#", "escape_char": "!", "double_quote": True, "newlines_in_values": False},
|
|
915
909
|
},
|
|
916
910
|
{
|
|
917
911
|
"name": "stream2",
|
|
918
|
-
"file_type": "csv",
|
|
919
912
|
"globs": ["b.csv"],
|
|
920
913
|
"validation_policy": "Emit Record",
|
|
921
914
|
"format": {
|
|
@@ -1055,7 +1048,7 @@ empty_schema_inference_scenario = (
|
|
|
1055
1048
|
"streams": [
|
|
1056
1049
|
{
|
|
1057
1050
|
"name": "stream1",
|
|
1058
|
-
"
|
|
1051
|
+
"format": {"filetype": "csv"},
|
|
1059
1052
|
"globs": ["*"],
|
|
1060
1053
|
"validation_policy": "Emit Record",
|
|
1061
1054
|
}
|
|
@@ -1096,7 +1089,7 @@ empty_schema_inference_scenario = (
|
|
|
1096
1089
|
]
|
|
1097
1090
|
}
|
|
1098
1091
|
)
|
|
1099
|
-
.set_parsers({
|
|
1092
|
+
.set_parsers({CsvFormat: EmptySchemaParser()})
|
|
1100
1093
|
.set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value)
|
|
1101
1094
|
.set_expected_records(
|
|
1102
1095
|
[
|
|
@@ -1130,7 +1123,7 @@ schemaless_csv_scenario = (
|
|
|
1130
1123
|
"streams": [
|
|
1131
1124
|
{
|
|
1132
1125
|
"name": "stream1",
|
|
1133
|
-
"
|
|
1126
|
+
"format": {"filetype": "csv"},
|
|
1134
1127
|
"globs": ["*"],
|
|
1135
1128
|
"validation_policy": "Skip Record",
|
|
1136
1129
|
"schemaless": True,
|
|
@@ -1225,14 +1218,14 @@ schemaless_csv_multi_stream_scenario = (
|
|
|
1225
1218
|
"streams": [
|
|
1226
1219
|
{
|
|
1227
1220
|
"name": "stream1",
|
|
1228
|
-
"
|
|
1221
|
+
"format": {"filetype": "csv"},
|
|
1229
1222
|
"globs": ["a.csv"],
|
|
1230
1223
|
"validation_policy": "Skip Record",
|
|
1231
1224
|
"schemaless": True,
|
|
1232
1225
|
},
|
|
1233
1226
|
{
|
|
1234
1227
|
"name": "stream2",
|
|
1235
|
-
"
|
|
1228
|
+
"format": {"filetype": "csv"},
|
|
1236
1229
|
"globs": ["b.csv"],
|
|
1237
1230
|
"validation_policy": "Skip Record",
|
|
1238
1231
|
},
|
|
@@ -1332,7 +1325,7 @@ schemaless_with_user_input_schema_fails_connection_check_scenario = (
|
|
|
1332
1325
|
"streams": [
|
|
1333
1326
|
{
|
|
1334
1327
|
"name": "stream1",
|
|
1335
|
-
"
|
|
1328
|
+
"format": {"filetype": "csv"},
|
|
1336
1329
|
"globs": ["*"],
|
|
1337
1330
|
"validation_policy": "Skip Record",
|
|
1338
1331
|
"input_schema": '{"col1": "string", "col2": "string", "col3": "string"}',
|
|
@@ -1396,7 +1389,7 @@ schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario =
|
|
|
1396
1389
|
"streams": [
|
|
1397
1390
|
{
|
|
1398
1391
|
"name": "stream1",
|
|
1399
|
-
"
|
|
1392
|
+
"format": {"filetype": "csv"},
|
|
1400
1393
|
"globs": ["a.csv"],
|
|
1401
1394
|
"validation_policy": "Skip Record",
|
|
1402
1395
|
"schemaless": True,
|
|
@@ -1404,7 +1397,7 @@ schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario =
|
|
|
1404
1397
|
},
|
|
1405
1398
|
{
|
|
1406
1399
|
"name": "stream2",
|
|
1407
|
-
"
|
|
1400
|
+
"format": {"filetype": "csv"},
|
|
1408
1401
|
"globs": ["b.csv"],
|
|
1409
1402
|
"validation_policy": "Skip Record",
|
|
1410
1403
|
},
|
|
@@ -1480,7 +1473,6 @@ csv_string_can_be_null_with_input_schemas_scenario = (
|
|
|
1480
1473
|
"streams": [
|
|
1481
1474
|
{
|
|
1482
1475
|
"name": "stream1",
|
|
1483
|
-
"file_type": "csv",
|
|
1484
1476
|
"globs": ["*"],
|
|
1485
1477
|
"validation_policy": "Emit Record",
|
|
1486
1478
|
"input_schema": '{"col1": "string", "col2": "string"}',
|
|
@@ -1549,7 +1541,6 @@ csv_string_are_not_null_if_strings_can_be_null_is_false_scenario = (
|
|
|
1549
1541
|
"streams": [
|
|
1550
1542
|
{
|
|
1551
1543
|
"name": "stream1",
|
|
1552
|
-
"file_type": "csv",
|
|
1553
1544
|
"globs": ["*"],
|
|
1554
1545
|
"validation_policy": "Emit Record",
|
|
1555
1546
|
"input_schema": '{"col1": "string", "col2": "string"}',
|
|
@@ -1619,7 +1610,6 @@ csv_string_not_null_if_no_null_values_scenario = (
|
|
|
1619
1610
|
"streams": [
|
|
1620
1611
|
{
|
|
1621
1612
|
"name": "stream1",
|
|
1622
|
-
"file_type": "csv",
|
|
1623
1613
|
"globs": ["*"],
|
|
1624
1614
|
"validation_policy": "Emit Record",
|
|
1625
1615
|
"format": {
|
|
@@ -1686,7 +1676,6 @@ csv_strings_can_be_null_not_quoted_scenario = (
|
|
|
1686
1676
|
"streams": [
|
|
1687
1677
|
{
|
|
1688
1678
|
"name": "stream1",
|
|
1689
|
-
"file_type": "csv",
|
|
1690
1679
|
"globs": ["*"],
|
|
1691
1680
|
"validation_policy": "Emit Record",
|
|
1692
1681
|
"format": {"filetype": "csv", "null_values": ["null"]},
|
|
@@ -1751,7 +1740,6 @@ csv_newline_in_values_quoted_value_scenario = (
|
|
|
1751
1740
|
"streams": [
|
|
1752
1741
|
{
|
|
1753
1742
|
"name": "stream1",
|
|
1754
|
-
"file_type": "csv",
|
|
1755
1743
|
"globs": ["*"],
|
|
1756
1744
|
"validation_policy": "Emit Record",
|
|
1757
1745
|
"format": {
|
|
@@ -1818,7 +1806,6 @@ csv_newline_in_values_not_quoted_scenario = (
|
|
|
1818
1806
|
"streams": [
|
|
1819
1807
|
{
|
|
1820
1808
|
"name": "stream1",
|
|
1821
|
-
"file_type": "csv",
|
|
1822
1809
|
"globs": ["*"],
|
|
1823
1810
|
"validation_policy": "Emit Record",
|
|
1824
1811
|
"format": {
|
|
@@ -1897,7 +1884,6 @@ csv_escape_char_is_set_scenario = (
|
|
|
1897
1884
|
"streams": [
|
|
1898
1885
|
{
|
|
1899
1886
|
"name": "stream1",
|
|
1900
|
-
"file_type": "csv",
|
|
1901
1887
|
"globs": ["*"],
|
|
1902
1888
|
"validation_policy": "Emit Record",
|
|
1903
1889
|
"format": {
|
|
@@ -1969,7 +1955,6 @@ csv_double_quote_is_set_scenario = (
|
|
|
1969
1955
|
"streams": [
|
|
1970
1956
|
{
|
|
1971
1957
|
"name": "stream1",
|
|
1972
|
-
"file_type": "csv",
|
|
1973
1958
|
"globs": ["*"],
|
|
1974
1959
|
"validation_policy": "Emit Record",
|
|
1975
1960
|
"format": {
|
|
@@ -2040,7 +2025,6 @@ csv_custom_delimiter_with_escape_char_scenario = (
|
|
|
2040
2025
|
"streams": [
|
|
2041
2026
|
{
|
|
2042
2027
|
"name": "stream1",
|
|
2043
|
-
"file_type": "csv",
|
|
2044
2028
|
"globs": ["*"],
|
|
2045
2029
|
"validation_policy": "Emit Record",
|
|
2046
2030
|
"format": {"filetype": "csv", "double_quotes": True, "quote_char": "@", "delimiter": "|", "escape_char": "+"},
|
|
@@ -2106,7 +2090,6 @@ csv_custom_delimiter_in_double_quotes_scenario = (
|
|
|
2106
2090
|
"streams": [
|
|
2107
2091
|
{
|
|
2108
2092
|
"name": "stream1",
|
|
2109
|
-
"file_type": "csv",
|
|
2110
2093
|
"globs": ["*"],
|
|
2111
2094
|
"validation_policy": "Emit Record",
|
|
2112
2095
|
"format": {
|
|
@@ -2176,7 +2159,6 @@ csv_skip_before_header_scenario = (
|
|
|
2176
2159
|
"streams": [
|
|
2177
2160
|
{
|
|
2178
2161
|
"name": "stream1",
|
|
2179
|
-
"file_type": "csv",
|
|
2180
2162
|
"globs": ["*"],
|
|
2181
2163
|
"validation_policy": "Emit Record",
|
|
2182
2164
|
"format": {"filetype": "csv", "skip_rows_before_header": 2},
|
|
@@ -2243,7 +2225,6 @@ csv_skip_after_header_scenario = (
|
|
|
2243
2225
|
"streams": [
|
|
2244
2226
|
{
|
|
2245
2227
|
"name": "stream1",
|
|
2246
|
-
"file_type": "csv",
|
|
2247
2228
|
"globs": ["*"],
|
|
2248
2229
|
"validation_policy": "Emit Record",
|
|
2249
2230
|
"format": {"filetype": "csv", "skip_rows_after_header": 2},
|
|
@@ -2310,7 +2291,6 @@ csv_skip_before_and_after_header_scenario = (
|
|
|
2310
2291
|
"streams": [
|
|
2311
2292
|
{
|
|
2312
2293
|
"name": "stream1",
|
|
2313
|
-
"file_type": "csv",
|
|
2314
2294
|
"globs": ["*"],
|
|
2315
2295
|
"validation_policy": "Emit Record",
|
|
2316
2296
|
"format": {
|
|
@@ -2381,7 +2361,6 @@ csv_autogenerate_column_names_scenario = (
|
|
|
2381
2361
|
"streams": [
|
|
2382
2362
|
{
|
|
2383
2363
|
"name": "stream1",
|
|
2384
|
-
"file_type": "csv",
|
|
2385
2364
|
"globs": ["*"],
|
|
2386
2365
|
"validation_policy": "Emit Record",
|
|
2387
2366
|
"format": {
|
|
@@ -2448,7 +2427,6 @@ csv_custom_bool_values_scenario = (
|
|
|
2448
2427
|
"streams": [
|
|
2449
2428
|
{
|
|
2450
2429
|
"name": "stream1",
|
|
2451
|
-
"file_type": "csv",
|
|
2452
2430
|
"globs": ["*"],
|
|
2453
2431
|
"validation_policy": "Emit Record",
|
|
2454
2432
|
"input_schema": '{"col1": "boolean", "col2": "boolean"}',
|
|
@@ -2518,7 +2496,6 @@ csv_custom_null_values_scenario = (
|
|
|
2518
2496
|
"streams": [
|
|
2519
2497
|
{
|
|
2520
2498
|
"name": "stream1",
|
|
2521
|
-
"file_type": "csv",
|
|
2522
2499
|
"globs": ["*"],
|
|
2523
2500
|
"validation_policy": "Emit Record",
|
|
2524
2501
|
"input_schema": '{"col1": "boolean", "col2": "string"}',
|
|
@@ -2587,7 +2564,7 @@ earlier_csv_scenario = (
|
|
|
2587
2564
|
"streams": [
|
|
2588
2565
|
{
|
|
2589
2566
|
"name": "stream1",
|
|
2590
|
-
"
|
|
2567
|
+
"format": {"filetype": "csv"},
|
|
2591
2568
|
"globs": ["*"],
|
|
2592
2569
|
"validation_policy": "Emit Record",
|
|
2593
2570
|
}
|
|
@@ -13,7 +13,7 @@ single_csv_input_state_is_earlier_scenario = (
|
|
|
13
13
|
"streams": [
|
|
14
14
|
{
|
|
15
15
|
"name": "stream1",
|
|
16
|
-
"
|
|
16
|
+
"format": {"filetype": "csv"},
|
|
17
17
|
"globs": ["*.csv"],
|
|
18
18
|
"validation_policy": "Emit Record",
|
|
19
19
|
}
|
|
@@ -100,7 +100,7 @@ single_csv_file_is_skipped_if_same_modified_at_as_in_history = (
|
|
|
100
100
|
"streams": [
|
|
101
101
|
{
|
|
102
102
|
"name": "stream1",
|
|
103
|
-
"
|
|
103
|
+
"format": {"filetype": "csv"},
|
|
104
104
|
"globs": ["*.csv"],
|
|
105
105
|
"validation_policy": "Emit Record",
|
|
106
106
|
}
|
|
@@ -184,7 +184,7 @@ single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history = (
|
|
|
184
184
|
"streams": [
|
|
185
185
|
{
|
|
186
186
|
"name": "stream1",
|
|
187
|
-
"
|
|
187
|
+
"format": {"filetype": "csv"},
|
|
188
188
|
"globs": ["*.csv"],
|
|
189
189
|
"validation_policy": "Emit Record",
|
|
190
190
|
}
|
|
@@ -270,7 +270,7 @@ single_csv_no_input_state_scenario = (
|
|
|
270
270
|
"streams": [
|
|
271
271
|
{
|
|
272
272
|
"name": "stream1",
|
|
273
|
-
"
|
|
273
|
+
"format": {"filetype": "csv"},
|
|
274
274
|
"globs": ["*.csv"],
|
|
275
275
|
"validation_policy": "Emit Record",
|
|
276
276
|
}
|
|
@@ -344,7 +344,7 @@ multi_csv_same_timestamp_scenario = (
|
|
|
344
344
|
"streams": [
|
|
345
345
|
{
|
|
346
346
|
"name": "stream1",
|
|
347
|
-
"
|
|
347
|
+
"format": {"filetype": "csv"},
|
|
348
348
|
"globs": ["*.csv"],
|
|
349
349
|
"validation_policy": "Emit Record",
|
|
350
350
|
}
|
|
@@ -435,7 +435,7 @@ single_csv_input_state_is_later_scenario = (
|
|
|
435
435
|
"streams": [
|
|
436
436
|
{
|
|
437
437
|
"name": "stream1",
|
|
438
|
-
"
|
|
438
|
+
"format": {"filetype": "csv"},
|
|
439
439
|
"globs": ["*.csv"],
|
|
440
440
|
"validation_policy": "Emit Record",
|
|
441
441
|
}
|
|
@@ -521,7 +521,7 @@ multi_csv_different_timestamps_scenario = (
|
|
|
521
521
|
"streams": [
|
|
522
522
|
{
|
|
523
523
|
"name": "stream1",
|
|
524
|
-
"
|
|
524
|
+
"format": {"filetype": "csv"},
|
|
525
525
|
"globs": ["*.csv"],
|
|
526
526
|
"validation_policy": "Emit Record",
|
|
527
527
|
}
|
|
@@ -620,7 +620,7 @@ multi_csv_per_timestamp_scenario = (
|
|
|
620
620
|
"streams": [
|
|
621
621
|
{
|
|
622
622
|
"name": "stream1",
|
|
623
|
-
"
|
|
623
|
+
"format": {"filetype": "csv"},
|
|
624
624
|
"globs": ["*.csv"],
|
|
625
625
|
"validation_policy": "Emit Record",
|
|
626
626
|
}
|
|
@@ -733,7 +733,7 @@ multi_csv_skip_file_if_already_in_history = (
|
|
|
733
733
|
"streams": [
|
|
734
734
|
{
|
|
735
735
|
"name": "stream1",
|
|
736
|
-
"
|
|
736
|
+
"format": {"filetype": "csv"},
|
|
737
737
|
"globs": ["*.csv"],
|
|
738
738
|
"validation_policy": "Emit Record",
|
|
739
739
|
}
|
|
@@ -855,7 +855,7 @@ multi_csv_include_missing_files_within_history_range = (
|
|
|
855
855
|
"streams": [
|
|
856
856
|
{
|
|
857
857
|
"name": "stream1",
|
|
858
|
-
"
|
|
858
|
+
"format": {"filetype": "csv"},
|
|
859
859
|
"globs": ["*.csv"],
|
|
860
860
|
"validation_policy": "Emit Record",
|
|
861
861
|
}
|
|
@@ -969,7 +969,7 @@ multi_csv_remove_old_files_if_history_is_full_scenario = (
|
|
|
969
969
|
"streams": [
|
|
970
970
|
{
|
|
971
971
|
"name": "stream1",
|
|
972
|
-
"
|
|
972
|
+
"format": {"filetype": "csv"},
|
|
973
973
|
"globs": ["*.csv"],
|
|
974
974
|
"validation_policy": "Emit Record",
|
|
975
975
|
}
|
|
@@ -1107,7 +1107,7 @@ multi_csv_same_timestamp_more_files_than_history_size_scenario = (
|
|
|
1107
1107
|
"streams": [
|
|
1108
1108
|
{
|
|
1109
1109
|
"name": "stream1",
|
|
1110
|
-
"
|
|
1110
|
+
"format": {"filetype": "csv"},
|
|
1111
1111
|
"globs": ["*.csv"],
|
|
1112
1112
|
"validation_policy": "Emit Record",
|
|
1113
1113
|
"days_to_sync_if_history_is_full": 3,
|
|
@@ -1225,7 +1225,7 @@ multi_csv_sync_recent_files_if_history_is_incomplete_scenario = (
|
|
|
1225
1225
|
"streams": [
|
|
1226
1226
|
{
|
|
1227
1227
|
"name": "stream1",
|
|
1228
|
-
"
|
|
1228
|
+
"format": {"filetype": "csv"},
|
|
1229
1229
|
"globs": ["*.csv"],
|
|
1230
1230
|
"validation_policy": "Emit Record",
|
|
1231
1231
|
"days_to_sync_if_history_is_full": 3,
|
|
@@ -1342,7 +1342,7 @@ multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_time
|
|
|
1342
1342
|
"streams": [
|
|
1343
1343
|
{
|
|
1344
1344
|
"name": "stream1",
|
|
1345
|
-
"
|
|
1345
|
+
"format": {"filetype": "csv"},
|
|
1346
1346
|
"globs": ["*.csv"],
|
|
1347
1347
|
"validation_policy": "Emit Record",
|
|
1348
1348
|
"days_to_sync_if_history_is_full": 3,
|
|
@@ -1465,7 +1465,7 @@ multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_differe
|
|
|
1465
1465
|
"streams": [
|
|
1466
1466
|
{
|
|
1467
1467
|
"name": "stream1",
|
|
1468
|
-
"
|
|
1468
|
+
"format": {"filetype": "csv"},
|
|
1469
1469
|
"globs": ["*.csv"],
|
|
1470
1470
|
"validation_policy": "Emit Record",
|
|
1471
1471
|
"days_to_sync_if_history_is_full": 3,
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
3
|
#
|
|
4
4
|
|
|
5
|
+
from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat
|
|
5
6
|
from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError
|
|
6
7
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
|
7
8
|
from unit_tests.sources.file_based.helpers import LowInferenceBytesJsonlParser, LowInferenceLimitDiscoveryPolicy
|
|
@@ -15,7 +16,7 @@ single_jsonl_scenario = (
|
|
|
15
16
|
"streams": [
|
|
16
17
|
{
|
|
17
18
|
"name": "stream1",
|
|
18
|
-
"
|
|
19
|
+
"format": {"filetype": "jsonl"},
|
|
19
20
|
"globs": ["*"],
|
|
20
21
|
"validation_policy": "Emit Record",
|
|
21
22
|
}
|
|
@@ -82,7 +83,7 @@ multi_jsonl_with_different_keys_scenario = (
|
|
|
82
83
|
"streams": [
|
|
83
84
|
{
|
|
84
85
|
"name": "stream1",
|
|
85
|
-
"
|
|
86
|
+
"format": {"filetype": "jsonl"},
|
|
86
87
|
"globs": ["*"],
|
|
87
88
|
"validation_policy": "Emit Record",
|
|
88
89
|
}
|
|
@@ -163,7 +164,7 @@ multi_jsonl_stream_n_file_exceeds_limit_for_inference = (
|
|
|
163
164
|
"streams": [
|
|
164
165
|
{
|
|
165
166
|
"name": "stream1",
|
|
166
|
-
"
|
|
167
|
+
"format": {"filetype": "jsonl"},
|
|
167
168
|
"globs": ["*"],
|
|
168
169
|
"validation_policy": "Emit Record",
|
|
169
170
|
}
|
|
@@ -241,7 +242,7 @@ multi_jsonl_stream_n_bytes_exceeds_limit_for_inference = (
|
|
|
241
242
|
"streams": [
|
|
242
243
|
{
|
|
243
244
|
"name": "stream1",
|
|
244
|
-
"
|
|
245
|
+
"format": {"filetype": "jsonl"},
|
|
245
246
|
"globs": ["*"],
|
|
246
247
|
"validation_policy": "Emit Record",
|
|
247
248
|
}
|
|
@@ -307,7 +308,7 @@ multi_jsonl_stream_n_bytes_exceeds_limit_for_inference = (
|
|
|
307
308
|
"_ab_source_file_url": "b.jsonl"}, "stream": "stream1"},
|
|
308
309
|
]
|
|
309
310
|
)
|
|
310
|
-
.set_parsers({
|
|
311
|
+
.set_parsers({JsonlFormat: LowInferenceBytesJsonlParser()})
|
|
311
312
|
).build()
|
|
312
313
|
|
|
313
314
|
|
|
@@ -319,7 +320,7 @@ invalid_jsonl_scenario = (
|
|
|
319
320
|
"streams": [
|
|
320
321
|
{
|
|
321
322
|
"name": "stream1",
|
|
322
|
-
"
|
|
323
|
+
"format": {"filetype": "jsonl"},
|
|
323
324
|
"globs": ["*"],
|
|
324
325
|
"validation_policy": "Emit Record",
|
|
325
326
|
}
|
|
@@ -390,13 +391,13 @@ jsonl_multi_stream_scenario = (
|
|
|
390
391
|
"streams": [
|
|
391
392
|
{
|
|
392
393
|
"name": "stream1",
|
|
393
|
-
"
|
|
394
|
+
"format": {"filetype": "jsonl"},
|
|
394
395
|
"globs": ["*.jsonl"],
|
|
395
396
|
"validation_policy": "Emit Record",
|
|
396
397
|
},
|
|
397
398
|
{
|
|
398
399
|
"name": "stream2",
|
|
399
|
-
"
|
|
400
|
+
"format": {"filetype": "jsonl"},
|
|
400
401
|
"globs": ["b.jsonl"],
|
|
401
402
|
"validation_policy": "Emit Record",
|
|
402
403
|
}
|
|
@@ -501,7 +502,7 @@ schemaless_jsonl_scenario = (
|
|
|
501
502
|
"streams": [
|
|
502
503
|
{
|
|
503
504
|
"name": "stream1",
|
|
504
|
-
"
|
|
505
|
+
"format": {"filetype": "jsonl"},
|
|
505
506
|
"globs": ["*"],
|
|
506
507
|
"validation_policy": "Skip Record",
|
|
507
508
|
"schemaless": True,
|
|
@@ -577,14 +578,14 @@ schemaless_jsonl_multi_stream_scenario = (
|
|
|
577
578
|
"streams": [
|
|
578
579
|
{
|
|
579
580
|
"name": "stream1",
|
|
580
|
-
"
|
|
581
|
+
"format": {"filetype": "jsonl"},
|
|
581
582
|
"globs": ["a.jsonl"],
|
|
582
583
|
"validation_policy": "Skip Record",
|
|
583
584
|
"schemaless": True,
|
|
584
585
|
},
|
|
585
586
|
{
|
|
586
587
|
"name": "stream2",
|
|
587
|
-
"
|
|
588
|
+
"format": {"filetype": "jsonl"},
|
|
588
589
|
"globs": ["b.jsonl"],
|
|
589
590
|
"validation_policy": "Skip Record",
|
|
590
591
|
}
|
|
@@ -678,7 +679,7 @@ jsonl_user_input_schema_scenario = (
|
|
|
678
679
|
"streams": [
|
|
679
680
|
{
|
|
680
681
|
"name": "stream1",
|
|
681
|
-
"
|
|
682
|
+
"format": {"filetype": "jsonl"},
|
|
682
683
|
"globs": ["*"],
|
|
683
684
|
"validation_policy": "Emit Record",
|
|
684
685
|
"input_schema": '{"col1": "integer", "col2": "string"}'
|
|
@@ -171,7 +171,7 @@ single_parquet_scenario = (
|
|
|
171
171
|
"streams": [
|
|
172
172
|
{
|
|
173
173
|
"name": "stream1",
|
|
174
|
-
"
|
|
174
|
+
"format": {"filetype": "parquet"},
|
|
175
175
|
"globs": ["*"],
|
|
176
176
|
"validation_policy": "Emit Record",
|
|
177
177
|
}
|
|
@@ -227,7 +227,7 @@ single_partitioned_parquet_scenario = (
|
|
|
227
227
|
"streams": [
|
|
228
228
|
{
|
|
229
229
|
"name": "stream1",
|
|
230
|
-
"
|
|
230
|
+
"format": {"filetype": "parquet"},
|
|
231
231
|
"globs": ["path_prefix/**/*"],
|
|
232
232
|
"validation_policy": "Emit Record",
|
|
233
233
|
}
|
|
@@ -289,7 +289,7 @@ multi_parquet_scenario = (
|
|
|
289
289
|
"streams": [
|
|
290
290
|
{
|
|
291
291
|
"name": "stream1",
|
|
292
|
-
"
|
|
292
|
+
"format": {"filetype": "parquet"},
|
|
293
293
|
"globs": ["*"],
|
|
294
294
|
"validation_policy": "Emit Record",
|
|
295
295
|
}
|
|
@@ -352,7 +352,7 @@ parquet_various_types_scenario = (
|
|
|
352
352
|
"streams": [
|
|
353
353
|
{
|
|
354
354
|
"name": "stream1",
|
|
355
|
-
"
|
|
355
|
+
"format": {"filetype": "parquet"},
|
|
356
356
|
"globs": ["*"],
|
|
357
357
|
"validation_policy": "Emit Record",
|
|
358
358
|
}
|
|
@@ -493,7 +493,7 @@ parquet_file_with_decimal_no_config_scenario = (
|
|
|
493
493
|
"streams": [
|
|
494
494
|
{
|
|
495
495
|
"name": "stream1",
|
|
496
|
-
"
|
|
496
|
+
"format": {"filetype": "parquet"},
|
|
497
497
|
"globs": ["*"],
|
|
498
498
|
"validation_policy": "Emit Record",
|
|
499
499
|
}
|
|
@@ -544,7 +544,6 @@ parquet_file_with_decimal_as_string_scenario = (
|
|
|
544
544
|
"streams": [
|
|
545
545
|
{
|
|
546
546
|
"name": "stream1",
|
|
547
|
-
"file_type": "parquet",
|
|
548
547
|
"globs": ["*"],
|
|
549
548
|
"validation_policy": "Emit Record",
|
|
550
549
|
"format": {
|
|
@@ -599,7 +598,6 @@ parquet_file_with_decimal_as_float_scenario = (
|
|
|
599
598
|
"streams": [
|
|
600
599
|
{
|
|
601
600
|
"name": "stream1",
|
|
602
|
-
"file_type": "parquet",
|
|
603
601
|
"globs": ["*"],
|
|
604
602
|
"validation_policy": "Emit Record",
|
|
605
603
|
"format": {
|
|
@@ -654,7 +652,6 @@ parquet_file_with_decimal_legacy_config_scenario = (
|
|
|
654
652
|
"streams": [
|
|
655
653
|
{
|
|
656
654
|
"name": "stream1",
|
|
657
|
-
"file_type": "parquet",
|
|
658
655
|
"format": {
|
|
659
656
|
"filetype": "parquet",
|
|
660
657
|
},
|
|
@@ -708,7 +705,6 @@ parquet_with_invalid_config_scenario = (
|
|
|
708
705
|
"streams": [
|
|
709
706
|
{
|
|
710
707
|
"name": "stream1",
|
|
711
|
-
"file_type": "parquet",
|
|
712
708
|
"globs": ["*"],
|
|
713
709
|
"validation_policy": "Emit Record",
|
|
714
710
|
"format": {
|
|
@@ -163,7 +163,7 @@ class TestScenarioBuilder:
|
|
|
163
163
|
self._expected_records = expected_records
|
|
164
164
|
return self
|
|
165
165
|
|
|
166
|
-
def set_parsers(self, parsers: Mapping[
|
|
166
|
+
def set_parsers(self, parsers: Mapping[Type[Any], FileTypeParser]) -> "TestScenarioBuilder":
|
|
167
167
|
self._parsers = parsers
|
|
168
168
|
return self
|
|
169
169
|
|