deltacat 1.1.36__py3-none-any.whl → 2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deltacat/__init__.py +42 -3
- deltacat/annotations.py +36 -0
- deltacat/api.py +168 -0
- deltacat/aws/s3u.py +4 -4
- deltacat/benchmarking/benchmark_engine.py +82 -0
- deltacat/benchmarking/benchmark_report.py +86 -0
- deltacat/benchmarking/benchmark_suite.py +11 -0
- deltacat/benchmarking/conftest.py +21 -0
- deltacat/benchmarking/data/random_row_generator.py +94 -0
- deltacat/benchmarking/data/row_generator.py +10 -0
- deltacat/benchmarking/test_benchmark_pipeline.py +106 -0
- deltacat/catalog/__init__.py +14 -0
- deltacat/catalog/delegate.py +199 -106
- deltacat/catalog/iceberg/__init__.py +4 -0
- deltacat/catalog/iceberg/iceberg_catalog_config.py +26 -0
- deltacat/catalog/iceberg/impl.py +368 -0
- deltacat/catalog/iceberg/overrides.py +74 -0
- deltacat/catalog/interface.py +273 -76
- deltacat/catalog/main/impl.py +720 -0
- deltacat/catalog/model/catalog.py +227 -20
- deltacat/catalog/model/properties.py +116 -0
- deltacat/catalog/model/table_definition.py +32 -1
- deltacat/compute/compactor/model/compaction_session_audit_info.py +7 -3
- deltacat/compute/compactor/model/delta_annotated.py +3 -3
- deltacat/compute/compactor/model/delta_file_envelope.py +3 -1
- deltacat/compute/compactor/model/delta_file_locator.py +3 -1
- deltacat/compute/compactor/model/round_completion_info.py +5 -5
- deltacat/compute/compactor/model/table_object_store.py +3 -2
- deltacat/compute/compactor/repartition_session.py +1 -1
- deltacat/compute/compactor/steps/dedupe.py +11 -4
- deltacat/compute/compactor/steps/hash_bucket.py +1 -1
- deltacat/compute/compactor/steps/materialize.py +6 -2
- deltacat/compute/compactor/utils/io.py +1 -1
- deltacat/compute/compactor/utils/sort_key.py +9 -2
- deltacat/compute/compactor_v2/compaction_session.py +5 -9
- deltacat/compute/compactor_v2/constants.py +1 -30
- deltacat/compute/compactor_v2/deletes/utils.py +3 -3
- deltacat/compute/compactor_v2/model/merge_input.py +1 -7
- deltacat/compute/compactor_v2/private/compaction_utils.py +5 -6
- deltacat/compute/compactor_v2/steps/merge.py +17 -126
- deltacat/compute/compactor_v2/utils/content_type_params.py +0 -17
- deltacat/compute/compactor_v2/utils/dedupe.py +1 -1
- deltacat/compute/compactor_v2/utils/io.py +1 -1
- deltacat/compute/compactor_v2/utils/merge.py +0 -1
- deltacat/compute/compactor_v2/utils/primary_key_index.py +3 -15
- deltacat/compute/compactor_v2/utils/task_options.py +23 -43
- deltacat/compute/converter/constants.py +4 -0
- deltacat/compute/converter/converter_session.py +143 -0
- deltacat/compute/converter/model/convert_input.py +69 -0
- deltacat/compute/converter/model/convert_input_files.py +61 -0
- deltacat/compute/converter/model/converter_session_params.py +99 -0
- deltacat/compute/converter/pyiceberg/__init__.py +0 -0
- deltacat/compute/converter/pyiceberg/catalog.py +75 -0
- deltacat/compute/converter/pyiceberg/overrides.py +135 -0
- deltacat/compute/converter/pyiceberg/update_snapshot_overrides.py +251 -0
- deltacat/compute/converter/steps/__init__.py +0 -0
- deltacat/compute/converter/steps/convert.py +211 -0
- deltacat/compute/converter/steps/dedupe.py +60 -0
- deltacat/compute/converter/utils/__init__.py +0 -0
- deltacat/compute/converter/utils/convert_task_options.py +88 -0
- deltacat/compute/converter/utils/converter_session_utils.py +109 -0
- deltacat/compute/converter/utils/iceberg_columns.py +82 -0
- deltacat/compute/converter/utils/io.py +43 -0
- deltacat/compute/converter/utils/s3u.py +133 -0
- deltacat/compute/resource_estimation/delta.py +1 -19
- deltacat/constants.py +47 -1
- deltacat/env.py +51 -0
- deltacat/examples/__init__.py +0 -0
- deltacat/examples/basic_logging.py +101 -0
- deltacat/examples/common/__init__.py +0 -0
- deltacat/examples/common/fixtures.py +15 -0
- deltacat/examples/hello_world.py +27 -0
- deltacat/examples/iceberg/__init__.py +0 -0
- deltacat/examples/iceberg/iceberg_bucket_writer.py +139 -0
- deltacat/examples/iceberg/iceberg_reader.py +149 -0
- deltacat/exceptions.py +51 -9
- deltacat/logs.py +4 -1
- deltacat/storage/__init__.py +118 -28
- deltacat/storage/iceberg/__init__.py +0 -0
- deltacat/storage/iceberg/iceberg_scan_planner.py +28 -0
- deltacat/storage/iceberg/impl.py +737 -0
- deltacat/storage/iceberg/model.py +709 -0
- deltacat/storage/interface.py +217 -134
- deltacat/storage/main/__init__.py +0 -0
- deltacat/storage/main/impl.py +2077 -0
- deltacat/storage/model/delta.py +118 -71
- deltacat/storage/model/interop.py +24 -0
- deltacat/storage/model/list_result.py +8 -0
- deltacat/storage/model/locator.py +93 -3
- deltacat/{aws/redshift → storage}/model/manifest.py +122 -98
- deltacat/storage/model/metafile.py +1316 -0
- deltacat/storage/model/namespace.py +34 -18
- deltacat/storage/model/partition.py +362 -37
- deltacat/storage/model/scan/__init__.py +0 -0
- deltacat/storage/model/scan/push_down.py +19 -0
- deltacat/storage/model/scan/scan_plan.py +10 -0
- deltacat/storage/model/scan/scan_task.py +34 -0
- deltacat/storage/model/schema.py +892 -0
- deltacat/storage/model/shard.py +47 -0
- deltacat/storage/model/sort_key.py +170 -13
- deltacat/storage/model/stream.py +208 -80
- deltacat/storage/model/table.py +123 -29
- deltacat/storage/model/table_version.py +322 -46
- deltacat/storage/model/transaction.py +757 -0
- deltacat/storage/model/transform.py +198 -61
- deltacat/storage/model/types.py +111 -13
- deltacat/storage/rivulet/__init__.py +11 -0
- deltacat/storage/rivulet/arrow/__init__.py +0 -0
- deltacat/storage/rivulet/arrow/serializer.py +75 -0
- deltacat/storage/rivulet/dataset.py +744 -0
- deltacat/storage/rivulet/dataset_executor.py +87 -0
- deltacat/storage/rivulet/feather/__init__.py +5 -0
- deltacat/storage/rivulet/feather/file_reader.py +136 -0
- deltacat/storage/rivulet/feather/serializer.py +35 -0
- deltacat/storage/rivulet/fs/__init__.py +0 -0
- deltacat/storage/rivulet/fs/file_provider.py +105 -0
- deltacat/storage/rivulet/fs/file_store.py +130 -0
- deltacat/storage/rivulet/fs/input_file.py +76 -0
- deltacat/storage/rivulet/fs/output_file.py +86 -0
- deltacat/storage/rivulet/logical_plan.py +105 -0
- deltacat/storage/rivulet/metastore/__init__.py +0 -0
- deltacat/storage/rivulet/metastore/delta.py +190 -0
- deltacat/storage/rivulet/metastore/json_sst.py +105 -0
- deltacat/storage/rivulet/metastore/sst.py +82 -0
- deltacat/storage/rivulet/metastore/sst_interval_tree.py +260 -0
- deltacat/storage/rivulet/mvp/Table.py +101 -0
- deltacat/storage/rivulet/mvp/__init__.py +5 -0
- deltacat/storage/rivulet/parquet/__init__.py +5 -0
- deltacat/storage/rivulet/parquet/data_reader.py +0 -0
- deltacat/storage/rivulet/parquet/file_reader.py +127 -0
- deltacat/storage/rivulet/parquet/serializer.py +37 -0
- deltacat/storage/rivulet/reader/__init__.py +0 -0
- deltacat/storage/rivulet/reader/block_scanner.py +378 -0
- deltacat/storage/rivulet/reader/data_reader.py +136 -0
- deltacat/storage/rivulet/reader/data_scan.py +63 -0
- deltacat/storage/rivulet/reader/dataset_metastore.py +178 -0
- deltacat/storage/rivulet/reader/dataset_reader.py +156 -0
- deltacat/storage/rivulet/reader/pyarrow_data_reader.py +121 -0
- deltacat/storage/rivulet/reader/query_expression.py +99 -0
- deltacat/storage/rivulet/reader/reader_type_registrar.py +84 -0
- deltacat/storage/rivulet/schema/__init__.py +0 -0
- deltacat/storage/rivulet/schema/datatype.py +128 -0
- deltacat/storage/rivulet/schema/schema.py +251 -0
- deltacat/storage/rivulet/serializer.py +40 -0
- deltacat/storage/rivulet/serializer_factory.py +42 -0
- deltacat/storage/rivulet/writer/__init__.py +0 -0
- deltacat/storage/rivulet/writer/dataset_writer.py +29 -0
- deltacat/storage/rivulet/writer/memtable_dataset_writer.py +294 -0
- deltacat/tests/_io/__init__.py +1 -0
- deltacat/tests/catalog/test_catalogs.py +324 -0
- deltacat/tests/catalog/test_default_catalog_impl.py +16 -8
- deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py +21 -21
- deltacat/tests/compute/compact_partition_rebase_test_cases.py +6 -6
- deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py +56 -56
- deltacat/tests/compute/compact_partition_test_cases.py +19 -53
- deltacat/tests/compute/compactor/steps/test_repartition.py +2 -2
- deltacat/tests/compute/compactor/utils/test_io.py +6 -8
- deltacat/tests/compute/compactor_v2/test_compaction_session.py +0 -466
- deltacat/tests/compute/compactor_v2/utils/test_task_options.py +1 -273
- deltacat/tests/compute/conftest.py +75 -0
- deltacat/tests/compute/converter/__init__.py +0 -0
- deltacat/tests/compute/converter/conftest.py +80 -0
- deltacat/tests/compute/converter/test_convert_session.py +478 -0
- deltacat/tests/compute/converter/utils.py +123 -0
- deltacat/tests/compute/resource_estimation/test_delta.py +0 -16
- deltacat/tests/compute/test_compact_partition_incremental.py +2 -42
- deltacat/tests/compute/test_compact_partition_multiple_rounds.py +5 -46
- deltacat/tests/compute/test_compact_partition_params.py +3 -3
- deltacat/tests/compute/test_compact_partition_rebase.py +1 -46
- deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py +5 -46
- deltacat/tests/compute/test_util_common.py +19 -12
- deltacat/tests/compute/test_util_create_table_deltas_repo.py +13 -22
- deltacat/tests/local_deltacat_storage/__init__.py +76 -103
- deltacat/tests/storage/__init__.py +0 -0
- deltacat/tests/storage/conftest.py +25 -0
- deltacat/tests/storage/main/__init__.py +0 -0
- deltacat/tests/storage/main/test_main_storage.py +1399 -0
- deltacat/tests/storage/model/__init__.py +0 -0
- deltacat/tests/storage/model/test_delete_parameters.py +21 -0
- deltacat/tests/storage/model/test_metafile_io.py +2535 -0
- deltacat/tests/storage/model/test_schema.py +308 -0
- deltacat/tests/storage/model/test_shard.py +22 -0
- deltacat/tests/storage/model/test_table_version.py +110 -0
- deltacat/tests/storage/model/test_transaction.py +308 -0
- deltacat/tests/storage/rivulet/__init__.py +0 -0
- deltacat/tests/storage/rivulet/conftest.py +149 -0
- deltacat/tests/storage/rivulet/fs/__init__.py +0 -0
- deltacat/tests/storage/rivulet/fs/test_file_location_provider.py +93 -0
- deltacat/tests/storage/rivulet/schema/__init__.py +0 -0
- deltacat/tests/storage/rivulet/schema/test_schema.py +241 -0
- deltacat/tests/storage/rivulet/test_dataset.py +406 -0
- deltacat/tests/storage/rivulet/test_manifest.py +67 -0
- deltacat/tests/storage/rivulet/test_sst_interval_tree.py +232 -0
- deltacat/tests/storage/rivulet/test_utils.py +122 -0
- deltacat/tests/storage/rivulet/writer/__init__.py +0 -0
- deltacat/tests/storage/rivulet/writer/test_dataset_write_then_read.py +341 -0
- deltacat/tests/storage/rivulet/writer/test_dataset_writer.py +79 -0
- deltacat/tests/storage/rivulet/writer/test_memtable_dataset_writer.py +75 -0
- deltacat/tests/test_deltacat_api.py +39 -0
- deltacat/tests/test_utils/filesystem.py +14 -0
- deltacat/tests/test_utils/message_pack_utils.py +54 -0
- deltacat/tests/test_utils/pyarrow.py +8 -15
- deltacat/tests/test_utils/storage.py +266 -3
- deltacat/tests/utils/test_daft.py +3 -3
- deltacat/tests/utils/test_pyarrow.py +0 -432
- deltacat/types/partial_download.py +1 -1
- deltacat/types/tables.py +1 -1
- deltacat/utils/export.py +59 -0
- deltacat/utils/filesystem.py +320 -0
- deltacat/utils/metafile_locator.py +73 -0
- deltacat/utils/pyarrow.py +36 -183
- deltacat-2.0.dist-info/METADATA +65 -0
- deltacat-2.0.dist-info/RECORD +347 -0
- deltacat/aws/redshift/__init__.py +0 -19
- deltacat/catalog/default_catalog_impl/__init__.py +0 -369
- deltacat/io/dataset.py +0 -73
- deltacat/io/read_api.py +0 -143
- deltacat/storage/model/delete_parameters.py +0 -40
- deltacat/storage/model/partition_spec.py +0 -71
- deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py +0 -253
- deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py +0 -45
- deltacat-1.1.36.dist-info/METADATA +0 -64
- deltacat-1.1.36.dist-info/RECORD +0 -219
- /deltacat/{aws/redshift/model → benchmarking/data}/__init__.py +0 -0
- /deltacat/{io/aws → catalog/main}/__init__.py +0 -0
- /deltacat/{io/aws/redshift → compute/converter}/__init__.py +0 -0
- /deltacat/{tests/io → compute/converter/model}/__init__.py +0 -0
- /deltacat/tests/{io → _io}/test_cloudpickle_bug_fix.py +0 -0
- /deltacat/tests/{io → _io}/test_file_object_store.py +0 -0
- /deltacat/tests/{io → _io}/test_memcached_object_store.py +0 -0
- /deltacat/tests/{io → _io}/test_ray_plasma_object_store.py +0 -0
- /deltacat/tests/{io → _io}/test_redis_object_store.py +0 -0
- /deltacat/tests/{io → _io}/test_s3_object_store.py +0 -0
- {deltacat-1.1.36.dist-info → deltacat-2.0.dist-info}/LICENSE +0 -0
- {deltacat-1.1.36.dist-info → deltacat-2.0.dist-info}/WHEEL +0 -0
- {deltacat-1.1.36.dist-info → deltacat-2.0.dist-info}/top_level.txt +0 -0
@@ -15,7 +15,7 @@ from deltacat.exceptions import ValidationError
|
|
15
15
|
|
16
16
|
from deltacat.storage import (
|
17
17
|
DeltaType,
|
18
|
-
|
18
|
+
EntryParams,
|
19
19
|
)
|
20
20
|
|
21
21
|
from deltacat.compute.compactor.model.compactor_version import CompactorVersion
|
@@ -36,15 +36,15 @@ class MultipleRoundsTestCaseParams:
|
|
36
36
|
Args:
|
37
37
|
primary_keys: Set[str] - argument for the primary_keys parameter in compact_partition. Also needed for table/delta creation
|
38
38
|
sort_keys: List[SortKey] - argument for the sort_keys parameter in compact_partition. Also needed for table/delta creation
|
39
|
-
|
40
|
-
|
39
|
+
partition_keys: List[PartitionKey] - argument for the partition_keys parameter. Needed for table/delta creation
|
40
|
+
partition_values: List[Optional[str]] - argument for the partition_valued parameter. Needed for table/delta creation
|
41
41
|
input_deltas: List[pa.Array] - argument required for delta creation during compact_partition test setup. Actual incoming deltas expressed as a PyArrow array (https://arrow.apache.org/docs/python/generated/pyarrow.array.html)
|
42
42
|
expected_terminal_compact_partition_result: pa.Table - expected PyArrow table after compaction (i.e,. the state of the table after applying all row UPDATES/DELETES/INSERTS)
|
43
43
|
expected_terminal_exception: BaseException - expected exception during compaction
|
44
44
|
expected_terminal_exception_message: Optional[str] - expected exception message if present.
|
45
45
|
do_create_placement_group: bool - toggles whether to create a placement group (https://docs.ray.io/en/latest/ray-core/scheduling/placement-group.html) or not
|
46
46
|
records_per_compacted_file: int - argument for the records_per_compacted_file parameter in compact_partition
|
47
|
-
|
47
|
+
hash_bucket_count: int - argument for the hash_bucket_count parameter in compact_partition
|
48
48
|
read_kwargs_provider: Optional[ReadKwargsProvider] - argument for read_kwargs_provider parameter in compact_partition. If None then no ReadKwargsProvider is provided to compact_partition_params
|
49
49
|
drop_duplicates: bool - argument for drop_duplicates parameter in compact_partition. Only recognized by compactor v2.
|
50
50
|
skip_enabled_compact_partition_drivers: List[CompactorVersion] - skip whatever enabled_compact_partition_drivers are included in this list
|
@@ -57,7 +57,7 @@ class MultipleRoundsTestCaseParams:
|
|
57
57
|
sort_keys: List[Optional[SortKey]]
|
58
58
|
partition_keys: Optional[List[PartitionKey]]
|
59
59
|
partition_values: List[Optional[str]]
|
60
|
-
input_deltas: Union[List[pa.Array], DeltaType
|
60
|
+
input_deltas: Union[List[pa.Array], DeltaType]
|
61
61
|
expected_terminal_compact_partition_result: pa.Table
|
62
62
|
expected_terminal_exception: BaseException
|
63
63
|
expected_terminal_exception_message: str
|
@@ -83,8 +83,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
83
83
|
"1-multiple-rounds-sanity": MultipleRoundsTestCaseParams(
|
84
84
|
primary_keys={"pk_col_1"},
|
85
85
|
sort_keys=[
|
86
|
-
SortKey.of(
|
87
|
-
SortKey.of(
|
86
|
+
SortKey.of(key=["sk_col_1"]),
|
87
|
+
SortKey.of(key=["sk_col_2"]),
|
88
88
|
],
|
89
89
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
90
90
|
partition_values=["1"],
|
@@ -177,8 +177,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
177
177
|
"2-multiple-rounds-unique-values": MultipleRoundsTestCaseParams(
|
178
178
|
primary_keys={"pk_col_1"},
|
179
179
|
sort_keys=[
|
180
|
-
SortKey.of(
|
181
|
-
SortKey.of(
|
180
|
+
SortKey.of(key=["sk_col_1"]),
|
181
|
+
SortKey.of(key=["sk_col_2"]),
|
182
182
|
],
|
183
183
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
184
184
|
partition_values=["1"],
|
@@ -270,8 +270,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
270
270
|
"3-num-rounds-greater-than-deltas-count": MultipleRoundsTestCaseParams(
|
271
271
|
primary_keys={"pk_col_1"},
|
272
272
|
sort_keys=[
|
273
|
-
SortKey.of(
|
274
|
-
SortKey.of(
|
273
|
+
SortKey.of(key=["sk_col_1"]),
|
274
|
+
SortKey.of(key=["sk_col_2"]),
|
275
275
|
],
|
276
276
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
277
277
|
partition_values=["1"],
|
@@ -364,8 +364,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
364
364
|
"4-multiple-rounds-hb-count-equals-1": MultipleRoundsTestCaseParams(
|
365
365
|
primary_keys={"pk_col_1"},
|
366
366
|
sort_keys=[
|
367
|
-
SortKey.of(
|
368
|
-
SortKey.of(
|
367
|
+
SortKey.of(key=["sk_col_1"]),
|
368
|
+
SortKey.of(key=["sk_col_2"]),
|
369
369
|
],
|
370
370
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
371
371
|
partition_values=["1"],
|
@@ -458,8 +458,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
458
458
|
"5-multiple-rounds-only-supports-rebase": MultipleRoundsTestCaseParams(
|
459
459
|
primary_keys={"pk_col_1"},
|
460
460
|
sort_keys=[
|
461
|
-
SortKey.of(
|
462
|
-
SortKey.of(
|
461
|
+
SortKey.of(key=["sk_col_1"]),
|
462
|
+
SortKey.of(key=["sk_col_2"]),
|
463
463
|
],
|
464
464
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
465
465
|
partition_values=["1"],
|
@@ -553,8 +553,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
553
553
|
"6-multiple-rounds-test-pgm": MultipleRoundsTestCaseParams(
|
554
554
|
primary_keys={"pk_col_1"},
|
555
555
|
sort_keys=[
|
556
|
-
SortKey.of(
|
557
|
-
SortKey.of(
|
556
|
+
SortKey.of(key=["sk_col_1"]),
|
557
|
+
SortKey.of(key=["sk_col_2"]),
|
558
558
|
],
|
559
559
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
560
560
|
partition_values=["1"],
|
@@ -690,7 +690,7 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
690
690
|
names=["pk_col_1", "col_1"],
|
691
691
|
),
|
692
692
|
DeltaType.DELETE,
|
693
|
-
|
693
|
+
EntryParams.of(equality_field_locators=["pk_col_1", "col_1"]),
|
694
694
|
),
|
695
695
|
],
|
696
696
|
rebase_expected_compact_partition_result=pa.Table.from_arrays(
|
@@ -758,7 +758,7 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
758
758
|
names=["pk_col_1", "col_1"],
|
759
759
|
),
|
760
760
|
DeltaType.DELETE,
|
761
|
-
|
761
|
+
EntryParams.of(equality_field_locators=["pk_col_1", "col_1"]),
|
762
762
|
),
|
763
763
|
(
|
764
764
|
pa.Table.from_arrays(
|
@@ -766,7 +766,7 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
766
766
|
names=["pk_col_1", "col_1"],
|
767
767
|
),
|
768
768
|
DeltaType.DELETE,
|
769
|
-
|
769
|
+
EntryParams.of(equality_field_locators=["pk_col_1", "col_1"]),
|
770
770
|
),
|
771
771
|
(
|
772
772
|
pa.Table.from_arrays(
|
@@ -897,7 +897,7 @@ MULTIPLE_ROUNDS_TEST_CASES = {
|
|
897
897
|
names=["pk_col_1", "col_1"],
|
898
898
|
),
|
899
899
|
DeltaType.DELETE,
|
900
|
-
|
900
|
+
EntryParams.of(["pk_col_1", "col_1"]),
|
901
901
|
),
|
902
902
|
],
|
903
903
|
rebase_expected_compact_partition_result=pa.Table.from_arrays(
|
@@ -41,8 +41,8 @@ REBASE_TEST_CASES = {
|
|
41
41
|
"1-rebase-sanity": RebaseCompactionTestCaseParams(
|
42
42
|
primary_keys={"pk_col_1"},
|
43
43
|
sort_keys=[
|
44
|
-
SortKey.of(
|
45
|
-
SortKey.of(
|
44
|
+
SortKey.of(key=["sk_col_1"]),
|
45
|
+
SortKey.of(key=["sk_col_2"]),
|
46
46
|
],
|
47
47
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
48
48
|
partition_values=["1"],
|
@@ -87,8 +87,8 @@ REBASE_TEST_CASES = {
|
|
87
87
|
"2-rebase-with-null-pk": RebaseCompactionTestCaseParams(
|
88
88
|
primary_keys={"pk_col_1"},
|
89
89
|
sort_keys=[
|
90
|
-
SortKey.of(
|
91
|
-
SortKey.of(
|
90
|
+
SortKey.of(key=["sk_col_1"]),
|
91
|
+
SortKey.of(key=["sk_col_2"]),
|
92
92
|
],
|
93
93
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
94
94
|
partition_values=["1"],
|
@@ -133,7 +133,7 @@ REBASE_TEST_CASES = {
|
|
133
133
|
"3-rebase-with-null-two-pk": RebaseCompactionTestCaseParams(
|
134
134
|
primary_keys={"pk_col_1", "pk_col_2"},
|
135
135
|
sort_keys=[
|
136
|
-
SortKey.of(
|
136
|
+
SortKey.of(key=["sk_col_1"]),
|
137
137
|
],
|
138
138
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
139
139
|
partition_values=["1"],
|
@@ -307,7 +307,7 @@ REBASE_TEST_CASES = {
|
|
307
307
|
"7-rebase-drop-duplicates-false": RebaseCompactionTestCaseParams(
|
308
308
|
primary_keys={"pk_col_1"},
|
309
309
|
sort_keys=[
|
310
|
-
SortKey.of(
|
310
|
+
SortKey.of(key=["sk_col_1"]),
|
311
311
|
],
|
312
312
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
313
313
|
partition_values=["1"],
|
@@ -17,6 +17,7 @@ from dataclasses import dataclass
|
|
17
17
|
|
18
18
|
from deltacat.storage import (
|
19
19
|
DeltaType,
|
20
|
+
EntryParams,
|
20
21
|
)
|
21
22
|
from deltacat.types.media import ContentType
|
22
23
|
|
@@ -38,7 +39,6 @@ from deltacat.tests.compute.compact_partition_test_cases import (
|
|
38
39
|
ZERO_VALUED_PRIMARY_KEY,
|
39
40
|
EMPTY_UTSV_PATH,
|
40
41
|
)
|
41
|
-
from deltacat.storage import DeleteParameters
|
42
42
|
from deltacat.exceptions import ValidationError
|
43
43
|
|
44
44
|
|
@@ -53,7 +53,7 @@ class RebaseThenIncrementalCompactionTestCaseParams(BaseCompactorTestCase):
|
|
53
53
|
rebase_expected_compact_partition_result: pa.Table - expected table after rebase compaction runs. An output that is asserted on in Rebase then Incremental unit tests
|
54
54
|
"""
|
55
55
|
|
56
|
-
incremental_deltas: List[Tuple[pa.Table, DeltaType, Optional[
|
56
|
+
incremental_deltas: List[Tuple[pa.Table, DeltaType, Optional[EntryParams]]]
|
57
57
|
rebase_expected_compact_partition_result: pa.Table
|
58
58
|
|
59
59
|
|
@@ -61,8 +61,8 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
61
61
|
"1-rebase-then-incremental-sanity": RebaseThenIncrementalCompactionTestCaseParams(
|
62
62
|
primary_keys={"pk_col_1"},
|
63
63
|
sort_keys=[
|
64
|
-
SortKey.of(
|
65
|
-
SortKey.of(
|
64
|
+
SortKey.of(key=["sk_col_1"]),
|
65
|
+
SortKey.of(key=["sk_col_2"]),
|
66
66
|
],
|
67
67
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
68
68
|
partition_values=["1"],
|
@@ -122,7 +122,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
122
122
|
"2-rebase-then-incremental-pk-multi": RebaseThenIncrementalCompactionTestCaseParams(
|
123
123
|
primary_keys={"pk_col_1", "pk_col_2"},
|
124
124
|
sort_keys=[
|
125
|
-
SortKey.of(
|
125
|
+
SortKey.of(key=["sk_col_1"]),
|
126
126
|
],
|
127
127
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
128
128
|
partition_values=["1"],
|
@@ -303,7 +303,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
303
303
|
"5-rebase-then-incremental-partial-deltas-on-incremental-deltas-2": RebaseThenIncrementalCompactionTestCaseParams(
|
304
304
|
primary_keys={"pk_col_1"},
|
305
305
|
sort_keys=[
|
306
|
-
SortKey.of(
|
306
|
+
SortKey.of(key=["sk_col_1"]),
|
307
307
|
],
|
308
308
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
309
309
|
partition_values=["1"],
|
@@ -359,8 +359,8 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
359
359
|
"6-rebase-then-incremental-hash-bucket-GT-records-per-compacted-file-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
|
360
360
|
primary_keys={"pk_col_1"},
|
361
361
|
sort_keys=[
|
362
|
-
SortKey.of(
|
363
|
-
SortKey.of(
|
362
|
+
SortKey.of(key=["sk_col_1"]),
|
363
|
+
SortKey.of(key=["sk_col_2"]),
|
364
364
|
],
|
365
365
|
partition_keys=[PartitionKey.of("day", PartitionKeyType.TIMESTAMP)],
|
366
366
|
partition_values=["2022-01-01T00:00:00.000Z"],
|
@@ -420,7 +420,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
420
420
|
"7-rebase-then-incremental-no-pk-compactor-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
|
421
421
|
primary_keys=ZERO_VALUED_PRIMARY_KEY,
|
422
422
|
sort_keys=[
|
423
|
-
SortKey.of(
|
423
|
+
SortKey.of(key=["sk_col_1"]),
|
424
424
|
],
|
425
425
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
426
426
|
partition_values=["1"],
|
@@ -529,8 +529,8 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
529
529
|
"9-rebase-then-incremental-single-hash-bucket": RebaseThenIncrementalCompactionTestCaseParams(
|
530
530
|
primary_keys={"pk_col_1"},
|
531
531
|
sort_keys=[
|
532
|
-
SortKey.of(
|
533
|
-
SortKey.of(
|
532
|
+
SortKey.of(key=["sk_col_1"]),
|
533
|
+
SortKey.of(key=["sk_col_2"]),
|
534
534
|
],
|
535
535
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
536
536
|
partition_values=["1"],
|
@@ -590,7 +590,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
590
590
|
"10-rebase-then-incremental-drop-duplicates-false-on-incremental-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
|
591
591
|
primary_keys={"pk_col_1"},
|
592
592
|
sort_keys=[
|
593
|
-
SortKey.of(
|
593
|
+
SortKey.of(key=["sk_col_1"]),
|
594
594
|
],
|
595
595
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
596
596
|
partition_values=["1"],
|
@@ -646,8 +646,8 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
646
646
|
"11-rebase-then-empty-incremental-delta": RebaseThenIncrementalCompactionTestCaseParams(
|
647
647
|
primary_keys={"pk_col_1"},
|
648
648
|
sort_keys=[
|
649
|
-
SortKey.of(
|
650
|
-
SortKey.of(
|
649
|
+
SortKey.of(key=["sk_col_1"]),
|
650
|
+
SortKey.of(key=["sk_col_2"]),
|
651
651
|
],
|
652
652
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
653
653
|
partition_values=["1"],
|
@@ -693,8 +693,8 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
693
693
|
"12-rebase-then-incremental-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
|
694
694
|
primary_keys={"pk_col_1"},
|
695
695
|
sort_keys=[
|
696
|
-
SortKey.of(
|
697
|
-
SortKey.of(
|
696
|
+
SortKey.of(key=["sk_col_1"]),
|
697
|
+
SortKey.of(key=["sk_col_2"]),
|
698
698
|
],
|
699
699
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
700
700
|
partition_values=["1"],
|
@@ -754,8 +754,8 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
754
754
|
"13-rebase-then-empty-incremental-delta-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
|
755
755
|
primary_keys={"pk_col_1"},
|
756
756
|
sort_keys=[
|
757
|
-
SortKey.of(
|
758
|
-
SortKey.of(
|
757
|
+
SortKey.of(key=["sk_col_1"]),
|
758
|
+
SortKey.of(key=["sk_col_2"]),
|
759
759
|
],
|
760
760
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
761
761
|
partition_values=["1"],
|
@@ -801,8 +801,8 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
|
|
801
801
|
"14-rebase-then-incremental-with-null-pk": RebaseThenIncrementalCompactionTestCaseParams(
|
802
802
|
primary_keys={"pk_col_1"},
|
803
803
|
sort_keys=[
|
804
|
-
SortKey.of(
|
805
|
-
SortKey.of(
|
804
|
+
SortKey.of(key=["sk_col_1"]),
|
805
|
+
SortKey.of(key=["sk_col_2"]),
|
806
806
|
],
|
807
807
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
808
808
|
partition_values=["1"],
|
@@ -900,7 +900,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
900
900
|
names=["pk_col_1", "col_1"],
|
901
901
|
),
|
902
902
|
DeltaType.DELETE,
|
903
|
-
|
903
|
+
EntryParams.of(["pk_col_1", "col_1"]),
|
904
904
|
),
|
905
905
|
(
|
906
906
|
pa.Table.from_arrays(
|
@@ -908,7 +908,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
908
908
|
names=["col_1"],
|
909
909
|
),
|
910
910
|
DeltaType.DELETE,
|
911
|
-
|
911
|
+
EntryParams.of(["col_1"]),
|
912
912
|
),
|
913
913
|
(
|
914
914
|
pa.Table.from_arrays(
|
@@ -916,7 +916,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
916
916
|
names=["col_1"],
|
917
917
|
),
|
918
918
|
DeltaType.DELETE,
|
919
|
-
|
919
|
+
EntryParams.of(["col_1"]),
|
920
920
|
),
|
921
921
|
(
|
922
922
|
pa.Table.from_arrays(
|
@@ -924,7 +924,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
924
924
|
names=["col_1"],
|
925
925
|
),
|
926
926
|
DeltaType.DELETE,
|
927
|
-
|
927
|
+
EntryParams.of(["col_1"]),
|
928
928
|
),
|
929
929
|
(
|
930
930
|
pa.Table.from_arrays(
|
@@ -932,7 +932,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
932
932
|
names=["pk_col_1", "col_1"],
|
933
933
|
),
|
934
934
|
DeltaType.DELETE,
|
935
|
-
|
935
|
+
EntryParams.of(["pk_col_1", "col_1"]),
|
936
936
|
),
|
937
937
|
(
|
938
938
|
pa.Table.from_arrays(
|
@@ -940,7 +940,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
940
940
|
names=["col_1"],
|
941
941
|
),
|
942
942
|
DeltaType.DELETE,
|
943
|
-
|
943
|
+
EntryParams.of(["col_1"]),
|
944
944
|
),
|
945
945
|
],
|
946
946
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -991,7 +991,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
991
991
|
names=["col_1"],
|
992
992
|
),
|
993
993
|
DeltaType.DELETE,
|
994
|
-
|
994
|
+
EntryParams.of(["col_1"]),
|
995
995
|
)
|
996
996
|
],
|
997
997
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1043,7 +1043,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1043
1043
|
names=["col_1"],
|
1044
1044
|
),
|
1045
1045
|
DeltaType.DELETE,
|
1046
|
-
|
1046
|
+
EntryParams.of(["col_1"]),
|
1047
1047
|
)
|
1048
1048
|
],
|
1049
1049
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1099,7 +1099,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1099
1099
|
names=["col_1"],
|
1100
1100
|
),
|
1101
1101
|
DeltaType.DELETE,
|
1102
|
-
|
1102
|
+
EntryParams.of(["col_1"]),
|
1103
1103
|
)
|
1104
1104
|
],
|
1105
1105
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1186,7 +1186,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1186
1186
|
names=["col_1"],
|
1187
1187
|
),
|
1188
1188
|
DeltaType.DELETE,
|
1189
|
-
|
1189
|
+
EntryParams.of(["col_1"]),
|
1190
1190
|
),
|
1191
1191
|
],
|
1192
1192
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1257,7 +1257,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1257
1257
|
names=["col_1"],
|
1258
1258
|
),
|
1259
1259
|
DeltaType.DELETE,
|
1260
|
-
|
1260
|
+
EntryParams.of(["col_1"]),
|
1261
1261
|
),
|
1262
1262
|
],
|
1263
1263
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1317,7 +1317,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1317
1317
|
names=["col_1"],
|
1318
1318
|
),
|
1319
1319
|
DeltaType.DELETE,
|
1320
|
-
|
1320
|
+
EntryParams.of(["col_1"]),
|
1321
1321
|
),
|
1322
1322
|
],
|
1323
1323
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1382,7 +1382,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1382
1382
|
names=["col_1"],
|
1383
1383
|
),
|
1384
1384
|
DeltaType.DELETE,
|
1385
|
-
|
1385
|
+
EntryParams.of(["col_1"]),
|
1386
1386
|
),
|
1387
1387
|
(
|
1388
1388
|
pa.Table.from_arrays(
|
@@ -1393,7 +1393,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1393
1393
|
names=["pk_col_1", "col_1"],
|
1394
1394
|
),
|
1395
1395
|
DeltaType.DELETE,
|
1396
|
-
|
1396
|
+
EntryParams.of(["col_1"]),
|
1397
1397
|
),
|
1398
1398
|
(
|
1399
1399
|
pa.Table.from_arrays(
|
@@ -1414,7 +1414,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1414
1414
|
names=["col_1"],
|
1415
1415
|
),
|
1416
1416
|
DeltaType.DELETE,
|
1417
|
-
|
1417
|
+
EntryParams.of(["col_1"]),
|
1418
1418
|
),
|
1419
1419
|
],
|
1420
1420
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1474,7 +1474,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1474
1474
|
names=["col_1"],
|
1475
1475
|
),
|
1476
1476
|
DeltaType.DELETE,
|
1477
|
-
|
1477
|
+
EntryParams.of(["col_1"]),
|
1478
1478
|
),
|
1479
1479
|
],
|
1480
1480
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1539,7 +1539,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1539
1539
|
names=["col_1"],
|
1540
1540
|
),
|
1541
1541
|
DeltaType.DELETE,
|
1542
|
-
|
1542
|
+
EntryParams.of(["col_1"]),
|
1543
1543
|
),
|
1544
1544
|
(
|
1545
1545
|
pa.Table.from_arrays(
|
@@ -1681,7 +1681,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1681
1681
|
),
|
1682
1682
|
),
|
1683
1683
|
DeltaType.DELETE,
|
1684
|
-
|
1684
|
+
EntryParams.of(["col_1"]),
|
1685
1685
|
),
|
1686
1686
|
],
|
1687
1687
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1741,7 +1741,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1741
1741
|
names=["col_1"],
|
1742
1742
|
),
|
1743
1743
|
DeltaType.DELETE,
|
1744
|
-
|
1744
|
+
EntryParams.of(["col_1"]),
|
1745
1745
|
),
|
1746
1746
|
],
|
1747
1747
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1790,7 +1790,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1790
1790
|
names=["col_1"],
|
1791
1791
|
),
|
1792
1792
|
DeltaType.DELETE,
|
1793
|
-
|
1793
|
+
EntryParams.of(["col_1"]),
|
1794
1794
|
),
|
1795
1795
|
(
|
1796
1796
|
pa.Table.from_arrays(
|
@@ -1800,7 +1800,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1800
1800
|
names=["pk_col_1"],
|
1801
1801
|
),
|
1802
1802
|
DeltaType.DELETE,
|
1803
|
-
|
1803
|
+
EntryParams.of(["pk_col_1"]),
|
1804
1804
|
),
|
1805
1805
|
(
|
1806
1806
|
pa.Table.from_arrays(
|
@@ -1834,8 +1834,8 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1834
1834
|
"28-rebase-then-incremental-delete-type-delta-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
|
1835
1835
|
primary_keys={"pk_col_1"},
|
1836
1836
|
sort_keys=[
|
1837
|
-
SortKey.of(
|
1838
|
-
SortKey.of(
|
1837
|
+
SortKey.of(key=["sk_col_1"]),
|
1838
|
+
SortKey.of(key=["sk_col_2"]),
|
1839
1839
|
],
|
1840
1840
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
1841
1841
|
partition_values=["1"],
|
@@ -1880,7 +1880,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1880
1880
|
names=["col_1"],
|
1881
1881
|
),
|
1882
1882
|
DeltaType.DELETE,
|
1883
|
-
|
1883
|
+
EntryParams.of(["col_1"]),
|
1884
1884
|
),
|
1885
1885
|
(
|
1886
1886
|
pa.Table.from_arrays(
|
@@ -1890,7 +1890,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1890
1890
|
names=["col_1"],
|
1891
1891
|
),
|
1892
1892
|
DeltaType.DELETE,
|
1893
|
-
|
1893
|
+
EntryParams.of(["col_1"]),
|
1894
1894
|
),
|
1895
1895
|
(
|
1896
1896
|
pa.Table.from_arrays(
|
@@ -1900,7 +1900,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1900
1900
|
names=["sk_col_1"],
|
1901
1901
|
),
|
1902
1902
|
DeltaType.DELETE,
|
1903
|
-
|
1903
|
+
EntryParams.of(["sk_col_1"]),
|
1904
1904
|
),
|
1905
1905
|
],
|
1906
1906
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -1925,7 +1925,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1925
1925
|
"29-rebase-then-incremental-delete-type-delta-no-pk-compactor": RebaseThenIncrementalCompactionTestCaseParams(
|
1926
1926
|
primary_keys=ZERO_VALUED_PRIMARY_KEY,
|
1927
1927
|
sort_keys=[
|
1928
|
-
SortKey.of(
|
1928
|
+
SortKey.of(key=["sk_col_1"]),
|
1929
1929
|
],
|
1930
1930
|
partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
|
1931
1931
|
partition_values=["1"],
|
@@ -1953,7 +1953,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1953
1953
|
names=["col_1"],
|
1954
1954
|
),
|
1955
1955
|
DeltaType.DELETE,
|
1956
|
-
|
1956
|
+
EntryParams.of(["col_1"]),
|
1957
1957
|
),
|
1958
1958
|
(
|
1959
1959
|
pa.Table.from_arrays(
|
@@ -1974,7 +1974,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
1974
1974
|
names=["sk_col_1"],
|
1975
1975
|
),
|
1976
1976
|
DeltaType.DELETE,
|
1977
|
-
|
1977
|
+
EntryParams.of(["sk_col_1"]),
|
1978
1978
|
),
|
1979
1979
|
],
|
1980
1980
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -2024,7 +2024,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
2024
2024
|
names=["pk_col_1", "col_1"],
|
2025
2025
|
),
|
2026
2026
|
DeltaType.DELETE,
|
2027
|
-
|
2027
|
+
EntryParams.of(["pk_col_1", "col_1"]),
|
2028
2028
|
),
|
2029
2029
|
],
|
2030
2030
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|
@@ -2082,7 +2082,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
2082
2082
|
names=["pk_col_1", "col_1"],
|
2083
2083
|
),
|
2084
2084
|
DeltaType.DELETE,
|
2085
|
-
|
2085
|
+
EntryParams.of(["pk_col_1", "col_1"]),
|
2086
2086
|
),
|
2087
2087
|
(
|
2088
2088
|
pa.Table.from_arrays(
|
@@ -2090,7 +2090,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
2090
2090
|
names=["pk_col_1"],
|
2091
2091
|
),
|
2092
2092
|
DeltaType.DELETE,
|
2093
|
-
|
2093
|
+
EntryParams.of(["pk_col_1"]),
|
2094
2094
|
),
|
2095
2095
|
(
|
2096
2096
|
pa.Table.from_arrays(
|
@@ -2098,7 +2098,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
2098
2098
|
names=["col_1"],
|
2099
2099
|
),
|
2100
2100
|
DeltaType.DELETE,
|
2101
|
-
|
2101
|
+
EntryParams.of(["col_1"]),
|
2102
2102
|
),
|
2103
2103
|
(
|
2104
2104
|
pa.Table.from_arrays(
|
@@ -2106,7 +2106,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
2106
2106
|
names=["col_1"],
|
2107
2107
|
),
|
2108
2108
|
DeltaType.DELETE,
|
2109
|
-
|
2109
|
+
EntryParams.of(["col_1"]),
|
2110
2110
|
),
|
2111
2111
|
(
|
2112
2112
|
pa.Table.from_arrays(
|
@@ -2114,7 +2114,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
2114
2114
|
names=["pk_col_1", "col_1"],
|
2115
2115
|
),
|
2116
2116
|
DeltaType.DELETE,
|
2117
|
-
|
2117
|
+
EntryParams.of(["pk_col_1", "col_1"]),
|
2118
2118
|
),
|
2119
2119
|
(
|
2120
2120
|
pa.Table.from_arrays(
|
@@ -2122,7 +2122,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
|
|
2122
2122
|
names=["col_1"],
|
2123
2123
|
),
|
2124
2124
|
DeltaType.DELETE,
|
2125
|
-
|
2125
|
+
EntryParams.of(["col_1"]),
|
2126
2126
|
),
|
2127
2127
|
],
|
2128
2128
|
expected_terminal_compact_partition_result=pa.Table.from_arrays(
|