deltacat 1.1.17__py3-none-any.whl → 1.1.18__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.17"
+__version__ = "1.1.18"
 
 
 __all__ = [

deltacat/aws/constants.py

@@ -1,7 +1,6 @@
 import botocore
 from typing import Set
 from daft.exceptions import DaftTransientError
-
 from deltacat.utils.common import env_integer, env_string
 
 

deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -27,8 +27,11 @@ def _append_sha1_hash_to_table(table: pa.Table, hash_column: pa.Array) -> pa.Tab
 
     result = []
     for hash_value in hash_column_np:
-        assert hash_value is not None, f"Expected non-null primary key"
-        result.append(hashlib.sha1(hash_value.encode("utf-8")).hexdigest())
+        if hash_value is None:
+            result.append(None)
+            logger.info("A primary key hash is null")
+        else:
+            result.append(hashlib.sha1(hash_value.encode("utf-8")).hexdigest())
 
     return sc.append_pk_hash_string_column(table, result)
 
@@ -191,7 +194,7 @@ def generate_pk_hash_column(
             pk_columns.append(sliced_string_cast(table[pk_name]))
 
         pk_columns.append(PK_DELIMITER)
-        hash_column = pc.binary_join_element_wise(*pk_columns)
+        hash_column = pc.binary_join_element_wise(*pk_columns, null_handling="replace")
         return hash_column
 
     def _generate_uuid(table: pa.Table) -> pa.Array:
@@ -345,8 +348,10 @@ def hash_group_index_to_hash_bucket_indices(
     return range(hb_group, num_buckets, num_groups)
 
 
-def pk_digest_to_hash_bucket_index(digest: str, num_buckets: int) -> int:
+def pk_digest_to_hash_bucket_index(digest: Optional[str], num_buckets: int) -> int:
     """
     Generates the hash bucket index from the given digest.
     """
+    if digest is None:
+        return 0
     return int(digest, 16) % num_buckets

deltacat/exceptions.py

@@ -299,7 +299,7 @@ def _categorize_tenacity_error(e: tenacity.RetryError):
 def _categorize_dependency_pyarrow_error(e: ArrowException):
     if isinstance(e, ArrowInvalid):
         raise DependencyPyarrowInvalidError(
-            f"Pyarrow Invalid error occurred. Reason: {e}"
+            f"Pyarrow Invalid error occurred. {e}"
         ) from e
     elif isinstance(e, ArrowCapacityError):
         raise DependencyPyarrowCapacityError("Pyarrow Capacity error occurred.") from e
@@ -308,9 +308,7 @@ def _categorize_dependency_pyarrow_error(e: ArrowException):
 
 
 def _categorize_assertion_error(e: BaseException):
-    raise ValidationError(
-        f"One of the assertions in DeltaCAT has failed. Reason: {e}"
-    ) from e
+    raise ValidationError(f"One of the assertions in DeltaCAT has failed. {e}") from e
 
 
 def _categorize_daft_error(e: DaftCoreException):

deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py

@@ -848,6 +848,83 @@ MULTIPLE_ROUNDS_TEST_CASES = {
         assert_compaction_audit=None,
         num_rounds=3,
     ),
+    # 4 input deltas (3 upsert, 1 delete delta), 2 rounds requested
+    # Expect to see a table that aggregates 10 records total
+    # (12 upserts - 2 deletes (null PK) = 10 records)
+    # (dropDuplicates = False)
+    "9-multiple-rounds-delete-deltas-with-null-pk": MultipleRoundsTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=ZERO_VALUED_SORT_KEY,
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=[
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([None, 11, 12, 13]),
+                        pa.array(["a", "b", "c", "d"]),
+                    ],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            ),
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([14, 15, 16, 17]),
+                        pa.array(["e", "f", "g", "h"]),
+                    ],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            ),
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([18, 19, 20, 21]),
+                        pa.array(["i", "j", "k", "l"]),
+                    ],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array([None, 11]), pa.array(["a", "b"])],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["pk_col_1", "col_1"]),
+            ),
+        ],
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(12, 22)]),
+                pa.array(["c", "d", "e", "f", "g", "h", "i", "j", "k", "l"]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(12, 22)]),
+                pa.array(["c", "d", "e", "f", "g", "h", "i", "j", "k", "l"]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=False,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+        num_rounds=2,
+    ),
 }
 
 MULTIPLE_ROUNDS_TEST_CASES = with_compactor_version_func_test_param(

deltacat/tests/compute/compact_partition_rebase_test_cases.py

@@ -84,6 +84,314 @@ REBASE_TEST_CASES = {
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
         assert_compaction_audit=None,
     ),
+    "2-rebase-with-null-pk": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[
+            SortKey.of(key_name="sk_col_1"),
+            SortKey.of(key_name="sk_col_2"),
+        ],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1]),
+                pa.array([1, 2, 3, 4, 5, 6]),
+                pa.array(["foo"] * 6),
+                pa.array([5, 6, 7, 8, 9, 10]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([None, 1, 2]),
+                pa.array([5, 6, 4]),
+                pa.array(["foo"] * 3),
+                pa.array([9, 10, 8]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([None, 1, 2]),
+                pa.array([5, 6, 4]),
+                pa.array(["foo"] * 3),
+                pa.array([7, 10, 8]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "3-rebase-with-null-two-pk": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2"},
+        sort_keys=[
+            SortKey.of(key_name="sk_col_1"),
+        ],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5]),
+                pa.array([1, None, 3, None, None, 1, 5]),
+                pa.array(["foo"] * 7),
+                pa.array([5, 6, 7, 8, 9, 10, 11]),
+            ],
+            names=["pk_col_1", "pk_col_2", "sk_col_1", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None]),
+                pa.array([1, None, 3, 5, None]),
+                pa.array(["foo"] * 5),
+                pa.array([10, 8, 7, 11, 9]),
+            ],
+            names=["pk_col_1", "pk_col_2", "sk_col_1", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None]),
+                pa.array([1, None, 3, 5, None]),
+                pa.array(["foo"] * 5),
+                pa.array([10, 8, 7, 11, 9]),
+            ],
+            names=["pk_col_1", "pk_col_2", "sk_col_1", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "4-rebase-with-null-multiple-pk-different-types": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2", "pk_col_3"},
+        sort_keys=[],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None, None]),
+                pa.array([1, None, 3, 5, None, None]),
+                pa.array(["a", "b", "c", "g", "e", None]),
+                pa.array([10, 8, 7, 11, 12, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None, None]),
+                pa.array([1, None, 3, 5, None, None]),
+                pa.array(["a", "b", "c", "g", "e", None]),
+                pa.array([10, 8, 7, 11, 12, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "5-rebase-with-null-multiple-pk-one-hash-bucket": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2", "pk_col_3"},
+        sort_keys=[],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None, None]),
+                pa.array([1, None, 3, 5, None, None]),
+                pa.array(["a", "b", "c", "g", "e", None]),
+                pa.array([10, 8, 7, 11, 12, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None, None]),
+                pa.array([1, None, 3, 5, None, None]),
+                pa.array(["a", "b", "c", "g", "e", None]),
+                pa.array([10, 8, 7, 11, 12, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=1,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "6-rebase-with-null-multiple-pk-drop-duplicates-false": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2", "pk_col_3"},
+        sort_keys=[],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=False,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "7-rebase-drop-duplicates-false": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[
+            SortKey.of(key_name="sk_col_1"),
+        ],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, 2, 3, 3, 1]),
+                pa.array([1, 2, 3, 4, 5, 6]),
+                pa.array(["a", "b", "c", "b", "e", "a"]),
+                pa.array([5, 6, 7, 8, 9, 10]),
+            ],
+            names=["pk_col_1", "sk_col_1", "col_1", "col_2"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, 2, 3, 3, 1]),
+                pa.array([1, 2, 3, 4, 5, 6]),
+                pa.array(["a", "b", "c", "b", "e", "a"]),
+                pa.array([5, 6, 7, 8, 9, 10]),
+            ],
+            names=["pk_col_1", "sk_col_1", "col_1", "col_2"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, 2, 3, 3, 1]),
+                pa.array([1, 2, 3, 4, 5, 6]),
+                pa.array(["a", "b", "c", "b", "e", "a"]),
+                pa.array([5, 6, 7, 8, 9, 10]),
+            ],
+            names=["pk_col_1", "sk_col_1", "col_1", "col_2"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=False,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "8-rebase-with-with-null-pk-duplicates-false-hash-bucket-1": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2", "pk_col_3"},
+        sort_keys=[],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=1,
+        hash_bucket_count=1,
+        read_kwargs_provider=None,
+        drop_duplicates=False,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
 }
 
 REBASE_TEST_CASES = with_compactor_version_func_test_param(REBASE_TEST_CASES)

deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py

@@ -798,6 +798,67 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         skip_enabled_compact_partition_drivers=None,
         assert_compaction_audit=None,
     ),
+    "14-rebase-then-incremental-with-null-pk": RebaseThenIncrementalCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[
+            SortKey.of(key_name="sk_col_1"),
+            SortKey.of(key_name="sk_col_2"),
+        ],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(9)] + [None]),
+                pa.array([i for i in range(0, 10)]),
+                pa.array(["foo"] * 10),
+                pa.array([i / 10 for i in range(10, 20)]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(9)] + [None]),
+                pa.array([i for i in range(0, 10)]),
+                pa.array(["foo"] * 10),
+                pa.array([i / 10 for i in range(10, 20)]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        incremental_deltas=[
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([str(i) for i in range(9)] + [None]),
+                        pa.array([i for i in range(20, 30)]),
+                        pa.array(["foo"] * 10),
+                        pa.array([i / 10 for i in range(40, 50)]),
+                    ],
+                    names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            )
+        ],
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(9)] + [None]),
+                pa.array([i for i in range(20, 30)]),
+                pa.array(["foo"] * 10),
+                pa.array([i / 10 for i in range(40, 50)]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
+    ),
 }
 
 REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
@@ -1983,6 +2044,104 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         skip_enabled_compact_partition_drivers=None,
         assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
+    "31-rebase-then-incremental-delete-delta-on-incremental-null-pk-delete-null": RebaseThenIncrementalCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=ZERO_VALUED_SORT_KEY,
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(11)] + [None]),
+                pa.array([str(i) for i in range(0, 12)]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(11)] + [None]),
+                pa.array([str(i) for i in range(0, 12)]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        incremental_deltas=[
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([10, 11, None, 13]),
+                        pa.array(["a", "b", "c", "d"]),
+                    ],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array([10, 11]), pa.array(["a", "b"])],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["pk_col_1", "col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array([None])],  # Support deleting null PK records
+                    names=["pk_col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["pk_col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array(["c"])],
+                    names=["col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array(["c"])],
+                    names=["col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array([10, 11]), pa.array(["a", "b"])],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["pk_col_1", "col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array(["c"])],
+                    names=["col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["col_1"]),
+            ),
+        ],
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(10)] + [13]),
+                pa.array([str(i) for i in range(0, 10)] + ["d"]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
+    ),
 }
 
 REBASE_THEN_INCREMENTAL_TEST_CASES = with_compactor_version_func_test_param(

deltacat/tests/compute/test_compact_partition_rebase.py

@@ -249,6 +249,7 @@ def test_compact_partition_rebase_same_source_and_destination(
         pgm = PlacementGroupManager(
             1, total_cpus, worker_instance_cpu, memory_per_bundle=4000000
         ).pgs[0]
+    last_stream_position_to_compact = source_partition.stream_position
     compact_partition_params = CompactPartitionParams.of(
         {
             "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
@@ -258,17 +259,19 @@ def test_compact_partition_rebase_same_source_and_destination(
             "deltacat_storage_kwargs": ds_mock_kwargs,
             "destination_partition_locator": rebased_partition.locator,
             "hash_bucket_count": hash_bucket_count_param,
-            "last_stream_position_to_compact": source_partition.stream_position,
+            "last_stream_position_to_compact": last_stream_position_to_compact,
             "list_deltas_kwargs": {**ds_mock_kwargs, **{"equivalent_table_types": []}},
             "object_store": RayPlasmaObjectStore(),
             "pg_config": pgm,
             "primary_keys": primary_keys,
             "read_kwargs_provider": read_kwargs_provider_param,
             "rebase_source_partition_locator": source_partition.locator,
+            "rebase_source_partition_high_watermark": rebased_partition.stream_position,
             "records_per_compacted_file": records_per_compacted_file_param,
             "s3_client_kwargs": {},
             "source_partition_locator": rebased_partition.locator,
             "sort_keys": sort_keys if sort_keys else None,
+            "drop_duplicates": drop_duplicates_param,
         }
     )
 
@@ -300,14 +303,20 @@ def test_compact_partition_rebase_same_source_and_destination(
     compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
         s3_resource, rcf_file_s3_uri
     )
+    assert (
+        compacted_delta_locator.stream_position == last_stream_position_to_compact
+    ), "Compacted delta locator must be equal to last stream position"
     tables = ds.download_delta(
         compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
     )
     actual_rebase_compacted_table = pa.concat_tables(tables)
     # if no primary key is specified then sort by sort_key for consistent assertion
-    sorting_cols: List[Any] = (
-        [(val, "ascending") for val in primary_keys] if primary_keys else sort_keys
-    )
+    sorting_cols: List[Any] = []
+    if primary_keys:
+        sorting_cols.extend([(val, "ascending") for val in primary_keys])
+    if sort_keys:
+        sorting_cols.extend(sort_keys)
+
     rebase_expected_compact_partition_result = (
         rebase_expected_compact_partition_result.combine_chunks().sort_by(sorting_cols)
     )

deltacat/tests/compute/test_util_create_table_deltas_repo.py

@@ -193,8 +193,10 @@ def create_src_w_deltas_destination_rebase_w_deltas_strategy(
     ds_mock_kwargs: Optional[Dict[str, Any]],
 ) -> Tuple[Stream, Stream, Optional[Stream]]:
     import deltacat.tests.local_deltacat_storage as ds
-    from deltacat.storage import Partition, Stream
+    from deltacat.storage import Delta
+    from deltacat.utils.common import current_time_ms
 
+    last_stream_position = current_time_ms()
     source_namespace, source_table_name, source_table_version = create_src_table(
         primary_keys, sort_keys, partition_keys, ds_mock_kwargs
     )
@@ -208,10 +210,12 @@ def create_src_w_deltas_destination_rebase_w_deltas_strategy(
     staged_partition: Partition = ds.stage_partition(
         source_table_stream, partition_values, **ds_mock_kwargs
     )
+    staged_delta: Delta = ds.stage_delta(
+        input_deltas, staged_partition, input_delta_type, **ds_mock_kwargs
+    )
+    staged_delta.locator.stream_position = last_stream_position
     ds.commit_delta(
-        ds.stage_delta(
-            input_deltas, staged_partition, input_delta_type, **ds_mock_kwargs
-        ),
+        staged_delta,
         **ds_mock_kwargs,
     )
     ds.commit_partition(staged_partition, **ds_mock_kwargs)
@@ -244,8 +248,12 @@ def create_src_w_deltas_destination_rebase_w_deltas_strategy(
     staged_partition: Partition = ds.stage_partition(
         rebasing_table_stream, partition_values, **ds_mock_kwargs
     )
+    staged_delta: Delta = ds.stage_delta(
+        input_deltas, staged_partition, **ds_mock_kwargs
+    )
+    staged_delta.locator.stream_position = last_stream_position
     ds.commit_delta(
-        ds.stage_delta(input_deltas, staged_partition, **ds_mock_kwargs),
+        staged_delta,
         **ds_mock_kwargs,
     )
     ds.commit_partition(staged_partition, **ds_mock_kwargs)

{deltacat-1.1.17.dist-info → deltacat-1.1.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.17
+Version: 1.1.18
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.17.dist-info → deltacat-1.1.18.dist-info}/RECORD

@@ -1,10 +1,10 @@
-deltacat/__init__.py,sha256=x9MIXX9uM_Gf7hvnQuf0dQwXgyXztDLmGSbM1F27e10,1778
+deltacat/__init__.py,sha256=-PrYkT-sQtWxMQFFAp4vobJJ8-dq-3EA5LjPiH5sFFQ,1778
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
-deltacat/exceptions.py,sha256=yWM4RXK7uRrQc1VgJv6Lv2UiNZWAx2wolLq7cBwjlkg,12770
+deltacat/exceptions.py,sha256=7sjk3BuMY5Oo-6OvAfHncZx_OcvtEL47BblWr2F7waE,12740
 deltacat/logs.py,sha256=_UAc_6GiQR3mxccys32Cp2CZOKOVZ9L-AkNUAlzepns,9091
 deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/clients.py,sha256=4eQvpkV1PzFfxog7EriuglOGGwNFHR5hbGYpjsNNPxk,6949
-deltacat/aws/constants.py,sha256=1HnDXrSokW-G3YA3qKEiv7fZVntDs1uSk6a7On-VG5k,1223
+deltacat/aws/constants.py,sha256=hcYAUot4ahq9GXCMClQiuYCtiDs5XaOebdUoKg4V84k,1222
 deltacat/aws/s3u.py,sha256=IdT0XqDXVOkPdo5Em5u3qAkV1UXFpXaE1rTkUDKv4f4,28578
 deltacat/aws/redshift/__init__.py,sha256=7SvjG-dqox8zZUhFicTsUvpG5vXYDl_QQ3ohlHOgTKc,342
 deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -76,7 +76,7 @@ deltacat/compute/compactor_v2/utils/dedupe.py,sha256=62tFCY2iRP7I3-45GCIYs6_SJsQ
 deltacat/compute/compactor_v2/utils/delta.py,sha256=I7Yvda8NVbpKXG3nM2Ku1utvR2r2OpHvUMqUL2ja3aw,3626
 deltacat/compute/compactor_v2/utils/io.py,sha256=5zwJEW_UHv9ttQ2exJ23ZnExwBQXn1KgN7FDx1MGYv0,5262
 deltacat/compute/compactor_v2/utils/merge.py,sha256=EV_iKhNc3WflgfLW1Q46dXUvyClx8VebWHGtninEfsI,5311
-deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=ghyIifjXtqXgi8lN3lfnVQ2vi8uk_ny0FE7hsQlLjRQ,11538
+deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=QOMwWxGhZ7VWa3oE6InM4thR5pbjmT7ttNXvx_IiKjo,11676
 deltacat/compute/compactor_v2/utils/task_options.py,sha256=VXvoVVUq5re8NiOoyrfz34qSRiOTB0IkxHJlMqKsBmk,14066
 deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
 deltacat/compute/merge_on_read/daft.py,sha256=1oC38u5ig_aTrq7EzyWBo8Ui54rb6yERYMk-vEFbpxM,1400
@@ -145,18 +145,18 @@ deltacat/tests/aws/test_s3u.py,sha256=FsYCH8K8DsDRPOtTp-w1Nu3ATqt4p1mqDo6aVJV-Sb
 deltacat/tests/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/catalog/test_default_catalog_impl.py,sha256=2l5uwmtLlUJ9yH1LDggtj81fa-pHqbE0-VBt6G4Hyc0,3180
 deltacat/tests/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py,sha256=vyqwuKYOb4FtRwC5r1SJf7kcZNYXoiGb-BUrBgr5_Xw,34852
-deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=NfcB1aTq6HeYMFrtooIIoifzLp5U0xFTN6F7Lpk8cYQ,3143
-deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=Kl5A7hoJ0pVOE-rZna_4XcuIjhuzQbJudvnfNYKHgGo,75436
+deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py,sha256=kWyIJQMyF9oBemvgOp3ngGhMpH9zjkznV-67ewELgHQ,37719
+deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=8HVr3EIFYFqNaJoqeCuj9xIBjM4Ch2bx-mJcO4BRrLo,16839
+deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=l_6-pAKOsRY3NbtfHsYmEaJEkq6IJueYuLsjyJxNgz4,81564
 deltacat/tests/compute/compact_partition_test_cases.py,sha256=R9eiKvxCLqcoHjAx3iOogdnXZEO9TvLbRf0wA7bcJN4,26170
 deltacat/tests/compute/test_compact_partition_incremental.py,sha256=Z0hyQGhMZjCaOn1Vk4qUbgDiS7HDhtdNeFQyG1PJhqA,14559
 deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=xhKCurTA29Y78_1eksUVJ0W35zNNZYm40rMpMM9ynvM,11853
 deltacat/tests/compute/test_compact_partition_params.py,sha256=Dm5eLyHo8oGMeO3XBbpj1rZqHtPZ1hAB7z2qvzc4Lxk,8497
-deltacat/tests/compute/test_compact_partition_rebase.py,sha256=p97zJmEoC2t6R12luSkCKjjBl50l4UGzh-IHdiQdpCs,11445
+deltacat/tests/compute/test_compact_partition_rebase.py,sha256=O_IwZ1Xeaff98V1XYOyVD8PoS_EpVXSQcHWz4In8bK4,11889
 deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=CHHfNFEJW8S1We7NE1Gg6EaoKEWnaOMRxWrLyirrahc,14643
 deltacat/tests/compute/test_util_common.py,sha256=oTkTuo6wscVN8hmoQASIKP_DJN-M0um_ySCOcXv9AJA,11699
 deltacat/tests/compute/test_util_constant.py,sha256=4o-W3E7r7jhFl1A3OFLLrdKnwcF46zx4lEIDY8ONJ3c,929
-deltacat/tests/compute/test_util_create_table_deltas_repo.py,sha256=k9lq_3r_kNMzruTSn4JE7yjdBBUT3Lh-l8khSYdYpYs,12945
+deltacat/tests/compute/test_util_create_table_deltas_repo.py,sha256=Q3HJj1fjoe2JwRUOW8KEjbTqPIIoP2o_T3ZGH6SJnCM,13244
 deltacat/tests/compute/compactor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/compute/compactor/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/compute/compactor/steps/test_repartition.py,sha256=0uRguPEKeLSYs746Jv8io-HZMWdyXNcOMBu8GO2mA0M,9305
@@ -220,8 +220,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.17.dist-info/METADATA,sha256=SxYHCGGCCaUP0Yej0eT0BMV3K8lQdOmodQj6VSnALcU,1734
-deltacat-1.1.17.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-deltacat-1.1.17.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.17.dist-info/RECORD,,
+deltacat-1.1.18.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.18.dist-info/METADATA,sha256=aMHqD2bxsU5-IHUUjn47PFrRP01jh1wlrDtOHeHGGaA,1734
+deltacat-1.1.18.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+deltacat-1.1.18.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.18.dist-info/RECORD,,