deltacat 1.1.12__py3-none-any.whl → 1.1.14__py3-none-any.whl

deltacat/compute/compactor_v2/utils/task_options.py

@@ -145,7 +145,6 @@ def hash_bucket_resource_options_provider(
     size_bytes = 0.0
     num_rows = 0
     total_pk_size = 0
-
     if not item.manifest or not item.manifest.entries:
         logger.debug(
             f"[Hash bucket task {index}]: No manifest entries, skipping memory allocation calculation"

deltacat/tests/compute/compact_partition_rebase_test_cases.py

@@ -82,6 +82,7 @@ REBASE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
     ),
 }
 

deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py

@@ -4,6 +4,8 @@ from deltacat.tests.compute.test_util_common import (
     offer_iso8601_timestamp_list,
     PartitionKey,
     PartitionKeyType,
+    assert_compaction_audit,
+    assert_compaction_audit_no_hash_bucket,
 )
 from deltacat.tests.compute.test_util_constant import (
     DEFAULT_MAX_RECORDS_PER_FILE,
@@ -115,6 +117,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "2-rebase-then-incremental-pk-multi": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
@@ -195,6 +198,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "3-rebase-then-incremental-no-sk-no-partition-key": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -244,6 +248,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "4-rebase-then-incremental-partial-deltas-on-incremental-deltas": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -293,6 +298,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "5-rebase-then-incremental-partial-deltas-on-incremental-deltas-2": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -348,6 +354,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "6-rebase-then-incremental-hash-bucket-GT-records-per-compacted-file-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -408,6 +415,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "7-rebase-then-incremental-no-pk-compactor-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys=ZERO_VALUED_PRIMARY_KEY,
@@ -459,6 +467,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "8-rebase-then-incremental-empty-csv-delta-case": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -515,6 +524,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=None,
     ),
     "9-rebase-then-incremental-single-hash-bucket": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -575,6 +585,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=None,
     ),
     "10-rebase-then-incremental-drop-duplicates-false-on-incremental-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -630,6 +641,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=False,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "11-rebase-then-empty-incremental-delta": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -676,6 +688,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "12-rebase-then-incremental-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -736,6 +749,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=None,
     ),
     "13-rebase-then-empty-incremental-delta-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -782,6 +796,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=None,
     ),
 }
 
@@ -882,6 +897,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "15-rebase-then-incremental-delete-type-delta-on-incremental-multi-pk": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
@@ -933,6 +949,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "16-rebase-then-incremental-delete-type-delta-on-incremental-multi-pk-delete-all": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
@@ -990,6 +1007,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "17-rebase-then-incremental-delete-type-delta-delete-entire-base-table": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1043,6 +1061,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "18-rebase-then-incremental-delete-type-delta-keep-base-table-drop-all-incremental": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1124,6 +1143,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "19-rebase-then-incremental-delete-type-delta-drop-only-from-base-table-keep-all-incremental": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1194,6 +1214,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "20-rebase-then-incremental-delete-type-delta-drop-all-base-table-drop-all-incremental": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1258,6 +1279,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "21-rebase-then-incremental-delete-type-delta-UDDUUDD": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1349,6 +1371,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "22-rebase-then-incremental-delete-type-delta-UD-affects-compacted-and-incremental": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1413,6 +1436,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "23-rebase-then-incremental-delete-type-delta-UDU-upsert-again": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1483,6 +1507,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "24-rebase-then-incremental-delete-type-no-delete-column-has-delete-deltas-expected-exception": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1547,6 +1572,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "25-rebase-then-incremental-delete-type-delta-has-delete-column-no-delete-records": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1612,6 +1638,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "26-rebase-then-incremental-delete-type-delta-UDU-duplicate-delete-records": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1671,6 +1698,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "27-rebase-then-incremental-delete-type-delta-DDU-deletes-then-upserts": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1740,6 +1768,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "28-rebase-then-incremental-delete-type-delta-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1830,6 +1859,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
     ),
     "29-rebase-then-incremental-delete-type-delta-no-pk-compactor": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys=ZERO_VALUED_PRIMARY_KEY,
@@ -1901,6 +1931,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "30-rebase-then-incremental-delete-type-delta-on-incremental-compactor-v1-v2": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1950,6 +1981,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
 }
 

deltacat/tests/compute/compact_partition_test_cases.py

@@ -4,6 +4,8 @@ from deltacat.tests.compute.test_util_common import (
     offer_iso8601_timestamp_list,
     PartitionKey,
     PartitionKeyType,
+    assert_compaction_audit,
+    assert_compaction_audit_no_hash_bucket,
 )
 from deltacat.tests.compute.test_util_constant import (
     DEFAULT_MAX_RECORDS_PER_FILE,
@@ -64,6 +66,7 @@ class BaseCompactorTestCase:
         read_kwargs_provider: Optional[ReadKwargsProvider] - argument for read_kwargs_provider parameter in compact_partition. If None then no ReadKwargsProvider is provided to compact_partition_params
         drop_duplicates: bool - argument for drop_duplicates parameter in compact_partition. Only recognized by compactor v2.
         skip_enabled_compact_partition_drivers: List[CompactorVersion] - skip whatever enabled_compact_partition_drivers are included in this list
+        assert_compaction_audit: Optional[Callable] - argument that asserts compaction_audit is updated only if compactor_version is v2.
     """
 
     primary_keys: Set[str]
@@ -81,6 +84,7 @@ class BaseCompactorTestCase:
     read_kwargs_provider: Optional[ReadKwargsProvider]
     drop_duplicates: bool
     skip_enabled_compact_partition_drivers: List[CompactorVersion]
+    assert_compaction_audit: Optional[Callable]
 
     # makes CompactorTestCase iterable which is required to build the list of pytest.param values to pass to pytest.mark.parametrize
     def __iter__(self):
@@ -127,8 +131,8 @@ def with_compactor_version_func_test_param(
             enriched_test_cases[f"{tc_name}_{compactor_version}"] = [
                 *tc_params,
                 compact_partition_func,
+                compactor_version,
             ]
-
     return enriched_test_cases
 
 
@@ -157,6 +161,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "2-incremental-pkstr-skstr-norcf": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -185,6 +190,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "3-incremental-pkstr-multiskstr-norcf": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -222,6 +228,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "4-incremental-duplicate-pk": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -258,6 +265,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "5-incremental-decimal-pk-simple": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -289,6 +297,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "6-incremental-integer-pk-simple": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -320,6 +329,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "7-incremental-timestamp-pk-simple": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -351,6 +361,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "8-incremental-decimal-timestamp-pk-multi": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
@@ -384,6 +395,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "9-incremental-decimal-pk-multi-dup": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -415,6 +427,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "10-incremental-decimal-pk-partitionless": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -446,6 +459,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "11-incremental-decimal-hash-bucket-single": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -477,6 +491,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "12-incremental-decimal-single-hash-bucket": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -508,6 +523,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "13-incremental-pkstr-skexists-isinplacecompacted": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -551,6 +567,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             )
         ],
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
     ),
     "14-incremental-pkstr-skexists-unhappy-hash-bucket-count-not-present": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -582,6 +599,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=False,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
     ),
 }
 

deltacat/tests/compute/test_compact_partition_incremental.py

@@ -13,6 +13,7 @@ from deltacat.types.media import StorageType
 from deltacat.tests.compute.test_util_common import (
     get_rcf,
 )
+from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 from deltacat.tests.test_utils.utils import read_s3_contents
 from deltacat.tests.compute.test_util_create_table_deltas_repo import (
     create_src_w_deltas_destination_plus_destination,
@@ -136,9 +137,11 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "read_kwargs_provider_param",
         "drop_duplicates_param",
         "skip_enabled_compact_partition_drivers",
+        "assert_compaction_audit",
         "is_inplace",
         "add_late_deltas",
         "compact_partition_func",
+        "compactor_version",
     ],
     [
         (
@@ -158,9 +161,11 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             drop_duplicates_param,
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
+            assert_compaction_audit,
             is_inplace,
             add_late_deltas,
             compact_partition_func,
+            compactor_version,
         )
         for test_name, (
             primary_keys,
@@ -178,9 +183,11 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             drop_duplicates_param,
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
+            assert_compaction_audit,
             is_inplace,
             add_late_deltas,
             compact_partition_func,
+            compactor_version,
         ) in INCREMENTAL_TEST_CASES.items()
     ],
     ids=[test_name for test_name in INCREMENTAL_TEST_CASES],
@@ -204,6 +211,8 @@ def test_compact_partition_incremental(
     drop_duplicates_param: bool,
     read_kwargs_provider_param: Any,
     skip_enabled_compact_partition_drivers,
+    assert_compaction_audit: Optional[Callable],
+    compactor_version: Optional[CompactorVersion],
     is_inplace: bool,
     add_late_deltas: Optional[List[Tuple[pa.Table, DeltaType]]],
     compact_partition_func: Callable,
@@ -339,6 +348,10 @@ def test_compact_partition_incremental(
         input_deltas
     ), "The input_records must be equal to total records in the input"
 
+    if assert_compaction_audit is not None:
+        if not assert_compaction_audit(compactor_version, compaction_audit):
+            assert False, "Compaction audit assertion failed"
+
     assert actual_compacted_table.equals(
         expected_terminal_compact_partition_result
     ), f"{actual_compacted_table} does not match {expected_terminal_compact_partition_result}"

deltacat/tests/compute/test_compact_partition_rebase.py

@@ -13,10 +13,17 @@ from deltacat.tests.compute.test_util_constant import (
     DEFAULT_NUM_WORKERS,
     DEFAULT_WORKER_INSTANCE_CPUS,
 )
+from deltacat.tests.compute.test_util_common import (
+    get_rcf,
+)
+from deltacat.tests.test_utils.utils import read_s3_contents
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 from deltacat.tests.compute.test_util_common import (
     get_compacted_delta_locator_from_rcf,
 )
+from deltacat.compute.compactor.model.compaction_session_audit_info import (
+    CompactionSessionAuditInfo,
+)
 from deltacat.tests.compute.test_util_create_table_deltas_repo import (
     create_src_w_deltas_destination_rebase_w_deltas_strategy,
 )
@@ -33,6 +40,9 @@ from deltacat.types.media import ContentType
 from deltacat.compute.compactor.model.compact_partition_params import (
     CompactPartitionParams,
 )
+from deltacat.compute.compactor import (
+    RoundCompletionInfo,
+)
 from deltacat.utils.placement import (
     PlacementGroupManager,
 )
@@ -121,8 +131,10 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "read_kwargs_provider_param",
         "drop_duplicates_param",
         "skip_enabled_compact_partition_drivers",
+        "assert_compaction_audit",
         "rebase_expected_compact_partition_result",
         "compact_partition_func",
+        "compactor_version",
     ],
     [
         (
@@ -142,8 +154,10 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             drop_duplicates_param,
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
+            assert_compaction_audit,
             rebase_expected_compact_partition_result,
             compact_partition_func,
+            compactor_version,
         )
         for test_name, (
             primary_keys,
@@ -161,8 +175,10 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             drop_duplicates_param,
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
+            assert_compaction_audit,
             rebase_expected_compact_partition_result,
             compact_partition_func,
+            compactor_version,
         ) in REBASE_TEST_CASES.items()
     ],
     ids=[test_name for test_name in REBASE_TEST_CASES],
@@ -188,6 +204,8 @@ def test_compact_partition_rebase_same_source_and_destination(
     read_kwargs_provider_param: Any,
     rebase_expected_compact_partition_result: pa.Table,
     skip_enabled_compact_partition_drivers: List[CompactorVersion],
+    assert_compaction_audit: Optional[Callable],
+    compactor_version: Optional[CompactorVersion],
     compact_partition_func: Callable,
     benchmark: BenchmarkFixture,
 ):
@@ -263,6 +281,18 @@ def test_compact_partition_rebase_same_source_and_destination(
     # execute
     rcf_file_s3_uri = compact_partition_func(compact_partition_params)
 
+    round_completion_info: RoundCompletionInfo = get_rcf(s3_resource, rcf_file_s3_uri)
+    audit_bucket, audit_key = RoundCompletionInfo.get_audit_bucket_name_and_key(
+        round_completion_info.compaction_audit_url
+    )
+
+    compaction_audit_obj: Dict[str, Any] = read_s3_contents(
+        s3_resource, audit_bucket, audit_key
+    )
+    compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
+        **compaction_audit_obj
+    )
+
     # Assert not in-place compacted
     assert (
         execute_compaction_result_spy.call_args.args[-1] is False
@@ -287,3 +317,7 @@ def test_compact_partition_rebase_same_source_and_destination(
     assert actual_rebase_compacted_table.equals(
         rebase_expected_compact_partition_result
     ), f"{actual_rebase_compacted_table} does not match {rebase_expected_compact_partition_result}"
+
+    if assert_compaction_audit is not None:
+        if not assert_compaction_audit(compactor_version, compaction_audit):
+            assert False, "Compaction audit assertion failed"

deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -136,9 +136,11 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "read_kwargs_provider_param",
         "drop_duplicates_param",
         "skip_enabled_compact_partition_drivers",
+        "assert_compaction_audit",
         "incremental_deltas",
         "rebase_expected_compact_partition_result",
         "compact_partition_func",
+        "compactor_version",
     ],
     [
         (
@@ -158,9 +160,11 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             drop_duplicates_param,
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
+            assert_compaction_audit,
             incremental_deltas,
             rebase_expected_compact_partition_result,
             compact_partition_func,
+            compactor_version,
         )
         for test_name, (
             primary_keys,
@@ -178,9 +182,11 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             drop_duplicates_param,
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
+            assert_compaction_audit,
             incremental_deltas,
             rebase_expected_compact_partition_result,
             compact_partition_func,
+            compactor_version,
         ) in REBASE_THEN_INCREMENTAL_TEST_CASES.items()
     ],
     ids=[test_name for test_name in REBASE_THEN_INCREMENTAL_TEST_CASES],
@@ -206,6 +212,8 @@ def test_compact_partition_rebase_then_incremental(
     incremental_deltas: List[Tuple[pa.Table, DeltaType, Optional[Dict[str, str]]]],
     rebase_expected_compact_partition_result: pa.Table,
     skip_enabled_compact_partition_drivers: List[CompactorVersion],
+    assert_compaction_audit: Optional[Callable],
+    compactor_version: Optional[CompactorVersion],
     compact_partition_func: Callable,
     benchmark: BenchmarkFixture,
 ):
@@ -381,4 +389,8 @@ def test_compact_partition_rebase_then_incremental(
     assert actual_compacted_table.equals(
         expected_terminal_compact_partition_result
     ), f"{actual_compacted_table} does not match {expected_terminal_compact_partition_result}"
+
+    if assert_compaction_audit is not None:
+        if not assert_compaction_audit(compactor_version, compaction_audit):
+            assert False, "Compaction audit assertion failed"
     return