deltacat 1.1.11__tar.gz → 1.1.13__tar.gz

{deltacat-1.1.11/deltacat.egg-info → deltacat-1.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.11
+Version: 1.1.13
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.11 → deltacat-1.1.13}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.11"
+__version__ = "1.1.13"
 
 
 __all__ = [

{deltacat-1.1.11 → deltacat-1.1.13}/deltacat/aws/clients.py

@@ -42,7 +42,7 @@ RETRYABLE_HTTP_STATUS_CODES = [
 
 boto_retry_wrapper = Retrying(
     wait=wait_random_exponential(multiplier=1, max=10),
-    stop=stop_after_delay(60 * 5),
+    stop=stop_after_delay(60 * 10),
     # CredentialRetrievalError can still be thrown due to throttling, even if IMDS health checks succeed.
     retry=retry_if_exception_type(CredentialRetrievalError),
 )

{deltacat-1.1.11 → deltacat-1.1.13}/deltacat/compute/compactor/model/compact_partition_params.py

@@ -185,6 +185,11 @@ class CompactPartitionParams(dict):
 
     @property
     def task_max_parallelism(self) -> int:
+        if self.pg_config:
+            cluster_resources = self.pg_config.resource
+            cluster_cpus = cluster_resources["CPU"]
+            self.task_max_parallelism = cluster_cpus
+            self["task_max_parallelism"] = self.task_max_parallelism
         return self["task_max_parallelism"]
 
     @task_max_parallelism.setter

deltacat-1.1.13/deltacat/compute/compactor_v2/compaction_session.py

@@ -0,0 +1,220 @@
+import numpy as np
+import importlib
+from contextlib import nullcontext
+import logging
+import time
+import ray
+
+import deltacat
+from deltacat.compute.compactor import (
+    PyArrowWriteResult,
+    RoundCompletionInfo,
+)
+from deltacat import logs
+from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
+    ExecutionCompactionResult,
+)
+from deltacat.compute.compactor.model.compactor_version import CompactorVersion
+from deltacat.compute.compactor.utils import round_completion_file as rcf
+from deltacat.compute.compactor import DeltaAnnotated
+from deltacat.compute.compactor_v2.deletes.delete_strategy import (
+    DeleteStrategy,
+)
+from deltacat.compute.compactor.model.materialize_result import MaterializeResult
+from deltacat.compute.compactor_v2.model.merge_result import MergeResult
+from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
+    DeleteFileEnvelope,
+)
+from deltacat.storage import (
+    Delta,
+    DeltaLocator,
+    Manifest,
+    Partition,
+)
+from deltacat.compute.compactor.model.compact_partition_params import (
+    CompactPartitionParams,
+)
+from deltacat.utils.resources import (
+    get_current_process_peak_memory_usage_in_bytes,
+)
+from deltacat.compute.compactor_v2.private.compaction_utils import (
+    _fetch_compaction_metadata,
+    _build_uniform_deltas,
+    _run_hash_and_merge,
+    _process_merge_results,
+    _upload_compaction_audit,
+    _write_new_round_completion_file,
+    _commit_compaction_result,
+)
+from deltacat.utils.metrics import metrics
+from deltacat.compute.compactor.model.compaction_session_audit_info import (
+    CompactionSessionAuditInfo,
+)
+
+from typing import List, Optional
+from deltacat.compute.compactor_v2.utils import io
+from deltacat.exceptions import categorize_errors
+from deltacat.compute.compactor_v2.constants import COMPACT_PARTITION_METRIC_PREFIX
+
+if importlib.util.find_spec("memray"):
+    import memray
+
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+@metrics(prefix=COMPACT_PARTITION_METRIC_PREFIX)
+@categorize_errors
+def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
+    assert (
+        params.hash_bucket_count is not None and params.hash_bucket_count >= 1
+    ), "hash_bucket_count is a required arg for compactor v2"
+
+    with memray.Tracker(
+        "compaction_partition.bin"
+    ) if params.enable_profiler else nullcontext():
+        execute_compaction_result: ExecutionCompactionResult = _execute_compaction(
+            params,
+            **kwargs,
+        )
+        _commit_compaction_result(params, execute_compaction_result)
+        return execute_compaction_result.round_completion_file_s3_url
+
+
+def _execute_compaction(
+    params: CompactPartitionParams, **kwargs
+) -> ExecutionCompactionResult:
+    compaction_start_time: float = time.monotonic()
+    # Fetch round completion info for previously compacted partition, if it exists
+    fetch_compaction_metadata_result: tuple[
+        Optional[Manifest], Optional[RoundCompletionInfo]
+    ] = _fetch_compaction_metadata(params)
+    (
+        previous_compacted_delta_manifest,
+        round_completion_info,
+    ) = fetch_compaction_metadata_result
+    rcf_source_partition_locator: rcf.PartitionLocator = (
+        params.rebase_source_partition_locator or params.source_partition_locator
+    )
+
+    base_audit_url: str = rcf_source_partition_locator.path(
+        f"s3://{params.compaction_artifact_s3_bucket}/compaction-audit"
+    )
+    audit_url: str = f"{base_audit_url}.json"
+    logger.info(f"Compaction audit will be written to {audit_url}")
+    compaction_audit: CompactionSessionAuditInfo = (
+        CompactionSessionAuditInfo(deltacat.__version__, ray.__version__, audit_url)
+        .set_hash_bucket_count(params.hash_bucket_count)
+        .set_compactor_version(CompactorVersion.V2.value)
+    )
+
+    if params.pg_config:
+        logger.info(
+            "pg_config specified. Tasks will be scheduled in a placement group."
+        )
+        cluster_resources = params.pg_config.resource
+        cluster_memory = cluster_resources["memory"]
+        compaction_audit.set_total_cluster_memory_bytes(cluster_memory)
+    high_watermark = (
+        round_completion_info.high_watermark if round_completion_info else None
+    )
+    audit_url = compaction_audit.audit_url if compaction_audit else None
+    # discover and build uniform deltas
+    delta_discovery_start = time.monotonic()
+    input_deltas: List[Delta] = io.discover_deltas(
+        params.source_partition_locator,
+        params.last_stream_position_to_compact,
+        params.rebase_source_partition_locator,
+        params.rebase_source_partition_high_watermark,
+        high_watermark,
+        params.deltacat_storage,
+        params.deltacat_storage_kwargs,
+        params.list_deltas_kwargs,
+    )
+    if not input_deltas:
+        logger.info("No input deltas found to compact.")
+        return ExecutionCompactionResult(None, None, None, False)
+    build_uniform_deltas_result: tuple[
+        List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope], Partition
+    ] = _build_uniform_deltas(
+        params, compaction_audit, input_deltas, delta_discovery_start
+    )
+    (
+        uniform_deltas,
+        delete_strategy,
+        delete_file_envelopes,
+    ) = build_uniform_deltas_result
+
+    # run merge
+    _run_hash_and_merge_result: tuple[
+        Optional[List[MergeResult]],
+        np.float64,
+        np.float64,
+        Partition,
+    ] = _run_hash_and_merge(
+        params,
+        uniform_deltas,
+        round_completion_info,
+        delete_strategy,
+        delete_file_envelopes,
+        compaction_audit,
+        previous_compacted_delta_manifest,
+    )
+    (
+        merge_results,
+        telemetry_time_hb,
+        telemetry_time_merge,
+        compacted_partition,
+    ) = _run_hash_and_merge_result
+    # process merge results
+    process_merge_results: tuple[
+        Delta, list[MaterializeResult], dict
+    ] = _process_merge_results(params, merge_results, compaction_audit)
+    merged_delta, mat_results, hb_id_to_entry_indices_range = process_merge_results
+    # Record information, logging, and return ExecutionCompactionResult
+    record_info_msg: str = f" Materialized records: {merged_delta.meta.record_count}"
+    logger.info(record_info_msg)
+    compacted_delta: Delta = params.deltacat_storage.commit_delta(
+        merged_delta,
+        properties=kwargs.get("properties", {}),
+        **params.deltacat_storage_kwargs,
+    )
+
+    logger.info(f"Committed compacted delta: {compacted_delta}")
+    compaction_end_time: float = time.monotonic()
+    compaction_audit.set_compaction_time_in_seconds(
+        compaction_end_time - compaction_start_time
+    )
+    new_compacted_delta_locator: DeltaLocator = DeltaLocator.of(
+        compacted_partition.locator,
+        compacted_delta.stream_position,
+    )
+    pyarrow_write_result: PyArrowWriteResult = PyArrowWriteResult.union(
+        [m.pyarrow_write_result for m in mat_results]
+    )
+
+    session_peak_memory = get_current_process_peak_memory_usage_in_bytes()
+    compaction_audit.set_peak_memory_used_bytes_by_compaction_session_process(
+        session_peak_memory
+    )
+
+    compaction_audit.save_round_completion_stats(
+        mat_results, telemetry_time_hb + telemetry_time_merge
+    )
+
+    _upload_compaction_audit(
+        params,
+        compaction_audit,
+        round_completion_info,
+    )
+    compaction_result: ExecutionCompactionResult = _write_new_round_completion_file(
+        params,
+        compaction_audit,
+        compacted_partition,
+        audit_url,
+        hb_id_to_entry_indices_range,
+        rcf_source_partition_locator,
+        new_compacted_delta_locator,
+        pyarrow_write_result,
+    )
+    return compaction_result

{deltacat-1.1.11 → deltacat-1.1.13}/deltacat/compute/compactor_v2/utils/task_options.py

@@ -145,7 +145,6 @@ def hash_bucket_resource_options_provider(
     size_bytes = 0.0
     num_rows = 0
     total_pk_size = 0
-
     if not item.manifest or not item.manifest.entries:
         logger.debug(
             f"[Hash bucket task {index}]: No manifest entries, skipping memory allocation calculation"

{deltacat-1.1.11 → deltacat-1.1.13}/deltacat/tests/compute/compact_partition_rebase_test_cases.py

@@ -82,6 +82,7 @@ REBASE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
     ),
 }
 

{deltacat-1.1.11 → deltacat-1.1.13}/deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py

@@ -4,6 +4,8 @@ from deltacat.tests.compute.test_util_common import (
     offer_iso8601_timestamp_list,
     PartitionKey,
     PartitionKeyType,
+    assert_compaction_audit,
+    assert_compaction_audit_no_hash_bucket,
 )
 from deltacat.tests.compute.test_util_constant import (
     DEFAULT_MAX_RECORDS_PER_FILE,
@@ -115,6 +117,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "2-rebase-then-incremental-pk-multi": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
@@ -195,6 +198,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "3-rebase-then-incremental-no-sk-no-partition-key": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -244,6 +248,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "4-rebase-then-incremental-partial-deltas-on-incremental-deltas": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -293,6 +298,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "5-rebase-then-incremental-partial-deltas-on-incremental-deltas-2": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -348,6 +354,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "6-rebase-then-incremental-hash-bucket-GT-records-per-compacted-file-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -408,6 +415,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "7-rebase-then-incremental-no-pk-compactor-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys=ZERO_VALUED_PRIMARY_KEY,
@@ -459,6 +467,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "8-rebase-then-incremental-empty-csv-delta-case": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -515,6 +524,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=None,
     ),
     "9-rebase-then-incremental-single-hash-bucket": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -575,6 +585,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=None,
     ),
     "10-rebase-then-incremental-drop-duplicates-false-on-incremental-v2-only": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -630,6 +641,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=False,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "11-rebase-then-empty-incremental-delta": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -676,6 +688,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "12-rebase-then-incremental-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -736,6 +749,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=None,
     ),
     "13-rebase-then-empty-incremental-delta-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -782,6 +796,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=None,
     ),
 }
 
@@ -882,6 +897,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "15-rebase-then-incremental-delete-type-delta-on-incremental-multi-pk": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
@@ -933,6 +949,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "16-rebase-then-incremental-delete-type-delta-on-incremental-multi-pk-delete-all": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
@@ -990,6 +1007,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "17-rebase-then-incremental-delete-type-delta-delete-entire-base-table": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1043,6 +1061,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "18-rebase-then-incremental-delete-type-delta-keep-base-table-drop-all-incremental": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1124,6 +1143,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "19-rebase-then-incremental-delete-type-delta-drop-only-from-base-table-keep-all-incremental": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1194,6 +1214,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "20-rebase-then-incremental-delete-type-delta-drop-all-base-table-drop-all-incremental": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1258,6 +1279,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "21-rebase-then-incremental-delete-type-delta-UDDUUDD": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1349,6 +1371,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "22-rebase-then-incremental-delete-type-delta-UD-affects-compacted-and-incremental": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1413,6 +1436,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "23-rebase-then-incremental-delete-type-delta-UDU-upsert-again": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1483,6 +1507,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "24-rebase-then-incremental-delete-type-no-delete-column-has-delete-deltas-expected-exception": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1547,6 +1572,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "25-rebase-then-incremental-delete-type-delta-has-delete-column-no-delete-records": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1612,6 +1638,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "26-rebase-then-incremental-delete-type-delta-UDU-duplicate-delete-records": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1671,6 +1698,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "27-rebase-then-incremental-delete-type-delta-DDU-deletes-then-upserts": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1740,6 +1768,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "28-rebase-then-incremental-delete-type-delta-hash-bucket-single": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1830,6 +1859,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
     ),
     "29-rebase-then-incremental-delete-type-delta-no-pk-compactor": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys=ZERO_VALUED_PRIMARY_KEY,
@@ -1901,6 +1931,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "30-rebase-then-incremental-delete-type-delta-on-incremental-compactor-v1-v2": RebaseThenIncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -1950,6 +1981,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
 }
 

{deltacat-1.1.11 → deltacat-1.1.13}/deltacat/tests/compute/compact_partition_test_cases.py

@@ -4,6 +4,8 @@ from deltacat.tests.compute.test_util_common import (
     offer_iso8601_timestamp_list,
     PartitionKey,
     PartitionKeyType,
+    assert_compaction_audit,
+    assert_compaction_audit_no_hash_bucket,
 )
 from deltacat.tests.compute.test_util_constant import (
     DEFAULT_MAX_RECORDS_PER_FILE,
@@ -64,6 +66,7 @@ class BaseCompactorTestCase:
         read_kwargs_provider: Optional[ReadKwargsProvider] - argument for read_kwargs_provider parameter in compact_partition. If None then no ReadKwargsProvider is provided to compact_partition_params
         drop_duplicates: bool - argument for drop_duplicates parameter in compact_partition. Only recognized by compactor v2.
         skip_enabled_compact_partition_drivers: List[CompactorVersion] - skip whatever enabled_compact_partition_drivers are included in this list
+        assert_compaction_audit: Optional[Callable] - argument that asserts compaction_audit is updated only if compactor_version is v2.
     """
 
     primary_keys: Set[str]
@@ -81,6 +84,7 @@ class BaseCompactorTestCase:
     read_kwargs_provider: Optional[ReadKwargsProvider]
     drop_duplicates: bool
     skip_enabled_compact_partition_drivers: List[CompactorVersion]
+    assert_compaction_audit: Optional[Callable]
 
     # makes CompactorTestCase iterable which is required to build the list of pytest.param values to pass to pytest.mark.parametrize
     def __iter__(self):
@@ -127,8 +131,8 @@ def with_compactor_version_func_test_param(
             enriched_test_cases[f"{tc_name}_{compactor_version}"] = [
                 *tc_params,
                 compact_partition_func,
+                compactor_version,
             ]
-
     return enriched_test_cases
 
 
@@ -157,6 +161,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "2-incremental-pkstr-skstr-norcf": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -185,6 +190,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "3-incremental-pkstr-multiskstr-norcf": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -222,6 +228,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "4-incremental-duplicate-pk": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -258,6 +265,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "5-incremental-decimal-pk-simple": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -289,6 +297,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "6-incremental-integer-pk-simple": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -320,6 +329,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "7-incremental-timestamp-pk-simple": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -351,6 +361,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "8-incremental-decimal-timestamp-pk-multi": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
@@ -384,6 +395,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "9-incremental-decimal-pk-multi-dup": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -415,6 +427,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "10-incremental-decimal-pk-partitionless": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -446,6 +459,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "11-incremental-decimal-hash-bucket-single": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -477,6 +491,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
     ),
     "12-incremental-decimal-single-hash-bucket": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -508,6 +523,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
     "13-incremental-pkstr-skexists-isinplacecompacted": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -551,6 +567,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             )
         ],
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
     ),
     "14-incremental-pkstr-skexists-unhappy-hash-bucket-count-not-present": IncrementalCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
@@ -582,6 +599,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         is_inplace=False,
         add_late_deltas=False,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
     ),
 }