deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/compactor/compaction_session.py

@@ -5,7 +5,8 @@ import logging
 import ray
 import time
 import json
-from deltacat.aws import s3u as s3_utils
+import posixpath
+from deltacat.utils.filesystem import resolve_path_and_filesystem
 import deltacat
 from deltacat import logs
 import pyarrow as pa
@@ -25,7 +26,7 @@ from deltacat.storage import (
     DeltaLocator,
     Partition,
     PartitionLocator,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat.compute.compactor.model.compact_partition_params import (
     CompactPartitionParams,
@@ -40,7 +41,7 @@ from deltacat.compute.compactor.steps import dedupe as dd
 from deltacat.compute.compactor.steps import hash_bucket as hb
 from deltacat.compute.compactor.steps import materialize as mat
 from deltacat.compute.compactor.utils import io
-from deltacat.compute.compactor.utils import round_completion_file as rcf
+from deltacat.compute.compactor.utils import round_completion_reader as rci
 
 from deltacat.types.media import ContentType
 from deltacat.utils.placement import PlacementGroupConfig
@@ -65,13 +66,37 @@ DEFAULT_DEDUPE_MAX_PARALLELISM_RATIO_ARG: int = 1
 DEFAULT_PROPERTIES_ARG: Dict[str, Any] = {}
 
 
+def _upload_audit_data(url: str, content: str, **kwargs) -> None:
+    """
+    Upload audit data to the specified URL using filesystem-agnostic operations.
+    """
+    try:
+        path, filesystem = resolve_path_and_filesystem(url)
+
+        # Create parent directories if they don't exist
+        parent_dir = posixpath.dirname(path)
+        if parent_dir:
+            try:
+                filesystem.create_dir(parent_dir, recursive=True)
+            except Exception as dir_error:
+                # Directory might already exist, which is fine
+                logger.debug(
+                    f"Directory creation warning for {parent_dir}: {dir_error}"
+                )
+
+        with filesystem.open_output_stream(path) as stream:
+            stream.write(content.encode("utf-8"))
+    except Exception as e:
+        logger.warning(f"Failed to upload audit data to {url}: {e}")
+
+
 def check_preconditions(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     sort_keys: List[SortKey],
     max_records_per_output_file: int,
     new_hash_bucket_count: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> int:
@@ -104,7 +129,7 @@ def compact_partition(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     primary_keys: Set[str],
-    compaction_artifact_s3_bucket: str,
+    compaction_artifact_path: str,
     last_stream_position_to_compact: int,
     *,
     hash_bucket_count: Optional[int] = None,
@@ -123,37 +148,29 @@ def compact_partition(
     metrics_config: Optional[MetricsConfig] = None,
     list_deltas_kwargs: Optional[Dict[str, Any]] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     object_store: Optional[IObjectStore] = RayPlasmaObjectStore(),
-    s3_client_kwargs: Optional[Dict[str, Any]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
-) -> Optional[str]:
+) -> None:
     if deltacat_storage_kwargs is None:
         deltacat_storage_kwargs = {}
     if not importlib.util.find_spec("memray"):
         logger.info(f"memray profiler not available, disabling all profiling")
         enable_profiler = False
 
-    if s3_client_kwargs is None:
-        s3_client_kwargs = {}
-
     # memray official documentation link:
     # https://bloomberg.github.io/memray/getting_started.html
     with memray.Tracker(
         f"compaction_partition.bin"
     ) if enable_profiler else nullcontext():
         partition = None
-        (
-            new_partition,
-            new_rci,
-            new_rcf_partition_locator,
-        ) = _execute_compaction_round(
+        (new_partition, new_rci,) = _execute_compaction_round(
             source_partition_locator,
             destination_partition_locator,
             primary_keys,
-            compaction_artifact_s3_bucket,
+            compaction_artifact_path,
             last_stream_position_to_compact,
             hash_bucket_count,
             sort_keys,
@@ -169,9 +186,8 @@ def compact_partition(
             metrics_config,
             list_deltas_kwargs,
             read_kwargs_provider,
-            s3_table_writer_kwargs,
+            table_writer_kwargs,
             object_store,
-            s3_client_kwargs,
             deltacat_storage,
             deltacat_storage_kwargs,
             **kwargs,
@@ -182,30 +198,23 @@ def compact_partition(
         logger.info(
             f"Partition-{source_partition_locator.partition_values}-> Compaction session data processing completed"
         )
-        round_completion_file_s3_url = None
         if partition:
             logger.info(f"Committing compacted partition to: {partition.locator}")
+            # Set the round completion info on the partition before committing
+            partition.compaction_round_completion_info = new_rci
             partition = deltacat_storage.commit_partition(
-                partition, **deltacat_storage_kwargs
+                partition,
+                **deltacat_storage_kwargs,
             )
             logger.info(f"Committed compacted partition: {partition}")
-
-            round_completion_file_s3_url = rcf.write_round_completion_file(
-                compaction_artifact_s3_bucket,
-                new_rcf_partition_locator,
-                partition.locator,
-                new_rci,
-                **s3_client_kwargs,
-            )
         logger.info(f"Completed compaction session for: {source_partition_locator}")
-        return round_completion_file_s3_url
 
 
 def _execute_compaction_round(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     primary_keys: Set[str],
-    compaction_artifact_s3_bucket: str,
+    compaction_artifact_path: str,
     last_stream_position_to_compact: int,
     hash_bucket_count: Optional[int],
     sort_keys: List[SortKey],
@@ -221,24 +230,25 @@ def _execute_compaction_round(
     metrics_config: Optional[MetricsConfig],
     list_deltas_kwargs: Optional[Dict[str, Any]],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    s3_table_writer_kwargs: Optional[Dict[str, Any]],
+    table_writer_kwargs: Optional[Dict[str, Any]],
     object_store: Optional[IObjectStore],
-    s3_client_kwargs: Optional[Dict[str, Any]],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
-) -> Tuple[Optional[Partition], Optional[RoundCompletionInfo], Optional[str]]:
+) -> Tuple[Optional[Partition], Optional[RoundCompletionInfo]]:
     if deltacat_storage_kwargs is None:
         deltacat_storage_kwargs = {}
-    rcf_source_partition_locator = (
+    rci_source_partition_locator = (
         rebase_source_partition_locator
         if rebase_source_partition_locator
         else source_partition_locator
     )
-    base_audit_url = rcf_source_partition_locator.path(
-        f"s3://{compaction_artifact_s3_bucket}/compaction-audit"
+    # Construct audit URL using filesystem-agnostic path joining
+    audit_url = posixpath.join(
+        compaction_artifact_path,
+        "compaction-audit.json",
+        f"{rci_source_partition_locator.hexdigest()}.json",
     )
-    audit_url = f"{base_audit_url}.json"
 
     logger.info(f"Compaction audit will be written to {audit_url}")
 
@@ -312,11 +322,11 @@ def _execute_compaction_round(
     # read the results from any previously completed compaction round
     round_completion_info = None
     if not rebase_source_partition_locator:
-        round_completion_info = rcf.read_round_completion_file(
-            compaction_artifact_s3_bucket,
-            source_partition_locator,
-            destination_partition_locator,
-            **s3_client_kwargs,
+        round_completion_info = rci.read_round_completion_info(
+            source_partition_locator=source_partition_locator,
+            destination_partition_locator=destination_partition_locator,
+            deltacat_storage=deltacat_storage,
+            deltacat_storage_kwargs=deltacat_storage_kwargs,
         )
         if not round_completion_info:
             logger.info(
@@ -363,15 +373,11 @@ def _execute_compaction_round(
         delta_discovery_end - delta_discovery_start
     )
 
-    s3_utils.upload(
-        compaction_audit.audit_url,
-        str(json.dumps(compaction_audit)),
-        **s3_client_kwargs,
-    )
+    _upload_audit_data(audit_url, json.dumps(compaction_audit))
 
     if not input_deltas:
         logger.info("No input deltas found to compact.")
-        return None, None, None
+        return None, None
 
     # limit the input deltas to fit on this cluster and convert them to
     # annotated deltas of equivalent size for easy parallel distribution
@@ -464,11 +470,7 @@ def _execute_compaction_round(
         hb_end - hb_start,
     )
 
-    s3_utils.upload(
-        compaction_audit.audit_url,
-        str(json.dumps(compaction_audit)),
-        **s3_client_kwargs,
-    )
+    _upload_audit_data(audit_url, json.dumps(compaction_audit))
 
     all_hash_group_idx_to_obj_id = defaultdict(list)
     for hb_result in hb_results:
@@ -485,9 +487,9 @@ def _execute_compaction_round(
     )
 
     compaction_audit.set_input_records(total_hb_record_count.item())
-    # TODO (pdames): when resources are freed during the last round of hash
-    #  bucketing, start running dedupe tasks that read existing dedupe
-    #  output from S3 then wait for hash bucketing to finish before continuing
+    # TODO(pdames): when resources are freed during the last round of hash bucketing,
+    #  start running dedupe tasks that read hash bucket output from storage then
+    #  wait for hash bucketing to finish before continuing
 
     # create a new stream for this round
     compacted_stream_locator = destination_partition_locator.stream_locator
@@ -497,6 +499,7 @@ def _execute_compaction_round(
         compacted_stream_locator.table_version,
         **deltacat_storage_kwargs,
     )
+
     partition = deltacat_storage.stage_partition(
         stream,
         destination_partition_locator.partition_values,
@@ -571,9 +574,9 @@ def _execute_compaction_round(
     logger.info(f"Materialize buckets created: " f"{len(all_mat_buckets_to_obj_id)}")
 
     compaction_audit.set_materialize_buckets(len(all_mat_buckets_to_obj_id))
-    # TODO(pdames): when resources are freed during the last round of deduping
+    # TODO(pdames): when resources are freed during the last round of deduping,
     #  start running materialize tasks that read materialization source file
-    #  tables from S3 then wait for deduping to finish before continuing
+    #  tables from storage then wait for deduping to finish before continuing
 
     # TODO(pdames): balance inputs to materialization tasks to ensure that each
     #  task has an approximately equal amount of input to materialize
@@ -584,11 +587,7 @@ def _execute_compaction_round(
     # parallel step 3:
     # materialize records to keep by index
 
-    s3_utils.upload(
-        compaction_audit.audit_url,
-        str(json.dumps(compaction_audit)),
-        **s3_client_kwargs,
-    )
+    _upload_audit_data(audit_url, json.dumps(compaction_audit))
 
     materialize_start = time.monotonic()
     mat_tasks_pending = invoke_parallel(
@@ -610,7 +609,7 @@ def _execute_compaction_round(
         enable_profiler=enable_profiler,
         metrics_config=metrics_config,
         read_kwargs_provider=read_kwargs_provider,
-        s3_table_writer_kwargs=s3_table_writer_kwargs,
+        table_writer_kwargs=table_writer_kwargs,
         object_store=object_store,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,
@@ -693,11 +692,7 @@ def _execute_compaction_round(
         telemetry_time_hb + telemetry_time_dd + telemetry_time_materialize
     )
 
-    s3_utils.upload(
-        compaction_audit.audit_url,
-        str(json.dumps(compaction_audit)),
-        **s3_client_kwargs,
-    )
+    _upload_audit_data(audit_url, json.dumps(compaction_audit))
 
     new_round_completion_info = RoundCompletionInfo.of(
         last_stream_position_compacted,
@@ -710,6 +705,7 @@ def _execute_compaction_round(
         hash_bucket_count,
         None,
         CompactorVersion.V1.value,
+        prev_source_partition_locator=rci_source_partition_locator,
     )
 
     logger.info(
@@ -721,17 +717,43 @@ def _execute_compaction_round(
     return (
         partition,
         new_round_completion_info,
-        rcf_source_partition_locator,
     )
 
 
 def compact_partition_from_request(
     compact_partition_params: CompactPartitionParams,
     *compact_partition_pos_args,
-) -> Optional[str]:
+) -> None:
     """
     Wrapper for compact_partition that allows for the compact_partition parameters to be
     passed in as a custom dictionary-like CompactPartitionParams object along with any compact_partition positional arguments.
     :param compact_partition_params:
     """
-    return compact_partition(*compact_partition_pos_args, **compact_partition_params)
+    # Extract required positional arguments
+    source_partition_locator = compact_partition_params.source_partition_locator
+    destination_partition_locator = (
+        compact_partition_params.destination_partition_locator
+    )
+    primary_keys = compact_partition_params.primary_keys
+    compaction_artifact_path = compact_partition_params.compaction_artifact_path
+    last_stream_position_to_compact = (
+        compact_partition_params.last_stream_position_to_compact
+    )
+
+    # Create a copy of params without the positional arguments
+    kwargs_params = dict(compact_partition_params)
+    kwargs_params.pop("source_partition_locator", None)
+    kwargs_params.pop("destination_partition_locator", None)
+    kwargs_params.pop("primary_keys", None)
+    kwargs_params.pop("last_stream_position_to_compact", None)
+    # Don't pop compaction_artifact_path as it's a computed property, not stored in the dict
+
+    compact_partition(
+        source_partition_locator,
+        destination_partition_locator,
+        primary_keys,
+        compaction_artifact_path,
+        last_stream_position_to_compact,
+        *compact_partition_pos_args,
+        **kwargs_params,
+    )

deltacat/compute/compactor/model/compact_partition_params.py

@@ -2,17 +2,19 @@ from __future__ import annotations
 import importlib
 import copy
 import json
-from typing import Any, Dict, List, Optional
+import posixpath
+from typing import Any, Dict, List, Optional, Set
 from deltacat.io.object_store import IObjectStore
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.types.media import ContentType
 from deltacat.utils.placement import PlacementGroupConfig
 from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
 from deltacat.storage import (
-    interface as unimplemented_deltacat_storage,
+    metastore,
     PartitionLocator,
     SortKey,
 )
+from deltacat.catalog.model.properties import CatalogProperties
 from deltacat.compute.resource_estimation import (
     ResourceEstimationMethod,
     EstimateResourcesParams,
@@ -52,11 +54,22 @@ class CompactPartitionParams(dict):
         assert (
             params.get("source_partition_locator") is not None
         ), "source_partition_locator is a required arg"
+        assert params.get("catalog") is not None, "catalog is a required arg"
         assert (
-            params.get("compaction_artifact_s3_bucket") is not None
-        ), "compaction_artifact_s3_bucket is a required arg"
+            params.get("all_column_names") is not None
+        ), "all_column_names is a required arg"
 
         result = CompactPartitionParams(params)
+        assert (
+            result.destination_partition_locator.partition_id
+        ), "destination_partition_locator must have a globally unique partition_id"
+        assert (
+            result.source_partition_locator.partition_id
+        ), "source_partition_locator must have a globally unique partition_id"
+        if result.rebase_source_partition_locator:
+            assert (
+                result.rebase_source_partition_locator.partition_id
+            ), "rebase_source_partition_locator must have a globally unique partition_id"
 
         result.records_per_compacted_file = params.get(
             "records_per_compacted_file", MAX_RECORDS_PER_COMPACTED_FILE
@@ -65,15 +78,18 @@ class CompactPartitionParams(dict):
             "compacted_file_content_type", ContentType.PARQUET
         )
         result.object_store = params.get("object_store", RayPlasmaObjectStore())
+        result.table_writer_kwargs = params.get("table_writer_kwargs", {})
 
         result.enable_profiler = params.get("enable_profiler", False)
-        result.deltacat_storage = params.get(
-            "deltacat_storage", unimplemented_deltacat_storage
-        )
-        result.s3_client_kwargs = params.get("s3_client_kwargs", {})
+        result.deltacat_storage = params.get("deltacat_storage", metastore)
+        result.catalog = params.get("catalog")
         result.deltacat_storage_kwargs = params.get("deltacat_storage_kwargs", {})
         result.list_deltas_kwargs = params.get("list_deltas_kwargs", {})
-        result.s3_table_writer_kwargs = params.get("s3_table_writer_kwargs", {})
+        result.all_column_names = params.get("all_column_names")
+
+        # Add catalog to deltacat_storage_kwargs
+        result.deltacat_storage_kwargs["catalog"] = result.catalog
+
         result.bit_width_of_sort_keys = validate_sort_keys(
             result.source_partition_locator,
             result.sort_keys,
@@ -133,6 +149,8 @@ class CompactPartitionParams(dict):
         if result.primary_keys:
             result.primary_keys = sorted(result.primary_keys)
 
+        result.original_fields = params.get("original_fields")
+
         # assertions
         assert (
             result.source_partition_locator.partition_values
@@ -177,21 +195,32 @@ class CompactPartitionParams(dict):
         self["source_partition_locator"] = locator
 
     @property
-    def compaction_artifact_s3_bucket(self) -> str:
-        return self["compaction_artifact_s3_bucket"]
-
-    @compaction_artifact_s3_bucket.setter
-    def compaction_artifact_s3_bucket(self, s3_bucket: str) -> None:
-        self["compaction_artifact_s3_bucket"] = s3_bucket
+    def compaction_artifact_path(self) -> str:
+        """
+        Returns the compaction artifact path based on catalog root.
+        """
+        return posixpath.join(self.catalog.root, "compute", "compactor")
 
     @property
-    def deltacat_storage(self) -> unimplemented_deltacat_storage:
+    def deltacat_storage(self) -> metastore:
         return self["deltacat_storage"]
 
     @deltacat_storage.setter
-    def deltacat_storage(self, storage: unimplemented_deltacat_storage) -> None:
+    def deltacat_storage(self, storage: metastore) -> None:
         self["deltacat_storage"] = storage
 
+    @property
+    def catalog(self) -> CatalogProperties:
+        return self["catalog"]
+
+    @catalog.setter
+    def catalog(self, catalog: CatalogProperties) -> None:
+        self["catalog"] = catalog
+        # Update deltacat_storage_kwargs when catalog is set
+        if "deltacat_storage_kwargs" not in self:
+            self["deltacat_storage_kwargs"] = {}
+        self["deltacat_storage_kwargs"]["catalog"] = catalog
+
     @property
     def object_store(self) -> IObjectStore:
         return self["object_store"]
@@ -286,14 +315,6 @@ class CompactPartitionParams(dict):
     def list_deltas_kwargs(self, kwargs: dict) -> None:
         self["list_deltas_kwargs"] = kwargs
 
-    @property
-    def s3_table_writer_kwargs(self) -> dict:
-        return self["s3_table_writer_kwargs"]
-
-    @s3_table_writer_kwargs.setter
-    def s3_table_writer_kwargs(self, kwargs: dict) -> None:
-        self["s3_table_writer_kwargs"] = kwargs
-
     @property
     def deltacat_storage_kwargs(self) -> dict:
         return self["deltacat_storage_kwargs"]
@@ -303,12 +324,12 @@ class CompactPartitionParams(dict):
         self["deltacat_storage_kwargs"] = kwargs
 
     @property
-    def s3_client_kwargs(self) -> dict:
-        return self["s3_client_kwargs"]
+    def all_column_names(self) -> List[str]:
+        return self.get("all_column_names")
 
-    @s3_client_kwargs.setter
-    def s3_client_kwargs(self, kwargs: dict) -> None:
-        self["s3_client_kwargs"] = kwargs
+    @all_column_names.setter
+    def all_column_names(self, column_names: List[str]) -> None:
+        self["all_column_names"] = column_names
 
     @property
     def records_per_compacted_file(self) -> int:
@@ -489,6 +510,30 @@ class CompactPartitionParams(dict):
             average_record_size_bytes=self.average_record_size_bytes,
         )
 
+    @property
+    def table_writer_kwargs(self) -> dict:
+        return self["table_writer_kwargs"]
+
+    @table_writer_kwargs.setter
+    def table_writer_kwargs(self, kwargs: dict) -> None:
+        self["table_writer_kwargs"] = kwargs
+
+    @property
+    def expected_previous_partition_id(self) -> Optional[str]:
+        return self.get("expected_previous_partition_id")
+
+    @expected_previous_partition_id.setter
+    def expected_previous_partition_id(self, partition_id: Optional[str]) -> None:
+        self["expected_previous_partition_id"] = partition_id
+
+    @property
+    def original_fields(self) -> Optional[Set[str]]:
+        return self.get("original_fields")
+
+    @original_fields.setter
+    def original_fields(self, fields: Optional[Set[str]]) -> None:
+        self["original_fields"] = fields
+
     @staticmethod
     def json_handler_for_compact_partition_params(obj):
         """

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -3,11 +3,13 @@ from __future__ import annotations
 from typing import Optional
 import pyarrow as pa
 import logging
+from pathlib import PosixPath
 from deltacat import logs
 from typing import List, Union
 from deltacat.compute.compactor.model.hash_bucket_result import HashBucketResult
 from deltacat.compute.compactor.model.dedupe_result import DedupeResult
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
+from deltacat.compute.compactor_v2.model.merge_result import MergeResult
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.resources import ClusterUtilization, get_size_of_object_in_bytes
 from deltacat.compute.compactor import PyArrowWriteResult
@@ -322,17 +324,6 @@ class CompactionSessionAuditInfo(dict):
         """
         return self.get("outputSizePyarrowBytes")
 
-    @property
-    def output_record_count(self) -> int:
-        """
-        The total number of records in the compacted output (includes untouched records).
-
-        Represents the final record count after compaction, including:
-            - Records that were processed and materialized
-            - Records that were untouched and copied by reference
-        """
-        return self.get("outputRecordCount")
-
     @property
     def total_cluster_memory_bytes(self) -> float:
         """
@@ -681,26 +672,13 @@ class CompactionSessionAuditInfo(dict):
         self, output_size_bytes: float
     ) -> CompactionSessionAuditInfo:
         self["outputSizeBytes"] = output_size_bytes
-        return output_size_bytes
-
-    def set_output_record_count(
-        self, output_records: int
-    ) -> CompactionSessionAuditInfo:
-        """
-        This includes both processed records and untouched records copied by reference.
-        """
-        if output_records < 0:
-            raise ValueError(
-                f"Output record count cannot be negative: {output_records}"
-            )
-        self["outputRecordCount"] = output_records
         return self
 
     def set_output_size_pyarrow_bytes(
         self, output_size_pyarrow_bytes: float
     ) -> CompactionSessionAuditInfo:
         self["outputSizePyarrowBytes"] = output_size_pyarrow_bytes
-        return output_size_pyarrow_bytes
+        return self
 
     def set_total_cluster_memory_bytes(
         self, total_cluster_memory_bytes: float
@@ -811,7 +789,10 @@ class CompactionSessionAuditInfo(dict):
         self,
         step_name: str,
         task_results: Union[
-            List[HashBucketResult], List[DedupeResult], List[MaterializeResult]
+            List[HashBucketResult],
+            List[DedupeResult],
+            List[MaterializeResult],
+            List[MergeResult],
         ],
         task_results_retrieved_at: float,
         invoke_time_in_seconds: float,
@@ -926,10 +907,6 @@ class CompactionSessionAuditInfo(dict):
         self.set_output_file_count(pyarrow_write_result.files)
         self.set_output_size_bytes(pyarrow_write_result.file_bytes)
         self.set_output_size_pyarrow_bytes(pyarrow_write_result.pyarrow_bytes)
-        # NOTE:  Aggregating untouched_record_count with records to get a total of record count in the compacted table
-        self.set_output_record_count(
-            pyarrow_write_result.records + untouched_file_record_count
-        )
 
         self.set_peak_memory_used_bytes_per_task(
             max(
@@ -943,3 +920,19 @@ class CompactionSessionAuditInfo(dict):
         )
 
         self.set_pyarrow_version(pa.__version__)
+
+    def to_serializable(self, catalog_root: str) -> CompactionSessionAuditInfo:
+        root_path = PosixPath(catalog_root)
+        target_path = PosixPath(self.audit_url)
+        if root_path == target_path:
+            raise ValueError(
+                "Target and root are identical, but expected target to be a child of root."
+            )
+        try:
+            relative_path = target_path.relative_to(root_path)
+            # Create a copy of the audit info with the relative path
+            audit_copy = CompactionSessionAuditInfo(**dict(self))
+            audit_copy["auditUrl"] = str(relative_path)
+            return audit_copy
+        except ValueError:
+            raise ValueError("Expected target to be a child of root.")

deltacat/compute/compactor/model/delta_annotated.py

@@ -11,10 +11,10 @@ from deltacat import logs
 from deltacat.storage import (
     Delta,
     DeltaType,
-    Manifest,
     ManifestEntry,
     ManifestEntryList,
 )
+from deltacat.storage.model.manifest import Manifest
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -107,7 +107,7 @@ class DeltaAnnotated(Delta):
             assert len(src_da_annotations) == len(src_da_entries), (
                 f"Unexpected Error: Length of delta annotations "
                 f"({len(src_da_annotations)}) doesn't mach the length of "
-                f"delta manifest entries ({len(src_da_entries)}).",
+                f"delta manifest entries ({len(src_da_entries)})."
             )
             for i, src_entry in enumerate(src_da_entries):
                 # create a new da group if src and dest has different delta locator
@@ -161,7 +161,7 @@ class DeltaAnnotated(Delta):
         assert len(src_da_annotations) == len(src_da_entries), (
             f"Unexpected Error: Length of delta annotations "
             f"({len(src_da_annotations)}) doesn't mach the length of "
-            f"delta manifest entries ({len(src_da_entries)}).",
+            f"delta manifest entries ({len(src_da_entries)})."
         )
         src_da_entries_length = len(src_da_entries)
         equal_length = src_da_entries_length // pieces