deltacat 1.1.8__py3-none-any.whl → 1.1.10__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.8"
+__version__ = "1.1.10"
 
 
 __all__ = [

deltacat/aws/constants.py

@@ -1,9 +1,14 @@
 import botocore
 from typing import Set
+from daft.exceptions import DaftTransientError
 
 from deltacat.utils.common import env_integer, env_string
 
+
 DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE", 8)
+DEFAULT_FILE_READ_TIMEOUT_MS = env_integer(
+    "DEFAULT_FILE_READ_TIMEOUT_MS", 300_000
+)  # 5 mins
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
 BOTO_TIMEOUT_ERROR_CODES: Set[str] = {"ReadTimeoutError", "ConnectTimeoutError"}
 BOTO_THROTTLING_ERROR_CODES: Set[str] = {"Throttling", "SlowDown"}
@@ -14,6 +19,7 @@ RETRYABLE_TRANSIENT_ERRORS = (
     botocore.exceptions.NoCredentialsError,
     botocore.exceptions.ConnectTimeoutError,
     botocore.exceptions.ReadTimeoutError,
+    DaftTransientError,
 )
 AWS_REGION = env_string("AWS_REGION", "us-east-1")
 UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY = env_integer(

deltacat/aws/redshift/model/manifest.py

@@ -99,6 +99,8 @@ class Manifest(dict):
         total_source_content_length = 0
         content_type = None
         content_encoding = None
+        partition_values_set = set()
+        partition_values = None
         if entries:
             content_type = entries[0].meta.content_type
             content_encoding = entries[0].meta.content_encoding
@@ -127,6 +129,12 @@ class Manifest(dict):
                 total_record_count += meta.record_count or 0
                 total_content_length += meta.content_length or 0
                 total_source_content_length += meta.source_content_length or 0
+                if len(partition_values_set) <= 1:
+                    partition_values_set.add(entry.meta.partition_values)
+
+        if len(partition_values_set) == 1:
+            partition_values = partition_values_set.pop()
+
         meta = ManifestMeta.of(
             total_record_count,
             total_content_length,
@@ -134,6 +142,7 @@ class Manifest(dict):
             content_encoding,
             total_source_content_length,
             entry_type=entry_type,
+            partition_values=partition_values,
         )
         manifest = Manifest._build_manifest(meta, entries, author, uuid, entry_type)
         return manifest
@@ -185,6 +194,7 @@ class ManifestMeta(dict):
         credentials: Optional[Dict[str, str]] = None,
         content_type_parameters: Optional[List[Dict[str, str]]] = None,
         entry_type: Optional[EntryType] = None,
+        partition_values: Optional[List[str]] = None,
     ) -> ManifestMeta:
         manifest_meta = ManifestMeta()
         if record_count is not None:
@@ -203,6 +213,8 @@ class ManifestMeta(dict):
             manifest_meta["credentials"] = credentials
         if entry_type is not None:
             manifest_meta["entry_type"] = entry_type.value
+        if partition_values is not None:
+            manifest_meta["partition_values"] = partition_values
         return manifest_meta
 
     @property
@@ -244,6 +256,10 @@ class ManifestMeta(dict):
             return EntryType(self["entry_type"])
         return val
 
+    @property
+    def partition_values(self) -> Optional[List[str]]:
+        return self.get("partition_values")
+
 
 class ManifestAuthor(dict):
     @staticmethod

deltacat/aws/s3u.py

@@ -21,19 +21,17 @@ from boto3.resources.base import ServiceResource
 from botocore.client import BaseClient
 from botocore.exceptions import ClientError
 from ray.data.block import Block, BlockAccessor, BlockMetadata
-from ray.data.datasource import BlockWritePathProvider
+from ray.data.datasource import FilenameProvider
 from ray.types import ObjectRef
 from tenacity import (
     Retrying,
     retry_if_exception_type,
-    retry_if_not_exception_type,
     stop_after_delay,
     wait_random_exponential,
 )
 from deltacat.utils.ray_utils.concurrency import invoke_parallel
 import deltacat.aws.clients as aws_utils
 from deltacat import logs
-from deltacat.exceptions import NonRetryableError, RetryableError
 from deltacat.storage import (
     DistributedDataset,
     LocalDataset,
@@ -55,14 +53,23 @@ from deltacat.types.tables import (
     DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC,
     get_table_length,
 )
+from deltacat.exceptions import (
+    RetryableError,
+    RetryableUploadTableError,
+    RetryableDownloadTableError,
+    RetryableDownloadFileError,
+    RetryableUploadFileError,
+    NonRetryableDownloadFileError,
+    NonRetryableUploadFileError,
+    NonRetryableUploadTableError,
+    NonRetryableDownloadTableError,
+)
 from deltacat.types.partial_download import PartialFileDownloadParams
 from deltacat.utils.common import ReadKwargsProvider
+from deltacat.exceptions import categorize_errors
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-# TODO(raghumdani): refactor redshift datasource to reuse the
-# same module for writing output files.
-
 
 class CapturedBlockWritePaths:
     def __init__(self):
@@ -90,12 +97,15 @@ class CapturedBlockWritePaths:
         return self._block_refs
 
 
-class UuidBlockWritePathProvider(BlockWritePathProvider):
+class UuidBlockWritePathProvider(FilenameProvider):
     """Block write path provider implementation that writes each
     dataset block out to a file of the form: {base_path}/{uuid}
     """
 
-    def __init__(self, capture_object: CapturedBlockWritePaths):
+    def __init__(
+        self, capture_object: CapturedBlockWritePaths, base_path: Optional[str] = None
+    ):
+        self.base_path = base_path
         self.write_paths: List[str] = []
         self.block_refs: List[ObjectRef[Block]] = []
         self.capture_object = capture_object
@@ -107,6 +117,19 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
                 self.block_refs,
             )
 
+    def get_filename_for_block(
+        self, block: Any, task_index: int, block_index: int
+    ) -> str:
+        if self.base_path is None:
+            raise ValueError(
+                "Base path must be provided to UuidBlockWritePathProvider",
+            )
+        return self._get_write_path_for_block(
+            base_path=self.base_path,
+            block=block,
+            block_index=block_index,
+        )
+
     def _get_write_path_for_block(
         self,
         base_path: str,
@@ -133,13 +156,6 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
         block_index: Optional[int] = None,
         file_format: Optional[str] = None,
     ) -> str:
-        """
-        TODO: BlockWritePathProvider is deprecated as of Ray version 2.20.0. Please use FilenameProvider.
-        See: https://docs.ray.io/en/master/data/api/doc/ray.data.datasource.FilenameProvider.html
-        Also See: https://github.com/ray-project/deltacat/issues/299
-
-        Hence, this class only works with Ray version 2.20.0 or lower when used in Ray Dataset.
-        """
         return self._get_write_path_for_block(
             base_path,
             filesystem=filesystem,
@@ -232,6 +248,7 @@ def filter_objects_by_prefix(
         more_objects_to_list = params["ContinuationToken"] is not None
 
 
+@categorize_errors
 def read_file(
     s3_url: str,
     content_type: ContentType,
@@ -263,15 +280,15 @@ def read_file(
             in BOTO_TIMEOUT_ERROR_CODES | BOTO_THROTTLING_ERROR_CODES
         ):
             # Timeout error not caught by botocore
-            raise RetryableError(
-                f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+            raise RetryableDownloadTableError(
+                f"Retry table download from: {s3_url} after receiving {type(e).__name__}",
             ) from e
-        raise NonRetryableError(
+        raise NonRetryableDownloadTableError(
             f"Failed table download from: {s3_url} after receiving {type(e).__name__}"
         ) from e
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
-            f"Retry upload for: {s3_url} after receiving {type(e).__name__}"
+        raise RetryableDownloadTableError(
+            f"Retry download for: {s3_url} after receiving {type(e).__name__}"
         ) from e
     except BaseException as e:
         logger.warn(
@@ -279,7 +296,10 @@ def read_file(
             f"and encoding={content_encoding}. Error: {e}",
             exc_info=True,
         )
-        raise e
+        raise NonRetryableDownloadTableError(
+            f"Read has failed for {s3_url} and content_type={content_type} "
+            f"and encoding={content_encoding}",
+        ) from e
 
 
 def upload_sliced_table(
@@ -378,29 +398,31 @@ def upload_table(
         except ClientError as e:
             if e.response["Error"]["Code"] == "NoSuchKey":
                 # s3fs may swallow S3 errors - we were probably throttled
-                raise RetryableError(
-                    f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+                raise RetryableUploadTableError(
+                    f"Retry table upload from: {s3_url} after receiving {type(e).__name__}",
                 ) from e
             if (
                 e.response["Error"]["Code"]
                 in BOTO_TIMEOUT_ERROR_CODES | BOTO_THROTTLING_ERROR_CODES
             ):
-                raise RetryableError(
-                    f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+                raise RetryableUploadTableError(
+                    f"Retry table upload from: {s3_url} after receiving {type(e).__name__}",
                 ) from e
-            raise NonRetryableError(
-                f"Failed table upload to: {s3_url} after receiving {type(e).__name__}"
+            raise NonRetryableUploadTableError(
+                f"Failed table upload to: {s3_url} after receiving {type(e).__name__}",
             ) from e
         except RETRYABLE_TRANSIENT_ERRORS as e:
-            raise RetryableError(
-                f"Retry upload for: {s3_url} after receiving {type(e).__name__}"
+            raise RetryableUploadTableError(
+                f"Retry upload for: {s3_url} after receiving {type(e).__name__}",
             ) from e
         except BaseException as e:
             logger.warn(
                 f"Upload has failed for {s3_url} and content_type={content_type}. Error: {e}",
                 exc_info=True,
             )
-            raise e
+            raise NonRetryableUploadTableError(
+                f"Upload has failed for {s3_url} and content_type={content_type} because of {type(e).__name__}",
+            ) from e
     return manifest_entries
 
 
@@ -443,7 +465,7 @@ def download_manifest_entry(
     retrying = Retrying(
         wait=wait_random_exponential(multiplier=1, max=60),
         stop=stop_after_delay(DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY),
-        retry=retry_if_not_exception_type(NonRetryableError),
+        retry=retry_if_exception_type(RetryableError),
     )
     table = retrying(
         read_file,
@@ -559,12 +581,15 @@ def _put_object(
         )
     except ClientError as e:
         if e.response["Error"]["Code"] in BOTO_THROTTLING_ERROR_CODES:
-            raise RetryableError(
-                f"Retry upload for: {bucket}/{key} after receiving {e.response['Error']['Code']}"
+            error_code = e.response["Error"]["Code"]
+            raise RetryableUploadFileError(
+                f"Retry upload for: {bucket}/{key} after receiving {error_code}",
             ) from e
-        raise NonRetryableError(f"Failed table upload to: {bucket}/{key}") from e
+        raise NonRetryableUploadFileError(
+            f"Failed table upload to: {bucket}/{key}"
+        ) from e
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
+        raise RetryableUploadFileError(
             f"Retry upload for: {bucket}/{key} after receiving {type(e).__name__}"
         ) from e
     except BaseException as e:
@@ -572,7 +597,9 @@ def _put_object(
             f"Upload has failed for {bucket}/{key}. Error: {type(e).__name__}",
             exc_info=True,
         )
-        raise NonRetryableError(f"Failed table upload to: {bucket}/{key}") from e
+        raise NonRetryableUploadFileError(
+            f"Failed table upload to: {bucket}/{key}"
+        ) from e
 
 
 def download(
@@ -604,12 +631,12 @@ def _get_object(s3_client, bucket: str, key: str, fail_if_not_found: bool = True
     except ClientError as e:
         if e.response["Error"]["Code"] == "NoSuchKey":
             if fail_if_not_found:
-                raise NonRetryableError(
+                raise NonRetryableDownloadFileError(
                     f"Failed get object from: {bucket}/{key}"
                 ) from e
             logger.info(f"file not found: {bucket}/{key}")
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
+        raise RetryableDownloadFileError(
             f"Retry get object: {bucket}/{key} after receiving {type(e).__name__}"
         ) from e
 

deltacat/compute/compactor/compaction_session.py

@@ -193,6 +193,7 @@ def compact_partition(
             round_completion_file_s3_url = rcf.write_round_completion_file(
                 compaction_artifact_s3_bucket,
                 new_rcf_partition_locator,
+                partition.locator,
                 new_rci,
                 **s3_client_kwargs,
             )
@@ -312,7 +313,10 @@ def _execute_compaction_round(
     round_completion_info = None
     if not rebase_source_partition_locator:
         round_completion_info = rcf.read_round_completion_file(
-            compaction_artifact_s3_bucket, source_partition_locator, **s3_client_kwargs
+            compaction_artifact_s3_bucket,
+            source_partition_locator,
+            destination_partition_locator,
+            **s3_client_kwargs,
         )
         if not round_completion_info:
             logger.info(

deltacat/compute/compactor/model/compact_partition_params.py

@@ -21,6 +21,7 @@ from deltacat.compute.compactor_v2.constants import (
     TASK_MAX_PARALLELISM,
     DROP_DUPLICATES,
     TOTAL_MEMORY_BUFFER_PERCENTAGE,
+    DEFAULT_DISABLE_COPY_BY_REFERENCE,
 )
 from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -50,7 +51,6 @@ class CompactPartitionParams(dict):
 
         result = CompactPartitionParams(params)
 
-        # TODO: move defaults to single file
         result.records_per_compacted_file = params.get(
             "records_per_compacted_file", MAX_RECORDS_PER_COMPACTED_FILE
         )
@@ -92,6 +92,9 @@ class CompactPartitionParams(dict):
         result.hash_group_count = params.get(
             "hash_group_count", result.hash_bucket_count
         )
+        result.disable_copy_by_reference = params.get(
+            "disable_copy_by_reference", DEFAULT_DISABLE_COPY_BY_REFERENCE
+        )
         result.drop_duplicates = params.get("drop_duplicates", DROP_DUPLICATES)
         result.ray_custom_resources = params.get("ray_custom_resources")
 
@@ -238,6 +241,14 @@ class CompactPartitionParams(dict):
     def enable_profiler(self, value: bool) -> None:
         self["enable_profiler"] = value
 
+    @property
+    def disable_copy_by_reference(self) -> bool:
+        return self["disable_copy_by_reference"]
+
+    @disable_copy_by_reference.setter
+    def disable_copy_by_reference(self, value: bool) -> None:
+        self["disable_copy_by_reference"] = value
+
     @property
     def list_deltas_kwargs(self) -> dict:
         return self["list_deltas_kwargs"]

deltacat/compute/compactor/model/materialize_result.py

@@ -55,10 +55,6 @@ class MaterializeResult(dict):
             self["paWriteResult"] = val = PyArrowWriteResult(val)
         return val
 
-    @property
-    def count_of_src_dfl_not_touched(self) -> int:
-        return self["countOfSrcFileNotTouched"]
-
     @property
     def referenced_pyarrow_write_result(self) -> PyArrowWriteResult:
         val: Dict[str, Any] = self.get("referencedPaWriteResult")

deltacat/compute/compactor/repartition_session.py

@@ -177,6 +177,7 @@ def repartition(
         s3_client_kwargs = {}
 
     return rcf.write_round_completion_file(
+        None,
         None,
         None,
         repartition_completion_info,

deltacat/compute/compactor/utils/round_completion_file.py

@@ -12,10 +12,17 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 def get_round_completion_file_s3_url(
-    bucket: str, source_partition_locator: PartitionLocator
+    bucket: str,
+    source_partition_locator: PartitionLocator,
+    destination_partition_locator: Optional[PartitionLocator] = None,
 ) -> str:
 
     base_url = source_partition_locator.path(f"s3://{bucket}")
+    if destination_partition_locator:
+        base_url = destination_partition_locator.path(
+            f"s3://{bucket}/{source_partition_locator.hexdigest()}"
+        )
+
     return f"{base_url}.json"
 
 
@@ -23,20 +30,41 @@ def get_round_completion_file_s3_url(
 def read_round_completion_file(
     bucket: str,
     source_partition_locator: PartitionLocator,
+    destination_partition_locator: Optional[PartitionLocator] = None,
     **s3_client_kwargs: Optional[Dict[str, Any]],
 ) -> RoundCompletionInfo:
 
-    round_completion_file_url = get_round_completion_file_s3_url(
+    all_uris = []
+    if destination_partition_locator:
+        round_completion_file_url_with_destination = get_round_completion_file_s3_url(
+            bucket,
+            source_partition_locator,
+            destination_partition_locator,
+        )
+        all_uris.append(round_completion_file_url_with_destination)
+
+    # Note: we read from RCF at two different URI for backward
+    # compatibility reasons.
+    round_completion_file_url_prev = get_round_completion_file_s3_url(
         bucket,
         source_partition_locator,
     )
-    logger.info(f"reading round completion file from: {round_completion_file_url}")
+
+    all_uris.append(round_completion_file_url_prev)
+
     round_completion_info = None
-    result = s3_utils.download(round_completion_file_url, False, **s3_client_kwargs)
-    if result:
-        json_str = result["Body"].read().decode("utf-8")
-        round_completion_info = RoundCompletionInfo(json.loads(json_str))
-        logger.info(f"read round completion info: {round_completion_info}")
+
+    for rcf_uri in all_uris:
+        logger.info(f"Reading round completion file from: {rcf_uri}")
+        result = s3_utils.download(rcf_uri, False, **s3_client_kwargs)
+        if result:
+            json_str = result["Body"].read().decode("utf-8")
+            round_completion_info = RoundCompletionInfo(json.loads(json_str))
+            logger.info(f"Read round completion info: {round_completion_info}")
+            break
+        else:
+            logger.warn(f"Round completion file not present at {rcf_uri}")
+
     return round_completion_info
 
 
@@ -44,8 +72,9 @@ def read_round_completion_file(
 def write_round_completion_file(
     bucket: Optional[str],
     source_partition_locator: Optional[PartitionLocator],
+    destination_partition_locator: Optional[PartitionLocator],
     round_completion_info: RoundCompletionInfo,
-    completion_file_s3_url: str = None,
+    completion_file_s3_url: Optional[str] = None,
     **s3_client_kwargs: Optional[Dict[str, Any]],
 ) -> str:
     if bucket is None and completion_file_s3_url is None:
@@ -56,6 +85,7 @@ def write_round_completion_file(
         completion_file_s3_url = get_round_completion_file_s3_url(
             bucket,
             source_partition_locator,
+            destination_partition_locator,
         )
     logger.info(f"writing round completion file to: {completion_file_s3_url}")
     s3_utils.upload(

deltacat/compute/compactor_v2/compaction_session.py

@@ -24,7 +24,7 @@ from deltacat.compute.compactor import (
 )
 from deltacat.compute.compactor_v2.model.merge_result import MergeResult
 from deltacat.compute.compactor_v2.model.hash_bucket_result import HashBucketResult
-from deltacat.compute.compactor_v2.model.compaction_session import (
+from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
     ExecutionCompactionResult,
 )
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
@@ -77,6 +77,8 @@ from deltacat.compute.compactor_v2.utils.task_options import (
     local_merge_resource_options_provider,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
+from deltacat.exceptions import categorize_errors
+from deltacat.compute.compactor_v2.constants import COMPACT_PARTITION_METRIC_PREFIX
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -85,7 +87,8 @@ if importlib.util.find_spec("memray"):
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-@metrics
+@metrics(prefix=COMPACT_PARTITION_METRIC_PREFIX)
+@categorize_errors
 def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
@@ -107,7 +110,6 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
             f"Partition-{params.source_partition_locator} -> "
             f"{compaction_session_type} Compaction session data processing completed"
         )
-        round_completion_file_s3_url: Optional[str] = None
         if execute_compaction_result.new_compacted_partition:
             previous_partition: Optional[Partition] = None
             if execute_compaction_result.is_inplace_compacted:
@@ -123,25 +125,19 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
                 f"Committing compacted partition to: {execute_compaction_result.new_compacted_partition.locator} "
                 f"using previous partition: {previous_partition.locator if previous_partition else None}"
             )
-            commited_partition: Partition = params.deltacat_storage.commit_partition(
+            committed_partition: Partition = params.deltacat_storage.commit_partition(
                 execute_compaction_result.new_compacted_partition,
                 previous_partition,
                 **params.deltacat_storage_kwargs,
             )
-            logger.info(f"Committed compacted partition: {commited_partition}")
-            round_completion_file_s3_url = rcf.write_round_completion_file(
-                params.compaction_artifact_s3_bucket,
-                execute_compaction_result.new_round_completion_file_partition_locator,
-                execute_compaction_result.new_round_completion_info,
-                **params.s3_client_kwargs,
-            )
+            logger.info(f"Committed compacted partition: {committed_partition}")
         else:
             logger.warning("No new partition was committed during compaction.")
 
         logger.info(
             f"Completed compaction session for: {params.source_partition_locator}"
         )
-        return round_completion_file_s3_url
+        return execute_compaction_result.round_completion_file_s3_url
 
 
 def _execute_compaction(
@@ -186,6 +182,7 @@ def _execute_compaction(
         round_completion_info = rcf.read_round_completion_file(
             params.compaction_artifact_s3_bucket,
             params.source_partition_locator,
+            params.destination_partition_locator,
             **params.s3_client_kwargs,
         )
         if not round_completion_info:
@@ -479,6 +476,7 @@ def _execute_compaction(
                     delete_strategy=delete_strategy,
                     delete_file_envelopes=delete_file_envelopes,
                     memory_logs_enabled=params.memory_logs_enabled,
+                    disable_copy_by_reference=params.disable_copy_by_reference,
                 )
             }
 
@@ -662,13 +660,16 @@ def _execute_compaction(
     )
 
     logger.info(
-        f"partition-{params.source_partition_locator.partition_values},"
+        f"Partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
+    logger.info(
+        f"Checking if partition {rcf_source_partition_locator} is inplace compacted against {params.destination_partition_locator}..."
+    )
     is_inplace_compacted: bool = (
-        params.source_partition_locator.partition_values
+        rcf_source_partition_locator.partition_values
         == params.destination_partition_locator.partition_values
-        and params.source_partition_locator.stream_id
+        and rcf_source_partition_locator.stream_id
         == params.destination_partition_locator.stream_id
     )
     if is_inplace_compacted:
@@ -678,9 +679,18 @@ def _execute_compaction(
             f"and rcf source partition_id of {rcf_source_partition_locator.partition_id}."
         )
         rcf_source_partition_locator = compacted_partition.locator
+
+    round_completion_file_s3_url = rcf.write_round_completion_file(
+        params.compaction_artifact_s3_bucket,
+        rcf_source_partition_locator,
+        compacted_partition.locator,
+        new_round_completion_info,
+        **params.s3_client_kwargs,
+    )
+
     return ExecutionCompactionResult(
         compacted_partition,
         new_round_completion_info,
-        rcf_source_partition_locator,
+        round_completion_file_s3_url,
         is_inplace_compacted,
     )

deltacat/compute/compactor_v2/constants.py

@@ -1,5 +1,3 @@
-from deltacat.utils.common import env_integer
-
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
 PK_DELIMITER = "L6kl7u5f"
@@ -43,15 +41,8 @@ DROP_DUPLICATES = True
 # size in metadata to pyarrow table size.
 PARQUET_TO_PYARROW_INFLATION = 4
 
-# A merge task will fail after this timeout
-# The default is currently double the observed maximum.
-# This timeout depends on total data processed per task.
-MERGE_TASK_TIMEOUT_IN_SECONDS = env_integer("MERGE_TASK_TIMEOUT_IN_SECONDS", 25 * 60)
-
-# A hash bucket task will fail after this timeout
-HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS = env_integer(
-    "HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS", 25 * 60
-)
+# By default, copy by reference is enabled
+DEFAULT_DISABLE_COPY_BY_REFERENCE = False
 
 # Metric Names
 # Time taken for a hash bucket task
@@ -77,3 +68,6 @@ DISCOVER_DELTAS_METRIC_PREFIX = "discover_deltas"
 
 # Metric prefix for prepare deletes
 PREPARE_DELETES_METRIC_PREFIX = "prepare_deletes"
+
+# Metric prefix for compact partition method
+COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"

deltacat/compute/compactor_v2/model/{compaction_session.py → evaluate_compaction_result.py}

@@ -2,7 +2,6 @@ from dataclasses import dataclass, fields
 
 from deltacat.storage import (
     Partition,
-    PartitionLocator,
 )
 from deltacat.compute.compactor import (
     RoundCompletionInfo,
@@ -14,7 +13,7 @@ from typing import Optional
 class ExecutionCompactionResult:
     new_compacted_partition: Optional[Partition]
     new_round_completion_info: Optional[RoundCompletionInfo]
-    new_round_completion_file_partition_locator: Optional[PartitionLocator]
+    round_completion_file_s3_url: Optional[str]
     is_inplace_compacted: bool
 
     def __iter__(self):