deltacat 1.1.8__py3-none-any.whl → 1.1.9__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.8"
+__version__ = "1.1.9"
 
 
 __all__ = [

deltacat/aws/constants.py

@@ -1,9 +1,14 @@
 import botocore
 from typing import Set
+from daft.exceptions import DaftTransientError
 
 from deltacat.utils.common import env_integer, env_string
 
+
 DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE", 8)
+DEFAULT_FILE_READ_TIMEOUT_MS = env_integer(
+    "DEFAULT_FILE_READ_TIMEOUT_MS", 300_000
+)  # 5 mins
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
 BOTO_TIMEOUT_ERROR_CODES: Set[str] = {"ReadTimeoutError", "ConnectTimeoutError"}
 BOTO_THROTTLING_ERROR_CODES: Set[str] = {"Throttling", "SlowDown"}
@@ -14,6 +19,7 @@ RETRYABLE_TRANSIENT_ERRORS = (
     botocore.exceptions.NoCredentialsError,
     botocore.exceptions.ConnectTimeoutError,
     botocore.exceptions.ReadTimeoutError,
+    DaftTransientError,
 )
 AWS_REGION = env_string("AWS_REGION", "us-east-1")
 UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY = env_integer(

deltacat/aws/s3u.py

@@ -26,14 +26,12 @@ from ray.types import ObjectRef
 from tenacity import (
     Retrying,
     retry_if_exception_type,
-    retry_if_not_exception_type,
     stop_after_delay,
     wait_random_exponential,
 )
 from deltacat.utils.ray_utils.concurrency import invoke_parallel
 import deltacat.aws.clients as aws_utils
 from deltacat import logs
-from deltacat.exceptions import NonRetryableError, RetryableError
 from deltacat.storage import (
     DistributedDataset,
     LocalDataset,
@@ -55,8 +53,20 @@ from deltacat.types.tables import (
     DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC,
     get_table_length,
 )
+from deltacat.exceptions import (
+    RetryableError,
+    RetryableUploadTableError,
+    RetryableDownloadTableError,
+    RetryableDownloadFileError,
+    RetryableUploadFileError,
+    NonRetryableDownloadFileError,
+    NonRetryableUploadFileError,
+    NonRetryableUploadTableError,
+    NonRetryableDownloadTableError,
+)
 from deltacat.types.partial_download import PartialFileDownloadParams
 from deltacat.utils.common import ReadKwargsProvider
+from deltacat.exceptions import categorize_errors
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -232,6 +242,7 @@ def filter_objects_by_prefix(
         more_objects_to_list = params["ContinuationToken"] is not None
 
 
+@categorize_errors
 def read_file(
     s3_url: str,
     content_type: ContentType,
@@ -263,15 +274,15 @@ def read_file(
             in BOTO_TIMEOUT_ERROR_CODES | BOTO_THROTTLING_ERROR_CODES
         ):
             # Timeout error not caught by botocore
-            raise RetryableError(
-                f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+            raise RetryableDownloadTableError(
+                f"Retry table download from: {s3_url} after receiving {type(e).__name__}",
             ) from e
-        raise NonRetryableError(
+        raise NonRetryableDownloadTableError(
             f"Failed table download from: {s3_url} after receiving {type(e).__name__}"
         ) from e
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
-            f"Retry upload for: {s3_url} after receiving {type(e).__name__}"
+        raise RetryableDownloadTableError(
+            f"Retry download for: {s3_url} after receiving {type(e).__name__}"
         ) from e
     except BaseException as e:
         logger.warn(
@@ -279,7 +290,10 @@ def read_file(
             f"and encoding={content_encoding}. Error: {e}",
             exc_info=True,
         )
-        raise e
+        raise NonRetryableDownloadTableError(
+            f"Read has failed for {s3_url} and content_type={content_type} "
+            f"and encoding={content_encoding}",
+        ) from e
 
 
 def upload_sliced_table(
@@ -378,29 +392,31 @@ def upload_table(
         except ClientError as e:
             if e.response["Error"]["Code"] == "NoSuchKey":
                 # s3fs may swallow S3 errors - we were probably throttled
-                raise RetryableError(
-                    f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+                raise RetryableUploadTableError(
+                    f"Retry table upload from: {s3_url} after receiving {type(e).__name__}",
                 ) from e
             if (
                 e.response["Error"]["Code"]
                 in BOTO_TIMEOUT_ERROR_CODES | BOTO_THROTTLING_ERROR_CODES
             ):
-                raise RetryableError(
-                    f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+                raise RetryableUploadTableError(
+                    f"Retry table upload from: {s3_url} after receiving {type(e).__name__}",
                 ) from e
-            raise NonRetryableError(
-                f"Failed table upload to: {s3_url} after receiving {type(e).__name__}"
+            raise NonRetryableUploadTableError(
+                f"Failed table upload to: {s3_url} after receiving {type(e).__name__}",
             ) from e
         except RETRYABLE_TRANSIENT_ERRORS as e:
-            raise RetryableError(
-                f"Retry upload for: {s3_url} after receiving {type(e).__name__}"
+            raise RetryableUploadTableError(
+                f"Retry upload for: {s3_url} after receiving {type(e).__name__}",
             ) from e
         except BaseException as e:
             logger.warn(
                 f"Upload has failed for {s3_url} and content_type={content_type}. Error: {e}",
                 exc_info=True,
             )
-            raise e
+            raise NonRetryableUploadTableError(
+                f"Upload has failed for {s3_url} and content_type={content_type} because of {type(e).__name__}",
+            ) from e
     return manifest_entries
 
 
@@ -443,7 +459,7 @@ def download_manifest_entry(
     retrying = Retrying(
         wait=wait_random_exponential(multiplier=1, max=60),
         stop=stop_after_delay(DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY),
-        retry=retry_if_not_exception_type(NonRetryableError),
+        retry=retry_if_exception_type(RetryableError),
     )
     table = retrying(
         read_file,
@@ -559,12 +575,15 @@ def _put_object(
         )
     except ClientError as e:
         if e.response["Error"]["Code"] in BOTO_THROTTLING_ERROR_CODES:
-            raise RetryableError(
-                f"Retry upload for: {bucket}/{key} after receiving {e.response['Error']['Code']}"
+            error_code = e.response["Error"]["Code"]
+            raise RetryableUploadFileError(
+                f"Retry upload for: {bucket}/{key} after receiving {error_code}",
             ) from e
-        raise NonRetryableError(f"Failed table upload to: {bucket}/{key}") from e
+        raise NonRetryableUploadFileError(
+            f"Failed table upload to: {bucket}/{key}"
+        ) from e
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
+        raise RetryableUploadFileError(
             f"Retry upload for: {bucket}/{key} after receiving {type(e).__name__}"
         ) from e
     except BaseException as e:
@@ -572,7 +591,9 @@ def _put_object(
             f"Upload has failed for {bucket}/{key}. Error: {type(e).__name__}",
             exc_info=True,
         )
-        raise NonRetryableError(f"Failed table upload to: {bucket}/{key}") from e
+        raise NonRetryableUploadFileError(
+            f"Failed table upload to: {bucket}/{key}"
+        ) from e
 
 
 def download(
@@ -604,12 +625,12 @@ def _get_object(s3_client, bucket: str, key: str, fail_if_not_found: bool = True
     except ClientError as e:
         if e.response["Error"]["Code"] == "NoSuchKey":
             if fail_if_not_found:
-                raise NonRetryableError(
+                raise NonRetryableDownloadFileError(
                     f"Failed get object from: {bucket}/{key}"
                 ) from e
             logger.info(f"file not found: {bucket}/{key}")
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
+        raise RetryableDownloadFileError(
             f"Retry get object: {bucket}/{key} after receiving {type(e).__name__}"
         ) from e
 

deltacat/compute/compactor/model/compact_partition_params.py

@@ -21,6 +21,7 @@ from deltacat.compute.compactor_v2.constants import (
     TASK_MAX_PARALLELISM,
     DROP_DUPLICATES,
     TOTAL_MEMORY_BUFFER_PERCENTAGE,
+    DEFAULT_DISABLE_COPY_BY_REFERENCE,
 )
 from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -50,7 +51,6 @@ class CompactPartitionParams(dict):
 
         result = CompactPartitionParams(params)
 
-        # TODO: move defaults to single file
         result.records_per_compacted_file = params.get(
             "records_per_compacted_file", MAX_RECORDS_PER_COMPACTED_FILE
         )
@@ -92,6 +92,9 @@ class CompactPartitionParams(dict):
         result.hash_group_count = params.get(
             "hash_group_count", result.hash_bucket_count
         )
+        result.disable_copy_by_reference = params.get(
+            "disable_copy_by_reference", DEFAULT_DISABLE_COPY_BY_REFERENCE
+        )
         result.drop_duplicates = params.get("drop_duplicates", DROP_DUPLICATES)
         result.ray_custom_resources = params.get("ray_custom_resources")
 
@@ -238,6 +241,14 @@ class CompactPartitionParams(dict):
     def enable_profiler(self, value: bool) -> None:
         self["enable_profiler"] = value
 
+    @property
+    def disable_copy_by_reference(self) -> bool:
+        return self["disable_copy_by_reference"]
+
+    @disable_copy_by_reference.setter
+    def disable_copy_by_reference(self, value: bool) -> None:
+        self["disable_copy_by_reference"] = value
+
     @property
     def list_deltas_kwargs(self) -> dict:
         return self["list_deltas_kwargs"]

deltacat/compute/compactor/model/materialize_result.py

@@ -55,10 +55,6 @@ class MaterializeResult(dict):
             self["paWriteResult"] = val = PyArrowWriteResult(val)
         return val
 
-    @property
-    def count_of_src_dfl_not_touched(self) -> int:
-        return self["countOfSrcFileNotTouched"]
-
     @property
     def referenced_pyarrow_write_result(self) -> PyArrowWriteResult:
         val: Dict[str, Any] = self.get("referencedPaWriteResult")

deltacat/compute/compactor_v2/compaction_session.py

@@ -77,6 +77,7 @@ from deltacat.compute.compactor_v2.utils.task_options import (
     local_merge_resource_options_provider,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
+from deltacat.exceptions import categorize_errors
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -86,6 +87,7 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 @metrics
+@categorize_errors
 def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
@@ -123,12 +125,12 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
                 f"Committing compacted partition to: {execute_compaction_result.new_compacted_partition.locator} "
                 f"using previous partition: {previous_partition.locator if previous_partition else None}"
             )
-            commited_partition: Partition = params.deltacat_storage.commit_partition(
+            committed_partition: Partition = params.deltacat_storage.commit_partition(
                 execute_compaction_result.new_compacted_partition,
                 previous_partition,
                 **params.deltacat_storage_kwargs,
             )
-            logger.info(f"Committed compacted partition: {commited_partition}")
+            logger.info(f"Committed compacted partition: {committed_partition}")
             round_completion_file_s3_url = rcf.write_round_completion_file(
                 params.compaction_artifact_s3_bucket,
                 execute_compaction_result.new_round_completion_file_partition_locator,
@@ -479,6 +481,7 @@ def _execute_compaction(
                     delete_strategy=delete_strategy,
                     delete_file_envelopes=delete_file_envelopes,
                     memory_logs_enabled=params.memory_logs_enabled,
+                    disable_copy_by_reference=params.disable_copy_by_reference,
                 )
             }
 
@@ -662,13 +665,16 @@ def _execute_compaction(
     )
 
     logger.info(
-        f"partition-{params.source_partition_locator.partition_values},"
+        f"Partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
+    logger.info(
+        f"Checking if partition {rcf_source_partition_locator} is inplace compacted against {params.destination_partition_locator}..."
+    )
     is_inplace_compacted: bool = (
-        params.source_partition_locator.partition_values
+        rcf_source_partition_locator.partition_values
         == params.destination_partition_locator.partition_values
-        and params.source_partition_locator.stream_id
+        and rcf_source_partition_locator.stream_id
         == params.destination_partition_locator.stream_id
     )
     if is_inplace_compacted:

deltacat/compute/compactor_v2/constants.py

@@ -1,5 +1,3 @@
-from deltacat.utils.common import env_integer
-
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
 PK_DELIMITER = "L6kl7u5f"
@@ -43,15 +41,8 @@ DROP_DUPLICATES = True
 # size in metadata to pyarrow table size.
 PARQUET_TO_PYARROW_INFLATION = 4
 
-# A merge task will fail after this timeout
-# The default is currently double the observed maximum.
-# This timeout depends on total data processed per task.
-MERGE_TASK_TIMEOUT_IN_SECONDS = env_integer("MERGE_TASK_TIMEOUT_IN_SECONDS", 25 * 60)
-
-# A hash bucket task will fail after this timeout
-HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS = env_integer(
-    "HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS", 25 * 60
-)
+# By default, copy by reference is enabled
+DEFAULT_DISABLE_COPY_BY_REFERENCE = False
 
 # Metric Names
 # Time taken for a hash bucket task

deltacat/compute/compactor_v2/model/merge_input.py

@@ -47,6 +47,7 @@ class MergeInput(Dict):
         deltacat_storage=unimplemented_deltacat_storage,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
+        disable_copy_by_reference: Optional[bool] = None,
     ) -> MergeInput:
 
         result = MergeInput()
@@ -69,6 +70,7 @@ class MergeInput(Dict):
         result["deltacat_storage"] = deltacat_storage
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
         result["memory_logs_enabled"] = memory_logs_enabled
+        result["disable_copy_by_reference"] = disable_copy_by_reference
         return result
 
     @property
@@ -148,3 +150,7 @@ class MergeInput(Dict):
     @property
     def delete_strategy(self) -> Optional[DeleteStrategy]:
         return self.get("delete_strategy")
+
+    @property
+    def disable_copy_by_reference(self) -> bool:
+        return self["disable_copy_by_reference"]

deltacat/compute/compactor_v2/steps/hash_bucket.py

@@ -29,14 +29,15 @@ from deltacat.utils.metrics import emit_timer_metrics, failure_metric, success_m
 from deltacat.utils.resources import (
     get_current_process_peak_memory_usage_in_bytes,
     ProcessUtilizationOverTimeRange,
-    timeout,
 )
 from deltacat.constants import BYTES_PER_GIBIBYTE
 from deltacat.compute.compactor_v2.constants import (
     HASH_BUCKET_TIME_IN_SECONDS,
     HASH_BUCKET_FAILURE_COUNT,
     HASH_BUCKET_SUCCESS_COUNT,
-    HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS,
+)
+from deltacat.exceptions import (
+    categorize_errors,
 )
 
 if importlib.util.find_spec("memray"):
@@ -79,7 +80,7 @@ def _group_file_records_by_pk_hash_bucket(
         logger.info("Grouping by pk hash bucket")
         group_start = time.monotonic()
         hash_bucket_to_table = group_by_pk_hash_bucket(
-            dfe.table, num_hash_buckets, primary_keys
+            table=dfe.table, num_buckets=num_hash_buckets, primary_keys=primary_keys
         )
         group_end = time.monotonic()
         logger.info(f"Grouping took: {group_end - group_start}")
@@ -98,12 +99,9 @@ def _group_file_records_by_pk_hash_bucket(
     return hb_to_delta_file_envelopes, total_record_count, total_size_bytes
 
 
-# TODO: use timeout parameter in ray.remote
-# https://github.com/ray-project/ray/issues/18916
-# Note: order of decorators is important
 @success_metric(name=HASH_BUCKET_SUCCESS_COUNT)
 @failure_metric(name=HASH_BUCKET_FAILURE_COUNT)
-@timeout(HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS)
+@categorize_errors
 def _timed_hash_bucket(input: HashBucketInput):
     task_id = get_current_ray_task_id()
     worker_id = get_current_ray_worker_id()

deltacat/compute/compactor_v2/steps/merge.py

@@ -28,7 +28,6 @@ from deltacat.utils.metrics import emit_timer_metrics, failure_metric, success_m
 from deltacat.utils.resources import (
     get_current_process_peak_memory_usage_in_bytes,
     ProcessUtilizationOverTimeRange,
-    timeout,
 )
 from deltacat.compute.compactor_v2.utils.primary_key_index import (
     generate_pk_hash_column,
@@ -47,9 +46,10 @@ from deltacat.compute.compactor_v2.constants import (
     MERGE_TIME_IN_SECONDS,
     MERGE_SUCCESS_COUNT,
     MERGE_FAILURE_COUNT,
-    MERGE_TASK_TIMEOUT_IN_SECONDS,
 )
-
+from deltacat.exceptions import (
+    categorize_errors,
+)
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -284,16 +284,19 @@ def _can_copy_by_reference(
     Can copy by reference only if there are no deletes to merge in
     and previous compacted stream id matches that of new stream
     """
-    return (
+    copy_by_ref = (
         not has_delete
         and not merge_file_group.dfe_groups
         and input.round_completion_info is not None
-        and (
-            input.write_to_partition.stream_id
-            == input.round_completion_info.compacted_delta_locator.stream_id
-        )
     )
 
+    if input.disable_copy_by_reference:
+        copy_by_ref = False
+
+    logger.info(f"Copy by reference is {copy_by_ref} for {merge_file_group.hb_index}")
+
+    return copy_by_ref
+
 
 def _flatten_dfe_list(
     df_envelopes_list: List[List[DeltaFileEnvelope]],
@@ -486,12 +489,9 @@ def _copy_manifests_from_hash_bucketing(
     return materialized_results
 
 
-# TODO: use timeout parameter in ray.remote
-# https://github.com/ray-project/ray/issues/18916
-# Note: order of decorators is important
 @success_metric(name=MERGE_SUCCESS_COUNT)
 @failure_metric(name=MERGE_FAILURE_COUNT)
-@timeout(MERGE_TASK_TIMEOUT_IN_SECONDS)
+@categorize_errors
 def _timed_merge(input: MergeInput) -> MergeResult:
     task_id = get_current_ray_task_id()
     worker_id = get_current_ray_worker_id()

deltacat/compute/compactor_v2/utils/merge.py

@@ -133,4 +133,5 @@ def generate_local_merge_input(
         deltacat_storage_kwargs=params.deltacat_storage_kwargs,
         delete_strategy=delete_strategy,
         delete_file_envelopes=delete_file_envelopes,
+        disable_copy_by_reference=params.disable_copy_by_reference,
     )

deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -157,7 +157,12 @@ def _optimized_group_record_batches_by_hash_bucket(
 def group_by_pk_hash_bucket(
     table: pa.Table, num_buckets: int, primary_keys: List[str]
 ) -> np.ndarray:
-    table = generate_pk_hash_column([table], primary_keys, requires_sha1=True)[0]
+    new_tables = generate_pk_hash_column([table], primary_keys, requires_hash=True)
+    assert (
+        len(new_tables) == 1
+    ), f"Expected only 1 table in the result but found {len(new_tables)}"
+
+    table = generate_pk_hash_column([table], primary_keys, requires_hash=True)[0]
 
     # group hash bucket record indices
     result = group_record_indices_by_hash_bucket(
@@ -171,7 +176,7 @@ def group_by_pk_hash_bucket(
 def generate_pk_hash_column(
     tables: List[pa.Table],
     primary_keys: Optional[List[str]] = None,
-    requires_sha1: bool = False,
+    requires_hash: bool = False,
 ) -> List[pa.Table]:
     """
     Returns a new table list after generating the primary key hash if desired.
@@ -203,12 +208,12 @@ def generate_pk_hash_column(
     if primary_keys:
         hash_column_list = [_generate_pk_hash(table) for table in tables]
 
-        can_sha1 = requires_sha1 or _is_sha1_desired(hash_column_list)
+        can_sha1 = requires_hash or _is_sha1_desired(hash_column_list)
     else:
         hash_column_list = [_generate_uuid(table) for table in tables]
 
     logger.info(
-        f"can_generate_sha1={can_sha1} for the table and requires_sha1={requires_sha1}"
+        f"can_generate_sha1={can_sha1} for the table and requires_sha1={requires_hash}"
     )
 
     result = []

deltacat/compute/compactor_v2/utils/task_options.py

@@ -1,6 +1,4 @@
-import botocore
 import logging
-import tenacity
 from typing import Dict, Optional, List, Tuple, Any
 from deltacat import logs
 from deltacat.compute.compactor_v2.model.merge_file_group import (
@@ -21,8 +19,7 @@ from deltacat.compute.compactor_v2.utils.primary_key_index import (
 from deltacat.compute.compactor_v2.constants import (
     PARQUET_TO_PYARROW_INFLATION,
 )
-from daft.exceptions import DaftTransientError
-
+from deltacat.exceptions import RetryableError
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -79,14 +76,7 @@ def get_task_options(
 
     # List of possible botocore exceptions are available at
     # https://github.com/boto/botocore/blob/develop/botocore/exceptions.py
-    task_opts["retry_exceptions"] = [
-        botocore.exceptions.ConnectionError,
-        botocore.exceptions.HTTPClientError,
-        ConnectionError,
-        TimeoutError,
-        DaftTransientError,
-        tenacity.RetryError,
-    ]
+    task_opts["retry_exceptions"] = [RetryableError]
 
     return task_opts