deltacat 1.1.7__tar.gz → 1.1.9__tar.gz

deltacat-1.1.9/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

{deltacat-1.1.7 → deltacat-1.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.7
+Version: 1.1.9
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -40,9 +40,8 @@ Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.7
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.7"
+__version__ = "1.1.9"
 
 
 __all__ = [

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/aws/constants.py

@@ -1,9 +1,14 @@
 import botocore
 from typing import Set
+from daft.exceptions import DaftTransientError
 
 from deltacat.utils.common import env_integer, env_string
 
+
 DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE", 8)
+DEFAULT_FILE_READ_TIMEOUT_MS = env_integer(
+    "DEFAULT_FILE_READ_TIMEOUT_MS", 300_000
+)  # 5 mins
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
 BOTO_TIMEOUT_ERROR_CODES: Set[str] = {"ReadTimeoutError", "ConnectTimeoutError"}
 BOTO_THROTTLING_ERROR_CODES: Set[str] = {"Throttling", "SlowDown"}
@@ -14,6 +19,7 @@ RETRYABLE_TRANSIENT_ERRORS = (
     botocore.exceptions.NoCredentialsError,
     botocore.exceptions.ConnectTimeoutError,
     botocore.exceptions.ReadTimeoutError,
+    DaftTransientError,
 )
 AWS_REGION = env_string("AWS_REGION", "us-east-1")
 UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY = env_integer(

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/aws/s3u.py

@@ -26,14 +26,12 @@ from ray.types import ObjectRef
 from tenacity import (
     Retrying,
     retry_if_exception_type,
-    retry_if_not_exception_type,
     stop_after_delay,
     wait_random_exponential,
 )
 from deltacat.utils.ray_utils.concurrency import invoke_parallel
 import deltacat.aws.clients as aws_utils
 from deltacat import logs
-from deltacat.exceptions import NonRetryableError, RetryableError
 from deltacat.storage import (
     DistributedDataset,
     LocalDataset,
@@ -55,8 +53,20 @@ from deltacat.types.tables import (
     DISTRIBUTED_DATASET_TYPE_TO_READER_FUNC,
     get_table_length,
 )
+from deltacat.exceptions import (
+    RetryableError,
+    RetryableUploadTableError,
+    RetryableDownloadTableError,
+    RetryableDownloadFileError,
+    RetryableUploadFileError,
+    NonRetryableDownloadFileError,
+    NonRetryableUploadFileError,
+    NonRetryableUploadTableError,
+    NonRetryableDownloadTableError,
+)
 from deltacat.types.partial_download import PartialFileDownloadParams
 from deltacat.utils.common import ReadKwargsProvider
+from deltacat.exceptions import categorize_errors
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -232,6 +242,7 @@ def filter_objects_by_prefix(
         more_objects_to_list = params["ContinuationToken"] is not None
 
 
+@categorize_errors
 def read_file(
     s3_url: str,
     content_type: ContentType,
@@ -263,15 +274,15 @@ def read_file(
             in BOTO_TIMEOUT_ERROR_CODES | BOTO_THROTTLING_ERROR_CODES
         ):
             # Timeout error not caught by botocore
-            raise RetryableError(
-                f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+            raise RetryableDownloadTableError(
+                f"Retry table download from: {s3_url} after receiving {type(e).__name__}",
             ) from e
-        raise NonRetryableError(
+        raise NonRetryableDownloadTableError(
             f"Failed table download from: {s3_url} after receiving {type(e).__name__}"
         ) from e
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
-            f"Retry upload for: {s3_url} after receiving {type(e).__name__}"
+        raise RetryableDownloadTableError(
+            f"Retry download for: {s3_url} after receiving {type(e).__name__}"
         ) from e
     except BaseException as e:
         logger.warn(
@@ -279,7 +290,10 @@ def read_file(
             f"and encoding={content_encoding}. Error: {e}",
             exc_info=True,
         )
-        raise e
+        raise NonRetryableDownloadTableError(
+            f"Read has failed for {s3_url} and content_type={content_type} "
+            f"and encoding={content_encoding}",
+        ) from e
 
 
 def upload_sliced_table(
@@ -378,29 +392,31 @@ def upload_table(
         except ClientError as e:
             if e.response["Error"]["Code"] == "NoSuchKey":
                 # s3fs may swallow S3 errors - we were probably throttled
-                raise RetryableError(
-                    f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+                raise RetryableUploadTableError(
+                    f"Retry table upload from: {s3_url} after receiving {type(e).__name__}",
                 ) from e
             if (
                 e.response["Error"]["Code"]
                 in BOTO_TIMEOUT_ERROR_CODES | BOTO_THROTTLING_ERROR_CODES
             ):
-                raise RetryableError(
-                    f"Retry table download from: {s3_url} after receiving {type(e).__name__}"
+                raise RetryableUploadTableError(
+                    f"Retry table upload from: {s3_url} after receiving {type(e).__name__}",
                 ) from e
-            raise NonRetryableError(
-                f"Failed table upload to: {s3_url} after receiving {type(e).__name__}"
+            raise NonRetryableUploadTableError(
+                f"Failed table upload to: {s3_url} after receiving {type(e).__name__}",
             ) from e
         except RETRYABLE_TRANSIENT_ERRORS as e:
-            raise RetryableError(
-                f"Retry upload for: {s3_url} after receiving {type(e).__name__}"
+            raise RetryableUploadTableError(
+                f"Retry upload for: {s3_url} after receiving {type(e).__name__}",
             ) from e
         except BaseException as e:
             logger.warn(
                 f"Upload has failed for {s3_url} and content_type={content_type}. Error: {e}",
                 exc_info=True,
             )
-            raise e
+            raise NonRetryableUploadTableError(
+                f"Upload has failed for {s3_url} and content_type={content_type} because of {type(e).__name__}",
+            ) from e
     return manifest_entries
 
 
@@ -443,7 +459,7 @@ def download_manifest_entry(
     retrying = Retrying(
         wait=wait_random_exponential(multiplier=1, max=60),
         stop=stop_after_delay(DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY),
-        retry=retry_if_not_exception_type(NonRetryableError),
+        retry=retry_if_exception_type(RetryableError),
     )
     table = retrying(
         read_file,
@@ -559,12 +575,15 @@ def _put_object(
         )
     except ClientError as e:
         if e.response["Error"]["Code"] in BOTO_THROTTLING_ERROR_CODES:
-            raise RetryableError(
-                f"Retry upload for: {bucket}/{key} after receiving {e.response['Error']['Code']}"
+            error_code = e.response["Error"]["Code"]
+            raise RetryableUploadFileError(
+                f"Retry upload for: {bucket}/{key} after receiving {error_code}",
             ) from e
-        raise NonRetryableError(f"Failed table upload to: {bucket}/{key}") from e
+        raise NonRetryableUploadFileError(
+            f"Failed table upload to: {bucket}/{key}"
+        ) from e
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
+        raise RetryableUploadFileError(
             f"Retry upload for: {bucket}/{key} after receiving {type(e).__name__}"
         ) from e
     except BaseException as e:
@@ -572,7 +591,9 @@ def _put_object(
             f"Upload has failed for {bucket}/{key}. Error: {type(e).__name__}",
             exc_info=True,
         )
-        raise NonRetryableError(f"Failed table upload to: {bucket}/{key}") from e
+        raise NonRetryableUploadFileError(
+            f"Failed table upload to: {bucket}/{key}"
+        ) from e
 
 
 def download(
@@ -604,12 +625,12 @@ def _get_object(s3_client, bucket: str, key: str, fail_if_not_found: bool = True
     except ClientError as e:
         if e.response["Error"]["Code"] == "NoSuchKey":
             if fail_if_not_found:
-                raise NonRetryableError(
+                raise NonRetryableDownloadFileError(
                     f"Failed get object from: {bucket}/{key}"
                 ) from e
             logger.info(f"file not found: {bucket}/{key}")
     except RETRYABLE_TRANSIENT_ERRORS as e:
-        raise RetryableError(
+        raise RetryableDownloadFileError(
             f"Retry get object: {bucket}/{key} after receiving {type(e).__name__}"
         ) from e
 

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/compute/compactor/model/compact_partition_params.py

@@ -21,6 +21,7 @@ from deltacat.compute.compactor_v2.constants import (
     TASK_MAX_PARALLELISM,
     DROP_DUPLICATES,
     TOTAL_MEMORY_BUFFER_PERCENTAGE,
+    DEFAULT_DISABLE_COPY_BY_REFERENCE,
 )
 from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -50,7 +51,6 @@ class CompactPartitionParams(dict):
 
         result = CompactPartitionParams(params)
 
-        # TODO: move defaults to single file
         result.records_per_compacted_file = params.get(
             "records_per_compacted_file", MAX_RECORDS_PER_COMPACTED_FILE
         )
@@ -92,6 +92,9 @@ class CompactPartitionParams(dict):
         result.hash_group_count = params.get(
             "hash_group_count", result.hash_bucket_count
         )
+        result.disable_copy_by_reference = params.get(
+            "disable_copy_by_reference", DEFAULT_DISABLE_COPY_BY_REFERENCE
+        )
         result.drop_duplicates = params.get("drop_duplicates", DROP_DUPLICATES)
         result.ray_custom_resources = params.get("ray_custom_resources")
 
@@ -238,6 +241,14 @@ class CompactPartitionParams(dict):
     def enable_profiler(self, value: bool) -> None:
         self["enable_profiler"] = value
 
+    @property
+    def disable_copy_by_reference(self) -> bool:
+        return self["disable_copy_by_reference"]
+
+    @disable_copy_by_reference.setter
+    def disable_copy_by_reference(self, value: bool) -> None:
+        self["disable_copy_by_reference"] = value
+
     @property
     def list_deltas_kwargs(self) -> dict:
         return self["list_deltas_kwargs"]

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/compute/compactor/model/materialize_result.py

@@ -55,10 +55,6 @@ class MaterializeResult(dict):
             self["paWriteResult"] = val = PyArrowWriteResult(val)
         return val
 
-    @property
-    def count_of_src_dfl_not_touched(self) -> int:
-        return self["countOfSrcFileNotTouched"]
-
     @property
     def referenced_pyarrow_write_result(self) -> PyArrowWriteResult:
         val: Dict[str, Any] = self.get("referencedPaWriteResult")

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/compute/compactor_v2/compaction_session.py

@@ -77,6 +77,7 @@ from deltacat.compute.compactor_v2.utils.task_options import (
     local_merge_resource_options_provider,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
+from deltacat.exceptions import categorize_errors
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -86,6 +87,7 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 @metrics
+@categorize_errors
 def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
@@ -123,12 +125,12 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
                 f"Committing compacted partition to: {execute_compaction_result.new_compacted_partition.locator} "
                 f"using previous partition: {previous_partition.locator if previous_partition else None}"
             )
-            commited_partition: Partition = params.deltacat_storage.commit_partition(
+            committed_partition: Partition = params.deltacat_storage.commit_partition(
                 execute_compaction_result.new_compacted_partition,
                 previous_partition,
                 **params.deltacat_storage_kwargs,
             )
-            logger.info(f"Committed compacted partition: {commited_partition}")
+            logger.info(f"Committed compacted partition: {committed_partition}")
             round_completion_file_s3_url = rcf.write_round_completion_file(
                 params.compaction_artifact_s3_bucket,
                 execute_compaction_result.new_round_completion_file_partition_locator,
@@ -479,6 +481,7 @@ def _execute_compaction(
                     delete_strategy=delete_strategy,
                     delete_file_envelopes=delete_file_envelopes,
                     memory_logs_enabled=params.memory_logs_enabled,
+                    disable_copy_by_reference=params.disable_copy_by_reference,
                 )
             }
 
@@ -662,13 +665,16 @@ def _execute_compaction(
     )
 
     logger.info(
-        f"partition-{params.source_partition_locator.partition_values},"
+        f"Partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
+    logger.info(
+        f"Checking if partition {rcf_source_partition_locator} is inplace compacted against {params.destination_partition_locator}..."
+    )
     is_inplace_compacted: bool = (
-        params.source_partition_locator.partition_values
+        rcf_source_partition_locator.partition_values
         == params.destination_partition_locator.partition_values
-        and params.source_partition_locator.stream_id
+        and rcf_source_partition_locator.stream_id
         == params.destination_partition_locator.stream_id
     )
     if is_inplace_compacted:

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/compute/compactor_v2/constants.py

@@ -1,5 +1,3 @@
-from deltacat.utils.common import env_integer
-
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
 PK_DELIMITER = "L6kl7u5f"
@@ -43,15 +41,8 @@ DROP_DUPLICATES = True
 # size in metadata to pyarrow table size.
 PARQUET_TO_PYARROW_INFLATION = 4
 
-# A merge task will fail after this timeout
-# The default is currently double the observed maximum.
-# This timeout depends on total data processed per task.
-MERGE_TASK_TIMEOUT_IN_SECONDS = env_integer("MERGE_TASK_TIMEOUT_IN_SECONDS", 25 * 60)
-
-# A hash bucket task will fail after this timeout
-HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS = env_integer(
-    "HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS", 25 * 60
-)
+# By default, copy by reference is enabled
+DEFAULT_DISABLE_COPY_BY_REFERENCE = False
 
 # Metric Names
 # Time taken for a hash bucket task

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/compute/compactor_v2/model/merge_input.py

@@ -47,6 +47,7 @@ class MergeInput(Dict):
         deltacat_storage=unimplemented_deltacat_storage,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
+        disable_copy_by_reference: Optional[bool] = None,
     ) -> MergeInput:
 
         result = MergeInput()
@@ -69,6 +70,7 @@ class MergeInput(Dict):
         result["deltacat_storage"] = deltacat_storage
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
         result["memory_logs_enabled"] = memory_logs_enabled
+        result["disable_copy_by_reference"] = disable_copy_by_reference
         return result
 
     @property
@@ -148,3 +150,7 @@ class MergeInput(Dict):
     @property
     def delete_strategy(self) -> Optional[DeleteStrategy]:
         return self.get("delete_strategy")
+
+    @property
+    def disable_copy_by_reference(self) -> bool:
+        return self["disable_copy_by_reference"]

{deltacat-1.1.7 → deltacat-1.1.9}/deltacat/compute/compactor_v2/steps/hash_bucket.py

@@ -29,14 +29,15 @@ from deltacat.utils.metrics import emit_timer_metrics, failure_metric, success_m
 from deltacat.utils.resources import (
     get_current_process_peak_memory_usage_in_bytes,
     ProcessUtilizationOverTimeRange,
-    timeout,
 )
 from deltacat.constants import BYTES_PER_GIBIBYTE
 from deltacat.compute.compactor_v2.constants import (
     HASH_BUCKET_TIME_IN_SECONDS,
     HASH_BUCKET_FAILURE_COUNT,
     HASH_BUCKET_SUCCESS_COUNT,
-    HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS,
+)
+from deltacat.exceptions import (
+    categorize_errors,
 )
 
 if importlib.util.find_spec("memray"):
@@ -79,7 +80,7 @@ def _group_file_records_by_pk_hash_bucket(
         logger.info("Grouping by pk hash bucket")
         group_start = time.monotonic()
         hash_bucket_to_table = group_by_pk_hash_bucket(
-            dfe.table, num_hash_buckets, primary_keys
+            table=dfe.table, num_buckets=num_hash_buckets, primary_keys=primary_keys
         )
         group_end = time.monotonic()
         logger.info(f"Grouping took: {group_end - group_start}")
@@ -98,12 +99,9 @@ def _group_file_records_by_pk_hash_bucket(
     return hb_to_delta_file_envelopes, total_record_count, total_size_bytes
 
 
-# TODO: use timeout parameter in ray.remote
-# https://github.com/ray-project/ray/issues/18916
-# Note: order of decorators is important
 @success_metric(name=HASH_BUCKET_SUCCESS_COUNT)
 @failure_metric(name=HASH_BUCKET_FAILURE_COUNT)
-@timeout(HASH_BUCKET_TASK_TIMEOUT_IN_SECONDS)
+@categorize_errors
 def _timed_hash_bucket(input: HashBucketInput):
     task_id = get_current_ray_task_id()
     worker_id = get_current_ray_worker_id()