deltacat 1.1.35__tar.gz → 1.1.36__tar.gz

{deltacat-1.1.35/deltacat.egg-info → deltacat-1.1.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.35
+Version: 1.1.36
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.35 → deltacat-1.1.36}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.35"
+__version__ = "1.1.36"
 
 
 __all__ = [

{deltacat-1.1.35 → deltacat-1.1.36}/deltacat/compute/compactor_v2/compaction_session.py

@@ -69,14 +69,17 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
     ), "hash_bucket_count is a required arg for compactor v2"
+    assert type(params.hash_bucket_count) is int, "Hash bucket count must be an integer"
     if params.num_rounds > 1:
         assert (
             not params.drop_duplicates
         ), "num_rounds > 1, drop_duplicates must be False but is True"
 
-    with memray.Tracker(
-        "compaction_partition.bin"
-    ) if params.enable_profiler else nullcontext():
+    with (
+        memray.Tracker("compaction_partition.bin")
+        if params.enable_profiler
+        else nullcontext()
+    ):
         execute_compaction_result: ExecutionCompactionResult = _execute_compaction(
             params,
             **kwargs,

{deltacat-1.1.35 → deltacat-1.1.36}/deltacat/compute/compactor_v2/model/merge_input.py

@@ -48,6 +48,7 @@ class MergeInput(Dict):
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
         disable_copy_by_reference: Optional[bool] = None,
+        hash_bucket_count: Optional[int] = None,
     ) -> MergeInput:
 
         result = MergeInput()
@@ -71,6 +72,7 @@ class MergeInput(Dict):
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
         result["memory_logs_enabled"] = memory_logs_enabled
         result["disable_copy_by_reference"] = disable_copy_by_reference
+        result["hash_bucket_count"] = hash_bucket_count
         return result
 
     @property
@@ -154,3 +156,7 @@ class MergeInput(Dict):
     @property
     def disable_copy_by_reference(self) -> bool:
         return self["disable_copy_by_reference"]
+
+    @property
+    def hash_bucket_count(self) -> int:
+        return self["hash_bucket_count"]

{deltacat-1.1.35 → deltacat-1.1.36}/deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -438,6 +438,7 @@ def _merge(
                 delete_file_envelopes=delete_file_envelopes,
                 memory_logs_enabled=params.memory_logs_enabled,
                 disable_copy_by_reference=params.disable_copy_by_reference,
+                hash_bucket_count=params.hash_bucket_count,
             )
         }
 

{deltacat-1.1.35 → deltacat-1.1.36}/deltacat/compute/compactor_v2/steps/merge.py

@@ -62,6 +62,10 @@ if importlib.util.find_spec("memray"):
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
+_EXISTING_VARIANT_LOG_PREFIX = "Existing variant "
+_INCREMENTAL_TABLE_LOG_PREFIX = "Incremental table "
+
+
 def _append_delta_type_column(table: pa.Table, value: np.bool_):
     return table.append_column(
         sc._DELTA_TYPE_COLUMN_FIELD,
@@ -112,6 +116,8 @@ def _merge_tables(
     table: pa.Table,
     primary_keys: List[str],
     can_drop_duplicates: bool,
+    hb_index: int,
+    num_buckets: int,
     compacted_table: Optional[pa.Table] = None,
 ) -> pa.Table:
     """
@@ -130,6 +136,20 @@ def _merge_tables(
 
     all_tables.append(table)
 
+    check_bucketing_spec = BUCKETING_SPEC_COMPLIANCE_PROFILE in [
+        BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
+        BUCKETING_SPEC_COMPLIANCE_ASSERT,
+    ]
+
+    if primary_keys and check_bucketing_spec:
+        _validate_bucketing_spec_compliance(
+            table=all_tables[incremental_idx],
+            num_buckets=num_buckets,
+            primary_keys=primary_keys,
+            hb_index=hb_index,
+            log_prefix=_INCREMENTAL_TABLE_LOG_PREFIX,
+        )
+
     if not primary_keys or not can_drop_duplicates:
         logger.info(
             f"Not dropping duplicates for primary keys={primary_keys} "
@@ -193,27 +213,40 @@ def _merge_tables(
 
 
 def _validate_bucketing_spec_compliance(
-    table: pa.Table, rcf: RoundCompletionInfo, hb_index: int, primary_keys: List[str]
+    table: pa.Table,
+    num_buckets: int,
+    hb_index: int,
+    primary_keys: List[str],
+    rcf: RoundCompletionInfo = None,
+    log_prefix=None,
 ) -> None:
+    if rcf is not None:
+        message_prefix = f"{log_prefix}{rcf.compacted_delta_locator.namespace}.{rcf.compacted_delta_locator.table_name}.{rcf.compacted_delta_locator.table_version}.{rcf.compacted_delta_locator.partition_id}.{rcf.compacted_delta_locator.partition_values}"
+    else:
+        message_prefix = f"{log_prefix}"
     pki_table = generate_pk_hash_column(
         [table], primary_keys=primary_keys, requires_hash=True
     )[0]
+    is_not_compliant: bool = False
     for index, hash_value in enumerate(sc.pk_hash_string_column_np(pki_table)):
-        hash_bucket = pk_digest_to_hash_bucket_index(hash_value, rcf.hash_bucket_count)
+        hash_bucket: int = pk_digest_to_hash_bucket_index(hash_value, num_buckets)
         if hash_bucket != hb_index:
+            is_not_compliant = True
             logger.info(
-                f"{rcf.compacted_delta_locator.namespace}.{rcf.compacted_delta_locator.table_name}"
-                f".{rcf.compacted_delta_locator.table_version}.{rcf.compacted_delta_locator.partition_id}"
-                f".{rcf.compacted_delta_locator.partition_values} has non-compliant bucketing spec. "
+                f"{message_prefix} has non-compliant bucketing spec at index: {index} "
                 f"Expected hash bucket is {hb_index} but found {hash_bucket}."
             )
             if BUCKETING_SPEC_COMPLIANCE_PROFILE == BUCKETING_SPEC_COMPLIANCE_ASSERT:
                 raise AssertionError(
-                    "Hash bucket drift detected. Expected hash bucket index"
+                    f"Hash bucket drift detected at index: {index}. Expected hash bucket index"
                     f" to be {hb_index} but found {hash_bucket}"
                 )
             # No further checks necessary
             break
+    if not is_not_compliant:
+        logger.debug(
+            f"{message_prefix} has compliant bucketing spec for hb_index: {hb_index}"
+        )
 
 
 def _download_compacted_table(
@@ -257,7 +290,12 @@ def _download_compacted_table(
     # Bucketing spec compliance isn't required without primary keys
     if primary_keys and check_bucketing_spec:
         _validate_bucketing_spec_compliance(
-            compacted_table, rcf, hb_index, primary_keys
+            compacted_table,
+            rcf.hash_bucket_count,
+            hb_index,
+            primary_keys,
+            rcf=rcf,
+            log_prefix=_EXISTING_VARIANT_LOG_PREFIX,
         )
     return compacted_table
 
@@ -462,12 +500,12 @@ def _compact_tables(
         _group_sequence_by_delta_type(reordered_all_dfes)
     ):
         if delta_type is DeltaType.UPSERT:
-            (
-                table,
-                incremental_len,
-                deduped_records,
-                merge_time,
-            ) = _apply_upserts(input, delta_type_sequence, hb_idx, table)
+            (table, incremental_len, deduped_records, merge_time,) = _apply_upserts(
+                input=input,
+                dfe_list=delta_type_sequence,
+                hb_idx=hb_idx,
+                prev_table=table,
+            )
             logger.info(
                 f" [Merge task index {input.merge_task_index}] Merged"
                 f" record count: {len(table)}, size={table.nbytes} took: {merge_time}s"
@@ -526,6 +564,8 @@ def _apply_upserts(
         primary_keys=input.primary_keys,
         can_drop_duplicates=input.drop_duplicates,
         compacted_table=prev_table,
+        hb_index=hb_idx,
+        num_buckets=input.hash_bucket_count,
     )
     deduped_records = hb_table_record_count - len(table)
     return table, incremental_len, deduped_records, merge_time

{deltacat-1.1.35 → deltacat-1.1.36}/deltacat/compute/compactor_v2/utils/merge.py

@@ -133,4 +133,5 @@ def generate_local_merge_input(
         delete_strategy=delete_strategy,
         delete_file_envelopes=delete_file_envelopes,
         disable_copy_by_reference=params.disable_copy_by_reference,
+        hash_bucket_count=params.hash_bucket_count,
     )

{deltacat-1.1.35 → deltacat-1.1.36}/deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -804,7 +804,7 @@ class TestCompactionSession:
             )
 
         assert (
-            "Hash bucket drift detected. Expected hash bucket index to be 1 but found 0"
+            "Hash bucket drift detected at index: 0. Expected hash bucket index to be 1 but found 0"
             in str(excinfo.value)
         )
 

{deltacat-1.1.35 → deltacat-1.1.36/deltacat.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.35
+Version: 1.1.36
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team