deltacat 1.1.37__tar.gz → 1.1.38__tar.gz

{deltacat-1.1.37/deltacat.egg-info → deltacat-1.1.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.37
+Version: 1.1.38
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.37 → deltacat-1.1.38}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.37"
+__version__ = "1.1.38"
 
 
 __all__ = [

{deltacat-1.1.37 → deltacat-1.1.38}/deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -322,6 +322,17 @@ class CompactionSessionAuditInfo(dict):
         """
         return self.get("outputSizePyarrowBytes")
 
+    @property
+    def output_record_count(self) -> int:
+        """
+        The total number of records in the compacted output (includes untouched records).
+
+        Represents the final record count after compaction, including:
+            - Records that were processed and materialized
+            - Records that were untouched and copied by reference
+        """
+        return self.get("outputRecordCount")
+
     @property
     def total_cluster_memory_bytes(self) -> float:
         """
@@ -672,6 +683,19 @@ class CompactionSessionAuditInfo(dict):
         self["outputSizeBytes"] = output_size_bytes
         return output_size_bytes
 
+    def set_output_record_count(
+        self, output_records: int
+    ) -> CompactionSessionAuditInfo:
+        """
+        This includes both processed records and untouched records copied by reference.
+        """
+        if output_records < 0:
+            raise ValueError(
+                f"Output record count cannot be negative: {output_records}"
+            )
+        self["outputRecordCount"] = output_records
+        return self
+
     def set_output_size_pyarrow_bytes(
         self, output_size_pyarrow_bytes: float
     ) -> CompactionSessionAuditInfo:
@@ -902,6 +926,10 @@ class CompactionSessionAuditInfo(dict):
         self.set_output_file_count(pyarrow_write_result.files)
         self.set_output_size_bytes(pyarrow_write_result.file_bytes)
         self.set_output_size_pyarrow_bytes(pyarrow_write_result.pyarrow_bytes)
+        # NOTE:  Aggregating untouched_record_count with records to get a total of record count in the compacted table
+        self.set_output_record_count(
+            pyarrow_write_result.records + untouched_file_record_count
+        )
 
         self.set_peak_memory_used_bytes_per_task(
             max(

{deltacat-1.1.37 → deltacat-1.1.38}/deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -93,7 +93,7 @@ def _append_table_by_hash_bucket(
         all_buckets = pc.unique(pyarrow_table[sc._HASH_BUCKET_IDX_COLUMN_NAME])
         assert (
             len(all_buckets) == 1
-        ), f"Only one hash bucket is allowed by found {len(all_buckets)}"
+        ), f"Only one hash bucket is allowed but found {len(all_buckets)}"
         assert (
             all_buckets[0].as_py() == hb_idx
         ), f"Hash bucket not equal, {all_buckets[0]} and {hb_idx}"
@@ -153,7 +153,6 @@ def _optimized_group_record_batches_by_hash_bucket(
         record_batches.append(record_batch)
 
     if record_batches:
-        print(f"{len(record_batches)} -- END")
         appended_len, append_latency = timed_invocation(
             _append_table_by_hash_bucket,
             pa.Table.from_batches(record_batches),

{deltacat-1.1.37 → deltacat-1.1.38}/deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -356,6 +356,7 @@ class TestCompactionSession:
         assert compaction_audit.hash_bucket_count == 2
         assert compaction_audit.input_file_count == 1
         assert compaction_audit.output_file_count == 2
+        assert compaction_audit.output_record_count == 4
         assert abs(compaction_audit.output_size_bytes - 1832) / 1832 <= self.ERROR_RATE
         assert abs(compaction_audit.input_size_bytes - 936) / 936 <= self.ERROR_RATE
 
@@ -423,9 +424,18 @@ class TestCompactionSession:
         assert compaction_audit.hash_bucket_count == 2
         assert compaction_audit.input_file_count == 3
         assert compaction_audit.output_file_count == 2
+        assert compaction_audit.output_record_count == 7
         assert abs(compaction_audit.output_size_bytes - 1843) / 1843 <= self.ERROR_RATE
         assert abs(compaction_audit.input_size_bytes - 2748) / 2748 <= self.ERROR_RATE
 
+        record_invariant = compaction_audit.output_record_count == (
+            compaction_audit.input_records
+            - compaction_audit.records_deduped
+            - compaction_audit.records_deleted
+            + compaction_audit.untouched_record_count
+        )
+        assert record_invariant is True
+
     def test_compact_partition_when_incremental_then_intelligent_estimation_sanity(
         self, s3_resource, local_deltacat_storage_kwargs
     ):
@@ -1006,3 +1016,81 @@ class TestCompactionSession:
 
         rcf = get_rcf(s3_resource, new_uri)
         assert rcf.hash_bucket_count == 4
+
+    def test_compaction_with_zero_records(
+        self, s3_resource, local_deltacat_storage_kwargs
+    ):
+        """
+        Test case where compaction results in 0 records.
+        Verify audit handles this correctly without crashing.
+        """
+        # setup - create empty source delta
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        # Create an empty table that will result in 0 records after compaction
+        empty_table = pa.table({"pk": pa.array([])})
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, pa_table=empty_table, **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 1,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        # verify - compaction should complete successfully with 0 records
+        assert rcf_url is not None
+        rcf = get_rcf(s3_resource, rcf_url)
+
+        _, compaction_audit_key = rcf.compaction_audit_url.strip("s3://").split("/", 1)
+        compaction_audit = CompactionSessionAuditInfo(
+            **read_s3_contents(
+                s3_resource, TEST_S3_RCF_BUCKET_NAME, compaction_audit_key
+            )
+        )
+
+        # Verify that audit handles zero records correctly
+        assert compaction_audit.input_records == 0
+        assert compaction_audit.output_record_count == 0
+        assert compaction_audit.records_deduped == 0
+        assert compaction_audit.records_deleted == 0
+        assert compaction_audit.untouched_record_count == 0
+        assert compaction_audit.output_file_count >= 0  # May still create empty files
+        record_invariant = compaction_audit.output_record_count == (
+            compaction_audit.input_records
+            - compaction_audit.records_deduped
+            - compaction_audit.records_deleted
+            + compaction_audit.untouched_record_count
+        )
+        assert record_invariant is True

{deltacat-1.1.37 → deltacat-1.1.38/deltacat.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.37
+Version: 1.1.38
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team