deltacat 1.1.37__py3-none-any.whl → 1.1.38__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.37"
+__version__ = "1.1.38"
 
 
 __all__ = [

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -322,6 +322,17 @@ class CompactionSessionAuditInfo(dict):
         """
         return self.get("outputSizePyarrowBytes")
 
+    @property
+    def output_record_count(self) -> int:
+        """
+        The total number of records in the compacted output (includes untouched records).
+
+        Represents the final record count after compaction, including:
+            - Records that were processed and materialized
+            - Records that were untouched and copied by reference
+        """
+        return self.get("outputRecordCount")
+
     @property
     def total_cluster_memory_bytes(self) -> float:
         """
@@ -672,6 +683,19 @@ class CompactionSessionAuditInfo(dict):
         self["outputSizeBytes"] = output_size_bytes
         return output_size_bytes
 
+    def set_output_record_count(
+        self, output_records: int
+    ) -> CompactionSessionAuditInfo:
+        """
+        This includes both processed records and untouched records copied by reference.
+        """
+        if output_records < 0:
+            raise ValueError(
+                f"Output record count cannot be negative: {output_records}"
+            )
+        self["outputRecordCount"] = output_records
+        return self
+
     def set_output_size_pyarrow_bytes(
         self, output_size_pyarrow_bytes: float
     ) -> CompactionSessionAuditInfo:
@@ -902,6 +926,10 @@ class CompactionSessionAuditInfo(dict):
         self.set_output_file_count(pyarrow_write_result.files)
         self.set_output_size_bytes(pyarrow_write_result.file_bytes)
         self.set_output_size_pyarrow_bytes(pyarrow_write_result.pyarrow_bytes)
+        # NOTE:  Aggregating untouched_record_count with records to get a total of record count in the compacted table
+        self.set_output_record_count(
+            pyarrow_write_result.records + untouched_file_record_count
+        )
 
         self.set_peak_memory_used_bytes_per_task(
             max(

deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -93,7 +93,7 @@ def _append_table_by_hash_bucket(
         all_buckets = pc.unique(pyarrow_table[sc._HASH_BUCKET_IDX_COLUMN_NAME])
         assert (
             len(all_buckets) == 1
-        ), f"Only one hash bucket is allowed by found {len(all_buckets)}"
+        ), f"Only one hash bucket is allowed but found {len(all_buckets)}"
         assert (
             all_buckets[0].as_py() == hb_idx
         ), f"Hash bucket not equal, {all_buckets[0]} and {hb_idx}"
@@ -153,7 +153,6 @@ def _optimized_group_record_batches_by_hash_bucket(
         record_batches.append(record_batch)
 
     if record_batches:
-        print(f"{len(record_batches)} -- END")
         appended_len, append_latency = timed_invocation(
             _append_table_by_hash_bucket,
             pa.Table.from_batches(record_batches),

deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -356,6 +356,7 @@ class TestCompactionSession:
         assert compaction_audit.hash_bucket_count == 2
         assert compaction_audit.input_file_count == 1
         assert compaction_audit.output_file_count == 2
+        assert compaction_audit.output_record_count == 4
         assert abs(compaction_audit.output_size_bytes - 1832) / 1832 <= self.ERROR_RATE
         assert abs(compaction_audit.input_size_bytes - 936) / 936 <= self.ERROR_RATE
 
@@ -423,9 +424,18 @@ class TestCompactionSession:
         assert compaction_audit.hash_bucket_count == 2
         assert compaction_audit.input_file_count == 3
         assert compaction_audit.output_file_count == 2
+        assert compaction_audit.output_record_count == 7
         assert abs(compaction_audit.output_size_bytes - 1843) / 1843 <= self.ERROR_RATE
         assert abs(compaction_audit.input_size_bytes - 2748) / 2748 <= self.ERROR_RATE
 
+        record_invariant = compaction_audit.output_record_count == (
+            compaction_audit.input_records
+            - compaction_audit.records_deduped
+            - compaction_audit.records_deleted
+            + compaction_audit.untouched_record_count
+        )
+        assert record_invariant is True
+
     def test_compact_partition_when_incremental_then_intelligent_estimation_sanity(
         self, s3_resource, local_deltacat_storage_kwargs
     ):
@@ -1006,3 +1016,81 @@ class TestCompactionSession:
 
         rcf = get_rcf(s3_resource, new_uri)
         assert rcf.hash_bucket_count == 4
+
+    def test_compaction_with_zero_records(
+        self, s3_resource, local_deltacat_storage_kwargs
+    ):
+        """
+        Test case where compaction results in 0 records.
+        Verify audit handles this correctly without crashing.
+        """
+        # setup - create empty source delta
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        # Create an empty table that will result in 0 records after compaction
+        empty_table = pa.table({"pk": pa.array([])})
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, pa_table=empty_table, **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 1,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        # verify - compaction should complete successfully with 0 records
+        assert rcf_url is not None
+        rcf = get_rcf(s3_resource, rcf_url)
+
+        _, compaction_audit_key = rcf.compaction_audit_url.strip("s3://").split("/", 1)
+        compaction_audit = CompactionSessionAuditInfo(
+            **read_s3_contents(
+                s3_resource, TEST_S3_RCF_BUCKET_NAME, compaction_audit_key
+            )
+        )
+
+        # Verify that audit handles zero records correctly
+        assert compaction_audit.input_records == 0
+        assert compaction_audit.output_record_count == 0
+        assert compaction_audit.records_deduped == 0
+        assert compaction_audit.records_deleted == 0
+        assert compaction_audit.untouched_record_count == 0
+        assert compaction_audit.output_file_count >= 0  # May still create empty files
+        record_invariant = compaction_audit.output_record_count == (
+            compaction_audit.input_records
+            - compaction_audit.records_deduped
+            - compaction_audit.records_deleted
+            + compaction_audit.untouched_record_count
+        )
+        assert record_invariant is True

{deltacat-1.1.37.dist-info → deltacat-1.1.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.37
+Version: 1.1.38
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.37.dist-info → deltacat-1.1.38.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=u00X92zHfZJzS08a-2kx3kCLcz40L-THm0HowDiBOiA,1778
+deltacat/__init__.py,sha256=N5dMIxvjmqQP9-zhdx6gAxQ4tZIS3t3PIPLXaIsxWjI,1778
 deltacat/constants.py,sha256=TUJLXUJ9xq1Ryil72yLkKR8EDH_Irp5wUg56QstbRNE,2181
 deltacat/exceptions.py,sha256=7sjk3BuMY5Oo-6OvAfHncZx_OcvtEL47BblWr2F7waE,12740
 deltacat/logs.py,sha256=EQSDin1deehzz5xlLV1_TrFJrO_IBZ9Ahp7MdL-4cK8,9363
@@ -25,7 +25,7 @@ deltacat/compute/compactor/compaction_session.py,sha256=YthBYNpj6qvr6SqfVfXTy5yl
 deltacat/compute/compactor/repartition_session.py,sha256=AAPwNZtPpC_Mtoja855_alBdXDA6efp7zcvkE-MANaQ,7254
 deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/model/compact_partition_params.py,sha256=jjvpUiHfGAw-Dy7s4wyTINtruf8Nk4EPMma7Y4KMF2U,19067
-deltacat/compute/compactor/model/compaction_session_audit_info.py,sha256=Jjt4YOEO8lc-kiV4fB7rOD_Xd17_BS6pRDzqbtZp0GI,31350
+deltacat/compute/compactor/model/compaction_session_audit_info.py,sha256=48DjIuJ6_Gv7GVoUfLSA5sjrCyZ4qY3tWnR-bYXh3oc,32440
 deltacat/compute/compactor/model/compactor_version.py,sha256=RwRvManiCxZmzjAWzm1OPDxjB1BEHu1d0fBJyGhXKxA,87
 deltacat/compute/compactor/model/dedupe_result.py,sha256=1OCV944qJdLQ_-8scisVKl45ej1eRv9OV539QYZtQ-U,292
 deltacat/compute/compactor/model/delta_annotated.py,sha256=bCE9H5mrBoHfd1lbL6tYWC4_dbAgucAlFLjOtyPLW14,12515
@@ -76,7 +76,7 @@ deltacat/compute/compactor_v2/utils/dedupe.py,sha256=Jz1QbBOdZJwT8K1vD9q01eOn7hd
 deltacat/compute/compactor_v2/utils/delta.py,sha256=I7Yvda8NVbpKXG3nM2Ku1utvR2r2OpHvUMqUL2ja3aw,3626
 deltacat/compute/compactor_v2/utils/io.py,sha256=Xjs7_D-0xKSetvllIe4o96aM1elfdjt1Ii7YfsHPvZs,6108
 deltacat/compute/compactor_v2/utils/merge.py,sha256=fAzEYwQYH2ia8MLdEFdZFivWHpi6qZu8AyyEK0H0vwE,5363
-deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=Qsn0BQrlBWSLqu4srd-LJUX8BaVqG6Wo1oAros7LYWw,12677
+deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=1IolwUoS9PkqG7yIASVHgZXIlIt8Px_8DIrqCajCqCs,12631
 deltacat/compute/compactor_v2/utils/task_options.py,sha256=0GoB_DLkCN1q8CVKTlWlDYt55qnpTDIa9fPyXJwB-cU,13801
 deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
 deltacat/compute/merge_on_read/daft.py,sha256=1oC38u5ig_aTrq7EzyWBo8Ui54rb6yERYMk-vEFbpxM,1400
@@ -152,7 +152,7 @@ deltacat/tests/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 deltacat/tests/compute/compactor/utils/test_io.py,sha256=st5mlU4cVU-eQl7B4mvPgNA3izuNwbVawYOp-NcoyrI,4326
 deltacat/tests/compute/compactor/utils/test_round_completion_file.py,sha256=LAQ4usiRF4oTx4cA85L0eOcBa_Z-febc-CuzUijSGrI,7439
 deltacat/tests/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compute/compactor_v2/test_compaction_session.py,sha256=F1DFaranHekHB7HSNH-0_hV5ovdR5HfF9JqTVDw6Vh8,42575
+deltacat/tests/compute/compactor_v2/test_compaction_session.py,sha256=R98elvDY9EwHI8vrC09VX4aFgKVtg8H4xFo7PNNNvZs,46446
 deltacat/tests/compute/compactor_v2/test_hashlib.py,sha256=8csF2hFWtBvY2MbX3-6iphCsVXxRp0zP1NTnKhfdmkg,328
 deltacat/tests/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py,sha256=eoiDuBUhgCmc3DYKCXL1g4QWtmROhZ0RJCQgePMY9as,9959
@@ -212,8 +212,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.37.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.37.dist-info/METADATA,sha256=iHlaZ9sS-CrQby0kxCrOigl1ZGZKpniwf9LyYbagwzI,1733
-deltacat-1.1.37.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-deltacat-1.1.37.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.37.dist-info/RECORD,,
+deltacat-1.1.38.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.38.dist-info/METADATA,sha256=ywP_JPvdLP0cpQUlwAHPSVjeu4nlFQyJGbGevs_RrWc,1733
+deltacat-1.1.38.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+deltacat-1.1.38.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.38.dist-info/RECORD,,