deltacat 1.1.15__py3-none-any.whl → 1.1.16__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.15"
+__version__ = "1.1.16"
 
 
 __all__ = [

deltacat/compute/compactor_v2/compaction_session.py

@@ -43,7 +43,6 @@ from deltacat.compute.compactor_v2.private.compaction_utils import (
     _stage_new_partition,
     _run_hash_and_merge,
     _process_merge_results,
-    _upload_compaction_audit,
     _write_new_round_completion_file,
     _commit_compaction_result,
 )
@@ -201,11 +200,6 @@ def _execute_compaction(
 
     compaction_audit.save_round_completion_stats(mat_results)
 
-    _upload_compaction_audit(
-        params,
-        compaction_audit,
-        round_completion_info,
-    )
     compaction_result: ExecutionCompactionResult = _write_new_round_completion_file(
         params,
         compaction_audit,
@@ -215,5 +209,6 @@ def _execute_compaction(
         rcf_source_partition_locator,
         new_compacted_delta_locator,
         pyarrow_write_result,
+        round_completion_info,
     )
     return compaction_result

deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -365,6 +365,7 @@ def _run_hash_and_merge(
         if mutable_compaction_audit.telemetry_time_in_seconds
         else 0.0
     )
+
     mutable_compaction_audit.set_telemetry_time_in_seconds(
         telemetry_this_round + previous_telemetry
     )
@@ -598,10 +599,10 @@ def _process_merge_results(
     return merged_delta, mat_results, hb_id_to_entry_indices_range
 
 
-def _upload_compaction_audit(
+def _update_and_upload_compaction_audit(
     params: CompactPartitionParams,
     mutable_compaction_audit: CompactionSessionAuditInfo,
-    round_completion_info: RoundCompletionInfo,
+    round_completion_info: Optional[RoundCompletionInfo] = None,
 ) -> None:
 
     # After all incremental delta related calculations, we update
@@ -637,6 +638,7 @@ def _write_new_round_completion_file(
     rcf_source_partition_locator: rcf.PartitionLocator,
     new_compacted_delta_locator: DeltaLocator,
     pyarrow_write_result: PyArrowWriteResult,
+    prev_round_completion_info: Optional[RoundCompletionInfo] = None,
 ) -> ExecutionCompactionResult:
     input_inflation = None
     input_average_record_size_bytes = None
@@ -664,6 +666,12 @@ def _write_new_round_completion_file(
         f" and average record size={input_average_record_size_bytes}"
     )
 
+    _update_and_upload_compaction_audit(
+        params,
+        mutable_compaction_audit,
+        prev_round_completion_info,
+    )
+
     new_round_completion_info = RoundCompletionInfo.of(
         high_watermark=params.last_stream_position_to_compact,
         compacted_delta_locator=new_compacted_delta_locator,

deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -87,6 +87,7 @@ class TestCompactionSession:
     INCREMENTAL_FILE_PATH = (
         "deltacat/tests/compute/compactor_v2/data/incremental_source_date_pk.csv"
     )
+    ERROR_RATE = 0.05
 
     def test_compact_partition_when_no_input_deltas_to_compact(
         self, local_deltacat_storage_kwargs
@@ -253,3 +254,148 @@ class TestCompactionSession:
         # as it should be running incremental
         assert compaction_audit.uniform_deltas_created == 1
         assert compaction_audit.input_records == 6
+
+    def test_compact_partition_when_incremental_then_rcf_stats_accurate(
+        self, s3_resource, local_deltacat_storage_kwargs
+    ):
+        """
+        A test case which asserts the RCF stats are correctly generated for
+        a rebase and incremental use-case.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 2,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        backfill_rcf = get_rcf(s3_resource, rcf_url)
+        _, compaction_audit_key = backfill_rcf.compaction_audit_url.strip(
+            "s3://"
+        ).split("/", 1)
+        compaction_audit = CompactionSessionAuditInfo(
+            **read_s3_contents(
+                s3_resource, TEST_S3_RCF_BUCKET_NAME, compaction_audit_key
+            )
+        )
+
+        assert abs(backfill_rcf.input_inflation - 0.05235042735042735) <= 1e-5
+        assert abs(backfill_rcf.input_average_record_size_bytes - 12.25) <= 1e-5
+
+        assert compaction_audit.input_records == 4
+        assert compaction_audit.records_deduped == 0
+        assert compaction_audit.records_deleted == 0
+        assert compaction_audit.untouched_file_count == 0
+        assert compaction_audit.untouched_record_count == 0
+        assert compaction_audit.untouched_size_bytes == 0
+        assert compaction_audit.untouched_file_ratio == 0
+        assert compaction_audit.uniform_deltas_created == 1
+        assert compaction_audit.hash_bucket_count == 2
+        assert compaction_audit.input_file_count == 1
+        assert compaction_audit.output_file_count == 2
+        assert abs(compaction_audit.output_size_bytes - 1832) / 1832 <= self.ERROR_RATE
+        assert abs(compaction_audit.input_size_bytes - 936) / 936 <= self.ERROR_RATE
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_destination_partition = ds.get_partition(
+            dest_partition.stream_locator, [], **local_deltacat_storage_kwargs
+        )
+
+        new_rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": new_destination_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 2,
+                    "last_stream_position_to_compact": new_source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": None,
+                    "rebase_source_partition_high_watermark": None,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": new_source_delta.partition_locator,
+                }
+            )
+        )
+
+        new_rcf = get_rcf(s3_resource, new_rcf_url)
+        _, compaction_audit_key = new_rcf.compaction_audit_url.strip("s3://").split(
+            "/", 1
+        )
+        compaction_audit = CompactionSessionAuditInfo(
+            **read_s3_contents(
+                s3_resource, TEST_S3_RCF_BUCKET_NAME, compaction_audit_key
+            )
+        )
+
+        # as it should be running incremental
+        assert abs(new_rcf.input_inflation - 0.027292576419213975) <= 1e-5
+        assert abs(new_rcf.input_average_record_size_bytes - 12.5) <= 1e-5
+
+        assert compaction_audit.input_records == 6
+        assert compaction_audit.records_deduped == 1
+        assert compaction_audit.records_deleted == 0
+        assert compaction_audit.untouched_file_count == 1
+        assert compaction_audit.untouched_record_count == 2
+        assert (
+            abs(compaction_audit.untouched_size_bytes - 916) / 916 <= self.ERROR_RATE
+        )  # 5% error
+        assert abs(compaction_audit.untouched_file_ratio - 50) <= 1e-5
+        assert compaction_audit.uniform_deltas_created == 1
+        assert compaction_audit.hash_bucket_count == 2
+        assert compaction_audit.input_file_count == 3
+        assert compaction_audit.output_file_count == 2
+        assert abs(compaction_audit.output_size_bytes - 1843) / 1843 <= self.ERROR_RATE
+        assert abs(compaction_audit.input_size_bytes - 2748) / 2748 <= self.ERROR_RATE

{deltacat-1.1.15.dist-info → deltacat-1.1.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.15
+Version: 1.1.16
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.15.dist-info → deltacat-1.1.16.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=B3feGTJ_HURh5NXPYw8eS-uft2b2FuwzVsZjfVXCa5c,1778
+deltacat/__init__.py,sha256=o3hcQ85nWUeDYkaXi3ADAbqwM_c5ajyxzlx-Z2jdKbI,1778
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=yWM4RXK7uRrQc1VgJv6Lv2UiNZWAx2wolLq7cBwjlkg,12770
 deltacat/logs.py,sha256=6g16VkEFidbaMjgenAjggE1r2l664drMVhreRs8B1IQ,8438
@@ -50,7 +50,7 @@ deltacat/compute/compactor/utils/round_completion_file.py,sha256=_rl8lBSO9KFW07Z
 deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZRG2NqBlCw1X3_FBc,2397
 deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
 deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/compaction_session.py,sha256=mRy7Rh2hIejKmfmVUXTbAD8ArUDRudDAY5YxlKV8dBM,8186
+deltacat/compute/compactor_v2/compaction_session.py,sha256=COtol2s63DRPbd-AN9KCiWr4exLX8x5Tvxea_7cOGEQ,8078
 deltacat/compute/compactor_v2/constants.py,sha256=4HkSebuRWlAzOnZ-_nYmMsf6d3koTwfrlBx9KxuoGe4,2417
 deltacat/compute/compactor_v2/deletes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/deletes/delete_file_envelope.py,sha256=AeuH9JRMwp6mvQf6P2cqL92hUEtResQq6qUTS0kIKac,3111
@@ -66,7 +66,7 @@ deltacat/compute/compactor_v2/model/merge_file_group.py,sha256=1o86t9lc3K6ZvtViV
 deltacat/compute/compactor_v2/model/merge_input.py,sha256=-SxTE0e67z2V7MiMEVz5aMu4E0k8h3-vqohvUUOC0do,5659
 deltacat/compute/compactor_v2/model/merge_result.py,sha256=_IZTCStpb4UKiRCJYA3g6EhAqjrw0t9vmoDAN8kIK-Y,436
 deltacat/compute/compactor_v2/private/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=B1i9y74gAhAQCu2vpjuN2muH425GHffnc2BhK-l6wQk,30061
+deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=HIr3ikF4iu_ztdy3FbtOw8vUKjc_RBP93ogH8EzMV64,30294
 deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=1R5xLUkl7GqL1nY-apAgY1czKDEHjIVYSRi9qLOMass,6726
 deltacat/compute/compactor_v2/steps/merge.py,sha256=LpktsDPfj7Of6RgUw9w1f3Y3OBkPDjvtyXjzFaIDoSo,21771
@@ -164,7 +164,7 @@ deltacat/tests/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 deltacat/tests/compute/compactor/utils/test_io.py,sha256=st5mlU4cVU-eQl7B4mvPgNA3izuNwbVawYOp-NcoyrI,4326
 deltacat/tests/compute/compactor/utils/test_round_completion_file.py,sha256=LAQ4usiRF4oTx4cA85L0eOcBa_Z-febc-CuzUijSGrI,7439
 deltacat/tests/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compute/compactor_v2/test_compaction_session.py,sha256=X1B47USzXXGt-b_BL6WyFENOXx2aqp6NLIdL_gCukNw,9641
+deltacat/tests/compute/compactor_v2/test_compaction_session.py,sha256=Ln3NxUy_2oC8cTfFSNy28lIRK8iNEabtxaqWzIqzyEY,16260
 deltacat/tests/compute/compactor_v2/test_hashlib.py,sha256=8csF2hFWtBvY2MbX3-6iphCsVXxRp0zP1NTnKhfdmkg,328
 deltacat/tests/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/compute/compactor_v2/utils/test_task_options.py,sha256=4fc5MJTLm3hFlFHK_-5MfyfzeZtOo8D2kBqDE2b8lh4,862
@@ -220,8 +220,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.15.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.15.dist-info/METADATA,sha256=KT4i-8YvQv2xdRV4DsIF7Htj5U-mJEsY68uornP81hg,1734
-deltacat-1.1.15.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-deltacat-1.1.15.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.15.dist-info/RECORD,,
+deltacat-1.1.16.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.16.dist-info/METADATA,sha256=EvOmjI60akKZ6BKEWQ2_KZJxr-Bp6wMZOU6-zV1EDos,1734
+deltacat-1.1.16.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+deltacat-1.1.16.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.16.dist-info/RECORD,,