deltacat 1.1.12__py3-none-any.whl → 1.1.14__py3-none-any.whl

deltacat/tests/compute/test_util_common.py

@@ -23,12 +23,16 @@ from deltacat.tests.compute.test_util_constant import (
 from deltacat.compute.compactor import (
     RoundCompletionInfo,
 )
+from deltacat.compute.compactor.model.compaction_session_audit_info import (
+    CompactionSessionAuditInfo,
+)
 
 from deltacat.storage.model.partition import PartitionLocator
 from deltacat.storage.model.stream import StreamLocator
 from deltacat.storage.model.table_version import TableVersionLocator
 from deltacat.storage.model.table import TableLocator
 from deltacat.storage.model.namespace import NamespaceLocator
+from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 
 
 class PartitionKeyType(str, Enum):
@@ -215,3 +219,100 @@ def offer_iso8601_timestamp_list(
             )
         )
     return res
+
+
+def assert_compaction_audit(
+    compactor_version: CompactorVersion, compaction_audit: CompactionSessionAuditInfo
+):
+    if compactor_version == CompactorVersion.V2:
+        audit_entries = [
+            compaction_audit.deltacat_version,
+            compaction_audit.ray_version,
+            compaction_audit.audit_url,
+            compaction_audit.hash_bucket_count,
+            compaction_audit.compactor_version,
+            compaction_audit.input_size_bytes,
+            compaction_audit.input_file_count,
+            compaction_audit.estimated_in_memory_size_bytes_during_discovery,
+            compaction_audit.uniform_deltas_created,
+            compaction_audit.delta_discovery_time_in_seconds,
+            compaction_audit.hash_bucket_time_in_seconds,
+            compaction_audit.hash_bucket_invoke_time_in_seconds,
+            compaction_audit.hash_bucket_result_size_bytes,
+            compaction_audit.total_cluster_object_store_memory_bytes,
+            compaction_audit.total_cluster_memory_bytes,
+            compaction_audit.total_object_store_memory_used_bytes,
+            compaction_audit.hash_bucket_post_object_store_memory_used_bytes,
+            compaction_audit.hash_bucket_result_wait_time_in_seconds,
+            compaction_audit.peak_memory_used_bytes_per_hash_bucket_task,
+            compaction_audit.hash_bucket_processed_size_bytes,
+            compaction_audit.input_records,
+            compaction_audit.merge_time_in_seconds,
+            compaction_audit.merge_invoke_time_in_seconds,
+            compaction_audit.merge_result_size,
+            compaction_audit.merge_post_object_store_memory_used_bytes,
+            compaction_audit.merge_result_wait_time_in_seconds,
+            compaction_audit.peak_memory_used_bytes_per_merge_task,
+            compaction_audit.records_deduped,
+            compaction_audit.records_deleted,
+            compaction_audit.compaction_time_in_seconds,
+            compaction_audit.peak_memory_used_bytes_by_compaction_session_process,
+            compaction_audit.untouched_file_count,
+            compaction_audit.untouched_file_ratio,
+            compaction_audit.untouched_record_count,
+            compaction_audit.untouched_size_bytes,
+            compaction_audit.output_file_count,
+            compaction_audit.output_size_bytes,
+            compaction_audit.output_size_pyarrow_bytes,
+            compaction_audit.peak_memory_used_bytes_per_task,
+            compaction_audit.pyarrow_version,
+            compaction_audit.telemetry_time_in_seconds,
+        ]
+        for entry in audit_entries:
+            assert entry is not None
+    return True
+
+
+def assert_compaction_audit_no_hash_bucket(
+    compactor_version: CompactorVersion, compaction_audit: CompactionSessionAuditInfo
+):
+    if compactor_version == CompactorVersion.V2:
+        audit_entries = [
+            compaction_audit.deltacat_version,
+            compaction_audit.ray_version,
+            compaction_audit.audit_url,
+            compaction_audit.hash_bucket_count,
+            compaction_audit.compactor_version,
+            compaction_audit.input_size_bytes,
+            compaction_audit.input_file_count,
+            compaction_audit.estimated_in_memory_size_bytes_during_discovery,
+            compaction_audit.uniform_deltas_created,
+            compaction_audit.delta_discovery_time_in_seconds,
+            compaction_audit.total_cluster_object_store_memory_bytes,
+            compaction_audit.total_cluster_memory_bytes,
+            compaction_audit.total_object_store_memory_used_bytes,
+            compaction_audit.input_records,
+            compaction_audit.merge_time_in_seconds,
+            compaction_audit.merge_invoke_time_in_seconds,
+            compaction_audit.merge_result_size,
+            compaction_audit.merge_post_object_store_memory_used_bytes,
+            compaction_audit.merge_result_wait_time_in_seconds,
+            compaction_audit.peak_memory_used_bytes_per_merge_task,
+            compaction_audit.records_deduped,
+            compaction_audit.records_deleted,
+            compaction_audit.compaction_time_in_seconds,
+            compaction_audit.peak_memory_used_bytes_by_compaction_session_process,
+            compaction_audit.untouched_file_count,
+            compaction_audit.untouched_file_ratio,
+            compaction_audit.untouched_record_count,
+            compaction_audit.untouched_size_bytes,
+            compaction_audit.output_file_count,
+            compaction_audit.output_size_bytes,
+            compaction_audit.output_size_pyarrow_bytes,
+            compaction_audit.peak_memory_used_bytes_per_task,
+            compaction_audit.pyarrow_version,
+            compaction_audit.telemetry_time_in_seconds,
+        ]
+        for entry in audit_entries:
+            assert entry is not None
+    return True

{deltacat-1.1.12.dist-info → deltacat-1.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.12
+Version: 1.1.14
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -15,20 +15,20 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aws-embedded-metrics ==3.2.0
-Requires-Dist: boto3 ~=1.34
-Requires-Dist: numpy ==1.21.5
-Requires-Dist: pandas ==1.3.5
-Requires-Dist: pyarrow ==12.0.1
-Requires-Dist: pydantic ==1.10.4
-Requires-Dist: ray >=2.20.0
-Requires-Dist: s3fs ==2024.5.0
-Requires-Dist: tenacity ==8.1.0
-Requires-Dist: typing-extensions ==4.4.0
-Requires-Dist: pymemcache ==4.0.0
-Requires-Dist: redis ==4.6.0
-Requires-Dist: getdaft ==0.2.31
-Requires-Dist: schedule ==1.2.0
+Requires-Dist: aws-embedded-metrics==3.2.0
+Requires-Dist: boto3~=1.34
+Requires-Dist: numpy==1.21.5
+Requires-Dist: pandas==1.3.5
+Requires-Dist: pyarrow==12.0.1
+Requires-Dist: pydantic==1.10.4
+Requires-Dist: ray>=2.20.0
+Requires-Dist: s3fs==2024.5.0
+Requires-Dist: tenacity==8.1.0
+Requires-Dist: typing-extensions==4.4.0
+Requires-Dist: pymemcache==4.0.0
+Requires-Dist: redis==4.6.0
+Requires-Dist: getdaft==0.2.31
+Requires-Dist: schedule==1.2.0
 
 # DeltaCAT
 

{deltacat-1.1.12.dist-info → deltacat-1.1.14.dist-info}/RECORD

@@ -1,9 +1,9 @@
-deltacat/__init__.py,sha256=jtrgQAzzZxq5YU_9RwtH0N4p8k4_ACd_vcjuWJf934Q,1778
+deltacat/__init__.py,sha256=dvd9BOMviyQgIHFPVSN_kQV6dAuyud4WZ6kUJyuO9go,1778
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=yWM4RXK7uRrQc1VgJv6Lv2UiNZWAx2wolLq7cBwjlkg,12770
 deltacat/logs.py,sha256=6g16VkEFidbaMjgenAjggE1r2l664drMVhreRs8B1IQ,8438
 deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/aws/clients.py,sha256=VgddlV3AEjlBGIFmhhHxokYzwJ-lXnmHAeprVyADduI,6948
+deltacat/aws/clients.py,sha256=4eQvpkV1PzFfxog7EriuglOGGwNFHR5hbGYpjsNNPxk,6949
 deltacat/aws/constants.py,sha256=1HnDXrSokW-G3YA3qKEiv7fZVntDs1uSk6a7On-VG5k,1223
 deltacat/aws/s3u.py,sha256=IdT0XqDXVOkPdo5Em5u3qAkV1UXFpXaE1rTkUDKv4f4,28578
 deltacat/aws/redshift/__init__.py,sha256=7SvjG-dqox8zZUhFicTsUvpG5vXYDl_QQ3ohlHOgTKc,342
@@ -24,7 +24,7 @@ deltacat/compute/compactor/__init__.py,sha256=ivpOPve1yKi3Vz3tVgp-eeFMNEeUSf-dlR
 deltacat/compute/compactor/compaction_session.py,sha256=sGlSpfZaC_RYG2n8NGpT9z2X8V521nlLJktYtzafpmg,27715
 deltacat/compute/compactor/repartition_session.py,sha256=AAPwNZtPpC_Mtoja855_alBdXDA6efp7zcvkE-MANaQ,7254
 deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor/model/compact_partition_params.py,sha256=yBBm4gk3y-ZdFz22KeBrVruF0GauHk9ZL3xPjuM4Kt4,15593
+deltacat/compute/compactor/model/compact_partition_params.py,sha256=RuyYQVuoxKlt4vYTbVpCgoM6K8S_-TFhVuOnPVkxLfE,15850
 deltacat/compute/compactor/model/compaction_session_audit_info.py,sha256=Aoae5KNmbdYt48utFQR_zrjHTkC9JNd4jPhQmyJji-A,30547
 deltacat/compute/compactor/model/compactor_version.py,sha256=RwRvManiCxZmzjAWzm1OPDxjB1BEHu1d0fBJyGhXKxA,87
 deltacat/compute/compactor/model/dedupe_result.py,sha256=1OCV944qJdLQ_-8scisVKl45ej1eRv9OV539QYZtQ-U,292
@@ -50,7 +50,7 @@ deltacat/compute/compactor/utils/round_completion_file.py,sha256=_rl8lBSO9KFW07Z
 deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZRG2NqBlCw1X3_FBc,2397
 deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
 deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/compaction_session.py,sha256=WUfWYZuQBCxA5RQ0qgAZjUE7tHORWemWZaUIwVwFsC8,27950
+deltacat/compute/compactor_v2/compaction_session.py,sha256=1V7EgtSF3gmqoaN6iRQ1Yw66Epvmlh2Hj6-y_rE83zI,7755
 deltacat/compute/compactor_v2/constants.py,sha256=RoVJb86km-0ghtEFB0z24H7yuCJCbQ2t7_-LuXFNYak,2304
 deltacat/compute/compactor_v2/deletes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/deletes/delete_file_envelope.py,sha256=AeuH9JRMwp6mvQf6P2cqL92hUEtResQq6qUTS0kIKac,3111
@@ -65,6 +65,8 @@ deltacat/compute/compactor_v2/model/hash_bucket_result.py,sha256=EsY9BPPywhmxlcL
 deltacat/compute/compactor_v2/model/merge_file_group.py,sha256=1o86t9lc3K6ZvtViVO1SVljCj6f0B3MfB3hqtGm2S0s,7410
 deltacat/compute/compactor_v2/model/merge_input.py,sha256=-SxTE0e67z2V7MiMEVz5aMu4E0k8h3-vqohvUUOC0do,5659
 deltacat/compute/compactor_v2/model/merge_result.py,sha256=_IZTCStpb4UKiRCJYA3g6EhAqjrw0t9vmoDAN8kIK-Y,436
+deltacat/compute/compactor_v2/private/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=3ukQx50xH810XJu1KzKdxY95lGuZGerHhHTJ89ns-jg,27622
 deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=1R5xLUkl7GqL1nY-apAgY1czKDEHjIVYSRi9qLOMass,6726
 deltacat/compute/compactor_v2/steps/merge.py,sha256=ukCn312igxq7jNiCn7a2Vzk309LKdYZ902HTcEZhjM4,21774
@@ -75,7 +77,7 @@ deltacat/compute/compactor_v2/utils/delta.py,sha256=8hjkDeIIkSX-gAQ2utQSp2sZcO2t
 deltacat/compute/compactor_v2/utils/io.py,sha256=autXlE3uHICdCCuJoS7mfdeJbRRiz2_xlz-3izlccB4,5264
 deltacat/compute/compactor_v2/utils/merge.py,sha256=7UHxm71iJ1dgRoz8v73CqoeylNzO36t90OJsVVBDFxk,5312
 deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=ghyIifjXtqXgi8lN3lfnVQ2vi8uk_ny0FE7hsQlLjRQ,11538
-deltacat/compute/compactor_v2/utils/task_options.py,sha256=XFvZ_8mCq3cDnFlopFG84IahcYEddilZDmU1PkKq-zg,14067
+deltacat/compute/compactor_v2/utils/task_options.py,sha256=VXvoVVUq5re8NiOoyrfz34qSRiOTB0IkxHJlMqKsBmk,14066
 deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
 deltacat/compute/merge_on_read/daft.py,sha256=1oC38u5ig_aTrq7EzyWBo8Ui54rb6yERYMk-vEFbpxM,1400
 deltacat/compute/merge_on_read/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -143,14 +145,14 @@ deltacat/tests/aws/test_s3u.py,sha256=FsYCH8K8DsDRPOtTp-w1Nu3ATqt4p1mqDo6aVJV-Sb
 deltacat/tests/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/catalog/test_default_catalog_impl.py,sha256=2l5uwmtLlUJ9yH1LDggtj81fa-pHqbE0-VBt6G4Hyc0,3180
 deltacat/tests/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=NCMgZ5d9YyTb4Irp8Rnx8GLbvod08TshpiblFkZ5ItE,3105
-deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=XP0rwMLgQRFt4aNblLT250tVjQqTYNtO2VXnD7CqugQ,73673
-deltacat/tests/compute/compact_partition_test_cases.py,sha256=H6CEXiRoHTIql3Z2_xMztY4YKoqqn9ZycpAnJyZooTU,25103
-deltacat/tests/compute/test_compact_partition_incremental.py,sha256=vH2z9F18QdjpQmNKTUk4IIrwRC5Uj7z41Ox7q1rFxZU,13995
+deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=NfcB1aTq6HeYMFrtooIIoifzLp5U0xFTN6F7Lpk8cYQ,3143
+deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=Kl5A7hoJ0pVOE-rZna_4XcuIjhuzQbJudvnfNYKHgGo,75436
+deltacat/tests/compute/compact_partition_test_cases.py,sha256=R9eiKvxCLqcoHjAx3iOogdnXZEO9TvLbRf0wA7bcJN4,26170
+deltacat/tests/compute/test_compact_partition_incremental.py,sha256=Z0hyQGhMZjCaOn1Vk4qUbgDiS7HDhtdNeFQyG1PJhqA,14559
 deltacat/tests/compute/test_compact_partition_params.py,sha256=Dm5eLyHo8oGMeO3XBbpj1rZqHtPZ1hAB7z2qvzc4Lxk,8497
-deltacat/tests/compute/test_compact_partition_rebase.py,sha256=qciMqIw-viNi6sdhSPWo2s3pK-cP00J6jz16h4vz2kY,10186
-deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=NvXjgrYAB5zq_SzwjYMYwsTqseAMPNlDjaA6LEBJRbg,14159
-deltacat/tests/compute/test_util_common.py,sha256=qarWBIrRK0daZ_p7O-SlSx2U-3ma5uOBQAyHuqreBTE,6703
+deltacat/tests/compute/test_compact_partition_rebase.py,sha256=hKMSQ-BQG6w0O_4TlzESY6pA5Ft5c5WSHLIFvj5hePI,11434
+deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=CHHfNFEJW8S1We7NE1Gg6EaoKEWnaOMRxWrLyirrahc,14643
+deltacat/tests/compute/test_util_common.py,sha256=oTkTuo6wscVN8hmoQASIKP_DJN-M0um_ySCOcXv9AJA,11699
 deltacat/tests/compute/test_util_constant.py,sha256=4o-W3E7r7jhFl1A3OFLLrdKnwcF46zx4lEIDY8ONJ3c,929
 deltacat/tests/compute/test_util_create_table_deltas_repo.py,sha256=oY7ZMrvooDVOkhoAJnfdLGO0ejQAwgkqKrt7UmgiObU,9037
 deltacat/tests/compute/compactor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -216,8 +218,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.12.dist-info/METADATA,sha256=fqP5NYc_4wrfWcsQXO1Aht8xM22LrmDZLHfMAqq2opQ,1748
-deltacat-1.1.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-deltacat-1.1.12.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.12.dist-info/RECORD,,
+deltacat-1.1.14.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.14.dist-info/METADATA,sha256=orKTHhgUb74RXGlgcKhu-M36EmYd3-41AH7V1IP2jEI,1734
+deltacat-1.1.14.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+deltacat-1.1.14.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.14.dist-info/RECORD,,

{deltacat-1.1.12.dist-info → deltacat-1.1.14.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any