deltacat 1.1.24__py3-none-any.whl → 1.1.26__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.24"
+__version__ = "1.1.26"
 
 
 __all__ = [

deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -227,7 +227,6 @@ def _run_hash_and_merge(
     previous_compacted_delta_manifest: Optional[Manifest],
     compacted_partition: Partition,
 ) -> List[MergeResult]:
-    created_obj_ids = set()
     telemetry_time_hb = 0
     total_input_records_count = np.int64(0)
     total_hb_record_count = np.int64(0)
@@ -289,7 +288,6 @@ def _run_hash_and_merge(
                 hb_result.hash_bucket_group_to_obj_id_tuple
             ):
                 if object_id_size_tuple:
-                    created_obj_ids.add(object_id_size_tuple[0])
                     all_hash_group_idx_to_obj_id[hash_group_index].append(
                         object_id_size_tuple[0],
                     )
@@ -369,10 +367,9 @@ def _run_hash_and_merge(
     )
     if params.num_rounds > 1:
         logger.info(
-            f"Detected number of rounds to be {params.num_rounds}, "
-            f"preparing to delete {len(created_obj_ids)} objects from object store..."
+            f"Detected number of rounds to be {params.num_rounds}, preparing to clear object store..."
         )
-        params.object_store.delete_many(list(created_obj_ids))
+        params.object_store.clear()
     else:
         logger.info(
             f"Detected number of rounds to be {params.num_rounds}, not cleaning up object store..."

deltacat/io/memcached_object_store.py

@@ -205,7 +205,8 @@ class MemcachedObjectStore(IObjectStore):
             except BaseException:
                 # if an exception is raised then all, some, or none of the keys may have been deleted
                 logger.warning(
-                    f"Failed to fully delete refs: {current_refs}", exc_info=True
+                    f"Failed to fully delete {len(current_refs)} refs for ip: {ip}",
+                    exc_info=True,
                 )
                 all_deleted = False
 
@@ -218,9 +219,13 @@ class MemcachedObjectStore(IObjectStore):
             f"The total time taken to attempt deleting {len(refs)} objects is: {end - start}"
         )
 
+        # We need to clear the client cache in case of multi-round compaction because client cannot be pickled
+        self.client_cache.clear()
+
         return all_deleted
 
     def clear(self) -> bool:
+        start = time.monotonic()
         flushed = all(
             [
                 self._get_client_by_ip(ip).flush_all(noreply=False)
@@ -228,10 +233,13 @@ class MemcachedObjectStore(IObjectStore):
             ]
         )
         self.client_cache.clear()
+        end = time.monotonic()
 
         if flushed:
             logger.info("Successfully cleared cache contents.")
 
+        logger.info(f"The total time taken to clear the cache is: {end - start}")
+
         return flushed
 
     def close(self) -> None:

deltacat/io/redis_object_store.py

@@ -115,6 +115,21 @@ class RedisObjectStore(IObjectStore):
 
         return num_deleted == len(refs)
 
+    def clear(self) -> bool:
+        start = time.monotonic()
+        current_ip = self._get_current_ip()
+        client = self._get_client_by_ip(current_ip)
+        flushed = client.flushall()
+        self.client_cache.clear()
+        end = time.monotonic()
+
+        if flushed:
+            logger.info("Successfully cleared cache contents.")
+
+        logger.info(f"The total time taken to clear the cache is: {end - start}")
+
+        return flushed
+
     def _get_client_by_ip(self, ip_address: str):
         if ip_address in self.client_cache:
             return self.client_cache[ip_address]

deltacat/tests/compute/test_compact_partition_multiple_rounds.py

@@ -290,7 +290,7 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
         execute_compaction_result_spy = mocker.spy(
             ExecutionCompactionResult, "__init__"
         )
-        object_store_delete_many_spy = mocker.spy(FileObjectStore, "delete_many")
+        object_store_clear_spy = mocker.spy(FileObjectStore, "clear")
 
         # execute
         rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
@@ -339,8 +339,5 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
         if assert_compaction_audit:
             if not assert_compaction_audit(compactor_version, compaction_audit):
                 assert False, "Compaction audit assertion failed"
-        assert os.listdir(test_dir) == []
-        assert (
-            object_store_delete_many_spy.call_count
-        ), "Object store was never cleaned up!"
+        assert object_store_clear_spy.call_count, "Object store was never cleaned up!"
         return

deltacat/tests/io/test_redis_object_store.py

@@ -121,3 +121,15 @@ class TestRedisObjectStore(unittest.TestCase):
 
         self.assertTrue(delete_success)
         self.assertEqual(2, mock_client.Redis.return_value.delete.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_clear_sanity(self, mock_client):
+        # setup
+        mock_client.Redis.return_value.flushall.side_effect = [True]
+
+        # action
+        clear_success = self.object_store.clear()
+
+        # assert
+        self.assertTrue(clear_success)
+        self.assertEqual(1, mock_client.Redis.return_value.flushall.call_count)

{deltacat-1.1.24.dist-info → deltacat-1.1.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.24
+Version: 1.1.26
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.24.dist-info → deltacat-1.1.26.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=rclFcR9SWLg7Fn_BcZPRhgFDZ6YfkZ7PZuKo0cS6vpU,1778
+deltacat/__init__.py,sha256=N7LrDYFJUaYdchJUVZ8VN_9QUJzuETzkz-oT833iEr4,1778
 deltacat/constants.py,sha256=TUJLXUJ9xq1Ryil72yLkKR8EDH_Irp5wUg56QstbRNE,2181
 deltacat/exceptions.py,sha256=7sjk3BuMY5Oo-6OvAfHncZx_OcvtEL47BblWr2F7waE,12740
 deltacat/logs.py,sha256=EQSDin1deehzz5xlLV1_TrFJrO_IBZ9Ahp7MdL-4cK8,9363
@@ -66,7 +66,7 @@ deltacat/compute/compactor_v2/model/merge_file_group.py,sha256=1o86t9lc3K6ZvtViV
 deltacat/compute/compactor_v2/model/merge_input.py,sha256=-SxTE0e67z2V7MiMEVz5aMu4E0k8h3-vqohvUUOC0do,5659
 deltacat/compute/compactor_v2/model/merge_result.py,sha256=_IZTCStpb4UKiRCJYA3g6EhAqjrw0t9vmoDAN8kIK-Y,436
 deltacat/compute/compactor_v2/private/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=AuzysedzCyapfNf1pfqsZe6mZw121lx6h6NTyLB-pyM,30930
+deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=e8pZFobq6KBCy67ZRn2z1CAwNVjPIJnAiD4HHDmDbCk,30757
 deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=1R5xLUkl7GqL1nY-apAgY1czKDEHjIVYSRi9qLOMass,6726
 deltacat/compute/compactor_v2/steps/merge.py,sha256=LpktsDPfj7Of6RgUw9w1f3Y3OBkPDjvtyXjzFaIDoSo,21771
@@ -100,11 +100,11 @@ deltacat/compute/stats/models/stats_result.py,sha256=XQAlmzhUqRmg4jzEMUAOqcYn1HU
 deltacat/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/dataset.py,sha256=pFU5UfK-fD9C4fIeffJtrA6yVQSgAx2UPbxzQ4GMFL8,3203
 deltacat/io/file_object_store.py,sha256=YoNL3Qla8uLOHaWnyBmIgotjSGAy3Td3Tumah0kk73Y,1868
-deltacat/io/memcached_object_store.py,sha256=PdOMIwv7agEW07mTSBpud6SWd8FjamcKH4_hVJEDPeI,10666
+deltacat/io/memcached_object_store.py,sha256=C96t77-4BQe0XZ4vC76Ygi2o1POUoMN4t4BiyPmulz0,10997
 deltacat/io/object_store.py,sha256=z3Crt8TLyLyoRunOuXAri373TQZKFoz66QHpxGOV82U,1910
 deltacat/io/ray_plasma_object_store.py,sha256=TyoUPWybE_cSISZ2SQa3YfD93QWMp0r82-6WnoVSmzk,905
 deltacat/io/read_api.py,sha256=BhkjL3xjY-fsa62AA9Yv20_88uTskn4_Bv2W6VmMXVA,7023
-deltacat/io/redis_object_store.py,sha256=ZXkJIrx7uHnnAayD-FG1BiB5xxDjMch9GO-YUkPVwqU,4410
+deltacat/io/redis_object_store.py,sha256=OkbQNq1DUVYA7eupmZTF-9OvXUDTOl6WtEifonA5teg,4862
 deltacat/io/s3_object_store.py,sha256=IxvLUvyQZ1w1oYwN9RvRgmKR0Dw56-GggYJw1UCyhBg,1911
 deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -138,7 +138,7 @@ deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=8HVr3EIFYFq
 deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=l_6-pAKOsRY3NbtfHsYmEaJEkq6IJueYuLsjyJxNgz4,81564
 deltacat/tests/compute/compact_partition_test_cases.py,sha256=R9eiKvxCLqcoHjAx3iOogdnXZEO9TvLbRf0wA7bcJN4,26170
 deltacat/tests/compute/test_compact_partition_incremental.py,sha256=Z0hyQGhMZjCaOn1Vk4qUbgDiS7HDhtdNeFQyG1PJhqA,14559
-deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=XOkpB5r-6lyDbqSIens8loaj86HG29PoNrHAOlIMqTM,12587
+deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=Qw74ajnKf41C3MCMvf4bIPXA6-ucKlPj_IeEqDm8rCg,12503
 deltacat/tests/compute/test_compact_partition_params.py,sha256=Dm5eLyHo8oGMeO3XBbpj1rZqHtPZ1hAB7z2qvzc4Lxk,8497
 deltacat/tests/compute/test_compact_partition_rebase.py,sha256=ztSiLgC2OpU4yz81vz-4xWzvZyrLGojtzomsW4q7Bl8,12626
 deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=CHHfNFEJW8S1We7NE1Gg6EaoKEWnaOMRxWrLyirrahc,14643
@@ -165,7 +165,7 @@ deltacat/tests/io/test_cloudpickle_bug_fix.py,sha256=qnYJg_S-nsLai77a4_I3Qs2Jtr_
 deltacat/tests/io/test_file_object_store.py,sha256=bjORXnHe7Ea733XUUO0S2Su_oqSwGuO84TlIfoNO6qA,3587
 deltacat/tests/io/test_memcached_object_store.py,sha256=0EIaU5MHiEmIEkA4x5qUXFY9TE6TJ7V2RGH827cu3AU,9512
 deltacat/tests/io/test_ray_plasma_object_store.py,sha256=-wJZP6lRtEOogR25wjEiIBGz_lpvWVihwlZ5GqandZU,1911
-deltacat/tests/io/test_redis_object_store.py,sha256=YpMsMFT6ltmJHlpAdmlxLK91KjCN8YFMaQrpJ6dcR6E,4595
+deltacat/tests/io/test_redis_object_store.py,sha256=4fCxb7PAqYixPbQZEPDwsDU3BEKfOkYxkhAI7V5Zdfc,4988
 deltacat/tests/io/test_s3_object_store.py,sha256=I8AbyrPfS32CAYvRHtn_OanL-XPpAnJeuCuhD-u9irQ,2270
 deltacat/tests/local_deltacat_storage/__init__.py,sha256=5T9ubNIS42-BotEH0yrUiWEU92feW7lkoSA1-wMeAnQ,40104
 deltacat/tests/local_deltacat_storage/exceptions.py,sha256=oxZ0psmrEO0M6P2r8gHQ2E8E-Y8UBfUCBUIwfuHcx38,251
@@ -210,8 +210,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.24.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.24.dist-info/METADATA,sha256=xOWZzt633xzTmkD_BrTUM0Kyy9Vf6WuEKXhRmpshHks,1733
-deltacat-1.1.24.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-deltacat-1.1.24.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.24.dist-info/RECORD,,
+deltacat-1.1.26.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.26.dist-info/METADATA,sha256=5p2qZYAkOXBNT_rc9PyfGJ5Id3zKfbTp3KhiqZWNxas,1733
+deltacat-1.1.26.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+deltacat-1.1.26.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.26.dist-info/RECORD,,