deltacat 1.1.24__py3-none-any.whl → 1.1.26__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
deltacat/__init__.py CHANGED
@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
44
44
 
45
45
  deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
46
46
 
47
- __version__ = "1.1.24"
47
+ __version__ = "1.1.26"
48
48
 
49
49
 
50
50
  __all__ = [
@@ -227,7 +227,6 @@ def _run_hash_and_merge(
227
227
  previous_compacted_delta_manifest: Optional[Manifest],
228
228
  compacted_partition: Partition,
229
229
  ) -> List[MergeResult]:
230
- created_obj_ids = set()
231
230
  telemetry_time_hb = 0
232
231
  total_input_records_count = np.int64(0)
233
232
  total_hb_record_count = np.int64(0)
@@ -289,7 +288,6 @@ def _run_hash_and_merge(
289
288
  hb_result.hash_bucket_group_to_obj_id_tuple
290
289
  ):
291
290
  if object_id_size_tuple:
292
- created_obj_ids.add(object_id_size_tuple[0])
293
291
  all_hash_group_idx_to_obj_id[hash_group_index].append(
294
292
  object_id_size_tuple[0],
295
293
  )
@@ -369,10 +367,9 @@ def _run_hash_and_merge(
369
367
  )
370
368
  if params.num_rounds > 1:
371
369
  logger.info(
372
- f"Detected number of rounds to be {params.num_rounds}, "
373
- f"preparing to delete {len(created_obj_ids)} objects from object store..."
370
+ f"Detected number of rounds to be {params.num_rounds}, preparing to clear object store..."
374
371
  )
375
- params.object_store.delete_many(list(created_obj_ids))
372
+ params.object_store.clear()
376
373
  else:
377
374
  logger.info(
378
375
  f"Detected number of rounds to be {params.num_rounds}, not cleaning up object store..."
@@ -205,7 +205,8 @@ class MemcachedObjectStore(IObjectStore):
205
205
  except BaseException:
206
206
  # if an exception is raised then all, some, or none of the keys may have been deleted
207
207
  logger.warning(
208
- f"Failed to fully delete refs: {current_refs}", exc_info=True
208
+ f"Failed to fully delete {len(current_refs)} refs for ip: {ip}",
209
+ exc_info=True,
209
210
  )
210
211
  all_deleted = False
211
212
 
@@ -218,9 +219,13 @@ class MemcachedObjectStore(IObjectStore):
218
219
  f"The total time taken to attempt deleting {len(refs)} objects is: {end - start}"
219
220
  )
220
221
 
222
+ # We need to clear the client cache in case of multi-round compaction because client cannot be pickled
223
+ self.client_cache.clear()
224
+
221
225
  return all_deleted
222
226
 
223
227
  def clear(self) -> bool:
228
+ start = time.monotonic()
224
229
  flushed = all(
225
230
  [
226
231
  self._get_client_by_ip(ip).flush_all(noreply=False)
@@ -228,10 +233,13 @@ class MemcachedObjectStore(IObjectStore):
228
233
  ]
229
234
  )
230
235
  self.client_cache.clear()
236
+ end = time.monotonic()
231
237
 
232
238
  if flushed:
233
239
  logger.info("Successfully cleared cache contents.")
234
240
 
241
+ logger.info(f"The total time taken to clear the cache is: {end - start}")
242
+
235
243
  return flushed
236
244
 
237
245
  def close(self) -> None:
@@ -115,6 +115,21 @@ class RedisObjectStore(IObjectStore):
115
115
 
116
116
  return num_deleted == len(refs)
117
117
 
118
+ def clear(self) -> bool:
119
+ start = time.monotonic()
120
+ current_ip = self._get_current_ip()
121
+ client = self._get_client_by_ip(current_ip)
122
+ flushed = client.flushall()
123
+ self.client_cache.clear()
124
+ end = time.monotonic()
125
+
126
+ if flushed:
127
+ logger.info("Successfully cleared cache contents.")
128
+
129
+ logger.info(f"The total time taken to clear the cache is: {end - start}")
130
+
131
+ return flushed
132
+
118
133
  def _get_client_by_ip(self, ip_address: str):
119
134
  if ip_address in self.client_cache:
120
135
  return self.client_cache[ip_address]
@@ -290,7 +290,7 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
290
290
  execute_compaction_result_spy = mocker.spy(
291
291
  ExecutionCompactionResult, "__init__"
292
292
  )
293
- object_store_delete_many_spy = mocker.spy(FileObjectStore, "delete_many")
293
+ object_store_clear_spy = mocker.spy(FileObjectStore, "clear")
294
294
 
295
295
  # execute
296
296
  rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
@@ -339,8 +339,5 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
339
339
  if assert_compaction_audit:
340
340
  if not assert_compaction_audit(compactor_version, compaction_audit):
341
341
  assert False, "Compaction audit assertion failed"
342
- assert os.listdir(test_dir) == []
343
- assert (
344
- object_store_delete_many_spy.call_count
345
- ), "Object store was never cleaned up!"
342
+ assert object_store_clear_spy.call_count, "Object store was never cleaned up!"
346
343
  return
@@ -121,3 +121,15 @@ class TestRedisObjectStore(unittest.TestCase):
121
121
 
122
122
  self.assertTrue(delete_success)
123
123
  self.assertEqual(2, mock_client.Redis.return_value.delete.call_count)
124
+
125
+ @mock.patch("deltacat.io.redis_object_store.redis")
126
+ def test_clear_sanity(self, mock_client):
127
+ # setup
128
+ mock_client.Redis.return_value.flushall.side_effect = [True]
129
+
130
+ # action
131
+ clear_success = self.object_store.clear()
132
+
133
+ # assert
134
+ self.assertTrue(clear_success)
135
+ self.assertEqual(1, mock_client.Redis.return_value.flushall.call_count)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deltacat
3
- Version: 1.1.24
3
+ Version: 1.1.26
4
4
  Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
5
5
  Home-page: https://github.com/ray-project/deltacat
6
6
  Author: Ray Team
@@ -1,4 +1,4 @@
1
- deltacat/__init__.py,sha256=rclFcR9SWLg7Fn_BcZPRhgFDZ6YfkZ7PZuKo0cS6vpU,1778
1
+ deltacat/__init__.py,sha256=N7LrDYFJUaYdchJUVZ8VN_9QUJzuETzkz-oT833iEr4,1778
2
2
  deltacat/constants.py,sha256=TUJLXUJ9xq1Ryil72yLkKR8EDH_Irp5wUg56QstbRNE,2181
3
3
  deltacat/exceptions.py,sha256=7sjk3BuMY5Oo-6OvAfHncZx_OcvtEL47BblWr2F7waE,12740
4
4
  deltacat/logs.py,sha256=EQSDin1deehzz5xlLV1_TrFJrO_IBZ9Ahp7MdL-4cK8,9363
@@ -66,7 +66,7 @@ deltacat/compute/compactor_v2/model/merge_file_group.py,sha256=1o86t9lc3K6ZvtViV
66
66
  deltacat/compute/compactor_v2/model/merge_input.py,sha256=-SxTE0e67z2V7MiMEVz5aMu4E0k8h3-vqohvUUOC0do,5659
67
67
  deltacat/compute/compactor_v2/model/merge_result.py,sha256=_IZTCStpb4UKiRCJYA3g6EhAqjrw0t9vmoDAN8kIK-Y,436
68
68
  deltacat/compute/compactor_v2/private/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
69
- deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=AuzysedzCyapfNf1pfqsZe6mZw121lx6h6NTyLB-pyM,30930
69
+ deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=e8pZFobq6KBCy67ZRn2z1CAwNVjPIJnAiD4HHDmDbCk,30757
70
70
  deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
71
71
  deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=1R5xLUkl7GqL1nY-apAgY1czKDEHjIVYSRi9qLOMass,6726
72
72
  deltacat/compute/compactor_v2/steps/merge.py,sha256=LpktsDPfj7Of6RgUw9w1f3Y3OBkPDjvtyXjzFaIDoSo,21771
@@ -100,11 +100,11 @@ deltacat/compute/stats/models/stats_result.py,sha256=XQAlmzhUqRmg4jzEMUAOqcYn1HU
100
100
  deltacat/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
101
101
  deltacat/io/dataset.py,sha256=pFU5UfK-fD9C4fIeffJtrA6yVQSgAx2UPbxzQ4GMFL8,3203
102
102
  deltacat/io/file_object_store.py,sha256=YoNL3Qla8uLOHaWnyBmIgotjSGAy3Td3Tumah0kk73Y,1868
103
- deltacat/io/memcached_object_store.py,sha256=PdOMIwv7agEW07mTSBpud6SWd8FjamcKH4_hVJEDPeI,10666
103
+ deltacat/io/memcached_object_store.py,sha256=C96t77-4BQe0XZ4vC76Ygi2o1POUoMN4t4BiyPmulz0,10997
104
104
  deltacat/io/object_store.py,sha256=z3Crt8TLyLyoRunOuXAri373TQZKFoz66QHpxGOV82U,1910
105
105
  deltacat/io/ray_plasma_object_store.py,sha256=TyoUPWybE_cSISZ2SQa3YfD93QWMp0r82-6WnoVSmzk,905
106
106
  deltacat/io/read_api.py,sha256=BhkjL3xjY-fsa62AA9Yv20_88uTskn4_Bv2W6VmMXVA,7023
107
- deltacat/io/redis_object_store.py,sha256=ZXkJIrx7uHnnAayD-FG1BiB5xxDjMch9GO-YUkPVwqU,4410
107
+ deltacat/io/redis_object_store.py,sha256=OkbQNq1DUVYA7eupmZTF-9OvXUDTOl6WtEifonA5teg,4862
108
108
  deltacat/io/s3_object_store.py,sha256=IxvLUvyQZ1w1oYwN9RvRgmKR0Dw56-GggYJw1UCyhBg,1911
109
109
  deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
110
110
  deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -138,7 +138,7 @@ deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=8HVr3EIFYFq
138
138
  deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=l_6-pAKOsRY3NbtfHsYmEaJEkq6IJueYuLsjyJxNgz4,81564
139
139
  deltacat/tests/compute/compact_partition_test_cases.py,sha256=R9eiKvxCLqcoHjAx3iOogdnXZEO9TvLbRf0wA7bcJN4,26170
140
140
  deltacat/tests/compute/test_compact_partition_incremental.py,sha256=Z0hyQGhMZjCaOn1Vk4qUbgDiS7HDhtdNeFQyG1PJhqA,14559
141
- deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=XOkpB5r-6lyDbqSIens8loaj86HG29PoNrHAOlIMqTM,12587
141
+ deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=Qw74ajnKf41C3MCMvf4bIPXA6-ucKlPj_IeEqDm8rCg,12503
142
142
  deltacat/tests/compute/test_compact_partition_params.py,sha256=Dm5eLyHo8oGMeO3XBbpj1rZqHtPZ1hAB7z2qvzc4Lxk,8497
143
143
  deltacat/tests/compute/test_compact_partition_rebase.py,sha256=ztSiLgC2OpU4yz81vz-4xWzvZyrLGojtzomsW4q7Bl8,12626
144
144
  deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=CHHfNFEJW8S1We7NE1Gg6EaoKEWnaOMRxWrLyirrahc,14643
@@ -165,7 +165,7 @@ deltacat/tests/io/test_cloudpickle_bug_fix.py,sha256=qnYJg_S-nsLai77a4_I3Qs2Jtr_
165
165
  deltacat/tests/io/test_file_object_store.py,sha256=bjORXnHe7Ea733XUUO0S2Su_oqSwGuO84TlIfoNO6qA,3587
166
166
  deltacat/tests/io/test_memcached_object_store.py,sha256=0EIaU5MHiEmIEkA4x5qUXFY9TE6TJ7V2RGH827cu3AU,9512
167
167
  deltacat/tests/io/test_ray_plasma_object_store.py,sha256=-wJZP6lRtEOogR25wjEiIBGz_lpvWVihwlZ5GqandZU,1911
168
- deltacat/tests/io/test_redis_object_store.py,sha256=YpMsMFT6ltmJHlpAdmlxLK91KjCN8YFMaQrpJ6dcR6E,4595
168
+ deltacat/tests/io/test_redis_object_store.py,sha256=4fCxb7PAqYixPbQZEPDwsDU3BEKfOkYxkhAI7V5Zdfc,4988
169
169
  deltacat/tests/io/test_s3_object_store.py,sha256=I8AbyrPfS32CAYvRHtn_OanL-XPpAnJeuCuhD-u9irQ,2270
170
170
  deltacat/tests/local_deltacat_storage/__init__.py,sha256=5T9ubNIS42-BotEH0yrUiWEU92feW7lkoSA1-wMeAnQ,40104
171
171
  deltacat/tests/local_deltacat_storage/exceptions.py,sha256=oxZ0psmrEO0M6P2r8gHQ2E8E-Y8UBfUCBUIwfuHcx38,251
@@ -210,8 +210,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
210
210
  deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
211
211
  deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
212
212
  deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
213
- deltacat-1.1.24.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
214
- deltacat-1.1.24.dist-info/METADATA,sha256=xOWZzt633xzTmkD_BrTUM0Kyy9Vf6WuEKXhRmpshHks,1733
215
- deltacat-1.1.24.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
216
- deltacat-1.1.24.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
217
- deltacat-1.1.24.dist-info/RECORD,,
213
+ deltacat-1.1.26.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
214
+ deltacat-1.1.26.dist-info/METADATA,sha256=5p2qZYAkOXBNT_rc9PyfGJ5Id3zKfbTp3KhiqZWNxas,1733
215
+ deltacat-1.1.26.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
216
+ deltacat-1.1.26.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
217
+ deltacat-1.1.26.dist-info/RECORD,,