deltacat 1.1.25__tar.gz → 1.1.27__tar.gz

{deltacat-1.1.25/deltacat.egg-info → deltacat-1.1.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.25
+Version: 1.1.27
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.25"
+__version__ = "1.1.27"
 
 
 __all__ = [

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -227,7 +227,6 @@ def _run_hash_and_merge(
     previous_compacted_delta_manifest: Optional[Manifest],
     compacted_partition: Partition,
 ) -> List[MergeResult]:
-    created_obj_ids = set()
     telemetry_time_hb = 0
     total_input_records_count = np.int64(0)
     total_hb_record_count = np.int64(0)
@@ -289,7 +288,6 @@ def _run_hash_and_merge(
                 hb_result.hash_bucket_group_to_obj_id_tuple
             ):
                 if object_id_size_tuple:
-                    created_obj_ids.add(object_id_size_tuple[0])
                     all_hash_group_idx_to_obj_id[hash_group_index].append(
                         object_id_size_tuple[0],
                     )
@@ -369,10 +367,9 @@ def _run_hash_and_merge(
     )
     if params.num_rounds > 1:
         logger.info(
-            f"Detected number of rounds to be {params.num_rounds}, "
-            f"preparing to delete {len(created_obj_ids)} objects from object store..."
+            f"Detected number of rounds to be {params.num_rounds}, preparing to clear object store..."
         )
-        params.object_store.delete_many(list(created_obj_ids))
+        params.object_store.clear()
     else:
         logger.info(
             f"Detected number of rounds to be {params.num_rounds}, not cleaning up object store..."
@@ -587,8 +584,11 @@ def _process_merge_results(
             f"Duplicate record count ({duplicate_hash_bucket_mat_results}) is as large "
             f"as or greater than params.num_rounds, which is {params.num_rounds}"
         )
+        # ensure start index is the first file index if task index is same
         hb_id_to_entry_indices_range[str(mat_result.task_index)] = (
-            file_index,
+            hb_id_to_entry_indices_range.get(str(mat_result.task_index), [file_index])[
+                0
+            ],
             file_index + mat_result.pyarrow_write_result.files,
         )
 

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/compute/resource_estimation/delta.py

@@ -188,7 +188,7 @@ def _estimate_resources_required_to_process_delta_using_file_sampling(
         sampled_on_disk_size += delta.manifest.entries[entry_index].meta.content_length
         sampled_num_rows += len(tbl)
 
-    if not sampled_on_disk_size:
+    if not sampled_on_disk_size or not sampled_in_memory_size:
         return EstimatedResources.of(
             memory_bytes=0,
             statistics=Statistics.of(

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/io/memcached_object_store.py

@@ -205,7 +205,8 @@ class MemcachedObjectStore(IObjectStore):
             except BaseException:
                 # if an exception is raised then all, some, or none of the keys may have been deleted
                 logger.warning(
-                    f"Failed to fully delete refs: {current_refs}", exc_info=True
+                    f"Failed to fully delete {len(current_refs)} refs for ip: {ip}",
+                    exc_info=True,
                 )
                 all_deleted = False
 
@@ -224,6 +225,7 @@ class MemcachedObjectStore(IObjectStore):
         return all_deleted
 
     def clear(self) -> bool:
+        start = time.monotonic()
         flushed = all(
             [
                 self._get_client_by_ip(ip).flush_all(noreply=False)
@@ -231,10 +233,13 @@ class MemcachedObjectStore(IObjectStore):
             ]
         )
         self.client_cache.clear()
+        end = time.monotonic()
 
         if flushed:
             logger.info("Successfully cleared cache contents.")
 
+        logger.info(f"The total time taken to clear the cache is: {end - start}")
+
         return flushed
 
     def close(self) -> None:

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/io/redis_object_store.py

@@ -115,6 +115,21 @@ class RedisObjectStore(IObjectStore):
 
         return num_deleted == len(refs)
 
+    def clear(self) -> bool:
+        start = time.monotonic()
+        current_ip = self._get_current_ip()
+        client = self._get_client_by_ip(current_ip)
+        flushed = client.flushall()
+        self.client_cache.clear()
+        end = time.monotonic()
+
+        if flushed:
+            logger.info("Successfully cleared cache contents.")
+
+        logger.info(f"The total time taken to clear the cache is: {end - start}")
+
+        return flushed
+
     def _get_client_by_ip(self, ip_address: str):
         if ip_address in self.client_cache:
             return self.client_cache[ip_address]

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/tests/compute/resource_estimation/test_delta.py

@@ -437,6 +437,43 @@ class TestEstimateResourcesRequiredToProcessDelta:
             == parquet_delta_with_manifest.meta.content_length
         )
 
+    def test_parquet_delta_when_file_sampling_and_arrow_size_zero(
+        self,
+        local_deltacat_storage_kwargs,
+        parquet_delta_with_manifest: Delta,
+        monkeypatch,
+    ):
+        params = EstimateResourcesParams.of(
+            resource_estimation_method=ResourceEstimationMethod.FILE_SAMPLING,
+            max_files_to_sample=2,
+        )
+
+        def mock_func(*args, **kwargs):
+            class MockedValue:
+                nbytes = 0
+
+                def __len__(self):
+                    return 0
+
+            return MockedValue()
+
+        monkeypatch.setattr(ds, "download_delta_manifest_entry", mock_func)
+
+        result = estimate_resources_required_to_process_delta(
+            delta=parquet_delta_with_manifest,
+            operation_type=OperationType.PYARROW_DOWNLOAD,
+            deltacat_storage=ds,
+            deltacat_storage_kwargs=local_deltacat_storage_kwargs,
+            estimate_resources_params=params,
+        )
+
+        assert parquet_delta_with_manifest.manifest is not None
+        assert result.memory_bytes == 0
+        assert (
+            result.statistics.on_disk_size_bytes
+            == parquet_delta_with_manifest.meta.content_length
+        )
+
     def test_delta_manifest_utsv_when_file_sampling(
         self, local_deltacat_storage_kwargs, utsv_delta_with_manifest: Delta
     ):

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/tests/compute/test_compact_partition_incremental.py

@@ -328,6 +328,16 @@ def test_compact_partition_incremental(
         **compaction_audit_obj
     )
 
+    # assert if RCF covers all files
+    if compactor_version != CompactorVersion.V1.value:
+        previous_end = None
+        for start, end in round_completion_info.hb_index_to_entry_range.values():
+            assert (previous_end is None and start == 0) or start == previous_end
+            previous_end = end
+        assert (
+            previous_end == round_completion_info.compacted_pyarrow_write_result.files
+        )
+
     tables = ds.download_delta(
         compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
     )

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/tests/compute/test_compact_partition_multiple_rounds.py

@@ -290,7 +290,7 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
         execute_compaction_result_spy = mocker.spy(
             ExecutionCompactionResult, "__init__"
         )
-        object_store_delete_many_spy = mocker.spy(FileObjectStore, "delete_many")
+        object_store_clear_spy = mocker.spy(FileObjectStore, "clear")
 
         # execute
         rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
@@ -309,6 +309,16 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
             **compaction_audit_obj
         )
 
+        # assert if RCF covers all files
+        # multiple rounds feature is only supported in V2 compactor
+        previous_end = None
+        for start, end in round_completion_info.hb_index_to_entry_range.values():
+            assert (previous_end is None and start == 0) or start == previous_end
+            previous_end = end
+        assert (
+            previous_end == round_completion_info.compacted_pyarrow_write_result.files
+        )
+
         # Assert not in-place compacted
         assert (
             execute_compaction_result_spy.call_args.args[-1] is False
@@ -339,8 +349,5 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
         if assert_compaction_audit:
             if not assert_compaction_audit(compactor_version, compaction_audit):
                 assert False, "Compaction audit assertion failed"
-        assert os.listdir(test_dir) == []
-        assert (
-            object_store_delete_many_spy.call_count
-        ), "Object store was never cleaned up!"
+        assert object_store_clear_spy.call_count, "Object store was never cleaned up!"
         return

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/tests/compute/test_compact_partition_rebase.py

@@ -299,6 +299,17 @@ def test_compact_partition_rebase_same_source_and_destination(
             round_completion_info.compaction_audit_url
         )
 
+        # assert if RCF covers all files
+        if compactor_version != CompactorVersion.V1.value:
+            previous_end = None
+            for start, end in round_completion_info.hb_index_to_entry_range.values():
+                assert (previous_end is None and start == 0) or start == previous_end
+                previous_end = end
+            assert (
+                previous_end
+                == round_completion_info.compacted_pyarrow_write_result.files
+            )
+
         compaction_audit_obj: Dict[str, Any] = read_s3_contents(
             s3_resource, audit_bucket, audit_key
         )

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -355,6 +355,16 @@ def test_compact_partition_rebase_then_incremental(
     compacted_delta_locator_incremental: DeltaLocator = (
         round_completion_info.compacted_delta_locator
     )
+    # assert if RCF covers all files
+    if compactor_version != CompactorVersion.V1.value:
+        previous_end = None
+        for start, end in round_completion_info.hb_index_to_entry_range.values():
+            assert (previous_end is None and start == 0) or start == previous_end
+            previous_end = end
+        assert (
+            previous_end == round_completion_info.compacted_pyarrow_write_result.files
+        )
+
     audit_bucket, audit_key = round_completion_info.compaction_audit_url.replace(
         "s3://", ""
     ).split("/", 1)

{deltacat-1.1.25 → deltacat-1.1.27}/deltacat/tests/io/test_redis_object_store.py

@@ -121,3 +121,15 @@ class TestRedisObjectStore(unittest.TestCase):
 
         self.assertTrue(delete_success)
         self.assertEqual(2, mock_client.Redis.return_value.delete.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_clear_sanity(self, mock_client):
+        # setup
+        mock_client.Redis.return_value.flushall.side_effect = [True]
+
+        # action
+        clear_success = self.object_store.clear()
+
+        # assert
+        self.assertTrue(clear_success)
+        self.assertEqual(1, mock_client.Redis.return_value.flushall.call_count)

{deltacat-1.1.25 → deltacat-1.1.27/deltacat.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.25
+Version: 1.1.27
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team