deltacat 1.1.22__py3-none-any.whl → 1.1.24__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.22"
+__version__ = "1.1.24"
 
 
 __all__ = [

deltacat/aws/s3u.py

@@ -291,7 +291,7 @@ def read_file(
             f"Retry download for: {s3_url} after receiving {type(e).__name__}"
         ) from e
     except BaseException as e:
-        logger.warn(
+        logger.warning(
             f"Read has failed for {s3_url} and content_type={content_type} "
             f"and encoding={content_encoding}. Error: {e}",
             exc_info=True,
@@ -416,7 +416,7 @@ def upload_table(
                 f"Retry upload for: {s3_url} after receiving {type(e).__name__}",
             ) from e
         except BaseException as e:
-            logger.warn(
+            logger.warning(
                 f"Upload has failed for {s3_url} and content_type={content_type}. Error: {e}",
                 exc_info=True,
             )

deltacat/compute/compactor/utils/round_completion_file.py

@@ -63,7 +63,7 @@ def read_round_completion_file(
             logger.info(f"Read round completion info: {round_completion_info}")
             break
         else:
-            logger.warn(f"Round completion file not present at {rcf_uri}")
+            logger.warning(f"Round completion file not present at {rcf_uri}")
 
     return round_completion_info
 

deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -227,6 +227,7 @@ def _run_hash_and_merge(
     previous_compacted_delta_manifest: Optional[Manifest],
     compacted_partition: Partition,
 ) -> List[MergeResult]:
+    created_obj_ids = set()
     telemetry_time_hb = 0
     total_input_records_count = np.int64(0)
     total_hb_record_count = np.int64(0)
@@ -288,6 +289,7 @@ def _run_hash_and_merge(
                 hb_result.hash_bucket_group_to_obj_id_tuple
             ):
                 if object_id_size_tuple:
+                    created_obj_ids.add(object_id_size_tuple[0])
                     all_hash_group_idx_to_obj_id[hash_group_index].append(
                         object_id_size_tuple[0],
                     )
@@ -365,6 +367,16 @@ def _run_hash_and_merge(
     mutable_compaction_audit.set_telemetry_time_in_seconds(
         telemetry_this_round + previous_telemetry
     )
+    if params.num_rounds > 1:
+        logger.info(
+            f"Detected number of rounds to be {params.num_rounds}, "
+            f"preparing to delete {len(created_obj_ids)} objects from object store..."
+        )
+        params.object_store.delete_many(list(created_obj_ids))
+    else:
+        logger.info(
+            f"Detected number of rounds to be {params.num_rounds}, not cleaning up object store..."
+        )
 
     return merge_results
 

deltacat/compute/compactor_v2/utils/content_type_params.py

@@ -97,7 +97,7 @@ def append_content_type_params(
     max_parquet_meta_size_bytes: Optional[int] = MAX_PARQUET_METADATA_SIZE,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
-) -> None:
+) -> bool:
     """
     This operation appends content type params into the delta entry. Note
     that this operation can be time consuming, hence we cache it in a Ray actor.
@@ -105,7 +105,7 @@ def append_content_type_params(
 
     if not delta.meta:
         logger.warning(f"Delta with locator {delta.locator} doesn't contain meta.")
-        return
+        return False
 
     entry_indices_to_download = []
     for entry_index, entry in enumerate(delta.manifest.entries):
@@ -120,7 +120,7 @@ def append_content_type_params(
         logger.info(
             f"No parquet type params to download for delta with locator {delta.locator}."
         )
-        return None
+        return False
 
     ray_namespace = ray.get_runtime_context().namespace
     logger.info(
@@ -147,7 +147,7 @@ def append_content_type_params(
                 f" {delta.locator} and digest {delta.locator.hexdigest()}."
             )
             delta.manifest = cached_value.manifest
-            return
+            return True
         logger.info(
             f"Cache doesn't contain parquet meta for delta with locator {delta.locator}."
         )
@@ -215,3 +215,5 @@ def append_content_type_params(
         )
         ray.get(cache.put.remote(delta.locator.hexdigest(), delta))
         assert ray.get(cache.get.remote(delta.locator.hexdigest())) is not None
+
+    return True

deltacat/compute/resource_estimation/delta.py

@@ -42,7 +42,11 @@ def _estimate_resources_required_to_process_delta_using_previous_inflation(
     in_memory_size = (
         delta.meta.content_length * estimate_resources_params.previous_inflation
     )
-    num_rows = int(in_memory_size / estimate_resources_params.average_record_size_bytes)
+    num_rows = 0
+    if estimate_resources_params.average_record_size_bytes is not None:
+        num_rows = int(
+            in_memory_size / estimate_resources_params.average_record_size_bytes
+        )
 
     return EstimatedResources.of(
         memory_bytes=in_memory_size,
@@ -68,6 +72,10 @@ def _estimate_resources_required_to_process_delta_using_type_params(
     ), "Number of rows can only be estimated for PYARROW_DOWNLOAD operation"
 
     if estimate_resources_params.parquet_to_pyarrow_inflation is None:
+        logger.debug(
+            "Could not estimate using type params as "
+            f"parquet_to_pyarrow_inflation is None for {delta.locator}"
+        )
         return None
 
     if not delta.manifest:
@@ -86,12 +94,18 @@ def _estimate_resources_required_to_process_delta_using_type_params(
             ),
         )
 
-    append_content_type_params(
+    appended = append_content_type_params(
         delta=delta,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,
     )
 
+    if not appended:
+        logger.debug(
+            f"Could not append content type params for {delta.locator}, returning None"
+        )
+        return None
+
     in_memory_size = 0.0
     num_rows = 0
 

deltacat/io/file_object_store.py

@@ -41,8 +41,23 @@ class FileObjectStore(IObjectStore):
                 serialized = f.read()
                 loaded = cloudpickle.loads(serialized)
                 result.append(loaded)
-            os.remove(ref)
         end = time.monotonic()
 
         logger.info(f"The total time taken to read all objects is: {end - start}")
         return result
+
+    def delete_many(self, refs: List[Any], *args, **kwargs) -> bool:
+        start = time.monotonic()
+        num_deleted = 0
+        for ref in refs:
+            try:
+                os.remove(ref)
+                num_deleted += 1
+            except Exception:
+                logger.warning(f"Failed to delete ref {ref}!", exc_info=True)
+        end = time.monotonic()
+
+        logger.info(
+            f"The total time taken to delete {num_deleted} out of {len(refs)} objects is: {end - start}"
+        )
+        return num_deleted == len(refs)

deltacat/io/memcached_object_store.py

@@ -100,16 +100,10 @@ class MemcachedObjectStore(IObjectStore):
 
     def get_many(self, refs: List[Any], *args, **kwargs) -> List[object]:
         result = []
-        refs_per_ip = defaultdict(lambda: [])
+        refs_per_ip = self._get_refs_per_ip(refs)
         chunks_by_refs = defaultdict(lambda: [])
 
         start = time.monotonic()
-        for ref in refs:
-            uid, ip, chunk_count = ref.split(self.SEPARATOR)
-            chunk_count = int(chunk_count)
-            for chunk_index in range(chunk_count):
-                current_ref = self._create_ref(uid, ip, chunk_index)
-                refs_per_ip[ip].append(current_ref)
 
         total_ref_count = 0
         for (ip, current_refs) in refs_per_ip.items():
@@ -193,6 +187,39 @@ class MemcachedObjectStore(IObjectStore):
 
         return cloudpickle.loads(serialized)
 
+    def delete_many(self, refs: List[Any], *args, **kwargs) -> bool:
+        refs_per_ip = self._get_refs_per_ip(refs)
+        all_deleted = True
+
+        start = time.monotonic()
+
+        total_refs = 0
+        fully_deleted_refs = 0
+        for (ip, current_refs) in refs_per_ip.items():
+            client = self._get_client_by_ip(ip)
+            total_refs += len(current_refs)
+            try:
+                # always returns true
+                client.delete_many(current_refs, noreply=self.noreply)
+                fully_deleted_refs += len(current_refs)
+            except BaseException:
+                # if an exception is raised then all, some, or none of the keys may have been deleted
+                logger.warning(
+                    f"Failed to fully delete refs: {current_refs}", exc_info=True
+                )
+                all_deleted = False
+
+        end = time.monotonic()
+
+        logger.info(
+            f"From {len(refs)} objects, found {total_refs} total chunk references, of which {fully_deleted_refs} were guaranteed to be successfully deleted."
+        )
+        logger.info(
+            f"The total time taken to attempt deleting {len(refs)} objects is: {end - start}"
+        )
+
+        return all_deleted
+
     def clear(self) -> bool:
         flushed = all(
             [
@@ -260,3 +287,14 @@ class MemcachedObjectStore(IObjectStore):
             self.current_ip = socket.gethostbyname(socket.gethostname())
 
         return self.current_ip
+
+    def _get_refs_per_ip(self, refs: List[Any]):
+        refs_per_ip = defaultdict(lambda: [])
+
+        for ref in refs:
+            uid, ip, chunk_count = ref.split(self.SEPARATOR)
+            chunk_count = int(chunk_count)
+            for chunk_index in range(chunk_count):
+                current_ref = self._create_ref(uid, ip, chunk_index)
+                refs_per_ip[ip].append(current_ref)
+        return refs_per_ip

deltacat/io/object_store.py

@@ -43,6 +43,19 @@ class IObjectStore:
         or may not return ordered results.
         """
 
+    def delete(self, ref: Any, *args, **kwargs) -> bool:
+        """
+        Delete a single object from the object store.
+        """
+        return self.delete_many([ref])
+
+    def delete_many(self, refs: List[Any], *args, **kwargs) -> bool:
+        ...
+
+        """
+        Delete many objects from the object store.
+        """
+
     def clear(self, *args, **kwargs) -> bool:
         ...
 
@@ -52,6 +65,7 @@ class IObjectStore:
 
     def close(self, *args, **kwargs) -> None:
         ...
+
         """
         Closes all the active connections to object store without clearing
         the data in the object store.

deltacat/io/redis_object_store.py

@@ -56,12 +56,9 @@ class RedisObjectStore(IObjectStore):
 
     def get_many(self, refs: List[Any], *args, **kwargs) -> List[object]:
         result = []
-        uid_per_ip = defaultdict(lambda: [])
+        uid_per_ip = self._get_uids_per_ip(refs)
 
         start = time.monotonic()
-        for ref in refs:
-            uid, ip = ref.split(self.SEPARATOR)
-            uid_per_ip[ip].append(uid)
 
         for (ip, uids) in uid_per_ip.items():
             client = self._get_client_by_ip(ip)
@@ -95,6 +92,29 @@ class RedisObjectStore(IObjectStore):
         serialized = client.get(uid)
         return cloudpickle.loads(serialized)
 
+    def delete_many(self, refs: List[Any], *args, **kwargs) -> bool:
+        uid_per_ip = self._get_uids_per_ip(refs)
+
+        start = time.monotonic()
+
+        num_deleted = 0
+        for (ip, uids) in uid_per_ip.items():
+            client = self._get_client_by_ip(ip)
+            num_keys_deleted = client.delete(*uids)
+            num_deleted += num_keys_deleted
+            if num_keys_deleted != len(uids):
+                logger.warning(
+                    f"Failed to delete {len(uids) - num_keys_deleted} out of {len(uids)} uids: {uids}"
+                )
+
+        end = time.monotonic()
+
+        logger.info(
+            f"The total time taken to delete {num_deleted} out of {len(refs)} objects is: {end - start}"
+        )
+
+        return num_deleted == len(refs)
+
     def _get_client_by_ip(self, ip_address: str):
         if ip_address in self.client_cache:
             return self.client_cache[ip_address]
@@ -112,3 +132,11 @@ class RedisObjectStore(IObjectStore):
 
     def _create_ref(self, uid, ip):
         return f"{uid}{self.SEPARATOR}{ip}"
+
+    def _get_uids_per_ip(self, refs: List[Any]):
+        uid_per_ip = defaultdict(lambda: [])
+
+        for ref in refs:
+            uid, ip = ref.split(self.SEPARATOR)
+            uid_per_ip[ip].append(uid)
+        return uid_per_ip

deltacat/io/s3_object_store.py

@@ -42,3 +42,20 @@ class S3ObjectStore(IObjectStore):
 
         logger.info(f"The total time taken to read all objects is: {end - start}")
         return result
+
+    def delete_many(self, refs: List[Any], *args, **kwargs) -> bool:
+        start = time.monotonic()
+        num_deleted = 0
+        for ref in refs:
+            try:
+                s3_utils.delete_files_by_prefix(self.bucket, str(ref))
+                num_deleted += 1
+            except BaseException:
+                logger.warning(f"Failed to delete ref {ref}!", exc_info=True)
+        end = time.monotonic()
+
+        logger.info(
+            f"The total time taken to delete {num_deleted} out of {len(refs)} objects is: {end - start}"
+        )
+
+        return num_deleted == len(refs)

deltacat/tests/compute/resource_estimation/test_delta.py

@@ -526,6 +526,30 @@ class TestEstimateResourcesRequiredToProcessDelta:
             == parquet_delta_with_manifest.meta.content_length
         )
 
+    def test_parquet_delta_when_default_v2_without_avg_record_size_and_sampling(
+        self, local_deltacat_storage_kwargs, parquet_delta_with_manifest: Delta
+    ):
+        params = EstimateResourcesParams.of(
+            resource_estimation_method=ResourceEstimationMethod.DEFAULT_V2,
+            previous_inflation=7,
+            parquet_to_pyarrow_inflation=1,
+        )
+
+        result = estimate_resources_required_to_process_delta(
+            delta=parquet_delta_with_manifest,
+            operation_type=OperationType.PYARROW_DOWNLOAD,
+            deltacat_storage=ds,
+            deltacat_storage_kwargs=local_deltacat_storage_kwargs,
+            estimate_resources_params=params,
+        )
+
+        assert parquet_delta_with_manifest.manifest is not None
+        assert result.memory_bytes is not None
+        assert (
+            result.statistics.on_disk_size_bytes
+            == parquet_delta_with_manifest.meta.content_length
+        )
+
     def test_parquet_delta_when_default_v2_and_files_to_sample_zero(
         self, local_deltacat_storage_kwargs, parquet_delta_with_manifest: Delta
     ):
@@ -578,6 +602,32 @@ class TestEstimateResourcesRequiredToProcessDelta:
             == utsv_delta_with_manifest.meta.content_length
         )
 
+    def test_utsv_delta_when_default_v2_without_avg_record_size(
+        self, local_deltacat_storage_kwargs, utsv_delta_with_manifest: Delta
+    ):
+        params = EstimateResourcesParams.of(
+            resource_estimation_method=ResourceEstimationMethod.DEFAULT_V2,
+            previous_inflation=7,
+            average_record_size_bytes=None,  # note
+            parquet_to_pyarrow_inflation=1,
+        )
+
+        result = estimate_resources_required_to_process_delta(
+            delta=utsv_delta_with_manifest,
+            operation_type=OperationType.PYARROW_DOWNLOAD,
+            deltacat_storage=ds,
+            deltacat_storage_kwargs=local_deltacat_storage_kwargs,
+            estimate_resources_params=params,
+        )
+
+        assert utsv_delta_with_manifest.manifest is not None
+        assert result.memory_bytes is not None
+        assert result.statistics.record_count == 0
+        assert (
+            result.statistics.on_disk_size_bytes
+            == utsv_delta_with_manifest.meta.content_length
+        )
+
     def test_parquet_delta_without_inflation_when_default_v2(
         self, local_deltacat_storage_kwargs, parquet_delta_with_manifest: Delta
     ):

deltacat/tests/compute/test_compact_partition_multiple_rounds.py

@@ -5,8 +5,9 @@ import pytest
 import boto3
 from boto3.resources.base import ServiceResource
 import pyarrow as pa
-from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
+from deltacat.io.file_object_store import FileObjectStore
 from pytest_benchmark.fixture import BenchmarkFixture
+import tempfile
 
 from deltacat.tests.compute.test_util_constant import (
     TEST_S3_RCF_BUCKET_NAME,
@@ -247,84 +248,99 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
         pgm = PlacementGroupManager(
             1, total_cpus, DEFAULT_WORKER_INSTANCE_CPUS, memory_per_bundle=4000000
         ).pgs[0]
-    compact_partition_params = CompactPartitionParams.of(
-        {
-            "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
-            "compacted_file_content_type": ContentType.PARQUET,
-            "dd_max_parallelism_ratio": 1.0,
-            "deltacat_storage": ds,
-            "deltacat_storage_kwargs": ds_mock_kwargs,
-            "destination_partition_locator": rebased_partition.locator,
-            "hash_bucket_count": hash_bucket_count_param,
-            "last_stream_position_to_compact": source_partition.stream_position,
-            "list_deltas_kwargs": {**ds_mock_kwargs, **{"equivalent_table_types": []}},
-            "object_store": RayPlasmaObjectStore(),
-            "pg_config": pgm,
-            "primary_keys": primary_keys,
-            "read_kwargs_provider": read_kwargs_provider_param,
-            "rebase_source_partition_locator": source_partition.locator,
-            "rebase_source_partition_high_watermark": rebased_partition.stream_position,
-            "records_per_compacted_file": records_per_compacted_file_param,
-            "s3_client_kwargs": {},
-            "source_partition_locator": rebased_partition.locator,
-            "sort_keys": sort_keys if sort_keys else None,
-            "num_rounds": num_rounds_param,
-            "drop_duplicates": drop_duplicates_param,
-            "min_delta_bytes": 560,
-        }
-    )
-    if expected_terminal_exception:
-        with pytest.raises(expected_terminal_exception) as exc_info:
-            benchmark(compact_partition_func, compact_partition_params)
-        assert expected_terminal_exception_message in str(exc_info.value)
-        return
-    from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
-        ExecutionCompactionResult,
-    )
+    with tempfile.TemporaryDirectory() as test_dir:
+        compact_partition_params = CompactPartitionParams.of(
+            {
+                "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                "compacted_file_content_type": ContentType.PARQUET,
+                "dd_max_parallelism_ratio": 1.0,
+                "deltacat_storage": ds,
+                "deltacat_storage_kwargs": ds_mock_kwargs,
+                "destination_partition_locator": rebased_partition.locator,
+                "hash_bucket_count": hash_bucket_count_param,
+                "last_stream_position_to_compact": source_partition.stream_position,
+                "list_deltas_kwargs": {
+                    **ds_mock_kwargs,
+                    **{"equivalent_table_types": []},
+                },
+                "object_store": FileObjectStore(test_dir),
+                "pg_config": pgm,
+                "primary_keys": primary_keys,
+                "read_kwargs_provider": read_kwargs_provider_param,
+                "rebase_source_partition_locator": source_partition.locator,
+                "rebase_source_partition_high_watermark": rebased_partition.stream_position,
+                "records_per_compacted_file": records_per_compacted_file_param,
+                "s3_client_kwargs": {},
+                "source_partition_locator": rebased_partition.locator,
+                "sort_keys": sort_keys if sort_keys else None,
+                "num_rounds": num_rounds_param,
+                "drop_duplicates": drop_duplicates_param,
+                "min_delta_bytes": 560,
+            }
+        )
+        if expected_terminal_exception:
+            with pytest.raises(expected_terminal_exception) as exc_info:
+                benchmark(compact_partition_func, compact_partition_params)
+            assert expected_terminal_exception_message in str(exc_info.value)
+            return
+        from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
+            ExecutionCompactionResult,
+        )
 
-    execute_compaction_result_spy = mocker.spy(ExecutionCompactionResult, "__init__")
+        execute_compaction_result_spy = mocker.spy(
+            ExecutionCompactionResult, "__init__"
+        )
+        object_store_delete_many_spy = mocker.spy(FileObjectStore, "delete_many")
 
-    # execute
-    rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
+        # execute
+        rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
 
-    round_completion_info: RoundCompletionInfo = get_rcf(s3_resource, rcf_file_s3_uri)
-    audit_bucket, audit_key = RoundCompletionInfo.get_audit_bucket_name_and_key(
-        round_completion_info.compaction_audit_url
-    )
+        round_completion_info: RoundCompletionInfo = get_rcf(
+            s3_resource, rcf_file_s3_uri
+        )
+        audit_bucket, audit_key = RoundCompletionInfo.get_audit_bucket_name_and_key(
+            round_completion_info.compaction_audit_url
+        )
 
-    compaction_audit_obj: Dict[str, Any] = read_s3_contents(
-        s3_resource, audit_bucket, audit_key
-    )
-    compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
-        **compaction_audit_obj
-    )
+        compaction_audit_obj: Dict[str, Any] = read_s3_contents(
+            s3_resource, audit_bucket, audit_key
+        )
+        compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
+            **compaction_audit_obj
+        )
 
-    # Assert not in-place compacted
-    assert (
-        execute_compaction_result_spy.call_args.args[-1] is False
-    ), "Table version erroneously marked as in-place compacted!"
-    compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
-        s3_resource, rcf_file_s3_uri
-    )
-    tables = ds.download_delta(
-        compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
-    )
-    actual_rebase_compacted_table = pa.concat_tables(tables)
-    # if no primary key is specified then sort by sort_key for consistent assertion
-    sorting_cols: List[Any] = (
-        [(val, "ascending") for val in primary_keys] if primary_keys else sort_keys
-    )
-    rebase_expected_compact_partition_result = (
-        rebase_expected_compact_partition_result.combine_chunks().sort_by(sorting_cols)
-    )
-    actual_rebase_compacted_table = (
-        actual_rebase_compacted_table.combine_chunks().sort_by(sorting_cols)
-    )
-    assert actual_rebase_compacted_table.equals(
-        rebase_expected_compact_partition_result
-    ), f"{actual_rebase_compacted_table} does not match {rebase_expected_compact_partition_result}"
+        # Assert not in-place compacted
+        assert (
+            execute_compaction_result_spy.call_args.args[-1] is False
+        ), "Table version erroneously marked as in-place compacted!"
+        compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
+            s3_resource, rcf_file_s3_uri
+        )
+        tables = ds.download_delta(
+            compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
+        )
+        actual_rebase_compacted_table = pa.concat_tables(tables)
+        # if no primary key is specified then sort by sort_key for consistent assertion
+        sorting_cols: List[Any] = (
+            [(val, "ascending") for val in primary_keys] if primary_keys else sort_keys
+        )
+        rebase_expected_compact_partition_result = (
+            rebase_expected_compact_partition_result.combine_chunks().sort_by(
+                sorting_cols
+            )
+        )
+        actual_rebase_compacted_table = (
+            actual_rebase_compacted_table.combine_chunks().sort_by(sorting_cols)
+        )
+        assert actual_rebase_compacted_table.equals(
+            rebase_expected_compact_partition_result
+        ), f"{actual_rebase_compacted_table} does not match {rebase_expected_compact_partition_result}"
 
-    if assert_compaction_audit:
-        if not assert_compaction_audit(compactor_version, compaction_audit):
-            assert False, "Compaction audit assertion failed"
-    return
+        if assert_compaction_audit:
+            if not assert_compaction_audit(compactor_version, compaction_audit):
+                assert False, "Compaction audit assertion failed"
+        assert os.listdir(test_dir) == []
+        assert (
+            object_store_delete_many_spy.call_count
+        ), "Object store was never cleaned up!"
+        return

deltacat/tests/compute/test_compact_partition_rebase.py

@@ -5,8 +5,9 @@ import pytest
 import boto3
 from boto3.resources.base import ServiceResource
 import pyarrow as pa
-from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
+from deltacat.io.file_object_store import FileObjectStore
 from pytest_benchmark.fixture import BenchmarkFixture
+import tempfile
 
 from deltacat.tests.compute.test_util_constant import (
     TEST_S3_RCF_BUCKET_NAME,
@@ -250,83 +251,97 @@ def test_compact_partition_rebase_same_source_and_destination(
             1, total_cpus, worker_instance_cpu, memory_per_bundle=4000000
         ).pgs[0]
     last_stream_position_to_compact = source_partition.stream_position
-    compact_partition_params = CompactPartitionParams.of(
-        {
-            "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
-            "compacted_file_content_type": ContentType.PARQUET,
-            "dd_max_parallelism_ratio": 1.0,
-            "deltacat_storage": ds,
-            "deltacat_storage_kwargs": ds_mock_kwargs,
-            "destination_partition_locator": rebased_partition.locator,
-            "hash_bucket_count": hash_bucket_count_param,
-            "last_stream_position_to_compact": last_stream_position_to_compact,
-            "list_deltas_kwargs": {**ds_mock_kwargs, **{"equivalent_table_types": []}},
-            "object_store": RayPlasmaObjectStore(),
-            "pg_config": pgm,
-            "primary_keys": primary_keys,
-            "read_kwargs_provider": read_kwargs_provider_param,
-            "rebase_source_partition_locator": source_partition.locator,
-            "rebase_source_partition_high_watermark": rebased_partition.stream_position,
-            "records_per_compacted_file": records_per_compacted_file_param,
-            "s3_client_kwargs": {},
-            "source_partition_locator": rebased_partition.locator,
-            "sort_keys": sort_keys if sort_keys else None,
-            "drop_duplicates": drop_duplicates_param,
-        }
-    )
+    with tempfile.TemporaryDirectory() as test_dir:
+        compact_partition_params = CompactPartitionParams.of(
+            {
+                "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                "compacted_file_content_type": ContentType.PARQUET,
+                "dd_max_parallelism_ratio": 1.0,
+                "deltacat_storage": ds,
+                "deltacat_storage_kwargs": ds_mock_kwargs,
+                "destination_partition_locator": rebased_partition.locator,
+                "hash_bucket_count": hash_bucket_count_param,
+                "last_stream_position_to_compact": last_stream_position_to_compact,
+                "list_deltas_kwargs": {
+                    **ds_mock_kwargs,
+                    **{"equivalent_table_types": []},
+                },
+                "object_store": FileObjectStore(test_dir),
+                "pg_config": pgm,
+                "primary_keys": primary_keys,
+                "read_kwargs_provider": read_kwargs_provider_param,
+                "rebase_source_partition_locator": source_partition.locator,
+                "rebase_source_partition_high_watermark": rebased_partition.stream_position,
+                "records_per_compacted_file": records_per_compacted_file_param,
+                "s3_client_kwargs": {},
+                "source_partition_locator": rebased_partition.locator,
+                "sort_keys": sort_keys if sort_keys else None,
+                "drop_duplicates": drop_duplicates_param,
+            }
+        )
 
-    from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
-        ExecutionCompactionResult,
-    )
+        from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
+            ExecutionCompactionResult,
+        )
 
-    execute_compaction_result_spy = mocker.spy(ExecutionCompactionResult, "__init__")
+        execute_compaction_result_spy = mocker.spy(
+            ExecutionCompactionResult, "__init__"
+        )
+        object_store_put_many_spy = mocker.spy(FileObjectStore, "put_many")
 
-    # execute
-    rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
+        # execute
+        rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
 
-    round_completion_info: RoundCompletionInfo = get_rcf(s3_resource, rcf_file_s3_uri)
-    audit_bucket, audit_key = RoundCompletionInfo.get_audit_bucket_name_and_key(
-        round_completion_info.compaction_audit_url
-    )
+        round_completion_info: RoundCompletionInfo = get_rcf(
+            s3_resource, rcf_file_s3_uri
+        )
+        audit_bucket, audit_key = RoundCompletionInfo.get_audit_bucket_name_and_key(
+            round_completion_info.compaction_audit_url
+        )
 
-    compaction_audit_obj: Dict[str, Any] = read_s3_contents(
-        s3_resource, audit_bucket, audit_key
-    )
-    compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
-        **compaction_audit_obj
-    )
+        compaction_audit_obj: Dict[str, Any] = read_s3_contents(
+            s3_resource, audit_bucket, audit_key
+        )
+        compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
+            **compaction_audit_obj
+        )
 
-    # Assert not in-place compacted
-    assert (
-        execute_compaction_result_spy.call_args.args[-1] is False
-    ), "Table version erroneously marked as in-place compacted!"
-    compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
-        s3_resource, rcf_file_s3_uri
-    )
-    assert (
-        compacted_delta_locator.stream_position == last_stream_position_to_compact
-    ), "Compacted delta locator must be equal to last stream position"
-    tables = ds.download_delta(
-        compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
-    )
-    actual_rebase_compacted_table = pa.concat_tables(tables)
-    # if no primary key is specified then sort by sort_key for consistent assertion
-    sorting_cols: List[Any] = []
-    if primary_keys:
-        sorting_cols.extend([(val, "ascending") for val in primary_keys])
-    if sort_keys:
-        sorting_cols.extend(sort_keys)
+        # Assert not in-place compacted
+        assert (
+            execute_compaction_result_spy.call_args.args[-1] is False
+        ), "Table version erroneously marked as in-place compacted!"
+        compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
+            s3_resource, rcf_file_s3_uri
+        )
+        assert (
+            compacted_delta_locator.stream_position == last_stream_position_to_compact
+        ), "Compacted delta locator must be equal to last stream position"
+        tables = ds.download_delta(
+            compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
+        )
+        actual_rebase_compacted_table = pa.concat_tables(tables)
+        # if no primary key is specified then sort by sort_key for consistent assertion
+        sorting_cols: List[Any] = []
+        if primary_keys:
+            sorting_cols.extend([(val, "ascending") for val in primary_keys])
+        if sort_keys:
+            sorting_cols.extend(sort_keys)
 
-    rebase_expected_compact_partition_result = (
-        rebase_expected_compact_partition_result.combine_chunks().sort_by(sorting_cols)
-    )
-    actual_rebase_compacted_table = (
-        actual_rebase_compacted_table.combine_chunks().sort_by(sorting_cols)
-    )
-    assert actual_rebase_compacted_table.equals(
-        rebase_expected_compact_partition_result
-    ), f"{actual_rebase_compacted_table} does not match {rebase_expected_compact_partition_result}"
+        rebase_expected_compact_partition_result = (
+            rebase_expected_compact_partition_result.combine_chunks().sort_by(
+                sorting_cols
+            )
+        )
+        actual_rebase_compacted_table = (
+            actual_rebase_compacted_table.combine_chunks().sort_by(sorting_cols)
+        )
+        assert actual_rebase_compacted_table.equals(
+            rebase_expected_compact_partition_result
+        ), f"{actual_rebase_compacted_table} does not match {rebase_expected_compact_partition_result}"
 
-    if assert_compaction_audit is not None:
-        if not assert_compaction_audit(compactor_version, compaction_audit):
-            assert False, "Compaction audit assertion failed"
+        if assert_compaction_audit is not None:
+            if not assert_compaction_audit(compactor_version, compaction_audit):
+                assert False, "Compaction audit assertion failed"
+        # We do not expect object store to be cleaned up when there's only one round
+        if object_store_put_many_spy.call_count:
+            assert os.listdir(test_dir) != []

deltacat/tests/io/test_file_object_store.py

@@ -27,11 +27,12 @@ class TestFileObjectStore(unittest.TestCase):
         new_callable=mock.mock_open,
         read_data="data",
     )
-    def test_put_many_sanity(self, mock_file):
+    @mock.patch("deltacat.io.file_object_store.cloudpickle.dumps")
+    def test_put_many_sanity(self, mock_dumps, mock_file):
         from deltacat.io.file_object_store import FileObjectStore
 
         object_store = FileObjectStore(dir_path="")
-        self.ray_mock.cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_dumps.return_value = self.TEST_VALUE
         result = object_store.put_many(["a", "b"])
 
         self.assertEqual(2, len(result))
@@ -42,11 +43,29 @@ class TestFileObjectStore(unittest.TestCase):
         new_callable=mock.mock_open,
         read_data="data",
     )
-    def test_get_many_sanity(self, mock_file):
+    @mock.patch("deltacat.io.file_object_store.cloudpickle.dumps")
+    def test_put_sanity(self, mock_dumps, mock_file):
         from deltacat.io.file_object_store import FileObjectStore
 
         object_store = FileObjectStore(dir_path="")
-        self.ray_mock.cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_dumps.return_value = self.TEST_VALUE
+
+        result = object_store.put("test")
+
+        self.assertIsNotNone(result)
+        self.assertEqual(1, mock_file.call_count)
+
+    @mock.patch(
+        "deltacat.io.file_object_store.open",
+        new_callable=mock.mock_open,
+        read_data="data",
+    )
+    @mock.patch("deltacat.io.file_object_store.cloudpickle.loads")
+    def test_get_many_sanity(self, mock_loads, mock_file):
+        from deltacat.io.file_object_store import FileObjectStore
+
+        object_store = FileObjectStore(dir_path="")
+        mock_loads.return_value = self.TEST_VALUE
 
         result = object_store.get_many(["test", "test"])
 
@@ -58,11 +77,12 @@ class TestFileObjectStore(unittest.TestCase):
         new_callable=mock.mock_open,
         read_data="data",
     )
-    def test_get_sanity(self, mock_file):
+    @mock.patch("deltacat.io.file_object_store.cloudpickle.loads")
+    def test_get_sanity(self, mock_loads, mock_file):
         from deltacat.io.file_object_store import FileObjectStore
 
         object_store = FileObjectStore(dir_path="")
-        self.ray_mock.cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_loads.return_value = self.TEST_VALUE
 
         result = object_store.get("test")
 
@@ -70,17 +90,27 @@ class TestFileObjectStore(unittest.TestCase):
         self.assertEqual(1, mock_file.call_count)
 
     @mock.patch(
-        "deltacat.io.file_object_store.open",
-        new_callable=mock.mock_open,
-        read_data="data",
+        "deltacat.io.file_object_store.os.remove",
     )
-    def test_put_sanity(self, mock_file):
+    def test_delete_sanity(self, mock_remove):
         from deltacat.io.file_object_store import FileObjectStore
 
         object_store = FileObjectStore(dir_path="")
-        self.ray_mock.cloudpickle.dumps.return_value = self.TEST_VALUE
 
-        result = object_store.put("test")
+        delete_success = object_store.delete("test")
 
-        self.assertIsNotNone(result)
-        self.assertEqual(1, mock_file.call_count)
+        self.assertTrue(delete_success)
+        self.assertEqual(1, mock_remove.call_count)
+
+    @mock.patch(
+        "deltacat.io.file_object_store.os.remove",
+    )
+    def test_delete_many_sanity(self, mock_remove):
+        from deltacat.io.file_object_store import FileObjectStore
+
+        object_store = FileObjectStore(dir_path="")
+
+        delete_success = object_store.delete_many(["test", "test"])
+
+        self.assertTrue(delete_success)
+        self.assertEqual(2, mock_remove.call_count)

deltacat/tests/io/test_memcached_object_store.py

@@ -28,6 +28,15 @@ class MockPyMemcacheClient:
     def get(self, key, *args, **kwargs):
         return self.store.get(key)
 
+    def delete(self, key, *args, **kwargs):
+        self.store.pop(key, None)
+        return True
+
+    def delete_many(self, keys, *args, **kwargs):
+        for key in keys:
+            self.store.pop(key, None)
+        return True
+
     def flush_all(self, *args, **kwargs):
         for key, value in self.store.items():
             self.store[key] = None
@@ -200,6 +209,37 @@ class TestMemcachedObjectStore(unittest.TestCase):
         result = self.object_store.get(ref)
         self.assertEqual(result, self.TEST_VALUE_LARGE)
 
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_delete_sanity(self, mock_retrying_client, mock_client):
+        mock_client.return_value = MockPyMemcacheClient()
+        mock_retrying_client.return_value = mock_client.return_value
+
+        # setup
+        ref = self.object_store.put(np.arange(100))
+
+        # action
+        delete_success = self.object_store.delete(ref)
+
+        # assert
+        self.assertTrue(delete_success)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_delete_many_sanity(self, mock_retrying_client, mock_client):
+        mock_client.return_value = MockPyMemcacheClient()
+        mock_retrying_client.return_value = mock_client.return_value
+
+        # setup
+        ref1 = self.object_store.put("a")
+        ref2 = self.object_store.put(np.arange(100))
+
+        # action
+        delete_success = self.object_store.delete_many([ref2, ref1])
+
+        # assert
+        self.assertTrue(delete_success)
+
     @mock.patch("deltacat.io.memcached_object_store.Client")
     @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
     def test_clear_sanity(self, mock_retrying_client, mock_client):

deltacat/tests/io/test_redis_object_store.py

@@ -101,3 +101,23 @@ class TestRedisObjectStore(unittest.TestCase):
             self.object_store.put("test_ip")
 
         self.assertEqual(1, mock_client.Redis.return_value.set.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_delete_sanity(self, mock_client):
+        mock_client.Redis.return_value.delete.return_value = 1
+
+        delete_success = self.object_store.delete("test_ip")
+
+        self.assertTrue(delete_success)
+        self.assertEqual(1, mock_client.Redis.return_value.delete.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_delete_many_sanity(self, mock_client):
+        mock_client.Redis.return_value.delete.side_effect = [2, 1]
+
+        delete_success = self.object_store.delete_many(
+            ["test_ip", "test_ip", "test_ip2"]
+        )
+
+        self.assertTrue(delete_success)
+        self.assertEqual(2, mock_client.Redis.return_value.delete.call_count)

deltacat/tests/io/test_s3_object_store.py

@@ -57,3 +57,12 @@ class TestS3ObjectStore(unittest.TestCase):
 
         self.assertIsNotNone(result)
         self.assertEqual(1, mock_upload.call_count)
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.delete_files_by_prefix")
+    def test_delete_many_sanity(self, mock_delete):
+        self.ray_mock.cloudpickle.loads.return_value = self.TEST_VALUE
+
+        delete_success = self.object_store.delete_many(["test", "test"])
+
+        self.assertTrue(delete_success)
+        self.assertEqual(2, mock_delete.call_count)

{deltacat-1.1.22.dist-info → deltacat-1.1.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.22
+Version: 1.1.24
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -27,7 +27,7 @@ Requires-Dist: tenacity==8.1.0
 Requires-Dist: typing-extensions==4.4.0
 Requires-Dist: pymemcache==4.0.0
 Requires-Dist: redis==4.6.0
-Requires-Dist: getdaft==0.3.4
+Requires-Dist: getdaft==0.3.6
 Requires-Dist: schedule==1.2.0
 
 # DeltaCAT

{deltacat-1.1.22.dist-info → deltacat-1.1.24.dist-info}/RECORD

@@ -1,11 +1,11 @@
-deltacat/__init__.py,sha256=DA3ai-LdX6TopktWC4tQPRs9GXGxAjSkEz-TeJbnWdE,1778
+deltacat/__init__.py,sha256=rclFcR9SWLg7Fn_BcZPRhgFDZ6YfkZ7PZuKo0cS6vpU,1778
 deltacat/constants.py,sha256=TUJLXUJ9xq1Ryil72yLkKR8EDH_Irp5wUg56QstbRNE,2181
 deltacat/exceptions.py,sha256=7sjk3BuMY5Oo-6OvAfHncZx_OcvtEL47BblWr2F7waE,12740
 deltacat/logs.py,sha256=EQSDin1deehzz5xlLV1_TrFJrO_IBZ9Ahp7MdL-4cK8,9363
 deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/clients.py,sha256=4eQvpkV1PzFfxog7EriuglOGGwNFHR5hbGYpjsNNPxk,6949
 deltacat/aws/constants.py,sha256=hcYAUot4ahq9GXCMClQiuYCtiDs5XaOebdUoKg4V84k,1222
-deltacat/aws/s3u.py,sha256=IdT0XqDXVOkPdo5Em5u3qAkV1UXFpXaE1rTkUDKv4f4,28578
+deltacat/aws/s3u.py,sha256=GRmYwE9If-JQAazowUo5BCCu2yRa5EeOwwLfOPIGeCc,28584
 deltacat/aws/redshift/__init__.py,sha256=7SvjG-dqox8zZUhFicTsUvpG5vXYDl_QQ3ohlHOgTKc,342
 deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/redshift/model/manifest.py,sha256=-ap44dxaG2bVNkVMzpJe-oIFHx0iBWCnA_sO-riQp0Y,13605
@@ -46,7 +46,7 @@ deltacat/compute/compactor/steps/repartition.py,sha256=_ITw4yvvnNv3wwOYxprzlIz5J
 deltacat/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/utils/io.py,sha256=S-JZdjETP_tHblK4j860jLHyX9S6A87BPz3Rl0jGbRM,17303
 deltacat/compute/compactor/utils/primary_key_index.py,sha256=ay2-7t4mP9I_l5gKkrv5h5_r8Icts8mBcbH7OJBknrY,2435
-deltacat/compute/compactor/utils/round_completion_file.py,sha256=_rl8lBSO9KFW07ZiicXTFBARwBex4JhSQ3aiVyhYeDQ,3378
+deltacat/compute/compactor/utils/round_completion_file.py,sha256=fFevhUuveCvrU3g_JhX_vPCuEv9Oku0ihbi-n9E6H74,3381
 deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZRG2NqBlCw1X3_FBc,2397
 deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
 deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -66,12 +66,12 @@ deltacat/compute/compactor_v2/model/merge_file_group.py,sha256=1o86t9lc3K6ZvtViV
 deltacat/compute/compactor_v2/model/merge_input.py,sha256=-SxTE0e67z2V7MiMEVz5aMu4E0k8h3-vqohvUUOC0do,5659
 deltacat/compute/compactor_v2/model/merge_result.py,sha256=_IZTCStpb4UKiRCJYA3g6EhAqjrw0t9vmoDAN8kIK-Y,436
 deltacat/compute/compactor_v2/private/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=QKGekJQWL_S1DifnENSQ7PQm5k7x27CoDT0m4QQWBIk,30416
+deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=AuzysedzCyapfNf1pfqsZe6mZw121lx6h6NTyLB-pyM,30930
 deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=1R5xLUkl7GqL1nY-apAgY1czKDEHjIVYSRi9qLOMass,6726
 deltacat/compute/compactor_v2/steps/merge.py,sha256=LpktsDPfj7Of6RgUw9w1f3Y3OBkPDjvtyXjzFaIDoSo,21771
 deltacat/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=Ftl8ZEroiKGmPkFY9bv0prpfrDtD-VK7vpQJbP1br70,7395
+deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=1P9CDpuWErsFcTTlRCeuUQHDokVI92he_MsL82uRAdA,7424
 deltacat/compute/compactor_v2/utils/dedupe.py,sha256=62tFCY2iRP7I3-45GCIYs6_SJsQl8C5lBEr8gbNfbsw,1932
 deltacat/compute/compactor_v2/utils/delta.py,sha256=I7Yvda8NVbpKXG3nM2Ku1utvR2r2OpHvUMqUL2ja3aw,3626
 deltacat/compute/compactor_v2/utils/io.py,sha256=3m4dorxj-WD6Yu9_3gRE6gz3C-eNJA7nn02sHKwo-J8,6018
@@ -85,7 +85,7 @@ deltacat/compute/merge_on_read/model/merge_on_read_params.py,sha256=Q51znagh8PtL
 deltacat/compute/merge_on_read/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/merge_on_read/utils/delta.py,sha256=e4BtOHa5XPpUnR4r0HqBKjXckBsTI8qBwdUWwpJfkWQ,1367
 deltacat/compute/resource_estimation/__init__.py,sha256=4bfBXcq-VAt9JCmjvj3yAmn0lEHVGdGsUCCoMGxjEqA,799
-deltacat/compute/resource_estimation/delta.py,sha256=mzq_0YUGwLMYcTsYhx1QFRyZljC-JYNLjD6WCqUVuXI,9045
+deltacat/compute/resource_estimation/delta.py,sha256=Ei4v9UYhtcT5P-wNEMAg0E4mYl0z5FpSkaTufVoGD18,9492
 deltacat/compute/resource_estimation/manifest.py,sha256=gSqOyIda-pYq3vRsKFq3IiZvwhV3mMqrWPtsmUH9dD8,13035
 deltacat/compute/resource_estimation/model.py,sha256=psyagFXdpLGt8DfDqy7c8DWiuXCacr0Swe5f0M7DdO4,5465
 deltacat/compute/resource_estimation/parquet.py,sha256=5_apma4EKbKcm-nfV73-qN2nfnCeyhFW23ZHX3jz0Kw,3158
@@ -99,13 +99,13 @@ deltacat/compute/stats/models/manifest_entry_stats.py,sha256=NCDAe2nPDEI4kOkuwNk
 deltacat/compute/stats/models/stats_result.py,sha256=XQAlmzhUqRmg4jzEMUAOqcYn1HUOBTMryBH1CCVlet8,3820
 deltacat/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/dataset.py,sha256=pFU5UfK-fD9C4fIeffJtrA6yVQSgAx2UPbxzQ4GMFL8,3203
-deltacat/io/file_object_store.py,sha256=HCFeXu9cWXPXVk54MHel_nw3-wIuzhMt2RI6jKzjRYM,1346
-deltacat/io/memcached_object_store.py,sha256=k9dXlnsK9YWtwtLKJqK9gT4O7xkC_dVsCxCCOi81pH4,9294
-deltacat/io/object_store.py,sha256=X6221ZuVx8NOyKUesz8LvjvQ_4vZ6p2RWV6VISL17AY,1576
+deltacat/io/file_object_store.py,sha256=YoNL3Qla8uLOHaWnyBmIgotjSGAy3Td3Tumah0kk73Y,1868
+deltacat/io/memcached_object_store.py,sha256=PdOMIwv7agEW07mTSBpud6SWd8FjamcKH4_hVJEDPeI,10666
+deltacat/io/object_store.py,sha256=z3Crt8TLyLyoRunOuXAri373TQZKFoz66QHpxGOV82U,1910
 deltacat/io/ray_plasma_object_store.py,sha256=TyoUPWybE_cSISZ2SQa3YfD93QWMp0r82-6WnoVSmzk,905
 deltacat/io/read_api.py,sha256=BhkjL3xjY-fsa62AA9Yv20_88uTskn4_Bv2W6VmMXVA,7023
-deltacat/io/redis_object_store.py,sha256=f54Qw-NMCDjUmKxrrok_swt0LkVDjfmaHdbtAujnxyA,3507
-deltacat/io/s3_object_store.py,sha256=aF-Mn7qbyz1AjdvcbXGZfuUge6vzkR6PrUMsq3sBxk4,1317
+deltacat/io/redis_object_store.py,sha256=ZXkJIrx7uHnnAayD-FG1BiB5xxDjMch9GO-YUkPVwqU,4410
+deltacat/io/s3_object_store.py,sha256=IxvLUvyQZ1w1oYwN9RvRgmKR0Dw56-GggYJw1UCyhBg,1911
 deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/storage/__init__.py,sha256=4sWa3oq89IC3YPclsnVc6ZhnlFM2MuSqshT2uW5cSEY,2158
@@ -138,9 +138,9 @@ deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=8HVr3EIFYFq
 deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=l_6-pAKOsRY3NbtfHsYmEaJEkq6IJueYuLsjyJxNgz4,81564
 deltacat/tests/compute/compact_partition_test_cases.py,sha256=R9eiKvxCLqcoHjAx3iOogdnXZEO9TvLbRf0wA7bcJN4,26170
 deltacat/tests/compute/test_compact_partition_incremental.py,sha256=Z0hyQGhMZjCaOn1Vk4qUbgDiS7HDhtdNeFQyG1PJhqA,14559
-deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=xhKCurTA29Y78_1eksUVJ0W35zNNZYm40rMpMM9ynvM,11853
+deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=XOkpB5r-6lyDbqSIens8loaj86HG29PoNrHAOlIMqTM,12587
 deltacat/tests/compute/test_compact_partition_params.py,sha256=Dm5eLyHo8oGMeO3XBbpj1rZqHtPZ1hAB7z2qvzc4Lxk,8497
-deltacat/tests/compute/test_compact_partition_rebase.py,sha256=O_IwZ1Xeaff98V1XYOyVD8PoS_EpVXSQcHWz4In8bK4,11889
+deltacat/tests/compute/test_compact_partition_rebase.py,sha256=ztSiLgC2OpU4yz81vz-4xWzvZyrLGojtzomsW4q7Bl8,12626
 deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=CHHfNFEJW8S1We7NE1Gg6EaoKEWnaOMRxWrLyirrahc,14643
 deltacat/tests/compute/test_util_common.py,sha256=0mEHo38bgH64y0XZ_zgUL_aZgQMgJOSTlOYvIJxG_MM,11825
 deltacat/tests/compute/test_util_constant.py,sha256=4o-W3E7r7jhFl1A3OFLLrdKnwcF46zx4lEIDY8ONJ3c,929
@@ -157,16 +157,16 @@ deltacat/tests/compute/compactor_v2/test_hashlib.py,sha256=8csF2hFWtBvY2MbX3-6ip
 deltacat/tests/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/compute/compactor_v2/utils/test_task_options.py,sha256=37DkR1u_XwhedV9cGed6FFuJTC0XmuiowHJIa_Op6uA,865
 deltacat/tests/compute/resource_estimation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compute/resource_estimation/test_delta.py,sha256=fE3UlfF7Oi07SxX7zFkeUFcgzUTWp3yUyaNe2QAKIFw,22520
+deltacat/tests/compute/resource_estimation/test_delta.py,sha256=LyzRitBrasQa35Bq7rHTQInaOelSWOSoC0_dyjgpNuE,24505
 deltacat/tests/compute/resource_estimation/test_manifest.py,sha256=yrMvqDjolExdRf6Vtg5XaKDuaKz9ok15PCZ7_aJOYrI,32893
 deltacat/tests/compute/resource_estimation/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/io/test_cloudpickle_bug_fix.py,sha256=qnYJg_S-nsLai77a4_I3Qs2Jtr_KWQJOxyl96f9PgHA,1376
-deltacat/tests/io/test_file_object_store.py,sha256=bHEJRleVHwvk-bbvAlNOFnOA_tbR8i0SxtsllMTb8w0,2559
-deltacat/tests/io/test_memcached_object_store.py,sha256=g2lOYSCH6JQvTzcrMOVvCabKcRQbCXOEzdlI5Sjre_E,8163
+deltacat/tests/io/test_file_object_store.py,sha256=bjORXnHe7Ea733XUUO0S2Su_oqSwGuO84TlIfoNO6qA,3587
+deltacat/tests/io/test_memcached_object_store.py,sha256=0EIaU5MHiEmIEkA4x5qUXFY9TE6TJ7V2RGH827cu3AU,9512
 deltacat/tests/io/test_ray_plasma_object_store.py,sha256=-wJZP6lRtEOogR25wjEiIBGz_lpvWVihwlZ5GqandZU,1911
-deltacat/tests/io/test_redis_object_store.py,sha256=sZrXrYjkw8u_XrvFilhBbLc8PPnZiuMKa1_Bt9ka5qs,3838
-deltacat/tests/io/test_s3_object_store.py,sha256=4b7PYEfQJnYGUz6fcLFWVVyRHTlH_yd8CIaCv9l33Gg,1900
+deltacat/tests/io/test_redis_object_store.py,sha256=YpMsMFT6ltmJHlpAdmlxLK91KjCN8YFMaQrpJ6dcR6E,4595
+deltacat/tests/io/test_s3_object_store.py,sha256=I8AbyrPfS32CAYvRHtn_OanL-XPpAnJeuCuhD-u9irQ,2270
 deltacat/tests/local_deltacat_storage/__init__.py,sha256=5T9ubNIS42-BotEH0yrUiWEU92feW7lkoSA1-wMeAnQ,40104
 deltacat/tests/local_deltacat_storage/exceptions.py,sha256=oxZ0psmrEO0M6P2r8gHQ2E8E-Y8UBfUCBUIwfuHcx38,251
 deltacat/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -210,8 +210,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.22.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.22.dist-info/METADATA,sha256=uaAF2wK6KSi4pooJGoqJERmH_zBIz7WZgUgORbQCnaA,1733
-deltacat-1.1.22.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-deltacat-1.1.22.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.22.dist-info/RECORD,,
+deltacat-1.1.24.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.24.dist-info/METADATA,sha256=xOWZzt633xzTmkD_BrTUM0Kyy9Vf6WuEKXhRmpshHks,1733
+deltacat-1.1.24.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+deltacat-1.1.24.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.24.dist-info/RECORD,,