deltacat 0.1.18b3__py3-none-any.whl → 0.1.18b7__py3-none-any.whl

deltacat/tests/io/test_memcached_object_store.py

@@ -0,0 +1,158 @@
+import unittest
+from unittest import mock
+
+
+@mock.patch("deltacat.io.memcached_object_store.cloudpickle")
+@mock.patch("deltacat.io.memcached_object_store.socket")
+class TestMemcachedObjectStore(unittest.TestCase):
+
+    TEST_VALUE = "test-value"
+
+    def setUp(self):
+        from deltacat.io.memcached_object_store import MemcachedObjectStore
+
+        self.object_store = MemcachedObjectStore()
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_put_many_sanity(
+        self,
+        mock_retrying_client,
+        mock_client,
+        mock_socket,
+        mock_cloudpickle,
+    ):
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_socket.gethostbyname.return_value = "0.0.0.0"
+        mock_socket.gethostname.return_value = "test-host"
+        mock_retrying_client.return_value = mock_client.return_value
+        mock_client.return_value.set_many.return_value = []
+
+        result = self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(2, len(result))
+        self.assertRegex(result[0], ".*_.*")
+        self.assertEqual(1, mock_client.return_value.set_many.call_count)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_put_many_when_cache_fails(
+        self, mock_retrying_client, mock_client, mock_socket, mock_cloudpickle
+    ):
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_socket.gethostbyname.return_value = "0.0.0.0"
+        mock_socket.gethostname.return_value = "test-host"
+        mock_retrying_client.return_value = mock_client.return_value
+        mock_client.return_value.set_many.return_value = ["abcd"]
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(1, mock_client.return_value.set_many.call_count)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_get_many_sanity(
+        self, mock_retrying_client, mock_client, mock_socket, mock_cloudpickle
+    ):
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_socket.gethostbyname.return_value = "0.0.0.0"
+        mock_socket.gethostname.return_value = "test-host"
+        mock_client.return_value.get_many.return_value = {
+            "key1": "value1",
+            "key2": "value2",
+        }
+        mock_retrying_client.return_value = mock_client.return_value
+
+        result = self.object_store.get_many(["test_ip", "test_ip"])
+
+        self.assertEqual(2, len(result))
+        self.assertEqual(1, mock_client.return_value.get_many.call_count)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_get_many_when_cache_expired(
+        self, mock_retrying_client, mock_client, mock_socket, mock_cloudpickle
+    ):
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_socket.gethostbyname.return_value = "0.0.0.0"
+        mock_socket.gethostname.return_value = "test-host"
+        mock_client.return_value.get_many.return_value = {"key1": "value1"}
+        mock_retrying_client.return_value = mock_client.return_value
+
+        with self.assertRaises(AssertionError):
+            self.object_store.get_many(["test_ip", "test_ip"])
+
+        self.assertEqual(1, mock_client.return_value.get_many.call_count)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_get_sanity(
+        self, mock_retrying_client, mock_client, mock_socket, mock_cloudpickle
+    ):
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_socket.gethostbyname.return_value = "0.0.0.0"
+        mock_socket.gethostname.return_value = "test-host"
+        mock_client.return_value.get.return_value = self.TEST_VALUE
+        mock_retrying_client.return_value = mock_client.return_value
+
+        result = self.object_store.get("test_ip")
+
+        self.assertEqual(self.TEST_VALUE, result)
+        self.assertEqual(1, mock_client.return_value.get.call_count)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_get_when_cache_fails(
+        self, mock_retrying_client, mock_client, mock_socket, mock_cloudpickle
+    ):
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_socket.gethostbyname.return_value = "0.0.0.0"
+        mock_socket.gethostname.return_value = "test-host"
+        mock_client.return_value.get.side_effect = RuntimeError()
+        mock_retrying_client.return_value = mock_client.return_value
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.get("test_ip")
+
+        self.assertEqual(1, mock_client.return_value.get.call_count)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_put_sanity(
+        self, mock_retrying_client, mock_client, mock_socket, mock_cloudpickle
+    ):
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_socket.gethostbyname.return_value = "0.0.0.0"
+        mock_socket.gethostname.return_value = "test-host"
+        mock_retrying_client.return_value = mock_client.return_value
+        mock_client.return_value.set.return_value = True
+
+        result = self.object_store.put("test")
+
+        self.assertIsNotNone(result)
+        self.assertEqual(1, mock_client.return_value.set.call_count)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_put_when_cache_fails(
+        self, mock_retrying_client, mock_client, mock_socket, mock_cloudpickle
+    ):
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+        mock_socket.gethostbyname.return_value = "0.0.0.0"
+        mock_socket.gethostname.return_value = "test-host"
+        mock_retrying_client.return_value = mock_client.return_value
+        mock_client.return_value.set.return_value = False
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.put("test_ip")
+
+        self.assertEqual(1, mock_client.return_value.set.call_count)

deltacat/tests/io/test_ray_plasma_object_store.py

@@ -0,0 +1,54 @@
+import unittest
+from unittest import mock
+
+
+class TestRayPlasmaObjectStore(unittest.TestCase):
+
+    TEST_VALUE = "test-value"
+
+    @classmethod
+    def setUpClass(cls):
+        from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
+
+        cls.object_store = RayPlasmaObjectStore()
+
+        super().setUpClass()
+
+    @mock.patch("deltacat.io.ray_plasma_object_store.ray")
+    @mock.patch("deltacat.io.ray_plasma_object_store.cloudpickle")
+    def test_put_many_sanity(self, mock_cloudpickle, mock_ray):
+        mock_ray.put.return_value = "c"
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+        result = self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(2, len(result))
+
+    @mock.patch("deltacat.io.ray_plasma_object_store.ray")
+    @mock.patch("deltacat.io.ray_plasma_object_store.cloudpickle")
+    def test_get_many_sanity(self, mock_cloudpickle, mock_ray):
+        mock_ray.get.return_value = ["a", "b"]
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+
+        result = self.object_store.get_many(["test", "test"])
+
+        self.assertEqual(2, len(result))
+
+    @mock.patch("deltacat.io.ray_plasma_object_store.ray")
+    @mock.patch("deltacat.io.ray_plasma_object_store.cloudpickle")
+    def test_get_sanity(self, mock_cloudpickle, mock_ray):
+        mock_ray.get.return_value = [self.TEST_VALUE]
+        mock_cloudpickle.loads.return_value = self.TEST_VALUE
+
+        result = self.object_store.get("test")
+
+        self.assertEqual(self.TEST_VALUE, result)
+
+    @mock.patch("deltacat.io.ray_plasma_object_store.ray")
+    @mock.patch("deltacat.io.ray_plasma_object_store.cloudpickle")
+    def test_put_sanity(self, mock_cloudpickle, mock_ray):
+        mock_ray.put.return_value = "c"
+        mock_cloudpickle.dumps.return_value = self.TEST_VALUE
+
+        result = self.object_store.put("test")
+
+        self.assertEqual(self.TEST_VALUE, result)

deltacat/tests/io/test_redis_object_store.py

@@ -0,0 +1,103 @@
+import unittest
+from unittest import mock
+
+
+class TestRedisObjectStore(unittest.TestCase):
+
+    TEST_VALUE = "test-value"
+
+    def setUp(self):
+        self.cloudpickle_patcher = mock.patch(
+            "deltacat.io.redis_object_store.cloudpickle"
+        )
+        self.cloudpickle_mock = self.cloudpickle_patcher.start()
+        self.socket_patcher = mock.patch("deltacat.io.redis_object_store.socket")
+        self.socket_mock = self.socket_patcher.start()
+
+        self.cloudpickle_mock.dumps.return_value = self.TEST_VALUE
+        self.cloudpickle_mock.loads.return_value = self.TEST_VALUE
+        self.socket_mock.gethostbyname.return_value = "0.0.0.0"
+        self.socket_mock.gethostname.return_value = "test-host"
+
+        from deltacat.io.redis_object_store import RedisObjectStore
+
+        self.object_store = RedisObjectStore()
+
+        super().setUpClass()
+
+    def tearDown(self) -> None:
+        self.cloudpickle_patcher.stop()
+        self.socket_patcher.stop()
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_put_many_sanity(self, mock_client):
+        mock_client.Redis.return_value.mset.return_value = ["a", "b"]
+
+        result = self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(2, len(result))
+        self.assertRegex(result[0], ".*_.*")
+        self.assertEqual(1, mock_client.Redis.return_value.mset.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_put_many_when_cache_fails(self, mock_client):
+        mock_client.Redis.return_value.mset.return_value = []
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(1, mock_client.Redis.return_value.mset.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_get_many_sanity(self, mock_client):
+        mock_client.Redis.return_value.mget.return_value = ["a", "b"]
+
+        result = self.object_store.get_many(["test_ip", "test_ip"])
+
+        self.assertEqual(2, len(result))
+        self.assertEqual(1, mock_client.Redis.return_value.mget.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_get_many_when_cache_expired(self, mock_client):
+        mock_client.Redis.return_value.mget.return_value = ["value1"]
+
+        with self.assertRaises(AssertionError):
+            self.object_store.get_many(["test_ip", "test_ip"])
+
+        self.assertEqual(1, mock_client.Redis.return_value.mget.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_get_sanity(self, mock_client):
+        mock_client.Redis.return_value.get.return_value = self.TEST_VALUE
+
+        result = self.object_store.get("test_ip")
+
+        self.assertEqual(self.TEST_VALUE, result)
+        self.assertEqual(1, mock_client.Redis.return_value.get.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_get_when_cache_fails(self, mock_client):
+        mock_client.Redis.return_value.get.side_effect = RuntimeError()
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.get("test_ip")
+
+        self.assertEqual(1, mock_client.Redis.return_value.get.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_put_sanity(self, mock_client):
+        mock_client.Redis.return_value.set.return_value = True
+
+        result = self.object_store.put("test")
+
+        self.assertIsNotNone(result)
+        self.assertEqual(1, mock_client.Redis.return_value.set.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_put_when_cache_fails(self, mock_client):
+        mock_client.Redis.return_value.set.return_value = False
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.put("test_ip")
+
+        self.assertEqual(1, mock_client.Redis.return_value.set.call_count)

deltacat/tests/io/test_s3_object_store.py

@@ -0,0 +1,59 @@
+import unittest
+from unittest import mock
+
+
+class TestS3ObjectStore(unittest.TestCase):
+
+    TEST_VALUE = "test-value"
+
+    @classmethod
+    def setUpClass(cls):
+        cls.ray_mock = mock.MagicMock()
+
+        cls.module_patcher = mock.patch.dict("sys.modules", {"ray": cls.ray_mock})
+        cls.module_patcher.start()
+
+        from deltacat.io.s3_object_store import S3ObjectStore
+
+        cls.object_store = S3ObjectStore(bucket_prefix="test")
+
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.module_patcher.stop()
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.upload")
+    def test_put_many_sanity(self, mock_upload):
+        self.ray_mock.cloudpickle.dumps.return_value = self.TEST_VALUE
+        result = self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(2, len(result))
+        self.assertEqual(2, mock_upload.call_count)
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.download")
+    def test_get_many_sanity(self, mock_download):
+        self.ray_mock.cloudpickle.loads.return_value = self.TEST_VALUE
+
+        result = self.object_store.get_many(["test", "test"])
+
+        self.assertEqual(2, len(result))
+        self.assertEqual(2, mock_download.call_count)
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.download")
+    def test_get_sanity(self, mock_download):
+        self.ray_mock.cloudpickle.loads.return_value = self.TEST_VALUE
+
+        result = self.object_store.get("test")
+
+        self.assertEqual(self.TEST_VALUE, result)
+        self.assertEqual(1, mock_download.call_count)
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.upload")
+    def test_put_sanity(self, mock_upload):
+        self.ray_mock.cloudpickle.dumps.return_value = self.TEST_VALUE
+
+        result = self.object_store.put("test")
+
+        self.assertIsNotNone(result)
+        self.assertEqual(1, mock_upload.call_count)

deltacat/tests/utils/test_record_batch_tables.py

@@ -232,7 +232,7 @@ class TestRecordBatchTables(unittest.TestCase):
 
         self.assertEqual(bt.batched_record_count, 8)
         self.assertEqual(bt.remaining_record_count, 4)
-        self.assertNotEquals(prev_remainder_records, bt.remaining_record_count)
+        self.assertNotEqual(prev_remainder_records, bt.remaining_record_count)
         self.assertTrue(_is_sorted(bt, self.column_names[0]))
 
         bt.clear_remaining()

deltacat/tests/utils/test_resources.py

@@ -1,5 +1,6 @@
 import unittest
 from unittest import mock
+import sys
 
 
 class TestGetCurrentClusterUtilization(unittest.TestCase):
@@ -20,8 +21,16 @@ class TestGetCurrentClusterUtilization(unittest.TestCase):
         cls.module_patcher = mock.patch.dict("sys.modules", {"ray": cls.ray_mock})
         cls.module_patcher.start()
 
+        # delete reference to reload from mocked ray
+        if "deltacat.utils.resources" in sys.modules:
+            del sys.modules["deltacat.utils.resources"]
+
         super().setUpClass()
 
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.module_patcher.stop()
+
     def test_sanity(self):
         from deltacat.utils.resources import ClusterUtilization
 

deltacat/utils/ray_utils/concurrency.py

@@ -7,7 +7,6 @@ from ray._private.ray_constants import MIN_RESOURCE_GRANULARITY
 from ray.types import ObjectRef
 
 from deltacat.utils.ray_utils.runtime import current_node_resource_key
-from deltacat.utils.resources import log_current_cluster_utilization
 
 
 def invoke_parallel(
@@ -47,7 +46,6 @@ def invoke_parallel(
     Returns:
         List of Ray object references returned from the submitted tasks.
     """
-    log_current_cluster_utilization(log_identifier=ray_task.__name__)
     if max_parallelism is not None and max_parallelism <= 0:
         raise ValueError(f"Max parallelism ({max_parallelism}) must be > 0.")
     pending_ids = []

deltacat/utils/resources.py

@@ -2,11 +2,15 @@
 from __future__ import annotations
 
 import ray
+import sys
 from typing import Dict, Any
 from dataclasses import dataclass
 from deltacat import logs
 import logging
-from deltacat.utils.performance import timed_invocation
+from resource import getrusage, RUSAGE_SELF
+import platform
+import psutil
+
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -52,21 +56,29 @@ class ClusterUtilization:
         )
 
 
-def log_current_cluster_utilization(log_identifier: str):
-    cluster_utilization, latency = timed_invocation(
-        ClusterUtilization.get_current_cluster_utilization
-    )
-    logger.info(f"Retrieved cluster utilization metrics. Took {latency}s")
+def get_current_node_peak_memory_usage_in_bytes():
+    """
+    Returns the peak memory usage of the node in bytes. This method works across
+    Windows, Darwin and Linux platforms.
+    """
+    current_platform = platform.system()
+    if current_platform != "Windows":
+        usage = getrusage(RUSAGE_SELF).ru_maxrss
+        if current_platform == "Linux":
+            usage = usage * 1024
+        return usage
+    else:
+        return psutil.Process().memory_info().peak_wset
+
 
-    logger.info(
-        f"Log ID={log_identifier} | Cluster Object store memory used: {cluster_utilization.used_object_store_memory_bytes} "
-        f"which is {cluster_utilization.used_object_store_memory_percent}%"
-    )
-    logger.info(
-        f"Log ID={log_identifier} | Total Cluster Memory used: {cluster_utilization.used_memory_bytes} which is "
-        f"{cluster_utilization.used_memory_percent}%"
-    )
-    logger.info(
-        f"Log ID={log_identifier} | Total Cluster CPU used: {cluster_utilization.used_cpu} which is "
-        f"{cluster_utilization.used_cpu_percent}%"
-    )
+def get_size_of_object_in_bytes(obj: object) -> float:
+    size = sys.getsizeof(obj)
+    if isinstance(obj, dict):
+        return (
+            size
+            + sum(map(get_size_of_object_in_bytes, obj.keys()))
+            + sum(map(get_size_of_object_in_bytes, obj.values()))
+        )
+    if isinstance(obj, (list, tuple, set, frozenset)):
+        return size + sum(map(get_size_of_object_in_bytes, obj))
+    return size

{deltacat-0.1.18b3.dist-info → deltacat-0.1.18b7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.1.18b3
+Version: 0.1.18b7
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -23,6 +23,8 @@ Requires-Dist: ray[default] (~=2.0)
 Requires-Dist: s3fs (==2022.2.0)
 Requires-Dist: tenacity (==8.1.0)
 Requires-Dist: typing-extensions (==4.4.0)
+Requires-Dist: pymemcache (==4.0.0)
+Requires-Dist: redis (==4.6.0)
 
 # DeltaCAT
 

{deltacat-0.1.18b3.dist-info → deltacat-0.1.18b7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=pO2phs6VUPzLj5RhW3ZxBJ0X7DU-Zu2UGOsa4waCmHk,1810
+deltacat/__init__.py,sha256=j2URX0ymVm1wouW1S1SUL7icyOO--sonM98V129b5_w,1810
 deltacat/constants.py,sha256=oMU8ypqvDBTG54-6MLGWrt9iJKTN-HKsSWxEWnWp77c,1969
 deltacat/exceptions.py,sha256=x7qem7FLujXf-DzPsNcQ-XYkW3cF3A0YGIbxkcpz0Mw,146
 deltacat/logs.py,sha256=yyve_6Y4bLWAdCOnxFOPrSR9FRXwZuh68_rRoPpmg08,5633
@@ -17,33 +17,35 @@ deltacat/catalog/model/catalog.py,sha256=-Ho7a3rV1hiOS9cSRCAor9AtXV9nJn9t_MDVql9
 deltacat/catalog/model/table_definition.py,sha256=tKrM1mmaQlvxqXrLt3QJVZK5BZfaJnhjTZ6KjybYlhE,727
 deltacat/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/__init__.py,sha256=kmWC-Qnw861k7mPhLH4fQEL6CaMeBql2AipHeFqJ2uI,1127
-deltacat/compute/compactor/compaction_session.py,sha256=m0i9Jt7FFuqC1UZr-fYqWsQ_rExHa55Mac7VSgLqwlY,21233
+deltacat/compute/compactor/compaction_session.py,sha256=pIm5SH-nIef5a75AFlD1wBTKhJECIUIl9sQzZ3Snj1E,25826
 deltacat/compute/compactor/repartition_session.py,sha256=7e-5exas25725aNh326wK8m9qDmzlcchS7GT6fh0a2o,6776
 deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor/model/dedupe_result.py,sha256=gc3E8aS39LKylb_USJeittCfwKZBT4lHctxAal33vN0,179
+deltacat/compute/compactor/model/compact_partition_params.py,sha256=QvjH10IsA8O6ufVzwPz-mcw326BT-Zbs29wFGCcGerA,5677
+deltacat/compute/compactor/model/compaction_session_audit_info.py,sha256=TKgFFdd38cplihdMtHja-cBTwk3dflEipc8smWtZlGg,25231
+deltacat/compute/compactor/model/dedupe_result.py,sha256=1OCV944qJdLQ_-8scisVKl45ej1eRv9OV539QYZtQ-U,292
 deltacat/compute/compactor/model/delta_annotated.py,sha256=0nlfu3CwMadBd2ENBvlmsCEuOb-CNrzjx-kzA6cjsHo,8353
-deltacat/compute/compactor/model/delta_file_envelope.py,sha256=wjJIjrbM2dhTh3YLFi7pICHtkWRYI40eduwxsKicNIg,2291
-deltacat/compute/compactor/model/delta_file_locator.py,sha256=Cc-YzxxyrXK6FlY8ek2L92XzfT0qkMCxs6yrC_FsEwU,1766
-deltacat/compute/compactor/model/hash_bucket_result.py,sha256=-IBtuwYWArDgJivZbc0ih2ZOEpgq7BqX-lPCiIJkRsI,162
-deltacat/compute/compactor/model/materialize_result.py,sha256=2IlDD-kDrvhfOZE4JvCuCOgI5hmI2MdZglssWP2WJRM,1738
+deltacat/compute/compactor/model/delta_file_envelope.py,sha256=vTChCiCE56tBLrFXTtD76v7fXalUvRYX1WQApvf632g,2534
+deltacat/compute/compactor/model/delta_file_locator.py,sha256=AmhPGPDsmahVhp91rohJMx4ByumcIY5feqRLZTrNu4s,1905
+deltacat/compute/compactor/model/hash_bucket_result.py,sha256=71qGmaT1Mks-r3-aatjNbn2x3yWIgT8RmV0bRWe6pdA,275
+deltacat/compute/compactor/model/materialize_result.py,sha256=Ngwg1khN55FK2iBKPnObfo9K-4vwdTznaqBvPo8Jqvo,2656
 deltacat/compute/compactor/model/primary_key_index.py,sha256=MT4zqwhzh3e9qZotWvZavT_MtWXm_81ojfcOCv1t17w,10459
 deltacat/compute/compactor/model/pyarrow_write_result.py,sha256=WYIa0DRcyaemR6yUS8_8RLQ2voTmCVNFUL99qxPmt70,1324
 deltacat/compute/compactor/model/repartition_result.py,sha256=HZy7Ls6toI4rXgVW2yIKMIkVS8o9kxvlIJPvo5_pCxA,140
-deltacat/compute/compactor/model/round_completion_info.py,sha256=2iayH0TGi-5ffo4tu4MuskwZlhG2a8VMnT015_3taB4,3413
+deltacat/compute/compactor/model/round_completion_info.py,sha256=FEeFzsqXfSrWGjWbwodrXGtqx-FMXvrgqURSF0TbIVU,3763
 deltacat/compute/compactor/model/sort_key.py,sha256=XDIoYrV18FciomV5yWxu1OaDsD78trmUUtseyRurIKo,4124
 deltacat/compute/compactor/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor/steps/dedupe.py,sha256=Bu-Yf5YQ3QJxNuzujjQ_PjO1Z0i0JwiYFNL-BU5GzBs,9592
-deltacat/compute/compactor/steps/hash_bucket.py,sha256=2aOdC91zTL50BiA94BZdpaROpdsf8AB0ziGXMlRaCm8,8781
-deltacat/compute/compactor/steps/materialize.py,sha256=_Hpq6QmsTeI11iZOlNxKfbbC5f72ecGf9bdaF3WHBoM,13522
+deltacat/compute/compactor/steps/dedupe.py,sha256=R6p43mOUWgA1t468FS8JU-Wlrr96tt0ccwa0uytuaRY,10063
+deltacat/compute/compactor/steps/hash_bucket.py,sha256=ZzJQWulSOMve7bDZX7ZRuYAl4bSC4U5SJzPhpeGpKB0,9769
+deltacat/compute/compactor/steps/materialize.py,sha256=mXxKSaPL7iYtqP-eiJlFwi8kuywFmiU5FLS2-DW5314,13964
 deltacat/compute/compactor/steps/repartition.py,sha256=lpvxhiTC27MKqUXPN70H5L-FcLA1-yCCElERQq74Zig,9487
 deltacat/compute/compactor/steps/rehash/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/steps/rehash/rehash_bucket.py,sha256=yh-sBuUI3hqw2vk_nK9o-KDrgSww4oSvAz2hBxTkv8s,1765
 deltacat/compute/compactor/steps/rehash/rewrite_index.py,sha256=-HVM08pk5ROHEgDP-FVty55-a_0dsGRiSnPlNJw7C6Q,1838
 deltacat/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor/utils/io.py,sha256=u7gNcPiaGoCyj3vfwWz7P9M9BwXJyzDUbu8FfxFXFec,15840
-deltacat/compute/compactor/utils/primary_key_index.py,sha256=taYw1AjGIFlD9c8OXyj9ps816a15B61aoV4I00EAUyo,12072
-deltacat/compute/compactor/utils/round_completion_file.py,sha256=nv_-pl8FRIWPWY5xWLuBXVJkGH4LYBhDwnrWKIaSRms,1935
-deltacat/compute/compactor/utils/system_columns.py,sha256=or9yqPk2QY6Ws3sq-G5JMDbizYO3MUZeFgLb5nCPrL0,7153
+deltacat/compute/compactor/utils/io.py,sha256=itraIfLGUFfVFrW-XHnsEEa9GNIJR4VCnav0LyjHons,16543
+deltacat/compute/compactor/utils/primary_key_index.py,sha256=Y8MBkDMS4N9xgJpuqWcdqpdNbfrfycIABrKlGZwfoRM,11359
+deltacat/compute/compactor/utils/round_completion_file.py,sha256=DmZfHeAXlQn0DDdcsIHZROHWfyBCKTY3pNUdHzalqkE,2284
+deltacat/compute/compactor/utils/system_columns.py,sha256=I36NAEGwRegv56ouVLwTCCisyoOupDCbbaxtoFDzYTE,8121
 deltacat/compute/metastats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/metastats/meta_stats.py,sha256=-Fb0yQAdUUgm2IShcWlPZto-qdivF-nK05sQqJu7K5s,18588
 deltacat/compute/metastats/stats.py,sha256=-aFFrh7b--PzvQWNJG5_PgdN7ZM1bmGMeha5khwxhNw,7285
@@ -69,16 +71,22 @@ deltacat/compute/stats/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 deltacat/compute/stats/utils/intervals.py,sha256=9ezOzIrBGU1fWBuAn1CorJ3uX5COU7vxrfA7kI1cB7I,3094
 deltacat/compute/stats/utils/io.py,sha256=ZXpntXqa41l5bxxAa2vcTW5mVpWeBIvd3QA9VWnX-aw,8573
 deltacat/compute/stats/utils/manifest_stats_file.py,sha256=PtqW5Zc5e09HcfiAgvoZHVMJ2gamGdwmynMXOJNJUaY,3693
-deltacat/io/__init__.py,sha256=5Al7BPSaQghEp1K3PfiKIJJ0HR6MUuaN7HTMyM_9lf4,154
+deltacat/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/dataset.py,sha256=8w9sPVDpGnjjGVDWB39YSKWxq4zRv9VEfDtj7PYwjqM,3755
+deltacat/io/file_object_store.py,sha256=HCFeXu9cWXPXVk54MHel_nw3-wIuzhMt2RI6jKzjRYM,1346
+deltacat/io/memcached_object_store.py,sha256=2fhC1WAY2qcFzAynDrSaIIaMCqC8kUGsmvGeR0pD4I0,3943
+deltacat/io/object_store.py,sha256=GX4pK-LY92s3uXRGcj8YsG2FFoiKfcJr2USIVz1ruGg,1380
+deltacat/io/ray_plasma_object_store.py,sha256=pupw7ulZY_EV5dERJDCCW_y_hzVx3Hl_uAvpQTNIh-E,705
 deltacat/io/read_api.py,sha256=BhkjL3xjY-fsa62AA9Yv20_88uTskn4_Bv2W6VmMXVA,7023
+deltacat/io/redis_object_store.py,sha256=f54Qw-NMCDjUmKxrrok_swt0LkVDjfmaHdbtAujnxyA,3507
+deltacat/io/s3_object_store.py,sha256=aF-Mn7qbyz1AjdvcbXGZfuUge6vzkR6PrUMsq3sBxk4,1317
 deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/aws/redshift/redshift_datasource.py,sha256=X183O4tgBqtaZOSFmMFvp-9mv8NX5kGvRvX0eoSX8rA,22599
 deltacat/storage/__init__.py,sha256=ElzZuG5zrX9nUIe7f0Sp21WDX7yBoclclq3TIL-doag,1371
 deltacat/storage/interface.py,sha256=czzC0iourcqteNNw_drMEyeOXsMOMLiBsSt--g5le8o,21143
 deltacat/storage/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/storage/model/delta.py,sha256=bmcG1rF6mwUdM3YHh6M9gLV6uqxbwHZVNS3WHkXFeDw,13734
+deltacat/storage/model/delta.py,sha256=2qhCiEWmrybV0mfBTKHi9VKlNTqJ6wkE5ssq4ozMMQQ,13757
 deltacat/storage/model/list_result.py,sha256=FgD6oYeKo0EPe8z7jC8T4pAFjBOuBwd4axxGrnYyBG4,2466
 deltacat/storage/model/locator.py,sha256=1S7szmDSx-R4Z3arFNILOvS4t7dF7_rJNV9fHyRc3G4,1296
 deltacat/storage/model/namespace.py,sha256=KI2umYWShXFTx1ykLwsQjuce078WYo_Hmavn3DDeBzE,2086
@@ -90,15 +98,22 @@ deltacat/storage/model/types.py,sha256=-9yPA5wjZf9jOd-iErf4sN-YD-6fbl2z8m8t1lGa0
 deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/test_repartition.py,sha256=xzqdfRzZS-bA1yBdPNxelecTFe2MtON5Lrd-jTGZ4Xk,7245
 deltacat/tests/compactor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/compactor/test_compact_partition_params.py,sha256=0h0cXNg-1NslQ98Nld7brD1WHHhzzBZR1x16kUd7MdA,8848
 deltacat/tests/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compactor/utils/test_io.py,sha256=FWguDt03ErItYluXBWaPOAUKwQKfUtGXJYDwP_O1cMM,2282
+deltacat/tests/compactor/utils/test_io.py,sha256=ioPyW0of5DdmK1NOe-UtKQYBOgMpA-yQfJVKoiuaXdE,3097
+deltacat/tests/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/io/test_file_object_store.py,sha256=bHEJRleVHwvk-bbvAlNOFnOA_tbR8i0SxtsllMTb8w0,2559
+deltacat/tests/io/test_memcached_object_store.py,sha256=ZViHYo5h-cThTXg3d46jvdU4C3e7WV5_kuIFKZigfxY,6907
+deltacat/tests/io/test_ray_plasma_object_store.py,sha256=-wJZP6lRtEOogR25wjEiIBGz_lpvWVihwlZ5GqandZU,1911
+deltacat/tests/io/test_redis_object_store.py,sha256=sZrXrYjkw8u_XrvFilhBbLc8PPnZiuMKa1_Bt9ka5qs,3838
+deltacat/tests/io/test_s3_object_store.py,sha256=4b7PYEfQJnYGUz6fcLFWVVyRHTlH_yd8CIaCv9l33Gg,1900
 deltacat/tests/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/stats/test_intervals.py,sha256=S92DgkALQ1WmbLWcxtvS7RlVGvL-XoPJKUUbkdn9_CQ,1955
 deltacat/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/test_utils/constants.py,sha256=zgqFmfIE5ZCtDw4NF-Y4ZEEnaPUP5nDY5768WPod0Fc,208
 deltacat/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/utils/test_record_batch_tables.py,sha256=yLExx5jZfi65uSjkdhOCGnP7Km6zWqKCzmULf1PEKA0,11322
-deltacat/tests/utils/test_resources.py,sha256=ubd2tSusagWLSuRXDA2L_2cWr5Xnt6UXpKp3NGGe1ww,1193
+deltacat/tests/utils/test_record_batch_tables.py,sha256=AkG1WyljQmjnl-AxhbFWyo5LnMIKRyLScfgC2B_ES-s,11321
+deltacat/tests/utils/test_resources.py,sha256=8oCGNofatgWuif9-Mcis6fFcRCWXXtJnQ9Ff16Gp14g,1456
 deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/types/media.py,sha256=py1BnfMqNpJlW1RKzHWwB0NmQ33oCk9qg1fz7alvi3E,2187
 deltacat/types/tables.py,sha256=yUzkzmUij8kssEYI_dfVDSLXf8HfMm_jpgWkPxDHAas,3893
@@ -110,15 +125,15 @@ deltacat/utils/pandas.py,sha256=eGOpiZE1zLznTtuwoN80j4PBp1_bUV8SE4c951r0a3o,9561
 deltacat/utils/performance.py,sha256=rC3CPfroZP3T5TbRNZXB9GRBr0F9i2KUeZYL45JBgCU,610
 deltacat/utils/placement.py,sha256=JE6OsW16VonlMhdH5B2IYuLJxItoYguaKpZNgbpMNLw,11066
 deltacat/utils/pyarrow.py,sha256=dgAruwOpWYSlnJ5w8iJz_NWpfQoZHA_iG-F7CBDieko,18245
-deltacat/utils/resources.py,sha256=gdw8_79GkYyD6FYLgB51bDbxHpviGAt8-Mhrt4lqG4I,2817
+deltacat/utils/resources.py,sha256=fA53NiJOd5rLMtwvuTnqTyq4g59deD6NCGDbX5yIlg8,2908
 deltacat/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/utils/ray_utils/collections.py,sha256=hj20s4D2RF2jZETU_44r6mFbsczA0JI_I_4kWKTmqes,1951
-deltacat/utils/ray_utils/concurrency.py,sha256=GmWjrpaB9Ad3i8miOJwdkmwIbOqR6KjaS8e5BYXFzAU,5262
+deltacat/utils/ray_utils/concurrency.py,sha256=AyL7hpvYjkmsz-KcpYjVgPpNsmu-x8-rlLyG0qXoV_c,5123
 deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=xOVkqL6o8qGsewGvzhMKxmCcqcFZDnNILuz5IGMgxSc,4991
-deltacat-0.1.18b3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-0.1.18b3.dist-info/METADATA,sha256=nBwFBXeWcqAe92M7yzEypKrDgJqhJyet1Gf_GrlkyQY,1475
-deltacat-0.1.18b3.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-deltacat-0.1.18b3.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-0.1.18b3.dist-info/RECORD,,
+deltacat-0.1.18b7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-0.1.18b7.dist-info/METADATA,sha256=o8gkDXBrItOzyPPs72dvXTSL9MEwP5G8J5LCirD6XdY,1542
+deltacat-0.1.18b7.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+deltacat-0.1.18b7.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-0.1.18b7.dist-info/RECORD,,