deltacat 0.1.18b4__py3-none-any.whl → 0.1.18b7__py3-none-any.whl

deltacat/tests/io/test_redis_object_store.py

@@ -0,0 +1,103 @@
+import unittest
+from unittest import mock
+
+
+class TestRedisObjectStore(unittest.TestCase):
+
+    TEST_VALUE = "test-value"
+
+    def setUp(self):
+        self.cloudpickle_patcher = mock.patch(
+            "deltacat.io.redis_object_store.cloudpickle"
+        )
+        self.cloudpickle_mock = self.cloudpickle_patcher.start()
+        self.socket_patcher = mock.patch("deltacat.io.redis_object_store.socket")
+        self.socket_mock = self.socket_patcher.start()
+
+        self.cloudpickle_mock.dumps.return_value = self.TEST_VALUE
+        self.cloudpickle_mock.loads.return_value = self.TEST_VALUE
+        self.socket_mock.gethostbyname.return_value = "0.0.0.0"
+        self.socket_mock.gethostname.return_value = "test-host"
+
+        from deltacat.io.redis_object_store import RedisObjectStore
+
+        self.object_store = RedisObjectStore()
+
+        super().setUpClass()
+
+    def tearDown(self) -> None:
+        self.cloudpickle_patcher.stop()
+        self.socket_patcher.stop()
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_put_many_sanity(self, mock_client):
+        mock_client.Redis.return_value.mset.return_value = ["a", "b"]
+
+        result = self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(2, len(result))
+        self.assertRegex(result[0], ".*_.*")
+        self.assertEqual(1, mock_client.Redis.return_value.mset.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_put_many_when_cache_fails(self, mock_client):
+        mock_client.Redis.return_value.mset.return_value = []
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(1, mock_client.Redis.return_value.mset.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_get_many_sanity(self, mock_client):
+        mock_client.Redis.return_value.mget.return_value = ["a", "b"]
+
+        result = self.object_store.get_many(["test_ip", "test_ip"])
+
+        self.assertEqual(2, len(result))
+        self.assertEqual(1, mock_client.Redis.return_value.mget.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_get_many_when_cache_expired(self, mock_client):
+        mock_client.Redis.return_value.mget.return_value = ["value1"]
+
+        with self.assertRaises(AssertionError):
+            self.object_store.get_many(["test_ip", "test_ip"])
+
+        self.assertEqual(1, mock_client.Redis.return_value.mget.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_get_sanity(self, mock_client):
+        mock_client.Redis.return_value.get.return_value = self.TEST_VALUE
+
+        result = self.object_store.get("test_ip")
+
+        self.assertEqual(self.TEST_VALUE, result)
+        self.assertEqual(1, mock_client.Redis.return_value.get.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_get_when_cache_fails(self, mock_client):
+        mock_client.Redis.return_value.get.side_effect = RuntimeError()
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.get("test_ip")
+
+        self.assertEqual(1, mock_client.Redis.return_value.get.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_put_sanity(self, mock_client):
+        mock_client.Redis.return_value.set.return_value = True
+
+        result = self.object_store.put("test")
+
+        self.assertIsNotNone(result)
+        self.assertEqual(1, mock_client.Redis.return_value.set.call_count)
+
+    @mock.patch("deltacat.io.redis_object_store.redis")
+    def test_put_when_cache_fails(self, mock_client):
+        mock_client.Redis.return_value.set.return_value = False
+
+        with self.assertRaises(RuntimeError):
+            self.object_store.put("test_ip")
+
+        self.assertEqual(1, mock_client.Redis.return_value.set.call_count)

deltacat/tests/io/test_s3_object_store.py

@@ -0,0 +1,59 @@
+import unittest
+from unittest import mock
+
+
+class TestS3ObjectStore(unittest.TestCase):
+
+    TEST_VALUE = "test-value"
+
+    @classmethod
+    def setUpClass(cls):
+        cls.ray_mock = mock.MagicMock()
+
+        cls.module_patcher = mock.patch.dict("sys.modules", {"ray": cls.ray_mock})
+        cls.module_patcher.start()
+
+        from deltacat.io.s3_object_store import S3ObjectStore
+
+        cls.object_store = S3ObjectStore(bucket_prefix="test")
+
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.module_patcher.stop()
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.upload")
+    def test_put_many_sanity(self, mock_upload):
+        self.ray_mock.cloudpickle.dumps.return_value = self.TEST_VALUE
+        result = self.object_store.put_many(["a", "b"])
+
+        self.assertEqual(2, len(result))
+        self.assertEqual(2, mock_upload.call_count)
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.download")
+    def test_get_many_sanity(self, mock_download):
+        self.ray_mock.cloudpickle.loads.return_value = self.TEST_VALUE
+
+        result = self.object_store.get_many(["test", "test"])
+
+        self.assertEqual(2, len(result))
+        self.assertEqual(2, mock_download.call_count)
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.download")
+    def test_get_sanity(self, mock_download):
+        self.ray_mock.cloudpickle.loads.return_value = self.TEST_VALUE
+
+        result = self.object_store.get("test")
+
+        self.assertEqual(self.TEST_VALUE, result)
+        self.assertEqual(1, mock_download.call_count)
+
+    @mock.patch("deltacat.io.s3_object_store.s3_utils.upload")
+    def test_put_sanity(self, mock_upload):
+        self.ray_mock.cloudpickle.dumps.return_value = self.TEST_VALUE
+
+        result = self.object_store.put("test")
+
+        self.assertIsNotNone(result)
+        self.assertEqual(1, mock_upload.call_count)

deltacat/tests/utils/test_resources.py

@@ -27,6 +27,10 @@ class TestGetCurrentClusterUtilization(unittest.TestCase):
 
         super().setUpClass()
 
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.module_patcher.stop()
+
     def test_sanity(self):
         from deltacat.utils.resources import ClusterUtilization
 

{deltacat-0.1.18b4.dist-info → deltacat-0.1.18b7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.1.18b4
+Version: 0.1.18b7
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -23,6 +23,8 @@ Requires-Dist: ray[default] (~=2.0)
 Requires-Dist: s3fs (==2022.2.0)
 Requires-Dist: tenacity (==8.1.0)
 Requires-Dist: typing-extensions (==4.4.0)
+Requires-Dist: pymemcache (==4.0.0)
+Requires-Dist: redis (==4.6.0)
 
 # DeltaCAT
 

{deltacat-0.1.18b4.dist-info → deltacat-0.1.18b7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=sDhoHzmA2ToJYKQJk6FecqlMqONL-i2KA7SSpXV8E64,1810
+deltacat/__init__.py,sha256=j2URX0ymVm1wouW1S1SUL7icyOO--sonM98V129b5_w,1810
 deltacat/constants.py,sha256=oMU8ypqvDBTG54-6MLGWrt9iJKTN-HKsSWxEWnWp77c,1969
 deltacat/exceptions.py,sha256=x7qem7FLujXf-DzPsNcQ-XYkW3cF3A0YGIbxkcpz0Mw,146
 deltacat/logs.py,sha256=yyve_6Y4bLWAdCOnxFOPrSR9FRXwZuh68_rRoPpmg08,5633
@@ -17,7 +17,7 @@ deltacat/catalog/model/catalog.py,sha256=-Ho7a3rV1hiOS9cSRCAor9AtXV9nJn9t_MDVql9
 deltacat/catalog/model/table_definition.py,sha256=tKrM1mmaQlvxqXrLt3QJVZK5BZfaJnhjTZ6KjybYlhE,727
 deltacat/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/__init__.py,sha256=kmWC-Qnw861k7mPhLH4fQEL6CaMeBql2AipHeFqJ2uI,1127
-deltacat/compute/compactor/compaction_session.py,sha256=7v2Qs5GYk8v_62nPxzhmOuh1volmnVnfdSAGPxms0vc,24572
+deltacat/compute/compactor/compaction_session.py,sha256=pIm5SH-nIef5a75AFlD1wBTKhJECIUIl9sQzZ3Snj1E,25826
 deltacat/compute/compactor/repartition_session.py,sha256=7e-5exas25725aNh326wK8m9qDmzlcchS7GT6fh0a2o,6776
 deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/model/compact_partition_params.py,sha256=QvjH10IsA8O6ufVzwPz-mcw326BT-Zbs29wFGCcGerA,5677
@@ -34,17 +34,17 @@ deltacat/compute/compactor/model/repartition_result.py,sha256=HZy7Ls6toI4rXgVW2y
 deltacat/compute/compactor/model/round_completion_info.py,sha256=FEeFzsqXfSrWGjWbwodrXGtqx-FMXvrgqURSF0TbIVU,3763
 deltacat/compute/compactor/model/sort_key.py,sha256=XDIoYrV18FciomV5yWxu1OaDsD78trmUUtseyRurIKo,4124
 deltacat/compute/compactor/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor/steps/dedupe.py,sha256=ZkLVCO6eonJDE1lSc-H7-SijfCIvOSzmqWf8uz-XWMw,10312
-deltacat/compute/compactor/steps/hash_bucket.py,sha256=OCDDUCYWJGXjvBV3jFZlQ0I9T__f10lTXcVqCwM1f9w,9604
-deltacat/compute/compactor/steps/materialize.py,sha256=CEiIokjAAGcoG4CaWYneqXAXj0e6KUlQusZtfyfjkkk,14451
+deltacat/compute/compactor/steps/dedupe.py,sha256=R6p43mOUWgA1t468FS8JU-Wlrr96tt0ccwa0uytuaRY,10063
+deltacat/compute/compactor/steps/hash_bucket.py,sha256=ZzJQWulSOMve7bDZX7ZRuYAl4bSC4U5SJzPhpeGpKB0,9769
+deltacat/compute/compactor/steps/materialize.py,sha256=mXxKSaPL7iYtqP-eiJlFwi8kuywFmiU5FLS2-DW5314,13964
 deltacat/compute/compactor/steps/repartition.py,sha256=lpvxhiTC27MKqUXPN70H5L-FcLA1-yCCElERQq74Zig,9487
 deltacat/compute/compactor/steps/rehash/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/steps/rehash/rehash_bucket.py,sha256=yh-sBuUI3hqw2vk_nK9o-KDrgSww4oSvAz2hBxTkv8s,1765
 deltacat/compute/compactor/steps/rehash/rewrite_index.py,sha256=-HVM08pk5ROHEgDP-FVty55-a_0dsGRiSnPlNJw7C6Q,1838
 deltacat/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/utils/io.py,sha256=itraIfLGUFfVFrW-XHnsEEa9GNIJR4VCnav0LyjHons,16543
-deltacat/compute/compactor/utils/primary_key_index.py,sha256=taYw1AjGIFlD9c8OXyj9ps816a15B61aoV4I00EAUyo,12072
-deltacat/compute/compactor/utils/round_completion_file.py,sha256=E6a1jRVGhEPWuDn3Q_yTNt_lqwqVPvNdfMH5Jf9f74w,2080
+deltacat/compute/compactor/utils/primary_key_index.py,sha256=Y8MBkDMS4N9xgJpuqWcdqpdNbfrfycIABrKlGZwfoRM,11359
+deltacat/compute/compactor/utils/round_completion_file.py,sha256=DmZfHeAXlQn0DDdcsIHZROHWfyBCKTY3pNUdHzalqkE,2284
 deltacat/compute/compactor/utils/system_columns.py,sha256=I36NAEGwRegv56ouVLwTCCisyoOupDCbbaxtoFDzYTE,8121
 deltacat/compute/metastats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/metastats/meta_stats.py,sha256=-Fb0yQAdUUgm2IShcWlPZto-qdivF-nK05sQqJu7K5s,18588
@@ -71,9 +71,15 @@ deltacat/compute/stats/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 deltacat/compute/stats/utils/intervals.py,sha256=9ezOzIrBGU1fWBuAn1CorJ3uX5COU7vxrfA7kI1cB7I,3094
 deltacat/compute/stats/utils/io.py,sha256=ZXpntXqa41l5bxxAa2vcTW5mVpWeBIvd3QA9VWnX-aw,8573
 deltacat/compute/stats/utils/manifest_stats_file.py,sha256=PtqW5Zc5e09HcfiAgvoZHVMJ2gamGdwmynMXOJNJUaY,3693
-deltacat/io/__init__.py,sha256=5Al7BPSaQghEp1K3PfiKIJJ0HR6MUuaN7HTMyM_9lf4,154
+deltacat/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/dataset.py,sha256=8w9sPVDpGnjjGVDWB39YSKWxq4zRv9VEfDtj7PYwjqM,3755
+deltacat/io/file_object_store.py,sha256=HCFeXu9cWXPXVk54MHel_nw3-wIuzhMt2RI6jKzjRYM,1346
+deltacat/io/memcached_object_store.py,sha256=2fhC1WAY2qcFzAynDrSaIIaMCqC8kUGsmvGeR0pD4I0,3943
+deltacat/io/object_store.py,sha256=GX4pK-LY92s3uXRGcj8YsG2FFoiKfcJr2USIVz1ruGg,1380
+deltacat/io/ray_plasma_object_store.py,sha256=pupw7ulZY_EV5dERJDCCW_y_hzVx3Hl_uAvpQTNIh-E,705
 deltacat/io/read_api.py,sha256=BhkjL3xjY-fsa62AA9Yv20_88uTskn4_Bv2W6VmMXVA,7023
+deltacat/io/redis_object_store.py,sha256=f54Qw-NMCDjUmKxrrok_swt0LkVDjfmaHdbtAujnxyA,3507
+deltacat/io/s3_object_store.py,sha256=aF-Mn7qbyz1AjdvcbXGZfuUge6vzkR6PrUMsq3sBxk4,1317
 deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/aws/redshift/redshift_datasource.py,sha256=X183O4tgBqtaZOSFmMFvp-9mv8NX5kGvRvX0eoSX8rA,22599
@@ -94,14 +100,20 @@ deltacat/tests/test_repartition.py,sha256=xzqdfRzZS-bA1yBdPNxelecTFe2MtON5Lrd-jT
 deltacat/tests/compactor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/compactor/test_compact_partition_params.py,sha256=0h0cXNg-1NslQ98Nld7brD1WHHhzzBZR1x16kUd7MdA,8848
 deltacat/tests/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compactor/utils/test_io.py,sha256=_8_ekwtIRh30uHOMWcxU6udnFCuhY059A8Sm6mj-hoY,3009
+deltacat/tests/compactor/utils/test_io.py,sha256=ioPyW0of5DdmK1NOe-UtKQYBOgMpA-yQfJVKoiuaXdE,3097
+deltacat/tests/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/io/test_file_object_store.py,sha256=bHEJRleVHwvk-bbvAlNOFnOA_tbR8i0SxtsllMTb8w0,2559
+deltacat/tests/io/test_memcached_object_store.py,sha256=ZViHYo5h-cThTXg3d46jvdU4C3e7WV5_kuIFKZigfxY,6907
+deltacat/tests/io/test_ray_plasma_object_store.py,sha256=-wJZP6lRtEOogR25wjEiIBGz_lpvWVihwlZ5GqandZU,1911
+deltacat/tests/io/test_redis_object_store.py,sha256=sZrXrYjkw8u_XrvFilhBbLc8PPnZiuMKa1_Bt9ka5qs,3838
+deltacat/tests/io/test_s3_object_store.py,sha256=4b7PYEfQJnYGUz6fcLFWVVyRHTlH_yd8CIaCv9l33Gg,1900
 deltacat/tests/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/stats/test_intervals.py,sha256=S92DgkALQ1WmbLWcxtvS7RlVGvL-XoPJKUUbkdn9_CQ,1955
 deltacat/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/test_utils/constants.py,sha256=zgqFmfIE5ZCtDw4NF-Y4ZEEnaPUP5nDY5768WPod0Fc,208
 deltacat/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/utils/test_record_batch_tables.py,sha256=AkG1WyljQmjnl-AxhbFWyo5LnMIKRyLScfgC2B_ES-s,11321
-deltacat/tests/utils/test_resources.py,sha256=EKlzgwjFYkQ_CxbgIjhSuMzURKdxdBHLs9p1BuGzL1Q,1368
+deltacat/tests/utils/test_resources.py,sha256=8oCGNofatgWuif9-Mcis6fFcRCWXXtJnQ9Ff16Gp14g,1456
 deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/types/media.py,sha256=py1BnfMqNpJlW1RKzHWwB0NmQ33oCk9qg1fz7alvi3E,2187
 deltacat/types/tables.py,sha256=yUzkzmUij8kssEYI_dfVDSLXf8HfMm_jpgWkPxDHAas,3893
@@ -120,8 +132,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=AyL7hpvYjkmsz-KcpYjVgPpNsmu-x8-rl
 deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=xOVkqL6o8qGsewGvzhMKxmCcqcFZDnNILuz5IGMgxSc,4991
-deltacat-0.1.18b4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-0.1.18b4.dist-info/METADATA,sha256=RFZsZ8S4h3_eNjHv-sA4pxoHWdzFSTJYQ61MYbYarj0,1475
-deltacat-0.1.18b4.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-deltacat-0.1.18b4.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-0.1.18b4.dist-info/RECORD,,
+deltacat-0.1.18b7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-0.1.18b7.dist-info/METADATA,sha256=o8gkDXBrItOzyPPs72dvXTSL9MEwP5G8J5LCirD6XdY,1542
+deltacat-0.1.18b7.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+deltacat-0.1.18b7.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-0.1.18b7.dist-info/RECORD,,