deltacat 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

deltacat/tests/compute/compactor/utils/test_io.py

@@ -1,6 +1,6 @@
 import unittest
 from unittest import mock
-from deltacat.tests.test_utils.constants import TEST_DELTA
+from deltacat.tests.test_utils.constants import TEST_UPSERT_DELTA
 from typing import Any, Dict
 
 DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
@@ -41,7 +41,7 @@ class TestFitInputDeltas(unittest.TestCase):
             high_watermark,
             require_multiple_rounds,
         ) = io.fit_input_deltas(
-            [TEST_DELTA],
+            [TEST_UPSERT_DELTA],
             {"CPU": 1, "memory": 20000000},
             self.COMPACTION_AUDIT,
             None,
@@ -68,7 +68,7 @@ class TestFitInputDeltas(unittest.TestCase):
             high_watermark,
             require_multiple_rounds,
         ) = io.fit_input_deltas(
-            [TEST_DELTA],
+            [TEST_UPSERT_DELTA],
             {"CPU": 1, "memory": 20000000},
             self.COMPACTION_AUDIT,
             20,
@@ -91,7 +91,7 @@ class TestFitInputDeltas(unittest.TestCase):
             high_watermark,
             require_multiple_rounds,
         ) = io.fit_input_deltas(
-            [TEST_DELTA],
+            [TEST_UPSERT_DELTA],
             {"CPU": 2, "memory": 10},
             self.COMPACTION_AUDIT,
             20,

deltacat/tests/compute/test_compact_partition_incremental.py

@@ -100,6 +100,7 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "input_deltas",
         "input_deltas_delta_type",
         "expected_terminal_compact_partition_result",
+        "expected_terminal_exception",
         "create_placement_group_param",
         "records_per_compacted_file_param",
         "hash_bucket_count_param",
@@ -118,6 +119,7 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             input_deltas_param,
             input_deltas_delta_type,
             expected_terminal_compact_partition_result,
+            expected_terminal_exception,
             create_placement_group_param,
             records_per_compacted_file_param,
             hash_bucket_count_param,
@@ -134,6 +136,7 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             input_deltas_param,
             input_deltas_delta_type,
             expected_terminal_compact_partition_result,
+            expected_terminal_exception,
             create_placement_group_param,
             records_per_compacted_file_param,
             hash_bucket_count_param,
@@ -157,6 +160,7 @@ def test_compact_partition_incremental(
     input_deltas: pa.Table,
     input_deltas_delta_type: str,
     expected_terminal_compact_partition_result: pa.Table,
+    expected_terminal_exception: BaseException,
     create_placement_group_param: bool,
     records_per_compacted_file_param: int,
     hash_bucket_count_param: int,

deltacat/tests/compute/test_compact_partition_params.py

@@ -72,6 +72,7 @@ class TestCompactPartitionParams(unittest.TestCase):
                 "partitionValues": [],
                 "partitionId": "79612ea39ac5493eae925abe60767d42",
             },
+            "memory_logs_enabled": True,
             "metrics_config": MetricsConfig("us-east-1", MetricsTarget.CLOUDWATCH_EMF),
         }
 
@@ -135,6 +136,10 @@ class TestCompactPartitionParams(unittest.TestCase):
             json.loads(serialized_params)["destination_partition_locator"]
             == params.destination_partition_locator
         )
+        assert (
+            json.loads(serialized_params)["memory_logs_enabled"]
+            == params.memory_logs_enabled
+        )
         assert (
             json.loads(serialized_params)["metrics_config"]["metrics_target"]
             == params.metrics_config.metrics_target

deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -5,6 +5,7 @@ import pytest
 import boto3
 from boto3.resources.base import ServiceResource
 import pyarrow as pa
+from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
 from pytest_benchmark.fixture import BenchmarkFixture
 
 from deltacat.tests.compute.test_util_constant import (
@@ -15,6 +16,7 @@ from deltacat.tests.compute.test_util_constant import (
     DEFAULT_NUM_WORKERS,
     DEFAULT_WORKER_INSTANCE_CPUS,
 )
+from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 from deltacat.tests.compute.test_util_common import (
     get_rcf,
 )
@@ -28,11 +30,14 @@ from deltacat.tests.compute.test_util_create_table_deltas_repo import (
 from deltacat.tests.compute.test_util_create_table_deltas_repo import (
     create_src_w_deltas_destination_rebase_w_deltas_strategy,
 )
-from deltacat.tests.compute.compact_partition_test_cases import (
+from deltacat.tests.compute.compact_partition_rebase_then_incremental_test_cases import (
     REBASE_THEN_INCREMENTAL_TEST_CASES,
 )
-from typing import Any, Callable, Dict, List, Optional, Set
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 from deltacat.types.media import StorageType
+from deltacat.storage import (
+    DeltaType,
+)
 
 DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
     "db_file_path",
@@ -89,7 +94,7 @@ FUNCTION scoped fixtures
 
 
 @pytest.fixture(scope="function")
-def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
+def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
     # see deltacat/tests/local_deltacat_storage/README.md for documentation
     kwargs_for_local_deltacat_storage: Dict[str, Any] = {
         DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
@@ -109,6 +114,7 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "input_deltas_param",
         "input_deltas_delta_type",
         "expected_terminal_compact_partition_result",
+        "expected_terminal_exception",
         "create_placement_group_param",
         "records_per_compacted_file_param",
         "hash_bucket_count_param",
@@ -116,7 +122,6 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "drop_duplicates_param",
         "skip_enabled_compact_partition_drivers",
         "incremental_deltas",
-        "incremental_deltas_delta_type",
         "rebase_expected_compact_partition_result",
         "compact_partition_func",
     ],
@@ -130,6 +135,7 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             input_deltas,
             input_deltas_delta_type,
             expected_terminal_compact_partition_result,
+            expected_terminal_exception,
             create_placement_group_param,
             records_per_compacted_file_param,
             hash_bucket_count_param,
@@ -137,7 +143,6 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
             incremental_deltas,
-            incremental_deltas_delta_type,
             rebase_expected_compact_partition_result,
             compact_partition_func,
         )
@@ -149,6 +154,7 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             input_deltas,
             input_deltas_delta_type,
             expected_terminal_compact_partition_result,
+            expected_terminal_exception,
             create_placement_group_param,
             records_per_compacted_file_param,
             hash_bucket_count_param,
@@ -156,17 +162,15 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
             incremental_deltas,
-            incremental_deltas_delta_type,
             rebase_expected_compact_partition_result,
             compact_partition_func,
         ) in REBASE_THEN_INCREMENTAL_TEST_CASES.items()
     ],
     ids=[test_name for test_name in REBASE_THEN_INCREMENTAL_TEST_CASES],
-    indirect=[],
 )
 def test_compact_partition_rebase_then_incremental(
     setup_s3_resource: ServiceResource,
-    offer_local_deltacat_storage_kwargs: Dict[str, Any],
+    local_deltacat_storage_kwargs: Dict[str, Any],
     test_name: str,
     primary_keys: Set[str],
     sort_keys: List[Optional[Any]],
@@ -175,15 +179,15 @@ def test_compact_partition_rebase_then_incremental(
     input_deltas_param: List[pa.Array],
     input_deltas_delta_type: str,
     expected_terminal_compact_partition_result: pa.Table,
+    expected_terminal_exception: BaseException,
     create_placement_group_param: bool,
     records_per_compacted_file_param: int,
     hash_bucket_count_param: int,
     drop_duplicates_param: bool,
     read_kwargs_provider_param: Any,
-    incremental_deltas: pa.Table,
-    incremental_deltas_delta_type: str,
+    incremental_deltas: List[Tuple[pa.Table, DeltaType, Optional[Dict[str, str]]]],
     rebase_expected_compact_partition_result: pa.Table,
-    skip_enabled_compact_partition_drivers,
+    skip_enabled_compact_partition_drivers: List[CompactorVersion],
     compact_partition_func: Callable,
     benchmark: BenchmarkFixture,
 ):
@@ -204,7 +208,7 @@ def test_compact_partition_rebase_then_incremental(
         CompactionSessionAuditInfo,
     )
 
-    ds_mock_kwargs = offer_local_deltacat_storage_kwargs
+    ds_mock_kwargs = local_deltacat_storage_kwargs
     ray.shutdown()
     ray.init(local_mode=True, ignore_reinit_error=True)
     """
@@ -258,6 +262,7 @@ def test_compact_partition_rebase_then_incremental(
             "hash_bucket_count": hash_bucket_count_param,
             "last_stream_position_to_compact": source_partition.stream_position,
             "list_deltas_kwargs": {**ds_mock_kwargs, **{"equivalent_table_types": []}},
+            "object_store": RayPlasmaObjectStore(),
             "pg_config": pgm,
             "primary_keys": primary_keys,
             "read_kwargs_provider": read_kwargs_provider_param,
@@ -296,6 +301,8 @@ def test_compact_partition_rebase_then_incremental(
     (
         source_partition_locator_w_deltas,
         new_delta,
+        incremental_delta_length,
+        has_delete_deltas,
     ) = create_incremental_deltas_on_source_table(
         BASE_TEST_SOURCE_NAMESPACE,
         BASE_TEST_SOURCE_TABLE_NAME,
@@ -303,7 +310,6 @@ def test_compact_partition_rebase_then_incremental(
         source_table_stream,
         partition_values_param,
         incremental_deltas,
-        incremental_deltas_delta_type,
         ds_mock_kwargs,
     )
     compact_partition_params = CompactPartitionParams.of(
@@ -318,6 +324,7 @@ def test_compact_partition_rebase_then_incremental(
             "hash_bucket_count": hash_bucket_count_param,
             "last_stream_position_to_compact": new_delta.stream_position,
             "list_deltas_kwargs": {**ds_mock_kwargs, **{"equivalent_table_types": []}},
+            "object_store": RayPlasmaObjectStore(),
             "pg_config": pgm,
             "primary_keys": primary_keys,
             "read_kwargs_provider": read_kwargs_provider_param,
@@ -329,6 +336,10 @@ def test_compact_partition_rebase_then_incremental(
             "sort_keys": sort_keys if sort_keys else None,
         }
     )
+    if expected_terminal_exception:
+        with pytest.raises(expected_terminal_exception):
+            compact_partition_func(compact_partition_params)
+        return
     rcf_file_s3_uri = compact_partition_func(compact_partition_params)
     round_completion_info = get_rcf(setup_s3_resource, rcf_file_s3_uri)
     compacted_delta_locator_incremental: DeltaLocator = (
@@ -358,13 +369,14 @@ def test_compact_partition_rebase_then_incremental(
     actual_compacted_table = actual_compacted_table.combine_chunks().sort_by(
         sorting_cols
     )
-
-    assert compaction_audit.input_records == (
-        len(incremental_deltas) if incremental_deltas else 0
-    ) + len(actual_rebase_compacted_table), (
-        "Total input records must be equal to incremental deltas"
-        "+ previous compacted table size"
-    )
+    # NOTE: if delete type-deltas are present this relationship no longer holds true
+    if not has_delete_deltas:
+        assert compaction_audit.input_records == (
+            incremental_delta_length if incremental_deltas else 0
+        ) + len(actual_rebase_compacted_table), (
+            " Total input records must be equal to incremental deltas"
+            " + previous compacted table size"
+        )
 
     assert actual_compacted_table.equals(
         expected_terminal_compact_partition_result

deltacat/tests/compute/test_util_create_table_deltas_repo.py

@@ -27,23 +27,36 @@ def create_incremental_deltas_on_source_table(
     source_table_version: str,
     source_table_stream: Stream,
     partition_values_param,
-    incremental_deltas: pa.Table,
-    incremental_delta_type: DeltaType,
+    incremental_deltas: List[Tuple[pa.Table, DeltaType, Optional[Dict[str, str]]]],
     ds_mock_kwargs: Optional[Dict[str, Any]] = None,
-) -> Tuple[PartitionLocator, Delta]:
+) -> Tuple[PartitionLocator, Delta, int, bool]:
     import deltacat.tests.local_deltacat_storage as ds
 
+    incremental_delta_length = 0
+    is_delete = False
     src_partition: Partition = ds.get_partition(
         source_table_stream.locator,
         partition_values_param,
         **ds_mock_kwargs,
     )
-    new_delta: Delta = ds.commit_delta(
-        ds.stage_delta(
-            incremental_deltas, src_partition, incremental_delta_type, **ds_mock_kwargs
-        ),
-        **ds_mock_kwargs,
-    )
+    for (
+        incremental_data,
+        incremental_delta_type,
+        incremental_delete_parameters,
+    ) in incremental_deltas:
+        if incremental_delta_type is DeltaType.DELETE:
+            is_delete = True
+        incremental_delta: Delta = ds.commit_delta(
+            ds.stage_delta(
+                incremental_data,
+                src_partition,
+                incremental_delta_type,
+                delete_parameters=incremental_delete_parameters,
+                **ds_mock_kwargs,
+            ),
+            **ds_mock_kwargs,
+        )
+        incremental_delta_length += len(incremental_data) if incremental_data else 0
     src_table_stream_after_committed_delta: Stream = ds.get_stream(
         source_namespace,
         source_table_name,
@@ -55,7 +68,12 @@ def create_incremental_deltas_on_source_table(
         partition_values_param,
         **ds_mock_kwargs,
     )
-    return src_partition_after_committed_delta.locator, new_delta
+    return (
+        src_partition_after_committed_delta.locator,
+        incremental_delta,
+        incremental_delta_length,
+        is_delete,
+    )
 
 
 def create_src_w_deltas_destination_plus_destination(

deltacat/tests/io/test_memcached_object_store.py

@@ -25,6 +25,10 @@ class MockPyMemcacheClient:
     def get(self, key, *args, **kwargs):
         return self.store.get(key)
 
+    def flush_all(self, *args, **kwargs):
+        for key, value in self.store.items():
+            self.store[key] = None
+
 
 class TestMemcachedObjectStore(unittest.TestCase):
 
@@ -192,3 +196,18 @@ class TestMemcachedObjectStore(unittest.TestCase):
         # assert
         result = self.object_store.get(ref)
         self.assertEqual(result, self.TEST_VALUE_LARGE)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_clear_sanity(self, mock_retrying_client, mock_client):
+        # setup
+        mock_client.return_value = MockPyMemcacheClient()
+        mock_retrying_client.return_value = mock_client.return_value
+
+        # action
+        ref = self.object_store.put(self.TEST_VALUE_LARGE)
+        self.object_store.clear()
+
+        # assert
+        with self.assertRaises(ValueError):
+            self.object_store.get(ref)

deltacat/tests/local_deltacat_storage/__init__.py

@@ -39,6 +39,7 @@ from deltacat.storage import (
     ManifestMeta,
     ManifestEntry,
     ManifestEntryList,
+    DeleteParameters,
 )
 from deltacat.types.media import (
     ContentType,
@@ -892,6 +893,7 @@ def stage_delta(
     properties: Optional[Dict[str, str]] = None,
     s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
+    delete_parameters: Optional[DeleteParameters] = None,
     *args,
     **kwargs,
 ) -> Delta:
@@ -958,6 +960,7 @@ def stage_delta(
         properties=properties,
         manifest=manifest,
         previous_stream_position=partition.stream_position,
+        delete_parameters=delete_parameters,
     )
 
     params = (uri, serialized_data)

deltacat/tests/test_utils/constants.py

@@ -3,5 +3,4 @@ from deltacat.storage import Delta
 
 test_delta_file = open("deltacat/tests/test_utils/resources/test_delta.json")
 test_delta_dict = json.load(test_delta_file)
-
-TEST_DELTA = Delta(test_delta_dict)
+TEST_UPSERT_DELTA = Delta(test_delta_dict)

deltacat/tests/test_utils/pyarrow.py

@@ -1,24 +1,29 @@
-from typing import List, Optional
+from typing import List, Optional, Union
 import pyarrow as pa
-from deltacat.storage import Delta, Partition, PartitionLocator
+from deltacat.storage import Delta, Partition, PartitionLocator, DeltaLocator
 import deltacat.tests.local_deltacat_storage as ds
+from deltacat.types.media import StorageType
 
 
 def create_delta_from_csv_file(
     namespace: str,
     file_paths: List[str],
     table_name: Optional[str] = None,
+    table_version: int = 1,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Delta:
     staged_partition = stage_partition_from_file_paths(
-        namespace, file_paths, *args, table_name=table_name, **kwargs
+        namespace,
+        file_paths,
+        *args,
+        table_name=table_name,
+        table_version=table_version,
+        **kwargs,
     )
-
     committed_delta = commit_delta_to_staged_partition(
         staged_partition, file_paths, *args, **kwargs
     )
-
     return committed_delta
 
 
@@ -26,14 +31,15 @@ def stage_partition_from_file_paths(
     namespace: str,
     file_paths: List[str],
     table_name: Optional[str] = None,
+    table_version: int = 1,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Partition:
     ds.create_namespace(namespace, {}, **kwargs)
     if table_name is None:
         table_name = "-".join(file_paths).replace("/", "_")
-    ds.create_table_version(namespace, table_name, "1", **kwargs)
-    stream = ds.get_stream(namespace, table_name, "1", **kwargs)
+    ds.create_table_version(namespace, table_name, str(table_version), **kwargs)
+    stream = ds.get_stream(namespace, table_name, str(table_version), **kwargs)
     staged_partition = ds.stage_partition(stream, [], **kwargs)
     return staged_partition
 
@@ -42,12 +48,23 @@ def commit_delta_to_staged_partition(
     staged_partition, file_paths: List[str], *args, **kwargs
 ) -> Delta:
     committed_delta = commit_delta_to_partition(
-        staged_partition, file_paths=file_paths, *args, **kwargs
+        staged_partition, *args, file_paths=file_paths, **kwargs
     )
     ds.commit_partition(staged_partition, **kwargs)
     return committed_delta
 
 
+def download_delta(delta_like: Union[Delta, DeltaLocator], *args, **kwargs) -> Delta:
+    return pa.concat_tables(
+        ds.download_delta(
+            delta_like,
+            storage_type=StorageType.LOCAL,
+            *args,
+            **kwargs,
+        )
+    )
+
+
 def commit_delta_to_partition(
     partition: Partition, file_paths: List[str], *args, **kwargs
 ) -> Delta:

deltacat/utils/pandas.py

@@ -126,7 +126,7 @@ def _add_column_kwargs(
         kwargs["usecols"] = include_columns
     else:
         if content_type in TABULAR_CONTENT_TYPES:
-            kwargs["columns"]: include_columns
+            kwargs["columns"] = include_columns
         else:
             if include_columns:
                 logger.warning(

deltacat/utils/ray_utils/runtime.py

@@ -37,7 +37,7 @@ def current_node_resource_key() -> str:
     actors on that node via:
     `foo.options(resources={get_current_node_resource_key(): 0.01}).remote()`
     """
-    current_node_id = ray.get_runtime_context().node_id.hex()
+    current_node_id = ray.get_runtime_context().get_node_id().hex()
     keys = node_resource_keys(lambda n: n["NodeID"] == current_node_id)
     assert (
         len(keys) <= 1
@@ -83,7 +83,7 @@ def other_live_node_resource_keys() -> List[str]:
 
     For example, invoking this function from your Ray application driver on the
     head node returns the resource keys of all live worker nodes."""
-    current_node_id = ray.get_runtime_context().node_id.hex()
+    current_node_id = ray.get_runtime_context().get_node_id().hex()
     return node_resource_keys(
         lambda n: n["NodeID"] != current_node_id and is_node_alive(n)
     )
@@ -97,7 +97,7 @@ def other_node_resource_keys() -> List[str]:
 
     For example, invoking this function from your Ray application driver on the
     head node returns the resource keys of all worker nodes."""
-    current_node_id = ray.get_runtime_context().node_id.hex()
+    current_node_id = ray.get_runtime_context().get_node_id().hex()
     return node_resource_keys(lambda n: n["NodeID"] != current_node_id)
 
 

deltacat/utils/resources.py

@@ -36,13 +36,15 @@ class ClusterUtilization:
                 used_resources[key] = cluster_resources[key] - available_resources[key]
 
         self.total_memory_bytes = cluster_resources.get("memory")
-        self.used_memory_bytes = used_resources.get("memory")
+        self.used_memory_bytes = used_resources.get("memory", 0.0)
         self.total_cpu = cluster_resources.get("CPU")
-        self.used_cpu = used_resources.get("CPU")
+        self.used_cpu = used_resources.get("CPU", 0)
         self.total_object_store_memory_bytes = cluster_resources.get(
             "object_store_memory"
         )
-        self.used_object_store_memory_bytes = used_resources.get("object_store_memory")
+        self.used_object_store_memory_bytes = used_resources.get(
+            "object_store_memory", 0.0
+        )
         self.used_memory_percent = (
             self.used_memory_bytes / self.total_memory_bytes
         ) * 100
@@ -92,7 +94,7 @@ class ClusterUtilizationOverTimeRange(AbstractContextManager):
     ) -> bool | None:
         if __exc_value:
             logger.error(
-                f"Error ocurred while calculating cluster resources: {__exc_value}"
+                f"Error occurred while calculating cluster resources: {__exc_value}"
             )
         self.stop_run_schedules.set()
         return super().__exit__(__exc_type, __exc_value, __traceback)
@@ -202,7 +204,7 @@ class ProcessUtilizationOverTimeRange(AbstractContextManager):
     ) -> bool | None:
         if __exc_value:
             logger.error(
-                f"Error ocurred while calculating process resources: {__exc_value}"
+                f"Error occurred while calculating process resources: {__exc_value}"
             )
         self.stop_run_schedules.set()
         return super().__exit__(__exc_type, __exc_value, __traceback)

{deltacat-1.0.2.dist-info → deltacat-1.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.0.2
+Version: 1.1.1
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team