deltacat 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.0"
+__version__ = "1.1.1"
 
 
 __all__ = [

deltacat/compute/compactor/model/compact_partition_params.py

@@ -20,6 +20,7 @@ from deltacat.compute.compactor_v2.constants import (
     AVERAGE_RECORD_SIZE_BYTES,
     TASK_MAX_PARALLELISM,
     DROP_DUPLICATES,
+    TOTAL_MEMORY_BUFFER_PERCENTAGE,
 )
 from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -85,12 +86,17 @@ class CompactPartitionParams(dict):
         result.average_record_size_bytes = params.get(
             "average_record_size_bytes", AVERAGE_RECORD_SIZE_BYTES
         )
+        result.total_memory_buffer_percentage = params.get(
+            "total_memory_buffer_percentage", TOTAL_MEMORY_BUFFER_PERCENTAGE
+        )
         result.hash_group_count = params.get(
             "hash_group_count", result.hash_bucket_count
         )
         result.drop_duplicates = params.get("drop_duplicates", DROP_DUPLICATES)
         result.ray_custom_resources = params.get("ray_custom_resources")
 
+        result.memory_logs_enabled = params.get("memory_logs_enabled", False)
+
         result.metrics_config = params.get("metrics_config")
 
         if not importlib.util.find_spec("memray"):
@@ -190,6 +196,16 @@ class CompactPartitionParams(dict):
     def average_record_size_bytes(self, average_record_size_bytes: float) -> None:
         self["average_record_size_bytes"] = average_record_size_bytes
 
+    @property
+    def total_memory_buffer_percentage(self) -> int:
+        return self["total_memory_buffer_percentage"]
+
+    @total_memory_buffer_percentage.setter
+    def total_memory_buffer_percentage(
+        self, total_memory_buffer_percentage: int
+    ) -> None:
+        self["total_memory_buffer_percentage"] = total_memory_buffer_percentage
+
     @property
     def min_files_in_batch(self) -> float:
         return self["min_files_in_batch"]
@@ -355,6 +371,14 @@ class CompactPartitionParams(dict):
     def sort_keys(self, keys: List[SortKey]) -> None:
         self["sort_keys"] = keys
 
+    @property
+    def memory_logs_enabled(self) -> bool:
+        return self.get("memory_logs_enabled")
+
+    @memory_logs_enabled.setter
+    def memory_logs_enabled(self, value: bool) -> None:
+        self["memory_logs_enabled"] = value
+
     @property
     def metrics_config(self) -> Optional[MetricsConfig]:
         return self.get("metrics_config")

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -84,6 +84,13 @@ class CompactionSessionAuditInfo(dict):
         """
         return self.get("recordsDeduped")
 
+    @property
+    def records_deleted(self) -> int:
+        """
+        The total count of deleted records in a compaction session if delete deltas are present.
+        """
+        return self.get("recordsDeleted")
+
     @property
     def input_size_bytes(self) -> float:
         """
@@ -461,6 +468,10 @@ class CompactionSessionAuditInfo(dict):
         self["recordsDeduped"] = records_deduped
         return self
 
+    def set_records_deleted(self, records_deleted: int) -> CompactionSessionAuditInfo:
+        self["recordsDeleted"] = records_deleted
+        return self
+
     def set_input_size_bytes(
         self, input_size_bytes: float
     ) -> CompactionSessionAuditInfo:

deltacat/compute/compactor_v2/compaction_session.py

@@ -62,6 +62,7 @@ from deltacat.utils.resources import (
 from deltacat.compute.compactor_v2.utils.task_options import (
     hash_bucket_resource_options_provider,
     merge_resource_options_provider,
+    local_merge_resource_options_provider,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 
@@ -258,8 +259,10 @@ def _execute_compaction(
         resource_amount_provider=hash_bucket_resource_options_provider,
         previous_inflation=params.previous_inflation,
         average_record_size_bytes=params.average_record_size_bytes,
+        total_memory_buffer_percentage=params.total_memory_buffer_percentage,
         primary_keys=params.primary_keys,
         ray_custom_resources=params.ray_custom_resources,
+        memory_logs_enabled=params.memory_logs_enabled,
     )
 
     total_input_records_count = np.int64(0)
@@ -275,7 +278,29 @@ def _execute_compaction(
             delete_strategy,
             delete_file_envelopes,
         )
-        local_merge_result = ray.get(mg.merge.remote(local_merge_input))
+        estimated_da_bytes = (
+            compaction_audit.estimated_in_memory_size_bytes_during_discovery
+        )
+        estimated_num_records = sum(
+            [
+                entry.meta.record_count
+                for delta in uniform_deltas
+                for entry in delta.manifest.entries
+            ]
+        )
+        local_merge_options = local_merge_resource_options_provider(
+            estimated_da_size=estimated_da_bytes,
+            estimated_num_rows=estimated_num_records,
+            total_memory_buffer_percentage=params.total_memory_buffer_percentage,
+            round_completion_info=round_completion_info,
+            compacted_delta_manifest=previous_compacted_delta_manifest,
+            ray_custom_resources=params.ray_custom_resources,
+            primary_keys=params.primary_keys,
+            memory_logs_enabled=params.memory_logs_enabled,
+        )
+        local_merge_result = ray.get(
+            mg.merge.options(**local_merge_options).remote(local_merge_input)
+        )
         total_input_records_count += local_merge_result.input_record_count
         merge_results = [local_merge_result]
         merge_invoke_end = time.monotonic()
@@ -296,6 +321,7 @@ def _execute_compaction(
                     object_store=params.object_store,
                     deltacat_storage=params.deltacat_storage,
                     deltacat_storage_kwargs=params.deltacat_storage_kwargs,
+                    memory_logs_enabled=params.memory_logs_enabled,
                 )
             }
 
@@ -382,12 +408,14 @@ def _execute_compaction(
             num_hash_groups=params.hash_group_count,
             hash_group_size_bytes=all_hash_group_idx_to_size_bytes,
             hash_group_num_rows=all_hash_group_idx_to_num_rows,
+            total_memory_buffer_percentage=params.total_memory_buffer_percentage,
             round_completion_info=round_completion_info,
             compacted_delta_manifest=previous_compacted_delta_manifest,
             primary_keys=params.primary_keys,
             deltacat_storage=params.deltacat_storage,
             deltacat_storage_kwargs=params.deltacat_storage_kwargs,
             ray_custom_resources=params.ray_custom_resources,
+            memory_logs_enabled=params.memory_logs_enabled,
         )
 
         def merge_input_provider(index, item):
@@ -417,6 +445,7 @@ def _execute_compaction(
                     deltacat_storage_kwargs=params.deltacat_storage_kwargs,
                     delete_strategy=delete_strategy,
                     delete_file_envelopes=delete_file_envelopes,
+                    memory_logs_enabled=params.memory_logs_enabled,
                 )
             }
 
@@ -438,11 +467,11 @@ def _execute_compaction(
     merge_end = time.monotonic()
 
     total_dd_record_count = sum([ddr.deduped_record_count for ddr in merge_results])
-    total_dropped_record_count = sum(
+    total_deleted_record_count = sum(
         [ddr.deleted_record_count for ddr in merge_results]
     )
     logger.info(
-        f"Deduped {total_dd_record_count} records and dropped {total_dropped_record_count} records..."
+        f"Deduped {total_dd_record_count} records and deleted {total_deleted_record_count} records..."
     )
 
     compaction_audit.set_input_records(total_input_records_count.item())
@@ -456,7 +485,7 @@ def _execute_compaction(
     )
 
     compaction_audit.set_records_deduped(total_dd_record_count.item())
-
+    compaction_audit.set_records_deleted(total_deleted_record_count.item())
     mat_results = []
     for merge_result in merge_results:
         mat_results.extend(merge_result.materialize_results)
@@ -503,7 +532,7 @@ def _execute_compaction(
     record_info_msg = (
         f"Hash bucket records: {total_hb_record_count},"
         f" Deduped records: {total_dd_record_count}, "
-        f" Dropped records: {total_dropped_record_count}, "
+        f" Deleted records: {total_deleted_record_count}, "
         f" Materialized records: {merged_delta.meta.record_count}"
     )
     logger.info(record_info_msg)
@@ -603,7 +632,6 @@ def _execute_compaction(
         f"partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
-
     return (
         compacted_partition,
         new_round_completion_info,

deltacat/compute/compactor_v2/model/hash_bucket_input.py

@@ -22,6 +22,7 @@ class HashBucketInput(Dict):
         object_store: Optional[IObjectStore] = None,
         deltacat_storage=unimplemented_deltacat_storage,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
+        memory_logs_enabled: Optional[bool] = None,
     ) -> HashBucketInput:
 
         result = HashBucketInput()
@@ -36,6 +37,7 @@ class HashBucketInput(Dict):
         result["object_store"] = object_store
         result["deltacat_storage"] = deltacat_storage
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
+        result["memory_logs_enabled"] = memory_logs_enabled
 
         return result
 
@@ -82,3 +84,7 @@ class HashBucketInput(Dict):
     @property
     def deltacat_storage_kwargs(self) -> Optional[Dict[str, Any]]:
         return self.get("deltacat_storage_kwargs")
+
+    @property
+    def memory_logs_enabled(self) -> Optional[bool]:
+        return self.get("memory_logs_enabled")

deltacat/compute/compactor_v2/model/merge_input.py

@@ -46,6 +46,7 @@ class MergeInput(Dict):
         delete_file_envelopes: Optional[List] = None,
         deltacat_storage=unimplemented_deltacat_storage,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
+        memory_logs_enabled: Optional[bool] = None,
     ) -> MergeInput:
 
         result = MergeInput()
@@ -67,6 +68,7 @@ class MergeInput(Dict):
         result["delete_strategy"] = delete_strategy
         result["deltacat_storage"] = deltacat_storage
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
+        result["memory_logs_enabled"] = memory_logs_enabled
         return result
 
     @property
@@ -133,6 +135,10 @@ class MergeInput(Dict):
     def deltacat_storage_kwargs(self) -> Optional[Dict[str, Any]]:
         return self.get("deltacat_storage_kwargs")
 
+    @property
+    def memory_logs_enabled(self) -> Optional[bool]:
+        return self.get("memory_logs_enabled")
+
     @property
     def delete_file_envelopes(
         self,

deltacat/compute/compactor_v2/steps/hash_bucket.py

@@ -142,7 +142,8 @@ def hash_bucket(input: HashBucketInput) -> HashBucketResult:
                 f"({process_util.max_memory/BYTES_PER_GIBIBYTE} GB)"
             )
 
-        process_util.schedule_callback(log_peak_memory, 10)
+        if input.memory_logs_enabled:
+            process_util.schedule_callback(log_peak_memory, 10)
 
         hash_bucket_result, duration = timed_invocation(
             func=_timed_hash_bucket, input=input

deltacat/compute/compactor_v2/steps/merge.py

@@ -12,6 +12,7 @@ from uuid import uuid4
 from deltacat import logs
 from typing import Callable, Iterator, List, Optional, Tuple
 from deltacat.compute.compactor_v2.model.merge_result import MergeResult
+from deltacat.compute.compactor_v2.model.merge_file_group import MergeFileGroup
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
 from deltacat.compute.compactor.model.pyarrow_write_result import PyArrowWriteResult
 from deltacat.compute.compactor import RoundCompletionInfo, DeltaFileEnvelope
@@ -269,6 +270,24 @@ def _has_previous_compacted_table(input: MergeInput, hb_idx: int) -> bool:
     )
 
 
+def _can_copy_by_reference(
+    has_delete: bool, merge_file_group: MergeFileGroup, input: MergeInput
+) -> bool:
+    """
+    Can copy by reference only if there are no deletes to merge in
+    and previous compacted stream id matches that of new stream
+    """
+    return (
+        not has_delete
+        and not merge_file_group.dfe_groups
+        and input.round_completion_info is not None
+        and (
+            input.write_to_partition.stream_id
+            == input.round_completion_info.compacted_delta_locator.stream_id
+        )
+    )
+
+
 def _flatten_dfe_list(
     df_envelopes_list: List[List[DeltaFileEnvelope]],
 ) -> List[DeltaFileEnvelope]:
@@ -349,7 +368,7 @@ def _compact_tables(
             1. The compacted PyArrow table.
             2. The total number of records in the incremental data.
             3. The total number of deduplicated records.
-            4. The total number of dropped records due to DELETE operations.
+            4. The total number of deleted records due to DELETE operations.
     """
     df_envelopes: List[DeltaFileEnvelope] = _flatten_dfe_list(dfe_list)
     delete_file_envelopes = input.delete_file_envelopes or []
@@ -479,10 +498,12 @@ def _timed_merge(input: MergeInput) -> MergeResult:
                 assert (
                     input.delete_strategy is not None
                 ), "Merge input missing delete_strategy"
-            if not has_delete and not merge_file_group.dfe_groups:
-                # Can copy by reference only if there are no deletes to merge in
+            if _can_copy_by_reference(
+                has_delete=has_delete, merge_file_group=merge_file_group, input=input
+            ):
                 hb_index_copy_by_ref_ids.append(merge_file_group.hb_index)
                 continue
+
             if _has_previous_compacted_table(input, merge_file_group.hb_index):
                 compacted_table = _download_compacted_table(
                     hb_index=merge_file_group.hb_index,
@@ -548,7 +569,8 @@ def merge(input: MergeInput) -> MergeResult:
                 f"({process_util.max_memory/BYTES_PER_GIBIBYTE} GB)"
             )
 
-        process_util.schedule_callback(log_peak_memory, 10)
+        if input.memory_logs_enabled:
+            process_util.schedule_callback(log_peak_memory, 10)
 
         merge_result, duration = timed_invocation(func=_timed_merge, input=input)
 

deltacat/compute/compactor_v2/utils/task_options.py

@@ -1,7 +1,10 @@
 import botocore
 import logging
-from typing import Dict, Optional, List, Tuple
+from typing import Dict, Optional, List, Tuple, Any
 from deltacat import logs
+from deltacat.compute.compactor_v2.model.merge_file_group import (
+    LocalMergeFileGroupsProvider,
+)
 from deltacat.types.media import ContentEncoding, ContentType
 from deltacat.types.partial_download import PartialParquetParameters
 from deltacat.storage import (
@@ -15,7 +18,6 @@ from deltacat.compute.compactor_v2.utils.primary_key_index import (
     hash_group_index_to_hash_bucket_indices,
 )
 from deltacat.compute.compactor_v2.constants import (
-    TOTAL_MEMORY_BUFFER_PERCENTAGE,
     PARQUET_TO_PYARROW_INFLATION,
 )
 
@@ -133,8 +135,10 @@ def hash_bucket_resource_options_provider(
     item: DeltaAnnotated,
     previous_inflation: float,
     average_record_size_bytes: float,
+    total_memory_buffer_percentage: int,
     primary_keys: List[str] = None,
     ray_custom_resources: Optional[Dict] = None,
+    memory_logs_enabled: Optional[bool] = None,
     **kwargs,
 ) -> Dict:
     debug_memory_params = {"hash_bucket_task_index": index}
@@ -189,10 +193,11 @@ def hash_bucket_resource_options_provider(
     debug_memory_params["average_record_size_bytes"] = average_record_size_bytes
 
     # Consider buffer
-    total_memory = total_memory * (1 + TOTAL_MEMORY_BUFFER_PERCENTAGE / 100.0)
+    total_memory = total_memory * (1 + total_memory_buffer_percentage / 100.0)
     debug_memory_params["total_memory_with_buffer"] = total_memory
-    logger.debug(
-        f"[Hash bucket task {index}]: Params used for calculating hash bucketing memory: {debug_memory_params}"
+    logger.debug_conditional(
+        f"[Hash bucket task {index}]: Params used for calculating hash bucketing memory: {debug_memory_params}",
+        memory_logs_enabled,
     )
 
     return get_task_options(0.01, total_memory, ray_custom_resources)
@@ -204,12 +209,14 @@ def merge_resource_options_provider(
     num_hash_groups: int,
     hash_group_size_bytes: Dict[int, int],
     hash_group_num_rows: Dict[int, int],
+    total_memory_buffer_percentage: int,
     round_completion_info: Optional[RoundCompletionInfo] = None,
     compacted_delta_manifest: Optional[Manifest] = None,
     ray_custom_resources: Optional[Dict] = None,
     primary_keys: Optional[List[str]] = None,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict] = {},
+    memory_logs_enabled: Optional[bool] = None,
     **kwargs,
 ) -> Dict:
     debug_memory_params = {"merge_task_index": index}
@@ -224,6 +231,84 @@ def merge_resource_options_provider(
     pk_size_bytes = data_size
     incremental_index_array_size = num_rows * 4
 
+    return get_merge_task_options(
+        index,
+        hb_group_idx,
+        data_size,
+        pk_size_bytes,
+        num_rows,
+        num_hash_groups,
+        total_memory_buffer_percentage,
+        incremental_index_array_size,
+        debug_memory_params,
+        ray_custom_resources,
+        round_completion_info=round_completion_info,
+        compacted_delta_manifest=compacted_delta_manifest,
+        primary_keys=primary_keys,
+        deltacat_storage=deltacat_storage,
+        deltacat_storage_kwargs=deltacat_storage_kwargs,
+        memory_logs_enabled=memory_logs_enabled,
+    )
+
+
+def local_merge_resource_options_provider(
+    estimated_da_size: float,
+    estimated_num_rows: int,
+    total_memory_buffer_percentage: int,
+    round_completion_info: Optional[RoundCompletionInfo] = None,
+    compacted_delta_manifest: Optional[Manifest] = None,
+    ray_custom_resources: Optional[Dict] = None,
+    primary_keys: Optional[List[str]] = None,
+    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage_kwargs: Optional[Dict] = {},
+    memory_logs_enabled: Optional[bool] = None,
+    **kwargs,
+) -> Dict:
+    index = hb_group_idx = LocalMergeFileGroupsProvider.LOCAL_HASH_BUCKET_INDEX
+    debug_memory_params = {"merge_task_index": index}
+
+    # upper bound for pk size of incremental
+    pk_size_bytes = estimated_da_size
+    incremental_index_array_size = estimated_num_rows * 4
+
+    return get_merge_task_options(
+        index=index,
+        hb_group_idx=hb_group_idx,
+        data_size=estimated_da_size,
+        pk_size_bytes=pk_size_bytes,
+        num_rows=estimated_num_rows,
+        num_hash_groups=1,
+        incremental_index_array_size=incremental_index_array_size,
+        total_memory_buffer_percentage=total_memory_buffer_percentage,
+        debug_memory_params=debug_memory_params,
+        ray_custom_resources=ray_custom_resources,
+        round_completion_info=round_completion_info,
+        compacted_delta_manifest=compacted_delta_manifest,
+        primary_keys=primary_keys,
+        deltacat_storage=deltacat_storage,
+        deltacat_storage_kwargs=deltacat_storage_kwargs,
+        memory_logs_enabled=memory_logs_enabled,
+    )
+
+
+def get_merge_task_options(
+    index: int,
+    hb_group_idx: int,
+    data_size: float,
+    pk_size_bytes: float,
+    num_rows: int,
+    num_hash_groups: int,
+    total_memory_buffer_percentage: int,
+    incremental_index_array_size: int,
+    debug_memory_params: Dict[str, Any],
+    ray_custom_resources: Optional[Dict],
+    round_completion_info: Optional[RoundCompletionInfo] = None,
+    compacted_delta_manifest: Optional[Manifest] = None,
+    primary_keys: Optional[List[str]] = None,
+    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage_kwargs: Optional[Dict] = {},
+    memory_logs_enabled: Optional[bool] = None,
+) -> Dict[str, Any]:
     if (
         round_completion_info
         and compacted_delta_manifest
@@ -296,10 +381,11 @@ def merge_resource_options_provider(
     debug_memory_params["incremental_index_array_size"] = incremental_index_array_size
     debug_memory_params["total_memory"] = total_memory
 
-    total_memory = total_memory * (1 + TOTAL_MEMORY_BUFFER_PERCENTAGE / 100.0)
+    total_memory = total_memory * (1 + total_memory_buffer_percentage / 100.0)
     debug_memory_params["total_memory_with_buffer"] = total_memory
-    logger.debug(
-        f"[Merge task {index}]: Params used for calculating merge memory: {debug_memory_params}"
+    logger.debug_conditional(
+        f"[Merge task {index}]: Params used for calculating merge memory: {debug_memory_params}",
+        memory_logs_enabled,
     )
 
     return get_task_options(0.01, total_memory, ray_custom_resources)

deltacat/io/memcached_object_store.py

@@ -181,15 +181,35 @@ class MemcachedObjectStore(IObjectStore):
         for chunk_index in range(chunk_count):
             ref = self._create_ref(uid, ip, chunk_index)
             chunk = client.get(ref)
+            if chunk is None:
+                raise ValueError(
+                    f"Expected uid: {uid}, chunk index: {chunk_index} from client ip: {ip}"
+                    f" to be non-empty."
+                )
             serialized.extend(chunk)
 
         return cloudpickle.loads(serialized)
 
+    def clear(self) -> bool:
+        flushed = all(
+            [
+                self._get_client_by_ip(ip).flush_all(noreply=False)
+                for ip in self.storage_node_ips
+            ]
+        )
+        self.client_cache.clear()
+
+        if flushed:
+            logger.info("Successfully cleared cache contents.")
+
+        return flushed
+
     def close(self) -> None:
         for client in self.client_cache.values():
             client.close()
 
         self.client_cache.clear()
+        logger.info("Successfully closed object store clients.")
 
     def _create_ref(self, uid, ip, chunk_index) -> str:
         return f"{uid}{self.SEPARATOR}{ip}{self.SEPARATOR}{chunk_index}"

deltacat/logs.py

@@ -2,7 +2,7 @@ import logging
 import os
 import pathlib
 from logging import FileHandler, Handler, Logger, LoggerAdapter, handlers
-from typing import Union
+from typing import Any, Dict, Optional, Union
 
 import ray
 from ray.runtime_context import RuntimeContext
@@ -26,7 +26,32 @@ DEFAULT_MAX_BYTES_PER_LOG = 2 ^ 20 * 256  # 256 MiB
 DEFAULT_BACKUP_COUNT = 0
 
 
-class RayRuntimeContextLoggerAdapter(logging.LoggerAdapter):
+class DeltaCATLoggerAdapter(logging.LoggerAdapter):
+    """
+    Logger Adapter class with additional functionality
+    """
+
+    def __init__(self, logger: Logger, extra: Optional[Dict[str, Any]] = {}):
+        super().__init__(logger, extra)
+
+    def debug_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.debug(msg, *args, **kwargs)
+
+    def info_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.info(msg, *args, **kwargs)
+
+    def warning_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.warning(msg, *args, **kwargs)
+
+    def error_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.error(msg, *args, **kwargs)
+
+
+class RayRuntimeContextLoggerAdapter(DeltaCATLoggerAdapter):
     """
     Logger Adapter for injecting Ray Runtime Context into logging messages.
     """
@@ -147,6 +172,8 @@ def _configure_logger(
         ray_runtime_ctx = ray.get_runtime_context()
         if ray_runtime_ctx.worker.connected:
             logger = RayRuntimeContextLoggerAdapter(logger, ray_runtime_ctx)
+    else:
+        logger = DeltaCATLoggerAdapter(logger)
 
     return logger
 

deltacat/tests/compute/test_compact_partition_params.py

@@ -72,6 +72,7 @@ class TestCompactPartitionParams(unittest.TestCase):
                 "partitionValues": [],
                 "partitionId": "79612ea39ac5493eae925abe60767d42",
             },
+            "memory_logs_enabled": True,
             "metrics_config": MetricsConfig("us-east-1", MetricsTarget.CLOUDWATCH_EMF),
         }
 
@@ -135,6 +136,10 @@ class TestCompactPartitionParams(unittest.TestCase):
             json.loads(serialized_params)["destination_partition_locator"]
             == params.destination_partition_locator
         )
+        assert (
+            json.loads(serialized_params)["memory_logs_enabled"]
+            == params.memory_logs_enabled
+        )
         assert (
             json.loads(serialized_params)["metrics_config"]["metrics_target"]
             == params.metrics_config.metrics_target

deltacat/tests/io/test_memcached_object_store.py

@@ -25,6 +25,10 @@ class MockPyMemcacheClient:
     def get(self, key, *args, **kwargs):
         return self.store.get(key)
 
+    def flush_all(self, *args, **kwargs):
+        for key, value in self.store.items():
+            self.store[key] = None
+
 
 class TestMemcachedObjectStore(unittest.TestCase):
 
@@ -192,3 +196,18 @@ class TestMemcachedObjectStore(unittest.TestCase):
         # assert
         result = self.object_store.get(ref)
         self.assertEqual(result, self.TEST_VALUE_LARGE)
+
+    @mock.patch("deltacat.io.memcached_object_store.Client")
+    @mock.patch("deltacat.io.memcached_object_store.RetryingClient")
+    def test_clear_sanity(self, mock_retrying_client, mock_client):
+        # setup
+        mock_client.return_value = MockPyMemcacheClient()
+        mock_retrying_client.return_value = mock_client.return_value
+
+        # action
+        ref = self.object_store.put(self.TEST_VALUE_LARGE)
+        self.object_store.clear()
+
+        # assert
+        with self.assertRaises(ValueError):
+            self.object_store.get(ref)

deltacat/utils/resources.py

@@ -36,13 +36,15 @@ class ClusterUtilization:
                 used_resources[key] = cluster_resources[key] - available_resources[key]
 
         self.total_memory_bytes = cluster_resources.get("memory")
-        self.used_memory_bytes = used_resources.get("memory")
+        self.used_memory_bytes = used_resources.get("memory", 0.0)
         self.total_cpu = cluster_resources.get("CPU")
-        self.used_cpu = used_resources.get("CPU")
+        self.used_cpu = used_resources.get("CPU", 0)
         self.total_object_store_memory_bytes = cluster_resources.get(
             "object_store_memory"
         )
-        self.used_object_store_memory_bytes = used_resources.get("object_store_memory")
+        self.used_object_store_memory_bytes = used_resources.get(
+            "object_store_memory", 0.0
+        )
         self.used_memory_percent = (
             self.used_memory_bytes / self.total_memory_bytes
         ) * 100

{deltacat-1.1.0.dist-info → deltacat-1.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.0
+Version: 1.1.1
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.0.dist-info → deltacat-1.1.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
-deltacat/__init__.py,sha256=jgsxhhooY-sexT3kAhQirASpDPtehwb4RDqDt_3rcaQ,1777
+deltacat/__init__.py,sha256=PeDnE7G_w2Gwkfg1VVp1mGIZm0_-WoyYbu7SlmoCFHs,1777
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=xqZf8CwysNYP2d39pf27OnXGStPREgBgIM-e2Tts-TI,199
-deltacat/logs.py,sha256=9XWuTBoWhhAF9rAL6t9veXmnAlJHsaqk0lTxteVPqyQ,5674
+deltacat/logs.py,sha256=YT26oj-oKZReyBEhiBQU5jc246aSu_d4B2bZcUVBHmE,6550
 deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/clients.py,sha256=WKgbgaq8CacH81nvvcqrjdshGtYvQicvBeOLg1m7wCA,6501
 deltacat/aws/constants.py,sha256=luXWMO_8eatq8f9NlFjNM7q362j77JwzTM2BEVS_8-8,353
@@ -24,8 +24,8 @@ deltacat/compute/compactor/__init__.py,sha256=ivpOPve1yKi3Vz3tVgp-eeFMNEeUSf-dlR
 deltacat/compute/compactor/compaction_session.py,sha256=bJpNBSTW7Raoa1gpojDpmVVqQGpvX0AwrusHQhUANcI,27612
 deltacat/compute/compactor/repartition_session.py,sha256=f5BTTGNv365qSuTioL7QUuVm-px_l8-zz-OC_p7gXt4,7240
 deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor/model/compact_partition_params.py,sha256=DdM6YsA5CSAMAwqQ5yscxR-NWadnItpgX18JikdJoQ4,14344
-deltacat/compute/compactor/model/compaction_session_audit_info.py,sha256=o8O0v3nOc7m1ZR4W0wQkTdsMyFL24LoMc9kzUo8i5uc,30174
+deltacat/compute/compactor/model/compact_partition_params.py,sha256=I5DHVv4a_Pb3hlfWf5pyUeKcxEtvMu8LLEWFRH9qdfE,15195
+deltacat/compute/compactor/model/compaction_session_audit_info.py,sha256=Aoae5KNmbdYt48utFQR_zrjHTkC9JNd4jPhQmyJji-A,30547
 deltacat/compute/compactor/model/compactor_version.py,sha256=RwRvManiCxZmzjAWzm1OPDxjB1BEHu1d0fBJyGhXKxA,87
 deltacat/compute/compactor/model/dedupe_result.py,sha256=1OCV944qJdLQ_-8scisVKl45ej1eRv9OV539QYZtQ-U,292
 deltacat/compute/compactor/model/delta_annotated.py,sha256=NERB9rOtYg-xzBwvqGJ7_hBOzBC7g6X5M9-Cq5pbdH8,12258
@@ -50,7 +50,7 @@ deltacat/compute/compactor/utils/round_completion_file.py,sha256=DmZfHeAXlQn0DDd
 deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZRG2NqBlCw1X3_FBc,2397
 deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
 deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/compaction_session.py,sha256=8z7fQCHkL1Ei70_HdBh_Ckkstb7B_FiG_rfglJRouLQ,23668
+deltacat/compute/compactor_v2/compaction_session.py,sha256=xAxiTOxHffRvPqkv4ObGM-NWdpxRRMyrS8WWZWrgTFQ,25141
 deltacat/compute/compactor_v2/constants.py,sha256=yZgzFD59wiXbXiTVgYPWRodZGpngiSBNFB2jmoZ4fps,1471
 deltacat/compute/compactor_v2/deletes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/deletes/delete_file_envelope.py,sha256=AeuH9JRMwp6mvQf6P2cqL92hUEtResQq6qUTS0kIKac,3111
@@ -59,14 +59,14 @@ deltacat/compute/compactor_v2/deletes/delete_strategy_equality_delete.py,sha256=
 deltacat/compute/compactor_v2/deletes/model.py,sha256=kW7kfRe4jVNMnsWQrl0nyKdDpvB9mbJND-MVzAajbAI,558
 deltacat/compute/compactor_v2/deletes/utils.py,sha256=g63imGECPpPK9eHdde-6qyCg8_RjXCLF2UdfckPk0h4,6468
 deltacat/compute/compactor_v2/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/model/hash_bucket_input.py,sha256=HS1BLgNCjQoqe7EuM9baQWE6U4BFqU2PVEWFEF725VE,2815
+deltacat/compute/compactor_v2/model/hash_bucket_input.py,sha256=iJy8kLi1dIpFIyfoAjkaAtZvg8Np1z7BsUNGAcWfFm4,3042
 deltacat/compute/compactor_v2/model/hash_bucket_result.py,sha256=EsY9BPPywhmxlcLKn3kGWzAX4s4BTR2vYyPUB-wAEOc,309
 deltacat/compute/compactor_v2/model/merge_file_group.py,sha256=1o86t9lc3K6ZvtViVO1SVljCj6f0B3MfB3hqtGm2S0s,7410
-deltacat/compute/compactor_v2/model/merge_input.py,sha256=EBfV2SNQjsEnbwKHFnumqGSfALzmz8vdMf41RxZNhZ0,5189
+deltacat/compute/compactor_v2/model/merge_input.py,sha256=ITRBR8gMbJpeRkZhRaVzGEEk3F2WqS2LwEkjd5zcaMQ,5416
 deltacat/compute/compactor_v2/model/merge_result.py,sha256=_IZTCStpb4UKiRCJYA3g6EhAqjrw0t9vmoDAN8kIK-Y,436
 deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=QUNgYrxEp3ODr8YjCpIvXNKDQAxMaGhINgB9_jG1jNs,6282
-deltacat/compute/compactor_v2/steps/merge.py,sha256=kYeLofIpYXF1FgNyXPIcRRr7SQYYK_DbkT8LPJJZrsw,20731
+deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=Kuhgh1y12t4V9rP67bdalQF6_sZiobfAK3tvOka8IWA,6324
+deltacat/compute/compactor_v2/steps/merge.py,sha256=dNGE_UtQmptYWj59_XQBoFrB6iNWYE37iup81HqWcAU,21394
 deltacat/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=rNKZisxGrLQOkwX8eHUQiFoTR1V-E66pMqWigtrs618,2156
 deltacat/compute/compactor_v2/utils/dedupe.py,sha256=62tFCY2iRP7I3-45GCIYs6_SJsQl8C5lBEr8gbNfbsw,1932
@@ -74,7 +74,7 @@ deltacat/compute/compactor_v2/utils/delta.py,sha256=8hjkDeIIkSX-gAQ2utQSp2sZcO2t
 deltacat/compute/compactor_v2/utils/io.py,sha256=jgIfwrfH2mTFUx1M0TgwZGGfrS4IXjP1PmqwaQmNAJM,5092
 deltacat/compute/compactor_v2/utils/merge.py,sha256=hK4Y7acrtgfvWWTz-fAGznEg6qn6dBYu8blQUQVHhs0,5244
 deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=MAscmL35WfwN7Is72aFlD_cGhxtZgjRwwR5kS9Yn2uU,11393
-deltacat/compute/compactor_v2/utils/task_options.py,sha256=Ndhff9F_zff6zX3mw4AztHkvSBgR4O8SgMUgwq3rvyM,10601
+deltacat/compute/compactor_v2/utils/task_options.py,sha256=MCY0Sz5NCgNMaY92W8p87FvvDB91mnPQ4AhL8ix3BiA,13780
 deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
 deltacat/compute/merge_on_read/daft.py,sha256=1oC38u5ig_aTrq7EzyWBo8Ui54rb6yERYMk-vEFbpxM,1400
 deltacat/compute/merge_on_read/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -109,7 +109,7 @@ deltacat/compute/stats/utils/manifest_stats_file.py,sha256=PtqW5Zc5e09HcfiAgvoZH
 deltacat/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/dataset.py,sha256=8w9sPVDpGnjjGVDWB39YSKWxq4zRv9VEfDtj7PYwjqM,3755
 deltacat/io/file_object_store.py,sha256=HCFeXu9cWXPXVk54MHel_nw3-wIuzhMt2RI6jKzjRYM,1346
-deltacat/io/memcached_object_store.py,sha256=dqAIh2hQrEHDv-UOO4n4nRpb9KapVWiev09ewSS2O90,8555
+deltacat/io/memcached_object_store.py,sha256=4Sc7Tue3giKBrRm6zM2G0ZrvDCtubnIp1UvsqA2i9M0,9185
 deltacat/io/object_store.py,sha256=X6221ZuVx8NOyKUesz8LvjvQ_4vZ6p2RWV6VISL17AY,1576
 deltacat/io/ray_plasma_object_store.py,sha256=TyoUPWybE_cSISZ2SQa3YfD93QWMp0r82-6WnoVSmzk,905
 deltacat/io/read_api.py,sha256=BhkjL3xjY-fsa62AA9Yv20_88uTskn4_Bv2W6VmMXVA,7023
@@ -141,7 +141,7 @@ deltacat/tests/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=SaYQI30d1sxOBVs-Qj9VsMtigp4tWaRFFPmsPdahILc,72082
 deltacat/tests/compute/compact_partition_test_cases.py,sha256=b6wVg1UP0pahtrS3dojk-_1tSXHjCWd95XBo4J6KEAU,20092
 deltacat/tests/compute/test_compact_partition_incremental.py,sha256=G_xUERQjtBzAVfipzucdR8ATJviaVwTsj-5rRWVgYJY,10271
-deltacat/tests/compute/test_compact_partition_params.py,sha256=MIzIcBscwFA1W-cfTTxVx0zcgbrs8D4bI9Hy4TF5eRo,8322
+deltacat/tests/compute/test_compact_partition_params.py,sha256=Dm5eLyHo8oGMeO3XBbpj1rZqHtPZ1hAB7z2qvzc4Lxk,8497
 deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=n6Qy-_STWgy3j02QyfZMUChIlHqlTfZjQ-G8yDppvyM,14077
 deltacat/tests/compute/test_util_common.py,sha256=Skz0ZfHzidArZhIzRDHOYt-5uGBwx6MRfKZpeBnzh9w,6055
 deltacat/tests/compute/test_util_constant.py,sha256=4o-W3E7r7jhFl1A3OFLLrdKnwcF46zx4lEIDY8ONJ3c,929
@@ -159,7 +159,7 @@ deltacat/tests/compute/compactor_v2/utils/test_task_options.py,sha256=4fc5MJTLm3
 deltacat/tests/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/io/test_cloudpickle_bug_fix.py,sha256=qnYJg_S-nsLai77a4_I3Qs2Jtr_KWQJOxyl96f9PgHA,1376
 deltacat/tests/io/test_file_object_store.py,sha256=bHEJRleVHwvk-bbvAlNOFnOA_tbR8i0SxtsllMTb8w0,2559
-deltacat/tests/io/test_memcached_object_store.py,sha256=gUVYycPkNpq9XxotdJwFZ2HOmNZqwoa1nAC57ZL3TBo,7392
+deltacat/tests/io/test_memcached_object_store.py,sha256=25SB5xTMAG0HKwZwsDLhMreJ0dA6zrCDmHuY955YeLA,8070
 deltacat/tests/io/test_ray_plasma_object_store.py,sha256=-wJZP6lRtEOogR25wjEiIBGz_lpvWVihwlZ5GqandZU,1911
 deltacat/tests/io/test_redis_object_store.py,sha256=sZrXrYjkw8u_XrvFilhBbLc8PPnZiuMKa1_Bt9ka5qs,3838
 deltacat/tests/io/test_s3_object_store.py,sha256=4b7PYEfQJnYGUz6fcLFWVVyRHTlH_yd8CIaCv9l33Gg,1900
@@ -193,7 +193,7 @@ deltacat/utils/pandas.py,sha256=GfwjYb8FUSEeoBdXZI1_NJkdjxPMbCCUhlyRfGbDkn8,9562
 deltacat/utils/performance.py,sha256=7ZLaMkS1ehPSIhT5uOQVBHvjC70iKHzoFquFo-KL0PI,645
 deltacat/utils/placement.py,sha256=S80CwD1eEK47lQNr0xTmF9kq092-z6lTTmOOBv8cW_o,11723
 deltacat/utils/pyarrow.py,sha256=gYcoRhQoBoAFo69WNijMobrLGta4VASg8VarWPiB34Y,28979
-deltacat/utils/resources.py,sha256=UhxEnRMP2ts5iFR-h9ORmHe4pmJlLbMWZT80GsLVPEk,8218
+deltacat/utils/resources.py,sha256=Ax1OgLLbZI4oYpp4Ki27OLaST-7I-AJgZwU87FVfY8g,8253
 deltacat/utils/s3fs.py,sha256=PmUJ5Fm1WmD-_zp_M6yd9VbXvIoJuBeK6ApOdJJApLE,662
 deltacat/utils/schema.py,sha256=m4Wm4ZQcpttzOUxex4dVneGlHy1_E36HspTcjNYzvVM,1564
 deltacat/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -202,8 +202,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=5eaBWTDm0IXVoc5Y6aacoVB-f0Mnv-K2ewyTSjHKHwM,5009
-deltacat-1.1.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.0.dist-info/METADATA,sha256=rRb66_RxRcgvMfnTnwLGsvubmhW83uhLLIDyNjLQX7E,1780
-deltacat-1.1.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-deltacat-1.1.0.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.0.dist-info/RECORD,,
+deltacat-1.1.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.1.dist-info/METADATA,sha256=XeXOc863rqN4olrgF3bK3Hm3Cozw7nlshK-hHILg-o4,1780
+deltacat-1.1.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+deltacat-1.1.1.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.1.dist-info/RECORD,,