deltacat 1.1.2__py3-none-any.whl → 1.1.4__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.2"
+__version__ = "1.1.4"
 
 
 __all__ = [

deltacat/aws/constants.py

@@ -6,7 +6,3 @@ DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
 TIMEOUT_ERROR_CODES: List[str] = ["ReadTimeoutError", "ConnectTimeoutError"]
 AWS_REGION = env_string("AWS_REGION", "us-east-1")
-
-# Metric Names
-DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX = "download_manifest_entry"
-UPLOAD_SLICED_TABLE_METRIC_PREFIX = "upload_sliced_table"

deltacat/aws/s3u.py

@@ -27,8 +27,6 @@ import deltacat.aws.clients as aws_utils
 from deltacat import logs
 from deltacat.aws.constants import (
     TIMEOUT_ERROR_CODES,
-    DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX,
-    UPLOAD_SLICED_TABLE_METRIC_PREFIX,
 )
 from deltacat.exceptions import NonRetryableError, RetryableError
 from deltacat.storage import (
@@ -54,7 +52,6 @@ from deltacat.types.tables import (
 )
 from deltacat.types.partial_download import PartialFileDownloadParams
 from deltacat.utils.common import ReadKwargsProvider
-from deltacat.utils.metrics import metrics
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -121,6 +118,32 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
             self.block_refs.append(block)
         return write_path
 
+    def __call__(
+        self,
+        base_path: str,
+        *,
+        filesystem: Optional[pa.filesystem.FileSystem] = None,
+        dataset_uuid: Optional[str] = None,
+        block: Optional[ObjectRef[Block]] = None,
+        block_index: Optional[int] = None,
+        file_format: Optional[str] = None,
+    ) -> str:
+        """
+        TODO: BlockWritePathProvider is deprecated as of Ray version 2.20.0. Please use FilenameProvider.
+        See: https://docs.ray.io/en/master/data/api/doc/ray.data.datasource.FilenameProvider.html
+        Also See: https://github.com/ray-project/deltacat/issues/299
+
+        Hence, this class only works with Ray version 2.20.0 or lower when used in Ray Dataset.
+        """
+        return self._get_write_path_for_block(
+            base_path,
+            filesystem=filesystem,
+            dataset_uuid=dataset_uuid,
+            block=block,
+            block_index=block_index,
+            file_format=file_format,
+        )
+
 
 class S3Url:
     def __init__(self, url: str):
@@ -243,7 +266,6 @@ def read_file(
         raise e
 
 
-@metrics(prefix=UPLOAD_SLICED_TABLE_METRIC_PREFIX)
 def upload_sliced_table(
     table: Union[LocalTable, DistributedDataset],
     s3_url_prefix: str,
@@ -352,7 +374,6 @@ def upload_table(
     return manifest_entries
 
 
-@metrics(prefix=DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX)
 def download_manifest_entry(
     manifest_entry: ManifestEntry,
     token_holder: Optional[Dict[str, Any]] = None,

deltacat/compute/compactor/utils/round_completion_file.py

@@ -6,6 +6,7 @@ from deltacat.compute.compactor import RoundCompletionInfo
 from deltacat.storage import PartitionLocator
 from deltacat.aws import s3u as s3_utils
 from typing import Optional
+from deltacat.utils.metrics import metrics
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -18,6 +19,7 @@ def get_round_completion_file_s3_url(
     return f"{base_url}.json"
 
 
+@metrics
 def read_round_completion_file(
     bucket: str,
     source_partition_locator: PartitionLocator,
@@ -38,6 +40,7 @@ def read_round_completion_file(
     return round_completion_info
 
 
+@metrics
 def write_round_completion_file(
     bucket: Optional[str],
     source_partition_locator: Optional[PartitionLocator],

deltacat/compute/compactor_v2/compaction_session.py

@@ -50,6 +50,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
 from deltacat.compute.compactor_v2.steps import hash_bucket as hb
 from deltacat.compute.compactor_v2.utils import io
 from deltacat.compute.compactor.utils import round_completion_file as rcf
+from deltacat.utils.metrics import metrics
 
 from typing import List, Optional, Tuple
 from collections import defaultdict
@@ -65,7 +66,6 @@ from deltacat.compute.compactor_v2.utils.task_options import (
     local_merge_resource_options_provider,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
-from deltacat.utils.metrics import MetricsActor, METRICS_CONFIG_ACTOR_NAME
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -74,6 +74,7 @@ if importlib.util.find_spec("memray"):
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
+@metrics
 def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
 
     assert (
@@ -119,15 +120,6 @@ def _execute_compaction(
     params: CompactPartitionParams, **kwargs
 ) -> Tuple[Optional[Partition], Optional[RoundCompletionInfo], Optional[str]]:
 
-    if params.metrics_config:
-        logger.info(
-            f"Setting metrics config with target: {params.metrics_config.metrics_target}"
-        )
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
-        ray.get(metrics_actor.set_metrics_config.remote(params.metrics_config))
-
     rcf_source_partition_locator = (
         params.rebase_source_partition_locator or params.source_partition_locator
     )

deltacat/compute/compactor_v2/constants.py

@@ -65,6 +65,3 @@ DISCOVER_DELTAS_METRIC_PREFIX = "discover_deltas"
 
 # Metric prefix for prepare deletes
 PREPARE_DELETES_METRIC_PREFIX = "prepare_deletes"
-
-# Metric prefix for materialize
-MATERIALIZE_METRIC_PREFIX = "delta_materialize"

deltacat/compute/compactor_v2/utils/merge.py

@@ -31,14 +31,11 @@ from deltacat.compute.compactor_v2.deletes.delete_strategy import (
 from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
     DeleteFileEnvelope,
 )
-from deltacat.utils.metrics import metrics
-from deltacat.compute.compactor_v2.constants import MATERIALIZE_METRIC_PREFIX
 
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-@metrics(prefix=MATERIALIZE_METRIC_PREFIX)
 def materialize(
     input: MergeInput,
     task_index: int,

deltacat/logs.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import json
 import pathlib
 from logging import FileHandler, Handler, Logger, LoggerAdapter, handlers
 from typing import Any, Dict, Optional, Union
@@ -19,13 +20,106 @@ from deltacat.constants import (
 )
 
 DEFAULT_LOG_LEVEL = "INFO"
-DEFAULT_LOG_FORMAT = (
-    "%(asctime)s\t%(levelname)s pid=%(process)d %(filename)s:%(lineno)s -- %(message)s"
-)
+DEFAULT_LOG_FORMAT = {
+    "level": "levelname",
+    "message": "message",
+    "loggerName": "name",
+    "processName": "processName",
+    "processID": "process",
+    "threadName": "threadName",
+    "timestamp": "asctime",
+    "filename": "filename",
+    "lineno": "lineno",
+}
 DEFAULT_MAX_BYTES_PER_LOG = 2 ^ 20 * 256  # 256 MiB
 DEFAULT_BACKUP_COUNT = 0
 
 
+class JsonFormatter(logging.Formatter):
+    """
+    Formatter that outputs JSON strings after parsing the LogRecord.
+
+    @param dict fmt_dict: Key: logging format attribute pairs. Defaults to {"message": "message"}.
+    @param str time_format: time.strftime() format string. Default: "%Y-%m-%dT%H:%M:%S"
+    @param str msec_format: Microsecond formatting. Appended at the end. Default: "%s.%03dZ"
+    """
+
+    def __init__(
+        self,
+        fmt_dict: dict = None,
+        time_format: str = "%Y-%m-%dT%H:%M:%S",
+        msec_format: str = "%s.%03dZ",
+    ):
+        self.fmt_dict = fmt_dict if fmt_dict is not None else {"message": "message"}
+        self.default_time_format = time_format
+        self.default_msec_format = msec_format
+        self.datefmt = None
+        if ray.is_initialized():
+            self.ray_runtime_ctx: RuntimeContext = ray.get_runtime_context()
+            self.context = {}
+            self.context["worker_id"] = self.ray_runtime_ctx.get_worker_id()
+            self.context["node_id"] = self.ray_runtime_ctx.get_node_id()
+            self.context["job_id"] = self.ray_runtime_ctx.get_job_id()
+        else:
+            self.ray_runtime_ctx = None
+            self.context = {}
+
+    def usesTime(self) -> bool:
+        """
+        Overwritten to look for the attribute in the format dict values instead of the fmt string.
+        """
+        return "asctime" in self.fmt_dict.values()
+
+    def formatMessage(self, record) -> dict:
+        """
+        Overwritten to return a dictionary of the relevant LogRecord attributes instead of a string.
+        KeyError is raised if an unknown attribute is provided in the fmt_dict.
+        """
+        return {
+            fmt_key: record.__dict__[fmt_val]
+            for fmt_key, fmt_val in self.fmt_dict.items()
+        }
+
+    def format(self, record) -> str:
+        """
+        Mostly the same as the parent's class method, the difference being that a dict is manipulated and dumped as JSON
+        instead of a string.
+        """
+        record.message = record.getMessage()
+
+        if self.usesTime():
+            record.asctime = self.formatTime(record, self.datefmt)
+
+        message_dict = self.formatMessage(record)
+
+        if record.exc_info:
+            # Cache the traceback text to avoid converting it multiple times
+            # (it's constant anyway)
+            if not record.exc_text:
+                record.exc_text = self.formatException(record.exc_info)
+
+        if record.exc_text:
+            message_dict["exc_info"] = record.exc_text
+
+        if record.stack_info:
+            message_dict["stack_info"] = self.formatStack(record.stack_info)
+
+        if self.ray_runtime_ctx:
+            # only workers will have task ID
+            if (
+                self.ray_runtime_ctx.worker
+                and self.ray_runtime_ctx.worker.mode == ray._private.worker.WORKER_MODE
+            ):
+                self.context["task_id"] = self.ray_runtime_ctx.get_task_id()
+                self.context[
+                    "assigned_resources"
+                ] = self.ray_runtime_ctx.get_assigned_resources()
+
+            message_dict["ray_runtime_context"] = self.context
+
+        return json.dumps(message_dict, default=str)
+
+
 class DeltaCATLoggerAdapter(logging.LoggerAdapter):
     """
     Logger Adapter class with additional functionality
@@ -51,54 +145,6 @@ class DeltaCATLoggerAdapter(logging.LoggerAdapter):
             self.error(msg, *args, **kwargs)
 
 
-class RayRuntimeContextLoggerAdapter(DeltaCATLoggerAdapter):
-    """
-    Logger Adapter for injecting Ray Runtime Context into logging messages.
-    """
-
-    def __init__(self, logger: Logger, runtime_context: RuntimeContext):
-        super().__init__(logger, {})
-        self.runtime_context = runtime_context
-
-    def process(self, msg, kwargs):
-        """
-        Injects Ray Runtime Context details into each log message.
-
-        This may include information such as the raylet node ID, task/actor ID, job ID,
-        placement group ID of the worker, and assigned resources to the task/actor.
-
-        Args:
-            msg: The original log message
-            kwargs: Keyword arguments for the log message
-
-        Returns: A log message with Ray Runtime Context details
-
-        """
-        runtime_context_dict = self.runtime_context.get()
-        runtime_context_dict[
-            "worker_id"
-        ] = self.runtime_context.worker.core_worker.get_worker_id()
-        if self.runtime_context.get_task_id() or self.runtime_context.get_actor_id():
-            runtime_context_dict[
-                "pg_id"
-            ] = self.runtime_context.get_placement_group_id()
-            runtime_context_dict[
-                "assigned_resources"
-            ] = self.runtime_context.get_assigned_resources()
-
-        return "(ray_runtime_context=%s) -- %s" % (runtime_context_dict, msg), kwargs
-
-    def __reduce__(self):
-        """
-        Used to unpickle the class during Ray object store transfer.
-        """
-
-        def deserializer(*args):
-            return RayRuntimeContextLoggerAdapter(args[0], ray.get_runtime_context())
-
-        return deserializer, (self.logger,)
-
-
 def _add_logger_handler(logger: Logger, handler: Handler) -> Logger:
 
     logger.setLevel(logging.getLevelName("DEBUG"))
@@ -109,10 +155,10 @@ def _add_logger_handler(logger: Logger, handler: Handler) -> Logger:
 def _create_rotating_file_handler(
     log_directory: str,
     log_base_file_name: str,
-    logging_level: str = DEFAULT_LOG_LEVEL,
+    logging_level: Union[str, int] = DEFAULT_LOG_LEVEL,
     max_bytes_per_log_file: int = DEFAULT_MAX_BYTES_PER_LOG,
     backup_count: int = DEFAULT_BACKUP_COUNT,
-    logging_format: str = DEFAULT_LOG_FORMAT,
+    logging_format: Union[str, dict] = DEFAULT_LOG_FORMAT,
 ) -> FileHandler:
 
     if type(logging_level) is str:
@@ -126,7 +172,12 @@ def _create_rotating_file_handler(
         maxBytes=max_bytes_per_log_file,
         backupCount=backup_count,
     )
-    handler.setFormatter(logging.Formatter(logging_format))
+
+    if type(logging_format) is str:
+        handler.setFormatter(logging.Formatter(logging_format))
+    else:
+        handler.setFormatter(JsonFormatter(logging_format))
+
     handler.setLevel(logging_level)
     return handler
 
@@ -135,7 +186,8 @@ def _file_handler_exists(logger: Logger, log_dir: str, log_base_file_name: str)
 
     handler_exists = False
     base_file_path = os.path.join(log_dir, log_base_file_name)
-    if len(logger.handlers) > 0:
+
+    if logger.handlers:
         norm_base_file_path = os.path.normpath(base_file_path)
         handler_exists = any(
             [
@@ -149,49 +201,54 @@ def _file_handler_exists(logger: Logger, log_dir: str, log_base_file_name: str)
 
 def _configure_logger(
     logger: Logger,
-    log_level: str,
+    log_level: int,
     log_dir: str,
     log_base_file_name: str,
     debug_log_base_file_name: str,
 ) -> Union[Logger, LoggerAdapter]:
+    # This maintains log level of rotating file handlers
     primary_log_level = log_level
     logger.propagate = False
-    if log_level.upper() == "DEBUG":
+    if log_level <= logging.getLevelName("DEBUG"):
         if not _file_handler_exists(logger, log_dir, debug_log_base_file_name):
             handler = _create_rotating_file_handler(
                 log_dir, debug_log_base_file_name, "DEBUG"
             )
             _add_logger_handler(logger, handler)
-            primary_log_level = "INFO"
+            primary_log_level = logging.getLevelName("INFO")
     if not _file_handler_exists(logger, log_dir, log_base_file_name):
         handler = _create_rotating_file_handler(
             log_dir, log_base_file_name, primary_log_level
         )
         _add_logger_handler(logger, handler)
-    if ray.is_initialized():
-        ray_runtime_ctx = ray.get_runtime_context()
-        if ray_runtime_ctx.worker.connected:
-            logger = RayRuntimeContextLoggerAdapter(logger, ray_runtime_ctx)
-    else:
-        logger = DeltaCATLoggerAdapter(logger)
 
-    return logger
+    return DeltaCATLoggerAdapter(logger)
+
 
+def configure_deltacat_logger(
+    logger: Logger, level: int = None
+) -> Union[Logger, LoggerAdapter]:
+    if level is None:
+        level = logging.getLevelName(DELTACAT_SYS_LOG_LEVEL)
 
-def configure_deltacat_logger(logger: Logger) -> Union[Logger, LoggerAdapter]:
     return _configure_logger(
         logger,
-        DELTACAT_SYS_LOG_LEVEL,
+        level,
         DELTACAT_SYS_LOG_DIR,
         DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
         DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
     )
 
 
-def configure_application_logger(logger: Logger) -> Union[Logger, LoggerAdapter]:
+def configure_application_logger(
+    logger: Logger, level: int = None
+) -> Union[Logger, LoggerAdapter]:
+    if level is None:
+        level = logging.getLevelName(DELTACAT_APP_LOG_LEVEL)
+
     return _configure_logger(
         logger,
-        DELTACAT_APP_LOG_LEVEL,
+        level,
         DELTACAT_APP_LOG_DIR,
         DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
         DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,

deltacat/tests/aws/test_s3u.py

@@ -0,0 +1,12 @@
+import unittest
+from deltacat.aws.s3u import UuidBlockWritePathProvider, CapturedBlockWritePaths
+
+
+class TestUuidBlockWritePathProvider(unittest.TestCase):
+    def test_uuid_block_write_provider_sanity(self):
+        capture_object = CapturedBlockWritePaths()
+        provider = UuidBlockWritePathProvider(capture_object=capture_object)
+
+        result = provider("base_path")
+
+        self.assertRegex(result, r"^base_path/[\w-]{36}$")

deltacat/tests/test_logs.py

@@ -0,0 +1,127 @@
+import unittest
+import json
+import ray
+from logging import LogRecord
+from deltacat.logs import JsonFormatter
+
+
+class TestJsonFormatter(unittest.TestCase):
+    def test_usesTime_sanity(self):
+
+        formatter = JsonFormatter()
+
+        self.assertFalse(formatter.usesTime())
+
+    def test_usesTime_success_case(self):
+
+        formatter = JsonFormatter(fmt_dict={"asctime": "asctime"})
+
+        self.assertTrue(formatter.usesTime())
+
+    def test_formatMessage_sanity(self):
+
+        formatter = JsonFormatter({"message": "msg"})
+
+        record = LogRecord(
+            level="INFO",
+            name="test",
+            pathname="test",
+            lineno=0,
+            message="test_message",
+            msg="test_message",
+            args=None,
+            exc_info=None,
+        )
+
+        result = formatter.formatMessage(record)
+
+        self.assertEqual({"message": "test_message"}, result)
+
+    def test_format_sanity(self):
+        formatter = JsonFormatter({"message": "msg"})
+
+        record = LogRecord(
+            level="INFO",
+            name="test",
+            pathname="test",
+            lineno=0,
+            message="test_message",
+            msg="test_message",
+            args=None,
+            exc_info=None,
+        )
+
+        result = formatter.format(record)
+
+        self.assertEqual({"message": "test_message"}, json.loads(result))
+        self.assertFalse(ray.is_initialized())
+        self.assertNotIn("ray_runtime_context", json.loads(result))
+
+    def test_format_when_ray_initialized(self):
+        ray.init(local_mode=True, ignore_reinit_error=True)
+
+        formatter = JsonFormatter({"message": "msg"})
+
+        record = LogRecord(
+            level="INFO",
+            name="test",
+            pathname="test",
+            lineno=0,
+            message="test_message",
+            msg="test_message",
+            args=None,
+            exc_info=None,
+        )
+
+        result = formatter.format(record)
+        result = json.loads(result)
+
+        self.assertEqual("test_message", result["message"])
+        self.assertTrue(ray.is_initialized())
+        self.assertIn("ray_runtime_context", result)
+        self.assertIn("job_id", result["ray_runtime_context"])
+        self.assertIn("node_id", result["ray_runtime_context"])
+        self.assertIn("worker_id", result["ray_runtime_context"])
+        self.assertNotIn(
+            "task_id",
+            result["ray_runtime_context"],
+            "We expect task ID not be present outside a remote task",
+        )
+        ray.shutdown()
+
+    def test_format_when_ray_initialized_in_task(self):
+        # worker mode is only true when local_mode is False
+        ray.init(local_mode=False, ignore_reinit_error=True)
+
+        @ray.remote
+        def ray_remote_task():
+            formatter = JsonFormatter({"message": "msg"})
+
+            record = LogRecord(
+                level="INFO",
+                name="test",
+                pathname="test",
+                lineno=0,
+                message="test_message",
+                msg="test_message",
+                args=None,
+                exc_info=None,
+            )
+
+            result = formatter.format(record)
+            result = json.loads(result)
+            return result
+
+        result = ray.get(ray_remote_task.remote())
+        self.assertEqual("test_message", result["message"])
+        self.assertTrue(ray.is_initialized())
+        self.assertIn("ray_runtime_context", result)
+        self.assertIn("job_id", result["ray_runtime_context"])
+        self.assertIn("node_id", result["ray_runtime_context"])
+        self.assertIn("worker_id", result["ray_runtime_context"])
+        self.assertIn(
+            "task_id",
+            result["ray_runtime_context"],
+            "We expect task ID to be present inside a remote task",
+        )
+        ray.shutdown()

deltacat/tests/utils/test_metrics.py

@@ -1,13 +1,11 @@
 import unittest
 from unittest.mock import MagicMock, ANY, call
-import ray
 from deltacat.utils.metrics import (
     metrics,
     success_metric,
     failure_metric,
     latency_metric,
-    MetricsActor,
-    METRICS_CONFIG_ACTOR_NAME,
+    MetricsConfigCache,
     MetricsConfig,
     MetricsTarget,
     METRICS_TARGET_TO_EMITTER_DICT,
@@ -43,11 +41,8 @@ class TestMetricsAnnotation(unittest.TestCase):
     def test_metrics_annotation_sanity(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         metrics_annotated_method(mock)
@@ -73,11 +68,8 @@ class TestMetricsAnnotation(unittest.TestCase):
     def test_metrics_annotation_when_error(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         self.assertRaises(ValueError, lambda: metrics_annotated_method_error(mock))
@@ -108,11 +100,8 @@ class TestMetricsAnnotation(unittest.TestCase):
     def test_metrics_with_prefix_annotation_sanity(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         metrics_with_prefix_annotated_method(mock)
@@ -134,11 +123,8 @@ class TestMetricsAnnotation(unittest.TestCase):
     def test_metrics_annotation_performance(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action with annotation
         _, actual_latency = timed_invocation(metrics_annotated_method, mock)
@@ -155,11 +141,8 @@ class TestMetricsAnnotation(unittest.TestCase):
     def test_metrics_with_prefix_annotation_when_error(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         self.assertRaises(
@@ -185,15 +168,10 @@ class TestMetricsAnnotation(unittest.TestCase):
             any_order=True,
         )
 
-    def test_metrics_with_prefix_annotation_without_actor(self):
+    def test_metrics_with_prefix_annotation_without_config(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
-
-        # explicitly making sure actor is killed
-        ray.kill(metrics_actor)
+        MetricsConfigCache.metrics_config = None
 
         # action
         self.assertRaises(
@@ -228,11 +206,8 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
     def test_annotation_sanity(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         latency_metric_annotated_method(mock)
@@ -253,11 +228,8 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
     def test_annotation_when_error(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         self.assertRaises(
@@ -280,11 +252,8 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
     def test_annotation_with_args_sanity(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         latency_metric_with_name_annotated_method(mock)
@@ -298,11 +267,8 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
     def test_annotation_with_args_when_error(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         self.assertRaises(
@@ -315,15 +281,10 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
             [call(metrics_name="test", metrics_config=ANY, value=ANY)], any_order=True
         )
 
-    def test_annotation_without_actor(self):
+    def test_annotation_without_config(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
-
-        # explicitly making sure actor is killed
-        ray.kill(metrics_actor)
+        MetricsConfigCache.metrics_config = None
 
         # action
         self.assertRaises(
@@ -358,11 +319,8 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
     def test_annotation_sanity(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         success_metric_annotated_method(mock)
@@ -383,11 +341,8 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
     def test_annotation_when_error(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         self.assertRaises(
@@ -400,11 +355,8 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
     def test_annotation_with_args_sanity(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         success_metric_with_name_annotated_method(mock)
@@ -418,11 +370,8 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
     def test_annotation_with_args_when_error(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         self.assertRaises(
@@ -432,15 +381,10 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
         mock.assert_not_called()
         mock_target.assert_not_called()
 
-    def test_annotation_without_actor(self):
+    def test_annotation_without_config(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
-
-        # explicitly making sure actor is killed
-        ray.kill(metrics_actor)
+        MetricsConfigCache.metrics_config = None
 
         # action
         success_metric_annotated_method(mock)
@@ -473,11 +417,8 @@ class TestFailureMetricAnnotation(unittest.TestCase):
     def test_annotation_sanity(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         failure_metric_annotated_method(mock)
@@ -488,11 +429,8 @@ class TestFailureMetricAnnotation(unittest.TestCase):
     def test_annotation_when_error(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         self.assertRaises(
@@ -520,11 +458,8 @@ class TestFailureMetricAnnotation(unittest.TestCase):
     def test_annotation_with_args_sanity(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         failure_metric_with_name_annotated_method(mock)
@@ -535,11 +470,8 @@ class TestFailureMetricAnnotation(unittest.TestCase):
     def test_annotation_with_args_when_error(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
         config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
-        ray.get(metrics_actor.set_metrics_config.remote(config))
+        MetricsConfigCache.metrics_config = config
 
         # action
         self.assertRaises(
@@ -556,15 +488,10 @@ class TestFailureMetricAnnotation(unittest.TestCase):
             any_order=True,
         )
 
-    def test_annotation_without_actor(self):
+    def test_annotation_without_config(self):
         mock, mock_target = MagicMock(), MagicMock()
         METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
-        metrics_actor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
-
-        # explicitly making sure actor is killed
-        ray.kill(metrics_actor)
+        MetricsConfigCache.metrics_config = None
 
         # action
         self.assertRaises(

deltacat/tests/utils/test_placement.py

@@ -0,0 +1,25 @@
+import unittest
+import ray
+from deltacat.utils.placement import (
+    PlacementGroupManager,
+    _get_available_resources_per_node,
+)
+
+
+class TestPlacementGroupManager(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        ray.init(local_mode=True, ignore_reinit_error=True)
+
+    def test_placement_group_manager_sanity(self):
+
+        pgm = PlacementGroupManager(1, 1, 1)
+
+        self.assertIsNotNone(pgm)
+
+    def test_ray_state_api_returns_correctly(self):
+
+        result = _get_available_resources_per_node()
+
+        self.assertIsNotNone(result)

deltacat/utils/metrics.py

@@ -2,7 +2,6 @@ from dataclasses import dataclass
 from typing import Optional
 
 import logging
-import ray
 import time
 import functools
 
@@ -22,7 +21,6 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 DEFAULT_DELTACAT_METRICS_NAMESPACE = "ray-deltacat-metrics"
 DEFAULT_DELTACAT_LOG_GROUP_NAME = "ray-deltacat-metrics-EMF-logs"
 DEFAULT_DELTACAT_LOG_STREAM_CALLABLE = get_node_ip_address
-METRICS_CONFIG_ACTOR_NAME = "metrics-config-actor"
 
 
 class MetricsTarget(str, Enum):
@@ -187,23 +185,13 @@ METRICS_TARGET_TO_EMITTER_DICT: Dict[str, Callable] = {
 }
 
 
-@ray.remote
-class MetricsActor:
+class MetricsConfigCache:
     metrics_config: MetricsConfig = None
 
-    def set_metrics_config(self, config: MetricsConfig) -> None:
-        self.metrics_config = config
-
-    def get_metrics_config(self) -> MetricsConfig:
-        return self.metrics_config
-
 
 def _emit_ignore_exceptions(name: Optional[str], value: Any):
     try:
-        config_cache: MetricsActor = MetricsActor.options(
-            name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
-        ).remote()
-        config = ray.get(config_cache.get_metrics_config.remote())
+        config = MetricsConfigCache.metrics_config
         if config:
             _emit_metrics(metrics_name=name, value=value, metrics_config=config)
     except BaseException as ex:
@@ -222,7 +210,14 @@ def emit_timer_metrics(metrics_name, value, metrics_config, **kwargs):
 def latency_metric(original_func=None, name: Optional[str] = None):
     """
     A decorator that emits latency metrics of a function
-    based on configured metrics config.
+    based on configured metrics config. Hence, make sure to set it in all worker processes:
+
+    def setup_cache():
+        from deltacat.utils.metrics import MetricsConfigCache
+        MetricsConfigCache.metrics_config = metrics_config
+
+    setup_cache();
+    ray.init(address="auto", runtime_env={"worker_process_setup_hook": setup_cache})
     """
 
     def _decorate(func):
@@ -248,7 +243,14 @@ def latency_metric(original_func=None, name: Optional[str] = None):
 def success_metric(original_func=None, name: Optional[str] = None):
     """
     A decorator that emits success metrics of a function
-    based on configured metrics config.
+    based on configured metrics config. Hence, make sure to set it in all worker processes:
+
+    def setup_cache():
+        from deltacat.utils.metrics import MetricsConfigCache
+        MetricsConfigCache.metrics_config = metrics_config
+
+    setup_cache();
+    ray.init(address="auto", runtime_env={"worker_process_setup_hook": setup_cache})
     """
 
     def _decorate(func):
@@ -271,7 +273,14 @@ def success_metric(original_func=None, name: Optional[str] = None):
 def failure_metric(original_func=None, name: Optional[str] = None):
     """
     A decorator that emits failure metrics of a function
-    based on configured metrics config.
+    based on configured metrics config. Hence, make sure to set it in all worker processes:
+
+    def setup_cache():
+        from deltacat.utils.metrics import MetricsConfigCache
+        MetricsConfigCache.metrics_config = metrics_config
+
+    setup_cache();
+    ray.init(address="auto", runtime_env={"worker_process_setup_hook": setup_cache})
     """
 
     def _decorate(func):
@@ -301,7 +310,15 @@ def failure_metric(original_func=None, name: Optional[str] = None):
 
 def metrics(original_func=None, prefix: Optional[str] = None):
     """
-    A decorator that emits all metrics for a function.
+    A decorator that emits all metrics for a function. This decorator depends
+    on a metrics config. Hence, make sure to set it in all worker processes:
+
+    def setup_cache():
+        from deltacat.utils.metrics import MetricsConfigCache
+        MetricsConfigCache.metrics_config = metrics_config
+
+    setup_cache();
+    ray.init(address="auto", runtime_env={"worker_process_setup_hook": setup_cache})
     """
 
     def _decorate(func):

deltacat/utils/placement.py

@@ -2,6 +2,7 @@ import logging
 import re
 import time
 from dataclasses import dataclass
+from packaging.version import Version
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 import ray
@@ -19,6 +20,16 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 # Issue: https://github.com/ray-project/ray/issues/29959
 
 
+def _get_available_resources_per_node():
+    # This API changed after this commit
+    # https://github.com/ray-project/ray/pull/43252
+    # TODO: Use this method from a durable State API once it's available
+    if Version(ray.__version__) >= Version("2.10.0"):
+        return ray._private.state.available_resources_per_node()
+    else:
+        return ray._private.state.state._available_resources_per_node()
+
+
 @dataclass
 class PlacementGroupConfig:
     def __init__(self, opts, resource, node_ips):
@@ -90,9 +101,7 @@ class NodeGroupManager:
         Returns:
                 group_res: a dict of resources, e.g., {'CPU':0,'memory':0,'object_store_memory':0}
         """
-        all_available_resources = (
-            ray._private.state.state._available_resources_per_node()
-        )
+        all_available_resources = _get_available_resources_per_node()
         group_keys = [x[0] for x in self.init_groups]
         group_res = {}
         for k in group_keys:
@@ -127,9 +136,7 @@ class NodeGroupManager:
         Returns:
                 group_res: dict of updated resource(cpu, memory, object store memory) for a given group
         """
-        all_available_resources = (
-            ray._private.state.state._available_resources_per_node()
-        )
+        all_available_resources = _get_available_resources_per_node()
         group_res = {"CPU": 0, "memory": 0, "object_store_memory": 0, "node_id": []}
         for v in all_available_resources.values():
             keys = v.keys()
@@ -285,7 +292,7 @@ def _config(
     for bd in bundles:
         node_ids.append(bd["node_id"])
     # query available resources given list of node id
-    all_nodes_available_res = ray._private.state.state._available_resources_per_node()
+    all_nodes_available_res = _get_available_resources_per_node()
     pg_res = {"CPU": 0, "memory": 0, "object_store_memory": 0}
     node_ips = []
     for node_id in node_ids:

{deltacat-1.1.2.dist-info → deltacat-1.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.2
+Version: 1.1.4
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.2.dist-info → deltacat-1.1.4.dist-info}/RECORD

@@ -1,11 +1,11 @@
-deltacat/__init__.py,sha256=1qjk3DyuuYXNfuzWlEqKVNNPDDpBkV8gwsSgJiMCq1s,1777
+deltacat/__init__.py,sha256=bZ2r6PnxfaWRxylN1GFZYEkxvWqx381U-7XuK3NEjq0,1777
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=xqZf8CwysNYP2d39pf27OnXGStPREgBgIM-e2Tts-TI,199
-deltacat/logs.py,sha256=YT26oj-oKZReyBEhiBQU5jc246aSu_d4B2bZcUVBHmE,6550
+deltacat/logs.py,sha256=6g16VkEFidbaMjgenAjggE1r2l664drMVhreRs8B1IQ,8438
 deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/clients.py,sha256=VgddlV3AEjlBGIFmhhHxokYzwJ-lXnmHAeprVyADduI,6948
-deltacat/aws/constants.py,sha256=Ti8D92AFY3lY2eoOwYBSI1eqS8gMykX5DHf4w3xPybE,492
-deltacat/aws/s3u.py,sha256=7-ZNz8fui_S5nprxtMO8PGa-G1YOXfidqcguzTTOvL8,24118
+deltacat/aws/constants.py,sha256=aAhOKeLVgtpekA3h9otHUrHqY2bLDWs2QlL7GrdI63g,352
+deltacat/aws/s3u.py,sha256=lgoE6es6N4xfzwyydxmVspROP1hrNfanB6JqjyBoRb4,24859
 deltacat/aws/redshift/__init__.py,sha256=7SvjG-dqox8zZUhFicTsUvpG5vXYDl_QQ3ohlHOgTKc,342
 deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/redshift/model/manifest.py,sha256=ThgpdwzaWz493Zz9e8HSWwuxEheA1nDuypM3pe4vozk,12987
@@ -46,12 +46,12 @@ deltacat/compute/compactor/steps/repartition.py,sha256=_ITw4yvvnNv3wwOYxprzlIz5J
 deltacat/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor/utils/io.py,sha256=S-JZdjETP_tHblK4j860jLHyX9S6A87BPz3Rl0jGbRM,17303
 deltacat/compute/compactor/utils/primary_key_index.py,sha256=ay2-7t4mP9I_l5gKkrv5h5_r8Icts8mBcbH7OJBknrY,2435
-deltacat/compute/compactor/utils/round_completion_file.py,sha256=DmZfHeAXlQn0DDdcsIHZROHWfyBCKTY3pNUdHzalqkE,2284
+deltacat/compute/compactor/utils/round_completion_file.py,sha256=-j6ZzhJBDrJ6Vz6WKEC-yKcNElkKRfO6S0P2JdJDQD0,2345
 deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZRG2NqBlCw1X3_FBc,2397
 deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
 deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/compaction_session.py,sha256=lXLoOiVjvsG-VEFD9afeKMjcExmOMg9I4-JTrTCVSo4,25576
-deltacat/compute/compactor_v2/constants.py,sha256=R5qvsaX-rDrSmSqUl0yLp6A3hVrAD4whYkV2NBkDR9Y,2199
+deltacat/compute/compactor_v2/compaction_session.py,sha256=hC0zde_3jq9W9bO8Z1XwsziK33qGeCFfyqFp1ZRWBJI,25193
+deltacat/compute/compactor_v2/constants.py,sha256=jGLEK5uS7AcnoVjPGUDIO4ljDbBYZlqzQleKJRKvnZM,2118
 deltacat/compute/compactor_v2/deletes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/deletes/delete_file_envelope.py,sha256=AeuH9JRMwp6mvQf6P2cqL92hUEtResQq6qUTS0kIKac,3111
 deltacat/compute/compactor_v2/deletes/delete_strategy.py,sha256=SMEJOxR-5r92kvKNqtu2w6HmwtmhljcZX1wcNEuS-4w,2833
@@ -72,7 +72,7 @@ deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=rNKZisxGrLQOkw
 deltacat/compute/compactor_v2/utils/dedupe.py,sha256=62tFCY2iRP7I3-45GCIYs6_SJsQl8C5lBEr8gbNfbsw,1932
 deltacat/compute/compactor_v2/utils/delta.py,sha256=8hjkDeIIkSX-gAQ2utQSp2sZcO2tWZHMTxpFusZwBHw,3635
 deltacat/compute/compactor_v2/utils/io.py,sha256=autXlE3uHICdCCuJoS7mfdeJbRRiz2_xlz-3izlccB4,5264
-deltacat/compute/compactor_v2/utils/merge.py,sha256=7p8lEmepxNekFFP5uVxgVLvAQRQnscOmUlSj8kUvW2c,5408
+deltacat/compute/compactor_v2/utils/merge.py,sha256=hK4Y7acrtgfvWWTz-fAGznEg6qn6dBYu8blQUQVHhs0,5244
 deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=MAscmL35WfwN7Is72aFlD_cGhxtZgjRwwR5kS9Yn2uU,11393
 deltacat/compute/compactor_v2/utils/task_options.py,sha256=MCY0Sz5NCgNMaY92W8p87FvvDB91mnPQ4AhL8ix3BiA,13780
 deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
@@ -133,8 +133,10 @@ deltacat/storage/model/table.py,sha256=IOu1ZOrdRkVDB-FOxYMRvnNf5TukIDfbdHWTqHYN_
 deltacat/storage/model/table_version.py,sha256=cOM9dN-YB_Hhi4h1CzFbldC5qRkm4C1rQ3rpKIZzCNs,7413
 deltacat/storage/model/types.py,sha256=hj7MmjjVmKT-R9sMUulOWG-FByGZKKaYXNnOWW32mP0,1608
 deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/test_logs.py,sha256=6BEMw8VApFg2msFwCAVosz8NWJYATtX5furHyz8UluM,3828
 deltacat/tests/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/aws/test_clients.py,sha256=23GMWfz27WWBDXSqphG9mfputsyS7j3I5P_HRk4YoKE,3790
+deltacat/tests/aws/test_s3u.py,sha256=QflXbR94o7WobGBm6jhQDK5lJJD2Pd9z2uvi4J7WEJg,437
 deltacat/tests/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/catalog/test_default_catalog_impl.py,sha256=9srCU5yQ159oZ9_PoJ_mWMzVUW5bKV0mnmPJc5zKCQQ,3125
 deltacat/tests/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -174,7 +176,8 @@ deltacat/tests/test_utils/utils.py,sha256=a32qEwcSSd1lvRi0aJJ4ZLnc1ZyXmoQF_K95za
 deltacat/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/utils/test_cloudpickle.py,sha256=J0pnBY3-PxlUh6MamZAN1PuquKQPr2iyzjiJ7-Rcl0o,1506
 deltacat/tests/utils/test_daft.py,sha256=Xal84zR42rXsWQI3lImdDYWOzewomKmhmiUQ59m67V0,6488
-deltacat/tests/utils/test_metrics.py,sha256=kznfueuChhTG3kKnfbVOtxks2uwIWrXlDjY3b2qQaNc,19429
+deltacat/tests/utils/test_metrics.py,sha256=Ym9nOz1EtB180pLmvugihj1sDTNDMb5opIjjr5Nmcls,16339
+deltacat/tests/utils/test_placement.py,sha256=g61wVOMkHe4YJeR9Oxg_BOVQ6bhHHbC3IBYv8YhUu94,597
 deltacat/tests/utils/test_pyarrow.py,sha256=eZAuYp9MUf8lmpIilH57JkURuNsTGZ3IAGC4Gm5hdrM,17307
 deltacat/tests/utils/test_record_batch_tables.py,sha256=AkG1WyljQmjnl-AxhbFWyo5LnMIKRyLScfgC2B_ES-s,11321
 deltacat/tests/utils/test_resources.py,sha256=HtpvDrfPZQNtGDXUlsIzc_yd7Vf1cDscZ3YbN0oTvO8,2560
@@ -188,11 +191,11 @@ deltacat/utils/arguments.py,sha256=5y1Xz4HSAD8M8Jt83i6gOEKoYjy_fMQe1V43IhIE4hY,1
 deltacat/utils/cloudpickle.py,sha256=XE7YDmQe56ksfl3NdYZkzOAhbHSuhNcBZGOehQpgZr0,1187
 deltacat/utils/common.py,sha256=RG_-enXNpLKaYrqyx1ne2lL10lxN9vK7F631oJP6SE8,1375
 deltacat/utils/daft.py,sha256=heg6J8NHnTI1UbcjXvsjGBbFKU6f7mvAio-KZuTSuFI,5506
-deltacat/utils/metrics.py,sha256=PZeBV4rhlA_7WjucXDe2DyZj5A8pYCQsYoKDwgyqb7E,9937
+deltacat/utils/metrics.py,sha256=HYKyZSrtVLu8gXezg_TMNUKJp4h1WWI0VEzn0Xlzf-I,10778
 deltacat/utils/numpy.py,sha256=ZiGREobTVT6IZXgPxkSUpLJFN2Hn8KEZcrqybLDXCIA,2027
 deltacat/utils/pandas.py,sha256=GfwjYb8FUSEeoBdXZI1_NJkdjxPMbCCUhlyRfGbDkn8,9562
 deltacat/utils/performance.py,sha256=7ZLaMkS1ehPSIhT5uOQVBHvjC70iKHzoFquFo-KL0PI,645
-deltacat/utils/placement.py,sha256=S80CwD1eEK47lQNr0xTmF9kq092-z6lTTmOOBv8cW_o,11723
+deltacat/utils/placement.py,sha256=Lj20fb-eq8rgMdm_M2MBMfDLwhDM1sS1nJj2DvIK56s,12060
 deltacat/utils/pyarrow.py,sha256=gYcoRhQoBoAFo69WNijMobrLGta4VASg8VarWPiB34Y,28979
 deltacat/utils/resources.py,sha256=Ax1OgLLbZI4oYpp4Ki27OLaST-7I-AJgZwU87FVfY8g,8253
 deltacat/utils/s3fs.py,sha256=PmUJ5Fm1WmD-_zp_M6yd9VbXvIoJuBeK6ApOdJJApLE,662
@@ -203,8 +206,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=5eaBWTDm0IXVoc5Y6aacoVB-f0Mnv-K2ewyTSjHKHwM,5009
-deltacat-1.1.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.2.dist-info/METADATA,sha256=r16g6Lc01Nlsb0Dr5DPona2hiOsWLIcscEXkQyfpD3o,1780
-deltacat-1.1.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-deltacat-1.1.2.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.2.dist-info/RECORD,,
+deltacat-1.1.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.4.dist-info/METADATA,sha256=q6kOOCDYRqloN7Fz1n87rkl3LlVRoJR32c1a5dBAkhQ,1780
+deltacat-1.1.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+deltacat-1.1.4.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.4.dist-info/RECORD,,