deltacat 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.3"
+__version__ = "1.1.5"
 
 
 __all__ = [

deltacat/aws/constants.py

@@ -6,7 +6,3 @@ DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
 TIMEOUT_ERROR_CODES: List[str] = ["ReadTimeoutError", "ConnectTimeoutError"]
 AWS_REGION = env_string("AWS_REGION", "us-east-1")
-
-# Metric Names
-DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX = "download_manifest_entry"
-UPLOAD_SLICED_TABLE_METRIC_PREFIX = "upload_sliced_table"

deltacat/aws/s3u.py

@@ -27,8 +27,6 @@ import deltacat.aws.clients as aws_utils
 from deltacat import logs
 from deltacat.aws.constants import (
     TIMEOUT_ERROR_CODES,
-    DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX,
-    UPLOAD_SLICED_TABLE_METRIC_PREFIX,
 )
 from deltacat.exceptions import NonRetryableError, RetryableError
 from deltacat.storage import (
@@ -54,7 +52,6 @@ from deltacat.types.tables import (
 )
 from deltacat.types.partial_download import PartialFileDownloadParams
 from deltacat.utils.common import ReadKwargsProvider
-from deltacat.utils.metrics import metrics
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -121,6 +118,32 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
             self.block_refs.append(block)
         return write_path
 
+    def __call__(
+        self,
+        base_path: str,
+        *,
+        filesystem: Optional[pa.filesystem.FileSystem] = None,
+        dataset_uuid: Optional[str] = None,
+        block: Optional[ObjectRef[Block]] = None,
+        block_index: Optional[int] = None,
+        file_format: Optional[str] = None,
+    ) -> str:
+        """
+        TODO: BlockWritePathProvider is deprecated as of Ray version 2.20.0. Please use FilenameProvider.
+        See: https://docs.ray.io/en/master/data/api/doc/ray.data.datasource.FilenameProvider.html
+        Also See: https://github.com/ray-project/deltacat/issues/299
+
+        Hence, this class only works with Ray version 2.20.0 or lower when used in Ray Dataset.
+        """
+        return self._get_write_path_for_block(
+            base_path,
+            filesystem=filesystem,
+            dataset_uuid=dataset_uuid,
+            block=block,
+            block_index=block_index,
+            file_format=file_format,
+        )
+
 
 class S3Url:
     def __init__(self, url: str):
@@ -243,7 +266,6 @@ def read_file(
         raise e
 
 
-@metrics(prefix=UPLOAD_SLICED_TABLE_METRIC_PREFIX)
 def upload_sliced_table(
     table: Union[LocalTable, DistributedDataset],
     s3_url_prefix: str,
@@ -352,7 +374,6 @@ def upload_table(
     return manifest_entries
 
 
-@metrics(prefix=DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX)
 def download_manifest_entry(
     manifest_entry: ManifestEntry,
     token_holder: Optional[Dict[str, Any]] = None,

deltacat/compute/compactor/utils/round_completion_file.py

@@ -6,6 +6,7 @@ from deltacat.compute.compactor import RoundCompletionInfo
 from deltacat.storage import PartitionLocator
 from deltacat.aws import s3u as s3_utils
 from typing import Optional
+from deltacat.utils.metrics import metrics
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -18,6 +19,7 @@ def get_round_completion_file_s3_url(
     return f"{base_url}.json"
 
 
+@metrics
 def read_round_completion_file(
     bucket: str,
     source_partition_locator: PartitionLocator,
@@ -38,6 +40,7 @@ def read_round_completion_file(
     return round_completion_info
 
 
+@metrics
 def write_round_completion_file(
     bucket: Optional[str],
     source_partition_locator: Optional[PartitionLocator],

deltacat/compute/compactor_v2/compaction_session.py

@@ -17,7 +17,11 @@ from deltacat.compute.compactor_v2.model.merge_input import MergeInput
 from deltacat.aws import s3u as s3_utils
 import deltacat
 from deltacat import logs
-from deltacat.compute.compactor import PyArrowWriteResult, RoundCompletionInfo
+from deltacat.compute.compactor import (
+    HighWatermark,
+    PyArrowWriteResult,
+    RoundCompletionInfo,
+)
 from deltacat.compute.compactor_v2.model.merge_result import MergeResult
 from deltacat.compute.compactor_v2.model.hash_bucket_result import HashBucketResult
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
@@ -37,6 +41,7 @@ from deltacat.compute.compactor_v2.deletes.utils import prepare_deletes
 from deltacat.storage import (
     Delta,
     DeltaLocator,
+    Manifest,
     Partition,
 )
 from deltacat.compute.compactor.model.compact_partition_params import (
@@ -50,6 +55,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
 from deltacat.compute.compactor_v2.steps import hash_bucket as hb
 from deltacat.compute.compactor_v2.utils import io
 from deltacat.compute.compactor.utils import round_completion_file as rcf
+from deltacat.utils.metrics import metrics
 
 from typing import List, Optional, Tuple
 from collections import defaultdict
@@ -73,6 +79,7 @@ if importlib.util.find_spec("memray"):
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
+@metrics
 def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
 
     assert (
@@ -94,7 +101,7 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
         round_completion_file_s3_url = None
         if new_partition:
             logger.info(f"Committing compacted partition to: {new_partition.locator}")
-            partition = params.deltacat_storage.commit_partition(
+            partition: Partition = params.deltacat_storage.commit_partition(
                 new_partition, **params.deltacat_storage_kwargs
             )
             logger.info(f"Committed compacted partition: {partition}")
@@ -148,9 +155,9 @@ def _execute_compaction(
         compaction_audit.set_total_cluster_memory_bytes(cluster_memory)
 
     # read the results from any previously completed compaction round
-    round_completion_info = None
-    high_watermark = None
-    previous_compacted_delta_manifest = None
+    round_completion_info: Optional[RoundCompletionInfo] = None
+    high_watermark: Optional[HighWatermark] = None
+    previous_compacted_delta_manifest: Optional[Manifest] = None
 
     if not params.rebase_source_partition_locator:
         round_completion_info = rcf.read_round_completion_file(
@@ -269,6 +276,7 @@ def _execute_compaction(
     total_hb_record_count = np.int64(0)
     telemetry_time_hb = 0
     if params.hash_bucket_count == 1:
+        logger.info("Hash bucket count set to 1. Running local merge")
         merge_start = time.monotonic()
         local_merge_input = generate_local_merge_input(
             params,
@@ -632,6 +640,19 @@ def _execute_compaction(
         f"partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
+    is_inplace_compacted: bool = (
+        params.source_partition_locator.partition_values
+        == params.destination_partition_locator.partition_values
+        and params.source_partition_locator.stream_id
+        == params.destination_partition_locator.stream_id
+    )
+    if is_inplace_compacted:
+        logger.info(
+            "Overriding round completion file source partition locator as in-place compacted. "
+            + f"Got compacted partition partition_id of {compacted_partition.locator.partition_id} "
+            f"and rcf source partition_id of {rcf_source_partition_locator.partition_id}."
+        )
+        rcf_source_partition_locator = compacted_partition.locator
     return (
         compacted_partition,
         new_round_completion_info,

deltacat/compute/compactor_v2/constants.py

@@ -65,6 +65,3 @@ DISCOVER_DELTAS_METRIC_PREFIX = "discover_deltas"
 
 # Metric prefix for prepare deletes
 PREPARE_DELETES_METRIC_PREFIX = "prepare_deletes"
-
-# Metric prefix for materialize
-MATERIALIZE_METRIC_PREFIX = "delta_materialize"

deltacat/compute/compactor_v2/utils/merge.py

@@ -31,14 +31,11 @@ from deltacat.compute.compactor_v2.deletes.delete_strategy import (
 from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
     DeleteFileEnvelope,
 )
-from deltacat.utils.metrics import metrics
-from deltacat.compute.compactor_v2.constants import MATERIALIZE_METRIC_PREFIX
 
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-@metrics(prefix=MATERIALIZE_METRIC_PREFIX)
 def materialize(
     input: MergeInput,
     task_index: int,

deltacat/compute/compactor_v2/utils/task_options.py

@@ -21,6 +21,9 @@ from deltacat.compute.compactor_v2.constants import (
     PARQUET_TO_PYARROW_INFLATION,
 )
 
+from daft.exceptions import DaftTransientError
+
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
@@ -76,6 +79,7 @@ def get_task_options(
         botocore.exceptions.HTTPClientError,
         ConnectionError,
         TimeoutError,
+        DaftTransientError,
     ]
 
     return task_opts

deltacat/logs.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import json
 import pathlib
 from logging import FileHandler, Handler, Logger, LoggerAdapter, handlers
 from typing import Any, Dict, Optional, Union
@@ -19,13 +20,106 @@ from deltacat.constants import (
 )
 
 DEFAULT_LOG_LEVEL = "INFO"
-DEFAULT_LOG_FORMAT = (
-    "%(asctime)s\t%(levelname)s pid=%(process)d %(filename)s:%(lineno)s -- %(message)s"
-)
+DEFAULT_LOG_FORMAT = {
+    "level": "levelname",
+    "message": "message",
+    "loggerName": "name",
+    "processName": "processName",
+    "processID": "process",
+    "threadName": "threadName",
+    "timestamp": "asctime",
+    "filename": "filename",
+    "lineno": "lineno",
+}
 DEFAULT_MAX_BYTES_PER_LOG = 2 ^ 20 * 256  # 256 MiB
 DEFAULT_BACKUP_COUNT = 0
 
 
+class JsonFormatter(logging.Formatter):
+    """
+    Formatter that outputs JSON strings after parsing the LogRecord.
+
+    @param dict fmt_dict: Key: logging format attribute pairs. Defaults to {"message": "message"}.
+    @param str time_format: time.strftime() format string. Default: "%Y-%m-%dT%H:%M:%S"
+    @param str msec_format: Microsecond formatting. Appended at the end. Default: "%s.%03dZ"
+    """
+
+    def __init__(
+        self,
+        fmt_dict: dict = None,
+        time_format: str = "%Y-%m-%dT%H:%M:%S",
+        msec_format: str = "%s.%03dZ",
+    ):
+        self.fmt_dict = fmt_dict if fmt_dict is not None else {"message": "message"}
+        self.default_time_format = time_format
+        self.default_msec_format = msec_format
+        self.datefmt = None
+        if ray.is_initialized():
+            self.ray_runtime_ctx: RuntimeContext = ray.get_runtime_context()
+            self.context = {}
+            self.context["worker_id"] = self.ray_runtime_ctx.get_worker_id()
+            self.context["node_id"] = self.ray_runtime_ctx.get_node_id()
+            self.context["job_id"] = self.ray_runtime_ctx.get_job_id()
+        else:
+            self.ray_runtime_ctx = None
+            self.context = {}
+
+    def usesTime(self) -> bool:
+        """
+        Overwritten to look for the attribute in the format dict values instead of the fmt string.
+        """
+        return "asctime" in self.fmt_dict.values()
+
+    def formatMessage(self, record) -> dict:
+        """
+        Overwritten to return a dictionary of the relevant LogRecord attributes instead of a string.
+        KeyError is raised if an unknown attribute is provided in the fmt_dict.
+        """
+        return {
+            fmt_key: record.__dict__[fmt_val]
+            for fmt_key, fmt_val in self.fmt_dict.items()
+        }
+
+    def format(self, record) -> str:
+        """
+        Mostly the same as the parent's class method, the difference being that a dict is manipulated and dumped as JSON
+        instead of a string.
+        """
+        record.message = record.getMessage()
+
+        if self.usesTime():
+            record.asctime = self.formatTime(record, self.datefmt)
+
+        message_dict = self.formatMessage(record)
+
+        if record.exc_info:
+            # Cache the traceback text to avoid converting it multiple times
+            # (it's constant anyway)
+            if not record.exc_text:
+                record.exc_text = self.formatException(record.exc_info)
+
+        if record.exc_text:
+            message_dict["exc_info"] = record.exc_text
+
+        if record.stack_info:
+            message_dict["stack_info"] = self.formatStack(record.stack_info)
+
+        if self.ray_runtime_ctx:
+            # only workers will have task ID
+            if (
+                self.ray_runtime_ctx.worker
+                and self.ray_runtime_ctx.worker.mode == ray._private.worker.WORKER_MODE
+            ):
+                self.context["task_id"] = self.ray_runtime_ctx.get_task_id()
+                self.context[
+                    "assigned_resources"
+                ] = self.ray_runtime_ctx.get_assigned_resources()
+
+            message_dict["ray_runtime_context"] = self.context
+
+        return json.dumps(message_dict, default=str)
+
+
 class DeltaCATLoggerAdapter(logging.LoggerAdapter):
     """
     Logger Adapter class with additional functionality
@@ -51,54 +145,6 @@ class DeltaCATLoggerAdapter(logging.LoggerAdapter):
             self.error(msg, *args, **kwargs)
 
 
-class RayRuntimeContextLoggerAdapter(DeltaCATLoggerAdapter):
-    """
-    Logger Adapter for injecting Ray Runtime Context into logging messages.
-    """
-
-    def __init__(self, logger: Logger, runtime_context: RuntimeContext):
-        super().__init__(logger, {})
-        self.runtime_context = runtime_context
-
-    def process(self, msg, kwargs):
-        """
-        Injects Ray Runtime Context details into each log message.
-
-        This may include information such as the raylet node ID, task/actor ID, job ID,
-        placement group ID of the worker, and assigned resources to the task/actor.
-
-        Args:
-            msg: The original log message
-            kwargs: Keyword arguments for the log message
-
-        Returns: A log message with Ray Runtime Context details
-
-        """
-        runtime_context_dict = self.runtime_context.get()
-        runtime_context_dict[
-            "worker_id"
-        ] = self.runtime_context.worker.core_worker.get_worker_id()
-        if self.runtime_context.get_task_id() or self.runtime_context.get_actor_id():
-            runtime_context_dict[
-                "pg_id"
-            ] = self.runtime_context.get_placement_group_id()
-            runtime_context_dict[
-                "assigned_resources"
-            ] = self.runtime_context.get_assigned_resources()
-
-        return "(ray_runtime_context=%s) -- %s" % (runtime_context_dict, msg), kwargs
-
-    def __reduce__(self):
-        """
-        Used to unpickle the class during Ray object store transfer.
-        """
-
-        def deserializer(*args):
-            return RayRuntimeContextLoggerAdapter(args[0], ray.get_runtime_context())
-
-        return deserializer, (self.logger,)
-
-
 def _add_logger_handler(logger: Logger, handler: Handler) -> Logger:
 
     logger.setLevel(logging.getLevelName("DEBUG"))
@@ -109,10 +155,10 @@ def _add_logger_handler(logger: Logger, handler: Handler) -> Logger:
 def _create_rotating_file_handler(
     log_directory: str,
     log_base_file_name: str,
-    logging_level: str = DEFAULT_LOG_LEVEL,
+    logging_level: Union[str, int] = DEFAULT_LOG_LEVEL,
     max_bytes_per_log_file: int = DEFAULT_MAX_BYTES_PER_LOG,
     backup_count: int = DEFAULT_BACKUP_COUNT,
-    logging_format: str = DEFAULT_LOG_FORMAT,
+    logging_format: Union[str, dict] = DEFAULT_LOG_FORMAT,
 ) -> FileHandler:
 
     if type(logging_level) is str:
@@ -126,7 +172,12 @@ def _create_rotating_file_handler(
         maxBytes=max_bytes_per_log_file,
         backupCount=backup_count,
     )
-    handler.setFormatter(logging.Formatter(logging_format))
+
+    if type(logging_format) is str:
+        handler.setFormatter(logging.Formatter(logging_format))
+    else:
+        handler.setFormatter(JsonFormatter(logging_format))
+
     handler.setLevel(logging_level)
     return handler
 
@@ -135,7 +186,8 @@ def _file_handler_exists(logger: Logger, log_dir: str, log_base_file_name: str)
 
     handler_exists = False
     base_file_path = os.path.join(log_dir, log_base_file_name)
-    if len(logger.handlers) > 0:
+
+    if logger.handlers:
         norm_base_file_path = os.path.normpath(base_file_path)
         handler_exists = any(
             [
@@ -149,49 +201,54 @@ def _file_handler_exists(logger: Logger, log_dir: str, log_base_file_name: str)
 
 def _configure_logger(
     logger: Logger,
-    log_level: str,
+    log_level: int,
     log_dir: str,
     log_base_file_name: str,
     debug_log_base_file_name: str,
 ) -> Union[Logger, LoggerAdapter]:
+    # This maintains log level of rotating file handlers
     primary_log_level = log_level
     logger.propagate = False
-    if log_level.upper() == "DEBUG":
+    if log_level <= logging.getLevelName("DEBUG"):
         if not _file_handler_exists(logger, log_dir, debug_log_base_file_name):
             handler = _create_rotating_file_handler(
                 log_dir, debug_log_base_file_name, "DEBUG"
             )
             _add_logger_handler(logger, handler)
-            primary_log_level = "INFO"
+            primary_log_level = logging.getLevelName("INFO")
     if not _file_handler_exists(logger, log_dir, log_base_file_name):
         handler = _create_rotating_file_handler(
             log_dir, log_base_file_name, primary_log_level
         )
         _add_logger_handler(logger, handler)
-    if ray.is_initialized():
-        ray_runtime_ctx = ray.get_runtime_context()
-        if ray_runtime_ctx.worker.connected:
-            logger = RayRuntimeContextLoggerAdapter(logger, ray_runtime_ctx)
-    else:
-        logger = DeltaCATLoggerAdapter(logger)
 
-    return logger
+    return DeltaCATLoggerAdapter(logger)
+
 
+def configure_deltacat_logger(
+    logger: Logger, level: int = None
+) -> Union[Logger, LoggerAdapter]:
+    if level is None:
+        level = logging.getLevelName(DELTACAT_SYS_LOG_LEVEL)
 
-def configure_deltacat_logger(logger: Logger) -> Union[Logger, LoggerAdapter]:
     return _configure_logger(
         logger,
-        DELTACAT_SYS_LOG_LEVEL,
+        level,
         DELTACAT_SYS_LOG_DIR,
         DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
         DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
     )
 
 
-def configure_application_logger(logger: Logger) -> Union[Logger, LoggerAdapter]:
+def configure_application_logger(
+    logger: Logger, level: int = None
+) -> Union[Logger, LoggerAdapter]:
+    if level is None:
+        level = logging.getLevelName(DELTACAT_APP_LOG_LEVEL)
+
     return _configure_logger(
         logger,
-        DELTACAT_APP_LOG_LEVEL,
+        level,
         DELTACAT_APP_LOG_DIR,
         DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
         DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,

deltacat/tests/aws/test_s3u.py

@@ -0,0 +1,12 @@
+import unittest
+from deltacat.aws.s3u import UuidBlockWritePathProvider, CapturedBlockWritePaths
+
+
+class TestUuidBlockWritePathProvider(unittest.TestCase):
+    def test_uuid_block_write_provider_sanity(self):
+        capture_object = CapturedBlockWritePaths()
+        provider = UuidBlockWritePathProvider(capture_object=capture_object)
+
+        result = provider("base_path")
+
+        self.assertRegex(result, r"^base_path/[\w-]{36}$")