deltacat 1.1.3__tar.gz → 1.1.5__tar.gz

{deltacat-1.1.3 → deltacat-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.3
+Version: 1.1.5
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.3 → deltacat-1.1.5}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.3"
+__version__ = "1.1.5"
 
 
 __all__ = [

{deltacat-1.1.3 → deltacat-1.1.5}/deltacat/aws/constants.py

@@ -6,7 +6,3 @@ DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
 TIMEOUT_ERROR_CODES: List[str] = ["ReadTimeoutError", "ConnectTimeoutError"]
 AWS_REGION = env_string("AWS_REGION", "us-east-1")
-
-# Metric Names
-DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX = "download_manifest_entry"
-UPLOAD_SLICED_TABLE_METRIC_PREFIX = "upload_sliced_table"

{deltacat-1.1.3 → deltacat-1.1.5}/deltacat/aws/s3u.py

@@ -27,8 +27,6 @@ import deltacat.aws.clients as aws_utils
 from deltacat import logs
 from deltacat.aws.constants import (
     TIMEOUT_ERROR_CODES,
-    DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX,
-    UPLOAD_SLICED_TABLE_METRIC_PREFIX,
 )
 from deltacat.exceptions import NonRetryableError, RetryableError
 from deltacat.storage import (
@@ -54,7 +52,6 @@ from deltacat.types.tables import (
 )
 from deltacat.types.partial_download import PartialFileDownloadParams
 from deltacat.utils.common import ReadKwargsProvider
-from deltacat.utils.metrics import metrics
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -121,6 +118,32 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
             self.block_refs.append(block)
         return write_path
 
+    def __call__(
+        self,
+        base_path: str,
+        *,
+        filesystem: Optional[pa.filesystem.FileSystem] = None,
+        dataset_uuid: Optional[str] = None,
+        block: Optional[ObjectRef[Block]] = None,
+        block_index: Optional[int] = None,
+        file_format: Optional[str] = None,
+    ) -> str:
+        """
+        TODO: BlockWritePathProvider is deprecated as of Ray version 2.20.0. Please use FilenameProvider.
+        See: https://docs.ray.io/en/master/data/api/doc/ray.data.datasource.FilenameProvider.html
+        Also See: https://github.com/ray-project/deltacat/issues/299
+
+        Hence, this class only works with Ray version 2.20.0 or lower when used in Ray Dataset.
+        """
+        return self._get_write_path_for_block(
+            base_path,
+            filesystem=filesystem,
+            dataset_uuid=dataset_uuid,
+            block=block,
+            block_index=block_index,
+            file_format=file_format,
+        )
+
 
 class S3Url:
     def __init__(self, url: str):
@@ -243,7 +266,6 @@ def read_file(
         raise e
 
 
-@metrics(prefix=UPLOAD_SLICED_TABLE_METRIC_PREFIX)
 def upload_sliced_table(
     table: Union[LocalTable, DistributedDataset],
     s3_url_prefix: str,
@@ -352,7 +374,6 @@ def upload_table(
     return manifest_entries
 
 
-@metrics(prefix=DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX)
 def download_manifest_entry(
     manifest_entry: ManifestEntry,
     token_holder: Optional[Dict[str, Any]] = None,

{deltacat-1.1.3 → deltacat-1.1.5}/deltacat/compute/compactor/utils/round_completion_file.py

@@ -6,6 +6,7 @@ from deltacat.compute.compactor import RoundCompletionInfo
 from deltacat.storage import PartitionLocator
 from deltacat.aws import s3u as s3_utils
 from typing import Optional
+from deltacat.utils.metrics import metrics
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -18,6 +19,7 @@ def get_round_completion_file_s3_url(
     return f"{base_url}.json"
 
 
+@metrics
 def read_round_completion_file(
     bucket: str,
     source_partition_locator: PartitionLocator,
@@ -38,6 +40,7 @@ def read_round_completion_file(
     return round_completion_info
 
 
+@metrics
 def write_round_completion_file(
     bucket: Optional[str],
     source_partition_locator: Optional[PartitionLocator],

{deltacat-1.1.3 → deltacat-1.1.5}/deltacat/compute/compactor_v2/compaction_session.py

@@ -17,7 +17,11 @@ from deltacat.compute.compactor_v2.model.merge_input import MergeInput
 from deltacat.aws import s3u as s3_utils
 import deltacat
 from deltacat import logs
-from deltacat.compute.compactor import PyArrowWriteResult, RoundCompletionInfo
+from deltacat.compute.compactor import (
+    HighWatermark,
+    PyArrowWriteResult,
+    RoundCompletionInfo,
+)
 from deltacat.compute.compactor_v2.model.merge_result import MergeResult
 from deltacat.compute.compactor_v2.model.hash_bucket_result import HashBucketResult
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
@@ -37,6 +41,7 @@ from deltacat.compute.compactor_v2.deletes.utils import prepare_deletes
 from deltacat.storage import (
     Delta,
     DeltaLocator,
+    Manifest,
     Partition,
 )
 from deltacat.compute.compactor.model.compact_partition_params import (
@@ -50,6 +55,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
 from deltacat.compute.compactor_v2.steps import hash_bucket as hb
 from deltacat.compute.compactor_v2.utils import io
 from deltacat.compute.compactor.utils import round_completion_file as rcf
+from deltacat.utils.metrics import metrics
 
 from typing import List, Optional, Tuple
 from collections import defaultdict
@@ -73,6 +79,7 @@ if importlib.util.find_spec("memray"):
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
+@metrics
 def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
 
     assert (
@@ -94,7 +101,7 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
         round_completion_file_s3_url = None
         if new_partition:
             logger.info(f"Committing compacted partition to: {new_partition.locator}")
-            partition = params.deltacat_storage.commit_partition(
+            partition: Partition = params.deltacat_storage.commit_partition(
                 new_partition, **params.deltacat_storage_kwargs
             )
             logger.info(f"Committed compacted partition: {partition}")
@@ -148,9 +155,9 @@ def _execute_compaction(
         compaction_audit.set_total_cluster_memory_bytes(cluster_memory)
 
     # read the results from any previously completed compaction round
-    round_completion_info = None
-    high_watermark = None
-    previous_compacted_delta_manifest = None
+    round_completion_info: Optional[RoundCompletionInfo] = None
+    high_watermark: Optional[HighWatermark] = None
+    previous_compacted_delta_manifest: Optional[Manifest] = None
 
     if not params.rebase_source_partition_locator:
         round_completion_info = rcf.read_round_completion_file(
@@ -269,6 +276,7 @@ def _execute_compaction(
     total_hb_record_count = np.int64(0)
     telemetry_time_hb = 0
     if params.hash_bucket_count == 1:
+        logger.info("Hash bucket count set to 1. Running local merge")
         merge_start = time.monotonic()
         local_merge_input = generate_local_merge_input(
             params,
@@ -632,6 +640,19 @@ def _execute_compaction(
         f"partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
+    is_inplace_compacted: bool = (
+        params.source_partition_locator.partition_values
+        == params.destination_partition_locator.partition_values
+        and params.source_partition_locator.stream_id
+        == params.destination_partition_locator.stream_id
+    )
+    if is_inplace_compacted:
+        logger.info(
+            "Overriding round completion file source partition locator as in-place compacted. "
+            + f"Got compacted partition partition_id of {compacted_partition.locator.partition_id} "
+            f"and rcf source partition_id of {rcf_source_partition_locator.partition_id}."
+        )
+        rcf_source_partition_locator = compacted_partition.locator
     return (
         compacted_partition,
         new_round_completion_info,

{deltacat-1.1.3 → deltacat-1.1.5}/deltacat/compute/compactor_v2/constants.py

@@ -65,6 +65,3 @@ DISCOVER_DELTAS_METRIC_PREFIX = "discover_deltas"
 
 # Metric prefix for prepare deletes
 PREPARE_DELETES_METRIC_PREFIX = "prepare_deletes"
-
-# Metric prefix for materialize
-MATERIALIZE_METRIC_PREFIX = "delta_materialize"

{deltacat-1.1.3 → deltacat-1.1.5}/deltacat/compute/compactor_v2/utils/merge.py

@@ -31,14 +31,11 @@ from deltacat.compute.compactor_v2.deletes.delete_strategy import (
 from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
     DeleteFileEnvelope,
 )
-from deltacat.utils.metrics import metrics
-from deltacat.compute.compactor_v2.constants import MATERIALIZE_METRIC_PREFIX
 
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-@metrics(prefix=MATERIALIZE_METRIC_PREFIX)
 def materialize(
     input: MergeInput,
     task_index: int,

{deltacat-1.1.3 → deltacat-1.1.5}/deltacat/compute/compactor_v2/utils/task_options.py

@@ -21,6 +21,9 @@ from deltacat.compute.compactor_v2.constants import (
     PARQUET_TO_PYARROW_INFLATION,
 )
 
+from daft.exceptions import DaftTransientError
+
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
@@ -76,6 +79,7 @@ def get_task_options(
         botocore.exceptions.HTTPClientError,
         ConnectionError,
         TimeoutError,
+        DaftTransientError,
     ]
 
     return task_opts

deltacat-1.1.5/deltacat/logs.py

@@ -0,0 +1,255 @@
+import logging
+import os
+import json
+import pathlib
+from logging import FileHandler, Handler, Logger, LoggerAdapter, handlers
+from typing import Any, Dict, Optional, Union
+
+import ray
+from ray.runtime_context import RuntimeContext
+
+from deltacat.constants import (
+    DELTACAT_APP_LOG_LEVEL,
+    DELTACAT_SYS_LOG_LEVEL,
+    DELTACAT_APP_LOG_DIR,
+    DELTACAT_SYS_LOG_DIR,
+    DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
+    DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
+    DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
+    DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
+)
+
+DEFAULT_LOG_LEVEL = "INFO"
+DEFAULT_LOG_FORMAT = {
+    "level": "levelname",
+    "message": "message",
+    "loggerName": "name",
+    "processName": "processName",
+    "processID": "process",
+    "threadName": "threadName",
+    "timestamp": "asctime",
+    "filename": "filename",
+    "lineno": "lineno",
+}
+DEFAULT_MAX_BYTES_PER_LOG = 2 ^ 20 * 256  # 256 MiB
+DEFAULT_BACKUP_COUNT = 0
+
+
+class JsonFormatter(logging.Formatter):
+    """
+    Formatter that outputs JSON strings after parsing the LogRecord.
+
+    @param dict fmt_dict: Key: logging format attribute pairs. Defaults to {"message": "message"}.
+    @param str time_format: time.strftime() format string. Default: "%Y-%m-%dT%H:%M:%S"
+    @param str msec_format: Microsecond formatting. Appended at the end. Default: "%s.%03dZ"
+    """
+
+    def __init__(
+        self,
+        fmt_dict: dict = None,
+        time_format: str = "%Y-%m-%dT%H:%M:%S",
+        msec_format: str = "%s.%03dZ",
+    ):
+        self.fmt_dict = fmt_dict if fmt_dict is not None else {"message": "message"}
+        self.default_time_format = time_format
+        self.default_msec_format = msec_format
+        self.datefmt = None
+        if ray.is_initialized():
+            self.ray_runtime_ctx: RuntimeContext = ray.get_runtime_context()
+            self.context = {}
+            self.context["worker_id"] = self.ray_runtime_ctx.get_worker_id()
+            self.context["node_id"] = self.ray_runtime_ctx.get_node_id()
+            self.context["job_id"] = self.ray_runtime_ctx.get_job_id()
+        else:
+            self.ray_runtime_ctx = None
+            self.context = {}
+
+    def usesTime(self) -> bool:
+        """
+        Overwritten to look for the attribute in the format dict values instead of the fmt string.
+        """
+        return "asctime" in self.fmt_dict.values()
+
+    def formatMessage(self, record) -> dict:
+        """
+        Overwritten to return a dictionary of the relevant LogRecord attributes instead of a string.
+        KeyError is raised if an unknown attribute is provided in the fmt_dict.
+        """
+        return {
+            fmt_key: record.__dict__[fmt_val]
+            for fmt_key, fmt_val in self.fmt_dict.items()
+        }
+
+    def format(self, record) -> str:
+        """
+        Mostly the same as the parent's class method, the difference being that a dict is manipulated and dumped as JSON
+        instead of a string.
+        """
+        record.message = record.getMessage()
+
+        if self.usesTime():
+            record.asctime = self.formatTime(record, self.datefmt)
+
+        message_dict = self.formatMessage(record)
+
+        if record.exc_info:
+            # Cache the traceback text to avoid converting it multiple times
+            # (it's constant anyway)
+            if not record.exc_text:
+                record.exc_text = self.formatException(record.exc_info)
+
+        if record.exc_text:
+            message_dict["exc_info"] = record.exc_text
+
+        if record.stack_info:
+            message_dict["stack_info"] = self.formatStack(record.stack_info)
+
+        if self.ray_runtime_ctx:
+            # only workers will have task ID
+            if (
+                self.ray_runtime_ctx.worker
+                and self.ray_runtime_ctx.worker.mode == ray._private.worker.WORKER_MODE
+            ):
+                self.context["task_id"] = self.ray_runtime_ctx.get_task_id()
+                self.context[
+                    "assigned_resources"
+                ] = self.ray_runtime_ctx.get_assigned_resources()
+
+            message_dict["ray_runtime_context"] = self.context
+
+        return json.dumps(message_dict, default=str)
+
+
+class DeltaCATLoggerAdapter(logging.LoggerAdapter):
+    """
+    Logger Adapter class with additional functionality
+    """
+
+    def __init__(self, logger: Logger, extra: Optional[Dict[str, Any]] = {}):
+        super().__init__(logger, extra)
+
+    def debug_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.debug(msg, *args, **kwargs)
+
+    def info_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.info(msg, *args, **kwargs)
+
+    def warning_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.warning(msg, *args, **kwargs)
+
+    def error_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.error(msg, *args, **kwargs)
+
+
+def _add_logger_handler(logger: Logger, handler: Handler) -> Logger:
+
+    logger.setLevel(logging.getLevelName("DEBUG"))
+    logger.addHandler(handler)
+    return logger
+
+
+def _create_rotating_file_handler(
+    log_directory: str,
+    log_base_file_name: str,
+    logging_level: Union[str, int] = DEFAULT_LOG_LEVEL,
+    max_bytes_per_log_file: int = DEFAULT_MAX_BYTES_PER_LOG,
+    backup_count: int = DEFAULT_BACKUP_COUNT,
+    logging_format: Union[str, dict] = DEFAULT_LOG_FORMAT,
+) -> FileHandler:
+
+    if type(logging_level) is str:
+        logging_level = logging.getLevelName(logging_level.upper())
+    assert log_base_file_name, "log file name is required"
+    assert log_directory, "log directory is required"
+    log_dir_path = pathlib.Path(log_directory)
+    log_dir_path.mkdir(parents=True, exist_ok=True)
+    handler = handlers.RotatingFileHandler(
+        os.path.join(log_directory, log_base_file_name),
+        maxBytes=max_bytes_per_log_file,
+        backupCount=backup_count,
+    )
+
+    if type(logging_format) is str:
+        handler.setFormatter(logging.Formatter(logging_format))
+    else:
+        handler.setFormatter(JsonFormatter(logging_format))
+
+    handler.setLevel(logging_level)
+    return handler
+
+
+def _file_handler_exists(logger: Logger, log_dir: str, log_base_file_name: str) -> bool:
+
+    handler_exists = False
+    base_file_path = os.path.join(log_dir, log_base_file_name)
+
+    if logger.handlers:
+        norm_base_file_path = os.path.normpath(base_file_path)
+        handler_exists = any(
+            [
+                isinstance(handler, logging.FileHandler)
+                and os.path.normpath(handler.baseFilename) == norm_base_file_path
+                for handler in logger.handlers
+            ]
+        )
+    return handler_exists
+
+
+def _configure_logger(
+    logger: Logger,
+    log_level: int,
+    log_dir: str,
+    log_base_file_name: str,
+    debug_log_base_file_name: str,
+) -> Union[Logger, LoggerAdapter]:
+    # This maintains log level of rotating file handlers
+    primary_log_level = log_level
+    logger.propagate = False
+    if log_level <= logging.getLevelName("DEBUG"):
+        if not _file_handler_exists(logger, log_dir, debug_log_base_file_name):
+            handler = _create_rotating_file_handler(
+                log_dir, debug_log_base_file_name, "DEBUG"
+            )
+            _add_logger_handler(logger, handler)
+            primary_log_level = logging.getLevelName("INFO")
+    if not _file_handler_exists(logger, log_dir, log_base_file_name):
+        handler = _create_rotating_file_handler(
+            log_dir, log_base_file_name, primary_log_level
+        )
+        _add_logger_handler(logger, handler)
+
+    return DeltaCATLoggerAdapter(logger)
+
+
+def configure_deltacat_logger(
+    logger: Logger, level: int = None
+) -> Union[Logger, LoggerAdapter]:
+    if level is None:
+        level = logging.getLevelName(DELTACAT_SYS_LOG_LEVEL)
+
+    return _configure_logger(
+        logger,
+        level,
+        DELTACAT_SYS_LOG_DIR,
+        DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
+        DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
+    )
+
+
+def configure_application_logger(
+    logger: Logger, level: int = None
+) -> Union[Logger, LoggerAdapter]:
+    if level is None:
+        level = logging.getLevelName(DELTACAT_APP_LOG_LEVEL)
+
+    return _configure_logger(
+        logger,
+        level,
+        DELTACAT_APP_LOG_DIR,
+        DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
+        DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
+    )

deltacat-1.1.5/deltacat/tests/aws/test_s3u.py

@@ -0,0 +1,12 @@
+import unittest
+from deltacat.aws.s3u import UuidBlockWritePathProvider, CapturedBlockWritePaths
+
+
+class TestUuidBlockWritePathProvider(unittest.TestCase):
+    def test_uuid_block_write_provider_sanity(self):
+        capture_object = CapturedBlockWritePaths()
+        provider = UuidBlockWritePathProvider(capture_object=capture_object)
+
+        result = provider("base_path")
+
+        self.assertRegex(result, r"^base_path/[\w-]{36}$")