deltacat 1.1.17__py3-none-any.whl → 1.1.19__py3-none-any.whl

deltacat/compute/resource_estimation/model.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+from enum import Enum
+from typing import Optional
+
+
+class ResourceEstimationMethod(str, Enum):
+    """
+    The default approach executes certain methods in a specific order until the size
+    is estimated by any. The order is as follows:
+    1. CONTENT_TYPE_META
+    2. PREVIOUS_INFLATION
+    This method expects previous inflation and average record bytes to be passed.
+    """
+
+    DEFAULT = "DEFAULT"
+
+    """
+    This approach combines intelligent estimation and inflation based methods
+    and runs them in the order specified below:
+    1. INTELLIGENT_ESTIMATION
+    2. FILE_SAMPLING
+    3. PREVIOUS_INFLATION
+    """
+    DEFAULT_V2 = "DEFAULT_V2"
+
+    """
+    This approach strictly uses previous inflation and average record size to arrive
+    at a resource estimate. It requires users to pass in previous inflation and average
+    record sizes.
+    """
+    PREVIOUS_INFLATION = "PREVIOUS_INFLATION"
+
+    """
+    This approach is similar to PREVIOUS_INFLATION, but it determines average record size
+    and previous inflation by sampling few files in the given set of files.
+    """
+    FILE_SAMPLING = "FILE_SAMPLING"
+
+    """
+    This approach leverages metadata present in content type params.
+    """
+    CONTENT_TYPE_META = "CONTENT_TYPE_META"
+
+    """
+    This approach leverages parquet metadata and granularly estimate resources for each column and
+    then aggregate to arrive at most accurate estimation.
+    """
+    INTELLIGENT_ESTIMATION = "INTELLIGENT_ESTIMATION"
+
+
+class EstimateResourcesParams(dict):
+    """
+    This class represents the parameters required for estimating resources.
+    """
+
+    @staticmethod
+    def of(
+        resource_estimation_method: ResourceEstimationMethod = ResourceEstimationMethod.DEFAULT,
+        previous_inflation: Optional[float] = None,
+        parquet_to_pyarrow_inflation: Optional[float] = None,
+        average_record_size_bytes: Optional[float] = None,
+        max_files_to_sample: Optional[int] = None,
+    ) -> EstimateResourcesParams:
+        result = EstimateResourcesParams()
+        result["previous_inflation"] = previous_inflation
+        result["parquet_to_pyarrow_inflation"] = parquet_to_pyarrow_inflation
+        result["resource_estimation_method"] = resource_estimation_method
+        result["max_files_to_sample"] = max_files_to_sample
+        result["average_record_size_bytes"] = average_record_size_bytes
+        return result
+
+    @property
+    def resource_estimation_method(self) -> ResourceEstimationMethod:
+        return self["resource_estimation_method"]
+
+    @property
+    def max_files_to_sample(self) -> Optional[int]:
+        """
+        Applicable only for FILE_SAMPLING method. This parameter controls the
+        number of files to sample to arrive at average record sizes and previous inflation.
+        """
+        return self.get("max_files_to_sample")
+
+    @property
+    def previous_inflation(self) -> Optional[float]:
+        """
+        This parameter is required for PREVIOUS_INFLATION method. The inflation factor determines
+        a ratio of in-memory size to the on-disk size.
+        """
+        return self.get("previous_inflation")
+
+    @property
+    def parquet_to_pyarrow_inflation(self) -> Optional[float]:
+        """
+        This parameter is required for INTELLIGENT_ESTIMATION or CONTENT_TYPE_META method.
+        This determines inflation factor for parquet estimated size to pyarrow in-memory table size.
+        """
+        return self.get("parquet_to_pyarrow_inflation")
+
+    @property
+    def average_record_size_bytes(self) -> Optional[float]:
+        """
+        This parameter is required for PREVIOUS_INFLATION method. This determines average size of
+        records in bytes in a given file or entity.
+        """
+        return self.get("average_record_size_bytes")
+
+
+class OperationType(str, Enum):
+    """
+    This operation type is used when user would download the given entities using pyarrow library.
+    """
+
+    PYARROW_DOWNLOAD = "DOWNLOAD"
+
+
+class EstimatedResources(dict):
+    """
+    This class represents the resource requirements for a certain type of operation.
+    For example, downloading a delta requires certain amount of memory.
+    """
+
+    @staticmethod
+    def of(memory_bytes: float, statistics: Statistics = None) -> EstimatedResources:
+        result = EstimatedResources()
+        result["memory_bytes"] = memory_bytes
+        result["statistics"] = statistics
+        return result
+
+    @property
+    def memory_bytes(self) -> float:
+        return self["memory_bytes"]
+
+    @property
+    def statistics(self) -> Optional[Statistics]:
+        return self.get("statistics")
+
+
+class Statistics(dict):
+    """
+    This class represents the statistics of underlying objects that was used
+    to estimate the resource required.
+    """
+
+    @staticmethod
+    def of(
+        in_memory_size_bytes: float, record_count: int, on_disk_size_bytes: float
+    ) -> Statistics:
+        result = Statistics()
+        result["in_memory_size_bytes"] = in_memory_size_bytes
+        result["record_count"] = record_count
+        result["on_disk_size_bytes"] = on_disk_size_bytes
+        return result
+
+    @property
+    def in_memory_size_bytes(self) -> float:
+        return self["in_memory_size_bytes"]
+
+    @property
+    def record_count(self) -> int:
+        return self["record_count"]
+
+    @property
+    def on_disk_size_bytes(self) -> float:
+        return self["on_disk_size_bytes"]

deltacat/compute/resource_estimation/parquet.py

@@ -0,0 +1,108 @@
+import logging
+from typing import Optional
+from deltacat import logs
+from pyarrow.parquet import ColumnChunkMetaData
+from deltacat.constants import NULL_SIZE_BYTES
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def _observed_string_size(min_value: str, max_value: str) -> float:
+    """
+    Pyarrow uses few additional bytes to store each string.
+    """
+    return (len(min_value) + len(max_value)) / 2 + 4
+
+
+def _int96_size_estimator(
+    column_chunk_metadata: ColumnChunkMetaData,
+) -> float:
+    return column_chunk_metadata.num_values * 12
+
+
+def _int64_size_estimator(
+    column_chunk_metadata: ColumnChunkMetaData,
+) -> float:
+    return column_chunk_metadata.num_values * 8
+
+
+def _int32_size_estimator(
+    column_chunk_metadata: ColumnChunkMetaData,
+) -> float:
+    return column_chunk_metadata.num_values * 4
+
+
+def _boolean_size_estimator(
+    column_chunk_metadata: ColumnChunkMetaData,
+) -> float:
+    return column_chunk_metadata.num_values
+
+
+def _double_size_estimator(
+    column_chunk_metadata: ColumnChunkMetaData,
+) -> float:
+    return column_chunk_metadata.num_values * 8
+
+
+def _float_size_estimator(
+    column_chunk_metadata: ColumnChunkMetaData,
+) -> float:
+    return column_chunk_metadata.num_values * 4
+
+
+def _byte_array_size_estimator(
+    column_chunk_metadata: ColumnChunkMetaData,
+) -> float:
+    uncompressed_size = column_chunk_metadata.total_uncompressed_size
+    if column_chunk_metadata.is_stats_set:
+        statistics = column_chunk_metadata.statistics
+        if (
+            statistics.has_min_max
+            and isinstance(statistics.min, str)
+            and isinstance(statistics.max, str)
+        ):
+            return max(
+                uncompressed_size,
+                (
+                    statistics.num_values
+                    * _observed_string_size(statistics.min, statistics.max)
+                    + statistics.null_count * NULL_SIZE_BYTES
+                ),
+            )
+        else:
+            # A case of decimal
+            return max(column_chunk_metadata.num_values * 16, uncompressed_size)
+    else:
+        return uncompressed_size
+
+
+def _fixed_len_byte_array_size_estimator(
+    column_chunk_metadata: ColumnChunkMetaData,
+) -> float:
+    return _byte_array_size_estimator(column_chunk_metadata)
+
+
+_PHYSICAL_TYPE_TO_SIZE_ESTIMATOR = {
+    "INT96": _int96_size_estimator,
+    "INT64": _int64_size_estimator,
+    "INT32": _int32_size_estimator,
+    "BOOLEAN": _boolean_size_estimator,
+    "DOUBLE": _double_size_estimator,
+    "FLOAT": _float_size_estimator,
+    "BYTE_ARRAY": _byte_array_size_estimator,
+    "FIXED_LEN_BYTE_ARRAY": _fixed_len_byte_array_size_estimator,
+}
+
+
+def parquet_column_chunk_size_estimator(
+    column_meta: ColumnChunkMetaData,
+) -> Optional[float]:
+    physical_type = column_meta.physical_type
+    if physical_type in _PHYSICAL_TYPE_TO_SIZE_ESTIMATOR:
+        return _PHYSICAL_TYPE_TO_SIZE_ESTIMATOR[physical_type](column_meta)
+    else:
+        logger.warning(
+            f"Unsupported physical type: {physical_type}. "
+            "Returning total_uncompressed_size."
+        )
+        return column_meta.total_uncompressed_size

deltacat/constants.py

@@ -28,6 +28,8 @@ DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME = env_string(
     "DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME",
     "application.debug.log",
 )
+# A json context which will be logged along with other context args.
+DELTACAT_LOGGER_CONTEXT = env_string("DELTACAT_LOGGER_CONTEXT", None)
 
 # Byte Units
 BYTES_PER_KIBIBYTE = 2**10
@@ -53,3 +55,6 @@ PYARROW_INFLATION_MULTIPLIER = 2.5
 PYARROW_INFLATION_MULTIPLIER_ALL_COLUMNS = 6
 
 MEMORY_TO_HASH_BUCKET_COUNT_RATIO = 0.0512 * BYTES_PER_TEBIBYTE
+
+# The number of bytes allocated to null values in string physical type in parquet
+NULL_SIZE_BYTES = 4

deltacat/exceptions.py

@@ -299,7 +299,7 @@ def _categorize_tenacity_error(e: tenacity.RetryError):
 def _categorize_dependency_pyarrow_error(e: ArrowException):
     if isinstance(e, ArrowInvalid):
         raise DependencyPyarrowInvalidError(
-            f"Pyarrow Invalid error occurred. Reason: {e}"
+            f"Pyarrow Invalid error occurred. {e}"
         ) from e
     elif isinstance(e, ArrowCapacityError):
         raise DependencyPyarrowCapacityError("Pyarrow Capacity error occurred.") from e
@@ -308,9 +308,7 @@ def _categorize_dependency_pyarrow_error(e: ArrowException):
 
 
 def _categorize_assertion_error(e: BaseException):
-    raise ValidationError(
-        f"One of the assertions in DeltaCAT has failed. Reason: {e}"
-    ) from e
+    raise ValidationError(f"One of the assertions in DeltaCAT has failed. {e}") from e
 
 
 def _categorize_daft_error(e: DaftCoreException):

deltacat/logs.py

@@ -17,6 +17,7 @@ from deltacat.constants import (
     DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
     DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
     DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
+    DELTACAT_LOGGER_CONTEXT,
 )
 
 DEFAULT_LOG_LEVEL = "INFO"
@@ -66,6 +67,13 @@ class JsonFormatter(logging.Formatter):
             self.ray_runtime_ctx = None
             self.context = {}
 
+        if DELTACAT_LOGGER_CONTEXT is not None:
+            try:
+                env_context = json.loads(DELTACAT_LOGGER_CONTEXT)
+                self.additional_context.update(env_context)
+            except Exception:
+                pass
+
     def usesTime(self) -> bool:
         """
         Overwritten to look for the attribute in the format dict values instead of the fmt string.

deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py

@@ -848,6 +848,83 @@ MULTIPLE_ROUNDS_TEST_CASES = {
         assert_compaction_audit=None,
         num_rounds=3,
     ),
+    # 4 input deltas (3 upsert, 1 delete delta), 2 rounds requested
+    # Expect to see a table that aggregates 10 records total
+    # (12 upserts - 2 deletes (null PK) = 10 records)
+    # (dropDuplicates = False)
+    "9-multiple-rounds-delete-deltas-with-null-pk": MultipleRoundsTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=ZERO_VALUED_SORT_KEY,
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=[
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([None, 11, 12, 13]),
+                        pa.array(["a", "b", "c", "d"]),
+                    ],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            ),
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([14, 15, 16, 17]),
+                        pa.array(["e", "f", "g", "h"]),
+                    ],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            ),
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([18, 19, 20, 21]),
+                        pa.array(["i", "j", "k", "l"]),
+                    ],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array([None, 11]), pa.array(["a", "b"])],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["pk_col_1", "col_1"]),
+            ),
+        ],
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(12, 22)]),
+                pa.array(["c", "d", "e", "f", "g", "h", "i", "j", "k", "l"]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(12, 22)]),
+                pa.array(["c", "d", "e", "f", "g", "h", "i", "j", "k", "l"]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=False,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+        num_rounds=2,
+    ),
 }
 
 MULTIPLE_ROUNDS_TEST_CASES = with_compactor_version_func_test_param(