deltacat 0.2.9__py3-none-any.whl → 0.2.11__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "0.2.9"
+__version__ = "0.2.11"
 
 
 __all__ = [

deltacat/aws/redshift/__init__.py

@@ -1,4 +1,6 @@
 from deltacat.aws.redshift.model.manifest import (
+    EntryFileParams,
+    EntryType,
     Manifest,
     ManifestAuthor,
     ManifestEntry,
@@ -7,6 +9,8 @@ from deltacat.aws.redshift.model.manifest import (
 )
 
 __all__ = [
+    "EntryFileParams",
+    "EntryType",
     "Manifest",
     "ManifestAuthor",
     "ManifestEntry",

deltacat/aws/redshift/model/manifest.py

@@ -5,12 +5,63 @@ import itertools
 import logging
 from typing import Any, Dict, List, Optional
 from uuid import uuid4
+from enum import Enum
 
 from deltacat import logs
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
+class EntryType(str, Enum):
+    """
+    Enum representing all possible content categories of an manifest entry file
+    """
+
+    DATA = "data"
+    POSITIONAL_DELETE = "positional_delete"
+    EQUALITY_DELETE = "equality_delete"
+
+    @classmethod
+    def get_default(cls):
+        return EntryType.DATA
+
+    @classmethod
+    def list(cls):
+        return [c.value for c in EntryType]
+
+
+class EntryFileParams(dict):
+    """
+    Represents parameters relevant to the underlying contents of manifest entry. Contains all parameters required to support DELETEs
+    equality_column_names: List of column names that would be used to determine row equality for equality deletes.  Relevant only to equality deletes
+    position: Ordinal position of a deleted row in the target data file identified by uri, starting at 0. Relevant only to positional deletes
+    """
+
+    @staticmethod
+    def of(
+        equality_column_names: Optional[List[str]] = None,
+        position: Optional[int] = None,
+    ) -> EntryFileParams:
+        entry_file_params = EntryFileParams()
+        if equality_column_names is not None:
+            entry_file_params["equality_column_names"] = equality_column_names
+        if position is not None:
+            entry_file_params["position"] = position
+        return entry_file_params
+
+    @property
+    def equality_column_names(self) -> Optional[List[str]]:
+        return self.get("equality_column_names")
+
+    @property
+    def url(self) -> Optional[str]:
+        return self.get("url")
+
+    @property
+    def position(self) -> Optional[int]:
+        return self.get("position")
+
+
 class Manifest(dict):
     @staticmethod
     def _build_manifest(
@@ -18,6 +69,7 @@ class Manifest(dict):
         entries: Optional[ManifestEntryList],
         author: Optional[ManifestAuthor] = None,
         uuid: str = None,
+        entry_type: Optional[EntryType] = None,
     ) -> Manifest:
         if not uuid:
             uuid = str(uuid4())
@@ -29,6 +81,8 @@ class Manifest(dict):
             manifest["entries"] = entries
         if author is not None:
             manifest["author"] = author
+        if entry_type is not None:
+            manifest["entry_type"] = entry_type.value
         return manifest
 
     @staticmethod
@@ -36,6 +90,7 @@ class Manifest(dict):
         entries: ManifestEntryList,
         author: Optional[ManifestAuthor] = None,
         uuid: str = None,
+        entry_type: Optional[EntryType] = None,
     ) -> Manifest:
         if not uuid:
             uuid = str(uuid4())
@@ -78,8 +133,9 @@ class Manifest(dict):
             content_type,
             content_encoding,
             total_source_content_length,
+            entry_type=entry_type,
         )
-        manifest = Manifest._build_manifest(meta, entries, author, uuid)
+        manifest = Manifest._build_manifest(meta, entries, author, uuid, entry_type)
         return manifest
 
     @staticmethod
@@ -128,6 +184,7 @@ class ManifestMeta(dict):
         source_content_length: Optional[int] = None,
         credentials: Optional[Dict[str, str]] = None,
         content_type_parameters: Optional[List[Dict[str, str]]] = None,
+        entry_type: Optional[EntryType] = None,
     ) -> ManifestMeta:
         manifest_meta = ManifestMeta()
         if record_count is not None:
@@ -144,6 +201,8 @@ class ManifestMeta(dict):
             manifest_meta["content_encoding"] = content_encoding
         if credentials is not None:
             manifest_meta["credentials"] = credentials
+        if entry_type is not None:
+            manifest_meta["entry_type"] = entry_type.value
         return manifest_meta
 
     @property
@@ -178,6 +237,13 @@ class ManifestMeta(dict):
     def credentials(self) -> Optional[Dict[str, str]]:
         return self.get("credentials")
 
+    @property
+    def entry_type(self) -> Optional[EntryType]:
+        val = self.get("entry_type")
+        if val is not None:
+            return EntryType(self["entry_type"])
+        return val
+
 
 class ManifestAuthor(dict):
     @staticmethod
@@ -206,6 +272,8 @@ class ManifestEntry(dict):
         mandatory: bool = True,
         uri: Optional[str] = None,
         uuid: Optional[str] = None,
+        entry_type: Optional[EntryType] = None,
+        entry_file_params: Optional[EntryFileParams] = None,
     ) -> ManifestEntry:
         manifest_entry = ManifestEntry()
         if not (uri or url):
@@ -222,6 +290,16 @@ class ManifestEntry(dict):
             manifest_entry["mandatory"] = mandatory
         if uuid is not None:
             manifest_entry["id"] = uuid
+        if entry_type is not None:
+            manifest_entry["entry_type"] = entry_type.value
+        if entry_file_params is not None:
+            if entry_file_params.get("url") != manifest_entry.get("url"):
+                msg = (
+                    f"Expected manifest entry url: {manifest_entry.url}"
+                    f" and entry_file_params: '{entry_file_params.url}' to match"
+                )
+                raise ValueError(msg)
+            manifest_entry["entry_file_params"] = entry_file_params
         return manifest_entry
 
     @staticmethod
@@ -268,6 +346,20 @@ class ManifestEntry(dict):
     def id(self) -> Optional[str]:
         return self.get("id")
 
+    @property
+    def entry_type(self) -> Optional[EntryType]:
+        val = self.get("entry_type")
+        if val is not None:
+            return EntryType(self["entry_type"])
+        return val
+
+    @property
+    def entry_file_params(self) -> Optional[EntryFileParams]:
+        val: Dict[str, Any] = self.get("entry_file_params")
+        if val is not None and not isinstance(val, EntryFileParams):
+            self["entry_file_params"] = val = EntryFileParams(val)
+        return val
+
 
 class ManifestEntryList(List[ManifestEntry]):
     @staticmethod

deltacat/compute/compactor_v2/utils/task_options.py

@@ -1,5 +1,7 @@
 import botocore
+import logging
 from typing import Dict, Optional, List, Tuple
+from deltacat import logs
 from deltacat.types.media import ContentEncoding, ContentType
 from deltacat.types.partial_download import PartialParquetParameters
 from deltacat.storage import (
@@ -17,6 +19,8 @@ from deltacat.compute.compactor_v2.constants import (
     PARQUET_TO_PYARROW_INFLATION,
 )
 
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
 
 def _get_parquet_type_params_if_exist(
     entry: ManifestEntry,
@@ -133,11 +137,15 @@ def hash_bucket_resource_options_provider(
     ray_custom_resources: Optional[Dict] = None,
     **kwargs,
 ) -> Dict:
+    debug_memory_params = {"hash_bucket_task_index": index}
     size_bytes = 0.0
     num_rows = 0
     total_pk_size = 0
 
     if not item.manifest or not item.manifest.entries:
+        logger.debug(
+            f"[Hash bucket task {index}]: No manifest entries, skipping memory allocation calculation"
+        )
         return {"CPU": 0.01}
 
     for entry in item.manifest.entries:
@@ -165,9 +173,17 @@ def hash_bucket_resource_options_provider(
     # total size + pk size + pk hash column + hash bucket index column
     # Refer to hash_bucket step for more details.
     total_memory = size_bytes + total_pk_size + num_rows * 20 + num_rows * 4
+    debug_memory_params["size_bytes"] = size_bytes
+    debug_memory_params["num_rows"] = num_rows
+    debug_memory_params["total_pk_size"] = total_pk_size
+    debug_memory_params["total_memory"] = total_memory
 
     # Consider buffer
     total_memory = total_memory * (1 + TOTAL_MEMORY_BUFFER_PERCENTAGE / 100.0)
+    debug_memory_params["total_memory_with_buffer"] = total_memory
+    logger.debug(
+        f"[Hash bucket task {index}]: Params used for calculating hash bucketing memory: {debug_memory_params}"
+    )
 
     return get_task_options(0.01, total_memory, ray_custom_resources)
 
@@ -186,10 +202,13 @@ def merge_resource_options_provider(
     deltacat_storage_kwargs: Optional[Dict] = {},
     **kwargs,
 ) -> Dict:
+    debug_memory_params = {"merge_task_index": index}
     hb_group_idx = item[0]
 
     data_size = hash_group_size_bytes.get(hb_group_idx, 0)
     num_rows = hash_group_num_rows.get(hb_group_idx, 0)
+    debug_memory_params["data_size_from_hash_group"] = data_size
+    debug_memory_params["num_rows_from_hash_group"] = num_rows
 
     # upper bound for pk size of incremental
     pk_size_bytes = data_size
@@ -205,10 +224,13 @@ def merge_resource_options_provider(
             round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
             / round_completion_info.compacted_pyarrow_write_result.file_bytes
         )
+        debug_memory_params["previous_inflation"] = previous_inflation
+
         average_record_size = (
             round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
             / round_completion_info.compacted_pyarrow_write_result.records
         )
+        debug_memory_params["average_record_size"] = average_record_size
 
         iterable = hash_group_index_to_hash_bucket_indices(
             hb_group_idx, round_completion_info.hash_bucket_count, num_hash_groups
@@ -256,7 +278,16 @@ def merge_resource_options_provider(
         + num_rows * 20
         + incremental_index_array_size
     )
+    debug_memory_params["data_size"] = data_size
+    debug_memory_params["num_rows"] = num_rows
+    debug_memory_params["pk_size_bytes"] = pk_size_bytes
+    debug_memory_params["incremental_index_array_size"] = incremental_index_array_size
+    debug_memory_params["total_memory"] = total_memory
 
     total_memory = total_memory * (1 + TOTAL_MEMORY_BUFFER_PERCENTAGE / 100.0)
+    debug_memory_params["total_memory_with_buffer"] = total_memory
+    logger.debug(
+        f"[Merge task {index}]: Params used for calculating merge memory: {debug_memory_params}"
+    )
 
     return get_task_options(0.01, total_memory, ray_custom_resources)

{deltacat-0.2.9.dist-info → deltacat-0.2.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.2.9
+Version: 0.2.11
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-0.2.9.dist-info → deltacat-0.2.11.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=9x12tKzGJVcmgVKVWjPCgZHxla7VH_PQf3HUvflyJZc,1777
+deltacat/__init__.py,sha256=35fh2bJ52CtfEDElc4OAwCeiXNY07uCDVhpQZzewvp8,1778
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=xqZf8CwysNYP2d39pf27OnXGStPREgBgIM-e2Tts-TI,199
 deltacat/logs.py,sha256=9XWuTBoWhhAF9rAL6t9veXmnAlJHsaqk0lTxteVPqyQ,5674
@@ -6,9 +6,9 @@ deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/clients.py,sha256=wWiqXyZPWXezdEbhQ7DLwEVnYV6KiitqzBc5B4UAwc0,6184
 deltacat/aws/constants.py,sha256=luXWMO_8eatq8f9NlFjNM7q362j77JwzTM2BEVS_8-8,353
 deltacat/aws/s3u.py,sha256=s2On5X3IQiCsCMKw4lpfV1GfKQVWOXNsdAmIJK5PEM0,18610
-deltacat/aws/redshift/__init__.py,sha256=fjuv3jWdPE8IgF4uSrL0YEqV3XUfqDULX3xV27ICceo,266
+deltacat/aws/redshift/__init__.py,sha256=7SvjG-dqox8zZUhFicTsUvpG5vXYDl_QQ3ohlHOgTKc,342
 deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/aws/redshift/model/manifest.py,sha256=N1RRGi1Rbou_9HQieoRCI_wE7eAf5eU_FTZ7dNPvUyY,9682
+deltacat/aws/redshift/model/manifest.py,sha256=ThgpdwzaWz493Zz9e8HSWwuxEheA1nDuypM3pe4vozk,12987
 deltacat/benchmarking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/benchmarking/benchmark_parquet_reads.py,sha256=2BctkvXAYcAxokLwMSTu4TQ6-HGqzkgYcVEAzPN2QQo,1709
 deltacat/benchmarking/conftest.py,sha256=6M9NJ71vnOpeMxG-Ly9UWRsgZmky5-1GTuoRD-OElng,1604
@@ -63,7 +63,7 @@ deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=rNKZisxGrLQOkw
 deltacat/compute/compactor_v2/utils/dedupe.py,sha256=62tFCY2iRP7I3-45GCIYs6_SJsQl8C5lBEr8gbNfbsw,1932
 deltacat/compute/compactor_v2/utils/io.py,sha256=jgIfwrfH2mTFUx1M0TgwZGGfrS4IXjP1PmqwaQmNAJM,5092
 deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=MAscmL35WfwN7Is72aFlD_cGhxtZgjRwwR5kS9Yn2uU,11393
-deltacat/compute/compactor_v2/utils/task_options.py,sha256=1-wIIXP0gDUJGdl8omMF5Q9kZs2oeu5WddgCnwBh3RE,8681
+deltacat/compute/compactor_v2/utils/task_options.py,sha256=LA1QbiDv3f9LJQwjKz3-YH3TpK3exL1c5acaGAOF57E,10210
 deltacat/compute/metastats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/metastats/meta_stats.py,sha256=78hN3aN5wLHUFJsZXuv2JLeqA35HZ8mLUWJDMslMj5Q,18731
 deltacat/compute/metastats/stats.py,sha256=8iUiSXOAjqiEeNP5RIb5gvhykBgpNHD5IKkB8zsPR0E,7363
@@ -181,8 +181,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=xOVkqL6o8qGsewGvzhMKxmCcqcFZDnNILuz5IGMgxSc,4991
-deltacat-0.2.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-0.2.9.dist-info/METADATA,sha256=XnXwpmM03bCIv-C-znj2rwE_6FDmI68H6zFL4icWMII,1779
-deltacat-0.2.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-deltacat-0.2.9.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-0.2.9.dist-info/RECORD,,
+deltacat-0.2.11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-0.2.11.dist-info/METADATA,sha256=paqGLkpxBHgQpjHWyXxTRSDC1WiiTkFCeVA64SHJg_Y,1780
+deltacat-0.2.11.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+deltacat-0.2.11.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-0.2.11.dist-info/RECORD,,