deltacat 1.1.36__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/compute/compactor_v2/utils/task_options.py

@@ -1,16 +1,11 @@
 import logging
 from typing import Dict, Optional, List, Tuple, Any
 from deltacat import logs
-from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
-from deltacat.compute.compactor_v2.constants import (
-    AVERAGE_RECORD_SIZE_BYTES as DEFAULT_AVERAGE_RECORD_SIZE_BYTES,
-)
 from deltacat.compute.compactor_v2.model.merge_file_group import (
     LocalMergeFileGroupsProvider,
 )
 from deltacat.storage import (
     Manifest,
-    ManifestEntry,
     interface as unimplemented_deltacat_storage,
 )
 from deltacat.compute.compactor.model.delta_annotated import DeltaAnnotated
@@ -86,27 +81,16 @@ def _get_merge_task_options(
         and compacted_delta_manifest
         and round_completion_info.hb_index_to_entry_range
     ):
-        logger.debug_conditional(
-            f"[Merge task {index}]: Using previous compaction rounds to calculate merge memory: {round_completion_info.compacted_pyarrow_write_result}",
-            memory_logs_enabled,
-        )
-        previous_inflation: float = (
-            (
-                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-                / round_completion_info.compacted_pyarrow_write_result.file_bytes
-            )
-            if round_completion_info.compacted_pyarrow_write_result.file_bytes
-            else PYARROW_INFLATION_MULTIPLIER
+
+        previous_inflation = (
+            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+            / round_completion_info.compacted_pyarrow_write_result.file_bytes
         )
         debug_memory_params["previous_inflation"] = previous_inflation
 
-        average_record_size: float = (
-            (
-                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-                / round_completion_info.compacted_pyarrow_write_result.records
-            )
-            if round_completion_info.compacted_pyarrow_write_result.records
-            else DEFAULT_AVERAGE_RECORD_SIZE_BYTES
+        average_record_size = (
+            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+            / round_completion_info.compacted_pyarrow_write_result.records
         )
         debug_memory_params["average_record_size"] = average_record_size
 
@@ -122,36 +106,31 @@ def _get_merge_task_options(
                 str(hb_idx)
             ]
             for entry_index in range(entry_start, entry_end):
-                entry: ManifestEntry = compacted_delta_manifest.entries[entry_index]
-                current_entry_size: float = (
-                    estimate_manifest_entry_size_bytes(
-                        entry=entry,
-                        operation_type=OperationType.PYARROW_DOWNLOAD,
-                        estimate_resources_params=estimate_resources_params,
-                    )
-                    or 0.0
+                entry = compacted_delta_manifest.entries[entry_index]
+
+                current_entry_size = estimate_manifest_entry_size_bytes(
+                    entry=entry,
+                    operation_type=OperationType.PYARROW_DOWNLOAD,
+                    estimate_resources_params=estimate_resources_params,
                 )
-                current_entry_rows: int = (
-                    estimate_manifest_entry_num_rows(
-                        entry=entry,
-                        operation_type=OperationType.PYARROW_DOWNLOAD,
-                        estimate_resources_params=estimate_resources_params,
-                    )
-                    or 0
+                current_entry_rows = estimate_manifest_entry_num_rows(
+                    entry=entry,
+                    operation_type=OperationType.PYARROW_DOWNLOAD,
+                    estimate_resources_params=estimate_resources_params,
                 )
-                # NOTE: We can treat the current_entry_size and current_entry_rows as 0 as a None estimated entry size implies a 0 value
+
                 data_size += current_entry_size
                 num_rows += current_entry_rows
+
                 if primary_keys:
-                    pk_size: Optional[
-                        float
-                    ] = estimate_manifest_entry_column_size_bytes(
+                    pk_size = estimate_manifest_entry_column_size_bytes(
                         entry=entry,
                         columns=primary_keys,
                         operation_type=OperationType.PYARROW_DOWNLOAD,
                         estimate_resources_params=estimate_resources_params,
                     )
-                    if not pk_size:
+
+                    if pk_size is None:
                         pk_size_bytes += current_entry_size
                     else:
                         pk_size_bytes += pk_size
@@ -180,6 +159,7 @@ def _get_merge_task_options(
         f"[Merge task {index}]: Params used for calculating merge memory: {debug_memory_params}",
         memory_logs_enabled,
     )
+
     return _get_task_options(0.01, total_memory, ray_custom_resources)
 
 

deltacat/compute/converter/constants.py

@@ -0,0 +1,4 @@
+DEFAULT_CONVERTER_TASK_MAX_PARALLELISM = 4096
+
+# Safe limit ONLY considering CPU limit, typically 32 for a 8x-large worker
+DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD = 30

deltacat/compute/converter/converter_session.py

@@ -0,0 +1,143 @@
+# from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties
+from deltacat.utils.ray_utils.concurrency import (
+    invoke_parallel,
+    task_resource_options_provider,
+)
+import ray
+import functools
+from deltacat.compute.converter.utils.convert_task_options import (
+    convert_resource_options_provider,
+)
+import logging
+from deltacat import logs
+from deltacat.compute.converter.model.converter_session_params import (
+    ConverterSessionParams,
+)
+
+
+from deltacat.compute.converter.constants import DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD
+from deltacat.compute.converter.steps.convert import convert
+from deltacat.compute.converter.model.convert_input import ConvertInput
+from deltacat.compute.converter.pyiceberg.overrides import (
+    fetch_all_bucket_files,
+    parquet_files_dict_to_iceberg_data_files,
+)
+from deltacat.compute.converter.utils.converter_session_utils import (
+    construct_iceberg_table_prefix,
+)
+from deltacat.compute.converter.pyiceberg.update_snapshot_overrides import (
+    commit_replace_snapshot,
+    commit_append_snapshot,
+)
+from deltacat.compute.converter.pyiceberg.catalog import load_table
+from deltacat.compute.converter.utils.converter_session_utils import (
+    group_all_files_to_each_bucket,
+)
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def converter_session(params: ConverterSessionParams, **kwargs):
+    """
+    Convert equality delete to position delete.
+    Compute and memory heavy work from downloading equality delete table and compute position deletes
+    will be executed on Ray remote tasks.
+    """
+
+    catalog = params.catalog
+    table_name = params.iceberg_table_name
+    iceberg_table = load_table(catalog, table_name)
+    enforce_primary_key_uniqueness = params.enforce_primary_key_uniqueness
+    data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(
+        iceberg_table
+    )
+    convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
+        data_file_dict=data_file_dict,
+        equality_delete_dict=equality_delete_dict,
+        pos_delete_dict=pos_delete_dict,
+    )
+    iceberg_warehouse_bucket_name = params.iceberg_warehouse_bucket_name
+    iceberg_namespace = params.iceberg_namespace
+    iceberg_table_warehouse_prefix = construct_iceberg_table_prefix(
+        iceberg_warehouse_bucket_name=iceberg_warehouse_bucket_name,
+        table_name=table_name,
+        iceberg_namespace=iceberg_namespace,
+    )
+    merge_keys = params.merge_keys
+    # Using table identifier fields as merge keys if merge keys not provided
+    if not merge_keys:
+        identifier_fields_set = iceberg_table.schema().identifier_field_names()
+        identifier_fields = list(identifier_fields_set)
+    else:
+        identifier_fields = merge_keys
+    if len(identifier_fields) > 1:
+        raise NotImplementedError(
+            f"Multiple identifier fields lookup not supported yet."
+        )
+    convert_options_provider = functools.partial(
+        task_resource_options_provider,
+        resource_amount_provider=convert_resource_options_provider,
+    )
+
+    # TODO (zyiqin): max_parallel_data_file_download should be determined by memory requirement for each bucket.
+    #  Specifically, for case when files for one bucket memory requirement exceed one worker node's memory limit, WITHOUT rebasing with larger hash bucket count,
+    #  1. We can control parallel files to download by adjusting max_parallel_data_file_download.
+    #  2. Implement two-layer converter tasks, with convert tasks to spin up child convert tasks.
+    #  Note that approach 2 will ideally require shared object store to avoid download equality delete files * number of child tasks times.
+    max_parallel_data_file_download = DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD
+
+    compact_small_files = params.compact_small_files
+    position_delete_for_multiple_data_files = (
+        params.position_delete_for_multiple_data_files
+    )
+    task_max_parallelism = params.task_max_parallelism
+
+    def convert_input_provider(index, item):
+        return {
+            "convert_input": ConvertInput.of(
+                files_for_each_bucket=item,
+                convert_task_index=index,
+                iceberg_table_warehouse_prefix=iceberg_table_warehouse_prefix,
+                identifier_fields=identifier_fields,
+                compact_small_files=compact_small_files,
+                enforce_primary_key_uniqueness=enforce_primary_key_uniqueness,
+                position_delete_for_multiple_data_files=position_delete_for_multiple_data_files,
+                max_parallel_data_file_download=max_parallel_data_file_download,
+            )
+        }
+
+    # Ray remote task: convert
+    # Assuming that memory consume by each bucket doesn't exceed one node's memory limit.
+    # TODO: Add split mechanism to split large buckets
+    convert_tasks_pending = invoke_parallel(
+        items=convert_input_files_for_all_buckets.items(),
+        ray_task=convert,
+        max_parallelism=task_max_parallelism,
+        options_provider=convert_options_provider,
+        kwargs_provider=convert_input_provider,
+    )
+    to_be_deleted_files_list = []
+    to_be_added_files_dict_list = []
+    convert_results = ray.get(convert_tasks_pending)
+    for convert_result in convert_results:
+        to_be_deleted_files_list.extend(convert_result[0].values())
+        to_be_added_files_dict_list.append(convert_result[1])
+
+    new_position_delete_files = parquet_files_dict_to_iceberg_data_files(
+        io=iceberg_table.io,
+        table_metadata=iceberg_table.metadata,
+        files_dict_list=to_be_added_files_dict_list,
+    )
+
+    if not to_be_deleted_files_list:
+        commit_append_snapshot(
+            iceberg_table=iceberg_table,
+            new_position_delete_files=new_position_delete_files,
+        )
+    else:
+        commit_replace_snapshot(
+            iceberg_table=iceberg_table,
+            # equality_delete_files + data file that all rows are deleted
+            to_be_deleted_files_list=to_be_deleted_files_list,
+            new_position_delete_files=new_position_delete_files,
+        )

deltacat/compute/converter/model/convert_input.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+from typing import Dict, List
+from deltacat.compute.converter.model.convert_input_files import ConvertInputFiles
+
+
+class ConvertInput(Dict):
+    @staticmethod
+    def of(
+        convert_input_files,
+        convert_task_index,
+        iceberg_table_warehouse_prefix,
+        identifier_fields,
+        compact_small_files,
+        enforce_primary_key_uniqueness,
+        position_delete_for_multiple_data_files,
+        max_parallel_data_file_download,
+        s3_file_system,
+    ) -> ConvertInput:
+
+        result = ConvertInput()
+        result["convert_input_files"] = convert_input_files
+        result["convert_task_index"] = convert_task_index
+        result["identifier_fields"] = identifier_fields
+        result["iceberg_table_warehouse_prefix"] = iceberg_table_warehouse_prefix
+        result["compact_small_files"] = compact_small_files
+        result["enforce_primary_key_uniqueness"] = enforce_primary_key_uniqueness
+        result[
+            "position_delete_for_multiple_data_files"
+        ] = position_delete_for_multiple_data_files
+        result["max_parallel_data_file_download"] = max_parallel_data_file_download
+        result["s3_file_system"] = s3_file_system
+
+        return result
+
+    @property
+    def convert_input_files(self) -> ConvertInputFiles:
+        return self["convert_input_files"]
+
+    @property
+    def identifier_fields(self) -> List[str]:
+        return self["identifier_fields"]
+
+    @property
+    def convert_task_index(self) -> int:
+        return self["convert_task_index"]
+
+    @property
+    def iceberg_table_warehouse_prefix(self) -> str:
+        return self["iceberg_table_warehouse_prefix"]
+
+    @property
+    def compact_small_files(self) -> bool:
+        return self["compact_small_files"]
+
+    @property
+    def enforce_primary_key_uniqueness(self) -> bool:
+        return self["enforce_primary_key_uniqueness"]
+
+    @property
+    def position_delete_for_multiple_data_files(self) -> bool:
+        return self["position_delete_for_multiple_data_files"]
+
+    @property
+    def max_parallel_data_file_download(self) -> int:
+        return self["max_parallel_data_file_download"]
+
+    @property
+    def s3_file_system(self):
+        return self["s3_file_system"]

deltacat/compute/converter/model/convert_input_files.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+from typing import Dict
+
+
+class ConvertInputFiles(Dict):
+    @staticmethod
+    def of(
+        partition_value,
+        all_data_files_for_dedupe=None,
+        applicable_data_files=None,
+        applicable_equality_delete_files=None,
+        existing_position_delete_files=None,
+    ) -> ConvertInputFiles:
+
+        result = ConvertInputFiles()
+        result["partition_value"] = partition_value
+        result["all_data_files_for_dedupe"] = all_data_files_for_dedupe
+        result["applicable_data_files"] = applicable_data_files
+        result["applicable_equality_delete_files"] = applicable_equality_delete_files
+        result["existing_position_delete_files"] = existing_position_delete_files
+        return result
+
+    @property
+    def partition_value(self):
+        return self["partition_value"]
+
+    @property
+    def all_data_files_for_dedupe(self):
+        return self["all_data_files_for_dedupe"]
+
+    @property
+    def applicable_data_files(self):
+        return self["applicable_data_files"]
+
+    @property
+    def applicable_equality_delete_files(self):
+        return self["applicable_equality_delete_files"]
+
+    @property
+    def existing_position_delete_files(self):
+        return self["existing_position_delete_files"]
+
+    @partition_value.setter
+    def partition_value(self, partition_value):
+        self["partition_value"] = partition_value
+
+    @all_data_files_for_dedupe.setter
+    def all_data_files_for_dedupe(self, all_data_files_for_dedupe):
+        self["all_data_files_for_dedupe"] = all_data_files_for_dedupe
+
+    @applicable_data_files.setter
+    def applicable_data_files(self, applicable_data_files):
+        self["applicable_data_files"] = applicable_data_files
+
+    @applicable_equality_delete_files.setter
+    def applicable_equality_delete_files(self, applicable_equality_delete_files):
+        self["applicable_equality_delete_files"] = applicable_equality_delete_files
+
+    @existing_position_delete_files.setter
+    def existing_position_delete_files(self, existing_position_delete_files):
+        self["existing_position_delete_files"] = existing_position_delete_files

deltacat/compute/converter/model/converter_session_params.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+from typing import Optional, Dict
+from deltacat.compute.converter.constants import DEFAULT_CONVERTER_TASK_MAX_PARALLELISM
+
+
+class ConverterSessionParams(dict):
+    """
+    This class represents the parameters passed to convert_ (deltacat/compute/compactor/compaction_session.py)
+    """
+
+    @staticmethod
+    def of(params: Optional[Dict]) -> ConverterSessionParams:
+        params = {} if params is None else params
+        assert params.get("catalog") is not None, "catalog is a required arg"
+        assert (
+            params.get("iceberg_table_name") is not None
+        ), "iceberg_table_name is a required arg"
+        assert (
+            params.get("iceberg_warehouse_bucket_name") is not None
+        ), "iceberg_warehouse_bucket_name is a required arg"
+        assert (
+            params.get("iceberg_namespace") is not None
+        ), "iceberg_namespace is a required arg"
+        result = ConverterSessionParams(params)
+
+        result.enforce_primary_key_uniqueness = params.get(
+            "enforce_primary_key_uniqueness", False
+        )
+        result.compact_small_files = params.get("compact_small_files", False)
+
+        # For Iceberg v3 spec, option to produce delete vector that can establish 1:1 mapping with data files.
+        result.position_delete_for_multiple_data_files = params.get(
+            "position_delete_for_multiple_data_files", True
+        )
+        result.task_max_parallelism = params.get(
+            "task_max_parallelism", DEFAULT_CONVERTER_TASK_MAX_PARALLELISM
+        )
+        result.merge_keys = params.get("merge_keys", None)
+        return result
+
+    @property
+    def catalog(self):
+        return self["catalog"]
+
+    @property
+    def iceberg_table_name(self) -> str:
+        return self["iceberg_table_name"]
+
+    @property
+    def iceberg_warehouse_bucket_name(self) -> str:
+        return self["iceberg_warehouse_bucket_name"]
+
+    @property
+    def iceberg_namespace(self) -> str:
+        return self["iceberg_namespace"]
+
+    @property
+    def enforce_primary_key_uniqueness(self) -> bool:
+        return self["enforce_primary_key_uniqueness"]
+
+    @enforce_primary_key_uniqueness.setter
+    def enforce_primary_key_uniqueness(self, enforce_primary_key_uniqueness) -> None:
+        self["enforce_primary_key_uniqueness"] = enforce_primary_key_uniqueness
+
+    @property
+    def compact_small_files(self) -> bool:
+        return self["compact_small_files"]
+
+    @compact_small_files.setter
+    def compact_small_files(self, compact_small_files) -> None:
+        self["compact_small_files"] = compact_small_files
+
+    @property
+    def position_delete_for_multiple_data_files(self) -> bool:
+        return self["position_delete_for_multiple_data_files"]
+
+    @position_delete_for_multiple_data_files.setter
+    def position_delete_for_multiple_data_files(
+        self, position_delete_for_multiple_data_files
+    ) -> None:
+        self[
+            "position_delete_for_multiple_data_files"
+        ] = position_delete_for_multiple_data_files
+
+    @property
+    def task_max_parallelism(self) -> str:
+        return self["task_max_parallelism"]
+
+    @task_max_parallelism.setter
+    def task_max_parallelism(self, task_max_parallelism) -> None:
+        self["task_max_parallelism"] = task_max_parallelism
+
+    @property
+    def merge_keys(self) -> str:
+        return self["merge_keys"]
+
+    @merge_keys.setter
+    def merge_keys(self, merge_keys) -> None:
+        self["merge_keys"] = merge_keys

deltacat/compute/converter/pyiceberg/catalog.py

@@ -0,0 +1,75 @@
+from typing import Optional
+
+
+def load_catalog(iceberg_catalog_name, iceberg_catalog_properties):
+    catalog = load_catalog(
+        name=iceberg_catalog_name,
+        **iceberg_catalog_properties,
+    )
+    return catalog
+
+
+def get_s3_path(
+    bucket_name: str,
+    database_name: Optional[str] = None,
+    table_name: Optional[str] = None,
+) -> str:
+    result_path = f"s3://{bucket_name}"
+    if database_name is not None:
+        result_path += f"/{database_name}.db"
+
+    if table_name is not None:
+        result_path += f"/{table_name}"
+    return result_path
+
+
+def get_bucket_name():
+    return "metadata-py4j-zyiqin1"
+
+
+def get_s3_prefix():
+    return get_s3_path(get_bucket_name())
+
+
+def get_credential():
+    import boto3
+
+    boto3_session = boto3.Session()
+    credentials = boto3_session.get_credentials()
+    return credentials
+
+
+def get_glue_catalog():
+    from pyiceberg.catalog import load_catalog
+
+    credential = get_credential()
+    # Credentials are refreshable, so accessing your access key / secret key
+    # separately can lead to a race condition. Use this to get an actual matched
+    # set.
+    credential = credential.get_frozen_credentials()
+    access_key_id = credential.access_key
+    secret_access_key = credential.secret_key
+    session_token = credential.token
+    s3_path = get_s3_prefix()
+    glue_catalog = load_catalog(
+        "glue",
+        **{
+            "warehouse": s3_path,
+            "type": "glue",
+            "aws_access_key_id": access_key_id,
+            "aws_secret_access_key": secret_access_key,
+            "aws_session_token": session_token,
+            "region_name": "us-east-1",
+            "s3.access-key-id": access_key_id,
+            "s3.secret-access-key": secret_access_key,
+            "s3.session-token": session_token,
+            "s3.region": "us-east-1",
+        },
+    )
+
+    return glue_catalog
+
+
+def load_table(catalog, table_name):
+    loaded_table = catalog.load_table(table_name)
+    return loaded_table