deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/converter/converter_session.py

@@ -1,4 +1,4 @@
-# from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties
+from deltacat.constants import DEFAULT_NAMESPACE
 from deltacat.utils.ray_utils.concurrency import (
     invoke_parallel,
     task_resource_options_provider,
@@ -13,14 +13,12 @@ from deltacat import logs
 from deltacat.compute.converter.model.converter_session_params import (
     ConverterSessionParams,
 )
-
-
+from typing import Dict, List, Any, Callable
 from deltacat.compute.converter.constants import DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD
 from deltacat.compute.converter.steps.convert import convert
 from deltacat.compute.converter.model.convert_input import ConvertInput
 from deltacat.compute.converter.pyiceberg.overrides import (
     fetch_all_bucket_files,
-    parquet_files_dict_to_iceberg_data_files,
 )
 from deltacat.compute.converter.utils.converter_session_utils import (
     construct_iceberg_table_prefix,
@@ -33,48 +31,112 @@ from deltacat.compute.converter.pyiceberg.catalog import load_table
 from deltacat.compute.converter.utils.converter_session_utils import (
     group_all_files_to_each_bucket,
 )
+from deltacat.compute.converter.model.convert_result import ConvertResult
+from deltacat.compute.converter.utils.converter_session_utils import (
+    _get_snapshot_action_description,
+    _determine_snapshot_type,
+    SnapshotType,
+)
+
+from pyiceberg.manifest import DataFile
+from pyiceberg.table.metadata import TableMetadata
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-def converter_session(params: ConverterSessionParams, **kwargs):
+def converter_session(params: ConverterSessionParams, **kwargs: Any) -> TableMetadata:
     """
-    Convert equality delete to position delete.
-    Compute and memory heavy work from downloading equality delete table and compute position deletes
-    will be executed on Ray remote tasks.
+    Convert equality deletes to position deletes with option to enforce primary key uniqueness.
+
+    This function processes Iceberg table files to convert equality delete files to position delete files.
+    It can optionally enforce primary key uniqueness by keeping only the latest version of each
+    primary key across all data files.
+
+    **Memory Requirements:**
+    - Minimum 512MB of free memory is required to run the converter
+
+    **Process Overview:**
+    1. Fetches all bucket files (data files, equality deletes, position deletes)
+    2. Groups files by bucket for parallel processing
+    3. Converts equality deletes to position deletes using Ray parallel tasks
+    4. Enforces primary key uniqueness if enabled
+    5. Commits appropriate snapshot (append, replace, or delete) to the Iceberg table
+
+
+    Args:
+        params: ConverterSessionParams containing all configuration parameters
+            - catalog: Iceberg catalog instance
+            - iceberg_table_name: Name of the target Iceberg table
+            - enforce_primary_key_uniqueness: Whether to enforce PK uniqueness
+            - iceberg_warehouse_bucket_name: S3 bucket for Iceberg warehouse
+            - iceberg_namespace: Iceberg namespace
+            - merge_keys: Optional list of merge key fields (uses table identifier fields if not provided)
+            - compact_previous_position_delete_files: Whether to compact existing position delete files
+            - task_max_parallelism: Maximum number of parallel Ray tasks
+            - s3_client_kwargs: Additional S3 client configuration
+            - s3_file_system: S3 file system instance
+            - location_provider_prefix_override: Optional prefix override for file locations
+            - position_delete_for_multiple_data_files: Whether to generate position deletes for multiple data files
+        **kwargs: Additional keyword arguments (currently unused)
+
+    Raises:
+        Exception: If snapshot commitment fails or other critical errors occur
+
     """
 
     catalog = params.catalog
     table_name = params.iceberg_table_name
-    iceberg_table = load_table(catalog, table_name)
+    if "." not in table_name:
+        iceberg_namespace = params.iceberg_namespace or DEFAULT_NAMESPACE
+        table_name = params.iceberg_table_name
+        table_identifier = f"{iceberg_namespace}.{table_name}"
+    else:
+        table_identifier = table_name
+        identifier_parts = table_identifier.split(".")
+        iceberg_namespace = identifier_parts[0]
+        table_name = identifier_parts[1]
+    iceberg_table = load_table(catalog, table_identifier)
     enforce_primary_key_uniqueness = params.enforce_primary_key_uniqueness
+    iceberg_warehouse_bucket_name = params.iceberg_warehouse_bucket_name
+    merge_keys = params.merge_keys
+    compact_previous_position_delete_files = (
+        params.compact_previous_position_delete_files
+    )
+    task_max_parallelism = params.task_max_parallelism
+    s3_client_kwargs = params.s3_client_kwargs
+    s3_file_system = params.filesystem
+    location_provider_prefix_override = params.location_provider_prefix_override
+    position_delete_for_multiple_data_files = (
+        params.position_delete_for_multiple_data_files
+    )
+
     data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(
         iceberg_table
     )
+
     convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
         data_file_dict=data_file_dict,
         equality_delete_dict=equality_delete_dict,
         pos_delete_dict=pos_delete_dict,
     )
-    iceberg_warehouse_bucket_name = params.iceberg_warehouse_bucket_name
-    iceberg_namespace = params.iceberg_namespace
-    iceberg_table_warehouse_prefix = construct_iceberg_table_prefix(
-        iceberg_warehouse_bucket_name=iceberg_warehouse_bucket_name,
-        table_name=table_name,
-        iceberg_namespace=iceberg_namespace,
-    )
-    merge_keys = params.merge_keys
+
+    if not location_provider_prefix_override:
+        iceberg_table_warehouse_prefix = construct_iceberg_table_prefix(
+            iceberg_warehouse_bucket_name=iceberg_warehouse_bucket_name,
+            table_name=table_name,
+            iceberg_namespace=iceberg_namespace,
+        )
+    else:
+        iceberg_table_warehouse_prefix = location_provider_prefix_override
+
     # Using table identifier fields as merge keys if merge keys not provided
     if not merge_keys:
         identifier_fields_set = iceberg_table.schema().identifier_field_names()
         identifier_fields = list(identifier_fields_set)
     else:
         identifier_fields = merge_keys
-    if len(identifier_fields) > 1:
-        raise NotImplementedError(
-            f"Multiple identifier fields lookup not supported yet."
-        )
-    convert_options_provider = functools.partial(
+
+    convert_options_provider: Callable = functools.partial(
         task_resource_options_provider,
         resource_amount_provider=convert_resource_options_provider,
     )
@@ -86,58 +148,151 @@ def converter_session(params: ConverterSessionParams, **kwargs):
     #  Note that approach 2 will ideally require shared object store to avoid download equality delete files * number of child tasks times.
     max_parallel_data_file_download = DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD
 
-    compact_small_files = params.compact_small_files
-    position_delete_for_multiple_data_files = (
-        params.position_delete_for_multiple_data_files
-    )
-    task_max_parallelism = params.task_max_parallelism
-
-    def convert_input_provider(index, item):
+    def convert_input_provider(index: int, item: Any) -> Dict[str, ConvertInput]:
+        task_opts = convert_options_provider(index, item)
         return {
             "convert_input": ConvertInput.of(
-                files_for_each_bucket=item,
+                convert_input_files=item,
                 convert_task_index=index,
                 iceberg_table_warehouse_prefix=iceberg_table_warehouse_prefix,
                 identifier_fields=identifier_fields,
-                compact_small_files=compact_small_files,
+                compact_previous_position_delete_files=compact_previous_position_delete_files,
+                table_io=iceberg_table.io,
+                table_metadata=iceberg_table.metadata,
                 enforce_primary_key_uniqueness=enforce_primary_key_uniqueness,
                 position_delete_for_multiple_data_files=position_delete_for_multiple_data_files,
                 max_parallel_data_file_download=max_parallel_data_file_download,
+                s3_client_kwargs=s3_client_kwargs,
+                filesystem=s3_file_system,
+                task_memory=task_opts["memory"],
             )
         }
 
+    logger.info(f"Getting remote convert tasks...")
     # Ray remote task: convert
-    # Assuming that memory consume by each bucket doesn't exceed one node's memory limit.
     # TODO: Add split mechanism to split large buckets
     convert_tasks_pending = invoke_parallel(
-        items=convert_input_files_for_all_buckets.items(),
+        items=convert_input_files_for_all_buckets,
         ray_task=convert,
         max_parallelism=task_max_parallelism,
         options_provider=convert_options_provider,
         kwargs_provider=convert_input_provider,
     )
-    to_be_deleted_files_list = []
-    to_be_added_files_dict_list = []
-    convert_results = ray.get(convert_tasks_pending)
+
+    to_be_deleted_files_list: List[List[DataFile]] = []
+    logger.info(f"Finished invoking {len(convert_tasks_pending)} convert tasks.")
+
+    convert_results: List[ConvertResult] = ray.get(convert_tasks_pending)
+    logger.info(f"Got {len(convert_tasks_pending)} convert tasks.")
+
+    total_position_delete_record_count = sum(
+        convert_result.position_delete_record_count
+        for convert_result in convert_results
+    )
+    total_input_data_file_record_count = sum(
+        convert_result.input_data_files_record_count
+        for convert_result in convert_results
+    )
+    total_data_file_hash_columns_in_memory_sizes = sum(
+        convert_result.input_data_files_hash_columns_in_memory_sizes
+        for convert_result in convert_results
+    )
+    total_position_delete_file_in_memory_sizes = sum(
+        convert_result.position_delete_in_memory_sizes
+        for convert_result in convert_results
+    )
+    total_position_delete_on_disk_sizes = sum(
+        convert_result.position_delete_on_disk_sizes
+        for convert_result in convert_results
+    )
+    total_input_data_files_on_disk_size = sum(
+        convert_result.input_data_files_on_disk_size
+        for convert_result in convert_results
+    )
+
+    # Calculate memory usage statistics
+    max_peak_memory_usage = max(
+        convert_result.peak_memory_usage_bytes for convert_result in convert_results
+    )
+    avg_memory_usage_percentage = sum(
+        convert_result.memory_usage_percentage for convert_result in convert_results
+    ) / len(convert_results)
+    max_memory_usage_percentage = max(
+        convert_result.memory_usage_percentage for convert_result in convert_results
+    )
+
+    logger.info(
+        f"Aggregated stats for {table_identifier}: "
+        f"total position delete record count: {total_position_delete_record_count}, "
+        f"total input data file record count: {total_input_data_file_record_count}, "
+        f"total data file hash columns in memory sizes: {total_data_file_hash_columns_in_memory_sizes}, "
+        f"total position delete file in memory sizes: {total_position_delete_file_in_memory_sizes}, "
+        f"total position delete file on disk sizes: {total_position_delete_on_disk_sizes}, "
+        f"total input data files on disk size: {total_input_data_files_on_disk_size}, "
+        f"max peak memory usage: {max_peak_memory_usage} bytes, "
+        f"average memory usage percentage: {avg_memory_usage_percentage:.2f}%, "
+        f"max memory usage percentage: {max_memory_usage_percentage:.2f}%"
+    )
+
+    to_be_added_files_list: List[DataFile] = []
     for convert_result in convert_results:
-        to_be_deleted_files_list.extend(convert_result[0].values())
-        to_be_added_files_dict_list.append(convert_result[1])
+        to_be_added_files = convert_result.to_be_added_files
+        to_be_deleted_files = convert_result.to_be_deleted_files
+
+        to_be_deleted_files_list.extend(to_be_deleted_files.values())
+        to_be_added_files_list.extend(to_be_added_files)
 
-    new_position_delete_files = parquet_files_dict_to_iceberg_data_files(
-        io=iceberg_table.io,
-        table_metadata=iceberg_table.metadata,
-        files_dict_list=to_be_added_files_dict_list,
+    logger.info(f"To be deleted files list length: {len(to_be_deleted_files_list)}")
+    logger.info(f"To be added files list length: {len(to_be_added_files_list)}")
+
+    # Determine snapshot type and commit
+    snapshot_type = _determine_snapshot_type(
+        to_be_deleted_files_list, to_be_added_files_list
     )
 
-    if not to_be_deleted_files_list:
-        commit_append_snapshot(
-            iceberg_table=iceberg_table,
-            new_position_delete_files=new_position_delete_files,
+    if snapshot_type == SnapshotType.NONE:
+        logger.info(
+            _get_snapshot_action_description(
+                snapshot_type, to_be_deleted_files_list, to_be_added_files_list
+            )
         )
-    else:
-        commit_replace_snapshot(
-            iceberg_table=iceberg_table,
-            # equality_delete_files + data file that all rows are deleted
-            to_be_deleted_files_list=to_be_deleted_files_list,
-            new_position_delete_files=new_position_delete_files,
+        return
+
+    logger.info(
+        f"Snapshot action: {_get_snapshot_action_description(snapshot_type, to_be_deleted_files_list, to_be_added_files_list)}"
+    )
+
+    try:
+        if snapshot_type == SnapshotType.APPEND:
+            logger.info(f"Committing append snapshot for {table_identifier}.")
+            updated_table_metadata = commit_append_snapshot(
+                iceberg_table=iceberg_table,
+                new_position_delete_files=to_be_added_files_list,
+            )
+        elif snapshot_type == SnapshotType.REPLACE:
+            logger.info(f"Committing replace snapshot for {table_identifier}.")
+            updated_table_metadata = commit_replace_snapshot(
+                iceberg_table=iceberg_table,
+                to_be_deleted_files=to_be_deleted_files_list,
+                new_position_delete_files=to_be_added_files_list,
+            )
+        elif snapshot_type == SnapshotType.DELETE:
+            logger.info(f"Committing delete snapshot for {table_identifier}.")
+            updated_table_metadata = commit_replace_snapshot(
+                iceberg_table=iceberg_table,
+                to_be_deleted_files=to_be_deleted_files_list,
+                new_position_delete_files=[],  # No new files to add
+            )
+        else:
+            logger.warning(f"Unexpected snapshot type: {snapshot_type}")
+            return
+
+        logger.info(
+            f"Committed new Iceberg snapshot for {table_identifier}: {updated_table_metadata.current_snapshot_id}"
         )
+
+        # Return the updated table metadata with the new snapshot
+        return updated_table_metadata
+    except Exception as e:
+        logger.error(f"Failed to commit snapshot for {table_identifier}: {str(e)}")
+        raise

deltacat/compute/converter/model/convert_input.py

@@ -1,20 +1,25 @@
 from __future__ import annotations
-from typing import Dict, List
+from typing import Dict, List, Any, Optional
 from deltacat.compute.converter.model.convert_input_files import ConvertInputFiles
+from fsspec import AbstractFileSystem
 
 
 class ConvertInput(Dict):
     @staticmethod
     def of(
-        convert_input_files,
-        convert_task_index,
-        iceberg_table_warehouse_prefix,
-        identifier_fields,
-        compact_small_files,
-        enforce_primary_key_uniqueness,
-        position_delete_for_multiple_data_files,
-        max_parallel_data_file_download,
-        s3_file_system,
+        convert_input_files: ConvertInputFiles,
+        convert_task_index: int,
+        iceberg_table_warehouse_prefix: str,
+        identifier_fields: List[str],
+        table_io: Any,
+        table_metadata: Any,
+        compact_previous_position_delete_files: bool,
+        enforce_primary_key_uniqueness: bool,
+        position_delete_for_multiple_data_files: bool,
+        max_parallel_data_file_download: int,
+        filesystem: Optional[AbstractFileSystem],
+        s3_client_kwargs: Optional[Dict[str, Any]],
+        task_memory: float,
     ) -> ConvertInput:
 
         result = ConvertInput()
@@ -22,13 +27,19 @@ class ConvertInput(Dict):
         result["convert_task_index"] = convert_task_index
         result["identifier_fields"] = identifier_fields
         result["iceberg_table_warehouse_prefix"] = iceberg_table_warehouse_prefix
-        result["compact_small_files"] = compact_small_files
+        result["table_io"] = table_io
+        result["table_metadata"] = table_metadata
+        result[
+            "compact_previous_position_delete_files"
+        ] = compact_previous_position_delete_files
         result["enforce_primary_key_uniqueness"] = enforce_primary_key_uniqueness
         result[
             "position_delete_for_multiple_data_files"
         ] = position_delete_for_multiple_data_files
         result["max_parallel_data_file_download"] = max_parallel_data_file_download
-        result["s3_file_system"] = s3_file_system
+        result["filesystem"] = filesystem
+        result["s3_client_kwargs"] = s3_client_kwargs
+        result["task_memory"] = task_memory
 
         return result
 
@@ -49,8 +60,16 @@ class ConvertInput(Dict):
         return self["iceberg_table_warehouse_prefix"]
 
     @property
-    def compact_small_files(self) -> bool:
-        return self["compact_small_files"]
+    def table_io(self) -> Any:
+        return self["table_io"]
+
+    @property
+    def table_metadata(self) -> Any:
+        return self["table_metadata"]
+
+    @property
+    def compact_previous_position_delete_files(self) -> bool:
+        return self["compact_previous_position_delete_files"]
 
     @property
     def enforce_primary_key_uniqueness(self) -> bool:
@@ -65,5 +84,13 @@ class ConvertInput(Dict):
         return self["max_parallel_data_file_download"]
 
     @property
-    def s3_file_system(self):
-        return self["s3_file_system"]
+    def filesystem(self) -> Optional[AbstractFileSystem]:
+        return self["filesystem"]
+
+    @property
+    def s3_client_kwargs(self) -> Optional[Dict[str, Any]]:
+        return self["s3_client_kwargs"]
+
+    @property
+    def task_memory(self) -> float:
+        return self["task_memory"]

deltacat/compute/converter/model/convert_input_files.py

@@ -1,15 +1,21 @@
 from __future__ import annotations
-from typing import Dict
+from typing import Dict, List, Any, Optional, Tuple
+from pyiceberg.manifest import DataFile
+
+# Type aliases to simplify nested types
+DataFileWithSequence = Tuple[int, DataFile]  # (sequence_number, data_file)
+DataFileList = List[DataFileWithSequence]  # List of data files with sequence numbers
+DataFileListGroup = List[DataFileList]  # Group of data file lists
 
 
 class ConvertInputFiles(Dict):
     @staticmethod
     def of(
-        partition_value,
-        all_data_files_for_dedupe=None,
-        applicable_data_files=None,
-        applicable_equality_delete_files=None,
-        existing_position_delete_files=None,
+        partition_value: Any,
+        all_data_files_for_dedupe: Optional[DataFileList] = None,
+        applicable_data_files: Optional[DataFileListGroup] = None,
+        applicable_equality_delete_files: Optional[DataFileListGroup] = None,
+        existing_position_delete_files: Optional[DataFileList] = None,
     ) -> ConvertInputFiles:
 
         result = ConvertInputFiles()
@@ -21,41 +27,52 @@ class ConvertInputFiles(Dict):
         return result
 
     @property
-    def partition_value(self):
+    def partition_value(self) -> Any:
         return self["partition_value"]
 
     @property
-    def all_data_files_for_dedupe(self):
+    def all_data_files_for_dedupe(self) -> Optional[DataFileList]:
         return self["all_data_files_for_dedupe"]
 
     @property
-    def applicable_data_files(self):
+    def applicable_data_files(self) -> Optional[DataFileListGroup]:
         return self["applicable_data_files"]
 
     @property
-    def applicable_equality_delete_files(self):
+    def applicable_equality_delete_files(
+        self,
+    ) -> Optional[DataFileListGroup]:
         return self["applicable_equality_delete_files"]
 
     @property
-    def existing_position_delete_files(self):
+    def existing_position_delete_files(self) -> Optional[DataFileList]:
         return self["existing_position_delete_files"]
 
     @partition_value.setter
-    def partition_value(self, partition_value):
+    def partition_value(self, partition_value: Any) -> None:
         self["partition_value"] = partition_value
 
     @all_data_files_for_dedupe.setter
-    def all_data_files_for_dedupe(self, all_data_files_for_dedupe):
+    def all_data_files_for_dedupe(
+        self, all_data_files_for_dedupe: Optional[DataFileList]
+    ) -> None:
         self["all_data_files_for_dedupe"] = all_data_files_for_dedupe
 
     @applicable_data_files.setter
-    def applicable_data_files(self, applicable_data_files):
+    def applicable_data_files(
+        self, applicable_data_files: Optional[DataFileListGroup]
+    ) -> None:
         self["applicable_data_files"] = applicable_data_files
 
     @applicable_equality_delete_files.setter
-    def applicable_equality_delete_files(self, applicable_equality_delete_files):
+    def applicable_equality_delete_files(
+        self,
+        applicable_equality_delete_files: Optional[DataFileListGroup],
+    ) -> None:
         self["applicable_equality_delete_files"] = applicable_equality_delete_files
 
     @existing_position_delete_files.setter
-    def existing_position_delete_files(self, existing_position_delete_files):
+    def existing_position_delete_files(
+        self, existing_position_delete_files: Optional[DataFileList]
+    ) -> None:
         self["existing_position_delete_files"] = existing_position_delete_files

deltacat/compute/converter/model/convert_result.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+from typing import Dict, List, Any
+from pyiceberg.manifest import DataFile
+
+
+class ConvertResult(Dict):
+    @staticmethod
+    def of(
+        convert_task_index: int,
+        to_be_added_files: List[DataFile],
+        to_be_deleted_files: Dict[Any, List[DataFile]],
+        position_delete_record_count: int,
+        input_data_files_record_count: int,
+        input_data_files_hash_columns_in_memory_sizes: int,
+        position_delete_in_memory_sizes: int,
+        position_delete_on_disk_sizes: int,
+        input_data_files_on_disk_size: int,
+        peak_memory_usage_bytes: int,
+        memory_usage_percentage: float,
+    ) -> ConvertResult:
+
+        result = ConvertResult()
+        result["convert_task_index"] = convert_task_index
+        result["to_be_added_files"] = to_be_added_files
+        result["to_be_deleted_files"] = to_be_deleted_files
+        result["position_delete_record_count"] = position_delete_record_count
+        result["input_data_files_record_count"] = input_data_files_record_count
+        result[
+            "input_data_files_hash_columns_in_memory_sizes"
+        ] = input_data_files_hash_columns_in_memory_sizes
+        result["position_delete_in_memory_sizes"] = position_delete_in_memory_sizes
+        result["position_delete_on_disk_sizes"] = position_delete_on_disk_sizes
+        result["input_data_files_on_disk_size"] = input_data_files_on_disk_size
+        result["peak_memory_usage_bytes"] = peak_memory_usage_bytes
+        result["memory_usage_percentage"] = memory_usage_percentage
+        return result
+
+    @property
+    def convert_task_index(self) -> int:
+        return self["convert_task_index"]
+
+    @property
+    def to_be_added_files(self) -> List[DataFile]:
+        return self["to_be_added_files"]
+
+    @property
+    def to_be_deleted_files(self) -> Dict[Any, List[DataFile]]:
+        return self["to_be_deleted_files"]
+
+    @property
+    def position_delete_record_count(self) -> int:
+        return self["position_delete_record_count"]
+
+    @property
+    def input_data_files_record_count(self) -> int:
+        return self["input_data_files_record_count"]
+
+    @property
+    def input_data_files_hash_columns_in_memory_sizes(self) -> int:
+        return self["input_data_files_hash_columns_in_memory_sizes"]
+
+    @property
+    def position_delete_in_memory_sizes(self) -> int:
+        return self["position_delete_in_memory_sizes"]
+
+    @property
+    def position_delete_on_disk_sizes(self) -> int:
+        return self["position_delete_on_disk_sizes"]
+
+    @property
+    def input_data_files_on_disk_size(self) -> int:
+        return self["input_data_files_on_disk_size"]
+
+    @property
+    def peak_memory_usage_bytes(self) -> int:
+        return self["peak_memory_usage_bytes"]
+
+    @property
+    def memory_usage_percentage(self) -> float:
+        return self["memory_usage_percentage"]