deltacat 2.0.0b11__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/compute/compactor_v2/utils/io.py

@@ -3,7 +3,7 @@ import functools
 from deltacat.storage import (
     PartitionLocator,
     Delta,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat import logs
 from deltacat.compute.compactor.utils import io as io_v1
@@ -38,7 +38,7 @@ def discover_deltas(
     rebase_source_partition_locator: Optional[PartitionLocator] = None,
     rebase_source_partition_high_watermark: Optional[int] = None,
     rcf_high_watermark: Optional[int] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[Delta]:
@@ -67,6 +67,11 @@ def discover_deltas(
         f"Length of input deltas from delta source table is {len(delta_source_incremental_deltas)}"
         f" from ({previous_compacted_high_watermark}, {last_stream_position_to_compact}]"
     )
+    logger.info(f"DEBUG: source_partition_locator = {source_partition_locator}")
+    logger.info(
+        f"DEBUG: source_partition_locator.partition_id = {getattr(source_partition_locator, 'partition_id', 'NO_PARTITION_ID')}"
+    )
+    logger.info(f"DEBUG: total input deltas found = {len(result)}")
 
     if rebase_source_partition_locator:
         previous_compacted_deltas = io_v1._discover_deltas(
@@ -93,7 +98,8 @@ def create_uniform_input_deltas(
     hash_bucket_count: int,
     compaction_audit: CompactionSessionAuditInfo,
     compact_partition_params: CompactPartitionParams,
-    deltacat_storage=unimplemented_deltacat_storage,
+    all_column_names: List[str],
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[DeltaAnnotated]:
 
@@ -101,7 +107,6 @@ def create_uniform_input_deltas(
     delta_manifest_entries_count = 0
     estimated_da_bytes = 0
     input_da_list = []
-
     for delta in input_deltas:
         if (
             compact_partition_params.enable_input_split
@@ -114,10 +119,12 @@ def create_uniform_input_deltas(
             )
             append_content_type_params(
                 delta=delta,
+                all_column_names=all_column_names,
                 deltacat_storage=deltacat_storage,
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
                 task_max_parallelism=compact_partition_params.task_max_parallelism,
                 max_parquet_meta_size_bytes=compact_partition_params.max_parquet_meta_size_bytes,
+                file_reader_kwargs_provider=compact_partition_params.read_kwargs_provider,
             )
 
         manifest_entries = delta.manifest.entries

deltacat/compute/compactor_v2/utils/merge.py

@@ -23,6 +23,7 @@ from deltacat.types.tables import TABLE_CLASS_TO_SIZE_FUNC
 
 from deltacat.utils.performance import timed_invocation
 from deltacat.storage import (
+    DeltaType,
     Partition,
 )
 from deltacat.compute.compactor_v2.deletes.delete_strategy import (
@@ -47,13 +48,21 @@ def materialize(
         # TODO (pdames): compare performance to pandas-native materialize path
         df = compacted_table.to_pandas(split_blocks=True, self_destruct=True)
         compacted_table = df
+    # Extract schema from table_writer_kwargs to pass as direct parameter
+    # This ensures schema_id is properly set in the manifest
+    schema = None
+    if input.table_writer_kwargs and "schema" in input.table_writer_kwargs:
+        schema = input.table_writer_kwargs["schema"]
+
     delta, stage_delta_time = timed_invocation(
         input.deltacat_storage.stage_delta,
         compacted_table,
         input.write_to_partition,
+        delta_type=DeltaType.APPEND,  # Compaction always produces APPEND deltas
         max_records_per_entry=input.max_records_per_output_file,
         content_type=input.compacted_file_content_type,
-        s3_table_writer_kwargs=input.s3_table_writer_kwargs,
+        schema=schema,  # Pass schema as direct parameter for schema_id extraction
+        table_writer_kwargs=input.table_writer_kwargs,
         **input.deltacat_storage_kwargs,
     )
     compacted_table_size = TABLE_CLASS_TO_SIZE_FUNC[type(compacted_table)](
@@ -112,6 +121,7 @@ def generate_local_merge_input(
     return MergeInput.of(
         merge_file_groups_provider=LocalMergeFileGroupsProvider(
             annotated_deltas,
+            all_column_names=params.all_column_names,
             read_kwargs_provider=params.read_kwargs_provider,
             deltacat_storage=params.deltacat_storage,
             deltacat_storage_kwargs=params.deltacat_storage_kwargs,
@@ -119,12 +129,13 @@ def generate_local_merge_input(
         write_to_partition=compacted_partition,
         compacted_file_content_type=params.compacted_file_content_type,
         primary_keys=params.primary_keys,
+        all_column_names=params.all_column_names,
         sort_keys=params.sort_keys,
         drop_duplicates=params.drop_duplicates,
         max_records_per_output_file=params.records_per_compacted_file,
         enable_profiler=params.enable_profiler,
         metrics_config=params.metrics_config,
-        s3_table_writer_kwargs=params.s3_table_writer_kwargs,
+        table_writer_kwargs=params.table_writer_kwargs,
         read_kwargs_provider=params.read_kwargs_provider,
         round_completion_info=round_completion_info,
         object_store=params.object_store,
@@ -133,4 +144,6 @@ def generate_local_merge_input(
         delete_strategy=delete_strategy,
         delete_file_envelopes=delete_file_envelopes,
         disable_copy_by_reference=params.disable_copy_by_reference,
+        hash_bucket_count=params.hash_bucket_count,
+        original_fields=params.original_fields,
     )

deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -10,6 +10,7 @@ from deltacat.compute.compactor_v2.constants import (
     TOTAL_BYTES_IN_SHA1_HASH,
     PK_DELIMITER,
     MAX_SIZE_OF_RECORD_BATCH_IN_GIB,
+    SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED,
 )
 import time
 from deltacat.compute.compactor.model.delta_file_envelope import DeltaFileEnvelope
@@ -48,6 +49,13 @@ def _is_sha1_desired(hash_columns: List[pa.Array]) -> bool:
         f"Found total length of hash column={total_len} and total_size={total_size}"
     )
 
+    if SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED:
+        logger.info(
+            f"SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED is True. "
+            f"Returning False for is_sha1_desired"
+        )
+        return False
+
     return total_size > TOTAL_BYTES_IN_SHA1_HASH * total_len
 
 
@@ -70,13 +78,25 @@ def _append_table_by_hash_bucket(
         f"Grouping a pki table of length {len(pki_table)} took {groupby_latency}s"
     )
 
+    hb_pk_grouped_by = hb_pk_grouped_by.sort_by(sc._HASH_BUCKET_IDX_COLUMN_NAME)
     group_count_array = hb_pk_grouped_by[f"{sc._HASH_BUCKET_IDX_COLUMN_NAME}_count"]
     hb_group_array = hb_pk_grouped_by[sc._HASH_BUCKET_IDX_COLUMN_NAME]
 
     result_len = 0
     for i, group_count in enumerate(group_count_array):
         hb_idx = hb_group_array[i].as_py()
-        pyarrow_table = hb_pk_table.slice(offset=result_len, length=group_count.as_py())
+        group_count_py = group_count.as_py()
+        pyarrow_table = hb_pk_table.slice(offset=result_len, length=group_count_py)
+        assert group_count_py == len(
+            pyarrow_table
+        ), f"Group count {group_count_py} not equal to {len(pyarrow_table)}"
+        all_buckets = pc.unique(pyarrow_table[sc._HASH_BUCKET_IDX_COLUMN_NAME])
+        assert (
+            len(all_buckets) == 1
+        ), f"Only one hash bucket is allowed but found {len(all_buckets)}"
+        assert (
+            all_buckets[0].as_py() == hb_idx
+        ), f"Hash bucket not equal, {all_buckets[0]} and {hb_idx}"
         pyarrow_table = pyarrow_table.drop(
             [sc._HASH_BUCKET_IDX_COLUMN_NAME, sc._PK_HASH_STRING_COLUMN_NAME]
         )
@@ -108,9 +128,10 @@ def _optimized_group_record_batches_by_hash_bucket(
     record_batches = []
     result_len = 0
     for record_batch in table_batches:
-        current_bytes += record_batch.nbytes
-        record_batches.append(record_batch)
-        if current_bytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB:
+        if (
+            record_batches
+            and current_bytes + record_batch.nbytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB
+        ):
             logger.info(
                 f"Total number of record batches without exceeding {MAX_SIZE_OF_RECORD_BATCH_IN_GIB} "
                 f"is {len(record_batches)} and size {current_bytes}"
@@ -128,6 +149,9 @@ def _optimized_group_record_batches_by_hash_bucket(
             current_bytes = 0
             record_batches.clear()
 
+        current_bytes += record_batch.nbytes
+        record_batches.append(record_batch)
+
     if record_batches:
         appended_len, append_latency = timed_invocation(
             _append_table_by_hash_bucket,

deltacat/compute/compactor_v2/utils/task_options.py

@@ -1,12 +1,17 @@
 import logging
 from typing import Dict, Optional, List, Tuple, Any
 from deltacat import logs
+from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
+from deltacat.compute.compactor_v2.constants import (
+    AVERAGE_RECORD_SIZE_BYTES as DEFAULT_AVERAGE_RECORD_SIZE_BYTES,
+)
 from deltacat.compute.compactor_v2.model.merge_file_group import (
     LocalMergeFileGroupsProvider,
 )
 from deltacat.storage import (
     Manifest,
-    interface as unimplemented_deltacat_storage,
+    ManifestEntry,
+    metastore,
 )
 from deltacat.compute.compactor.model.delta_annotated import DeltaAnnotated
 from deltacat.compute.compactor.model.round_completion_info import RoundCompletionInfo
@@ -72,8 +77,6 @@ def _get_merge_task_options(
     round_completion_info: Optional[RoundCompletionInfo] = None,
     compacted_delta_manifest: Optional[Manifest] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
 ) -> Dict[str, Any]:
     if (
@@ -81,16 +84,27 @@ def _get_merge_task_options(
         and compacted_delta_manifest
         and round_completion_info.hb_index_to_entry_range
     ):
-
-        previous_inflation = (
-            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-            / round_completion_info.compacted_pyarrow_write_result.file_bytes
+        logger.debug_conditional(
+            f"[Merge task {index}]: Using previous compaction rounds to calculate merge memory: {round_completion_info.compacted_pyarrow_write_result}",
+            memory_logs_enabled,
+        )
+        previous_inflation: float = (
+            (
+                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+                / round_completion_info.compacted_pyarrow_write_result.file_bytes
+            )
+            if round_completion_info.compacted_pyarrow_write_result.file_bytes
+            else PYARROW_INFLATION_MULTIPLIER
         )
         debug_memory_params["previous_inflation"] = previous_inflation
 
-        average_record_size = (
-            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-            / round_completion_info.compacted_pyarrow_write_result.records
+        average_record_size: float = (
+            (
+                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+                / round_completion_info.compacted_pyarrow_write_result.records
+            )
+            if round_completion_info.compacted_pyarrow_write_result.records
+            else DEFAULT_AVERAGE_RECORD_SIZE_BYTES
         )
         debug_memory_params["average_record_size"] = average_record_size
 
@@ -106,31 +120,36 @@ def _get_merge_task_options(
                 str(hb_idx)
             ]
             for entry_index in range(entry_start, entry_end):
-                entry = compacted_delta_manifest.entries[entry_index]
-
-                current_entry_size = estimate_manifest_entry_size_bytes(
-                    entry=entry,
-                    operation_type=OperationType.PYARROW_DOWNLOAD,
-                    estimate_resources_params=estimate_resources_params,
+                entry: ManifestEntry = compacted_delta_manifest.entries[entry_index]
+                current_entry_size: float = (
+                    estimate_manifest_entry_size_bytes(
+                        entry=entry,
+                        operation_type=OperationType.PYARROW_DOWNLOAD,
+                        estimate_resources_params=estimate_resources_params,
+                    )
+                    or 0.0
                 )
-                current_entry_rows = estimate_manifest_entry_num_rows(
-                    entry=entry,
-                    operation_type=OperationType.PYARROW_DOWNLOAD,
-                    estimate_resources_params=estimate_resources_params,
+                current_entry_rows: int = (
+                    estimate_manifest_entry_num_rows(
+                        entry=entry,
+                        operation_type=OperationType.PYARROW_DOWNLOAD,
+                        estimate_resources_params=estimate_resources_params,
+                    )
+                    or 0
                 )
-
+                # NOTE: We can treat the current_entry_size and current_entry_rows as 0 as a None estimated entry size implies a 0 value
                 data_size += current_entry_size
                 num_rows += current_entry_rows
-
                 if primary_keys:
-                    pk_size = estimate_manifest_entry_column_size_bytes(
+                    pk_size: Optional[
+                        float
+                    ] = estimate_manifest_entry_column_size_bytes(
                         entry=entry,
                         columns=primary_keys,
                         operation_type=OperationType.PYARROW_DOWNLOAD,
                         estimate_resources_params=estimate_resources_params,
                     )
-
-                    if pk_size is None:
+                    if not pk_size:
                         pk_size_bytes += current_entry_size
                     else:
                         pk_size_bytes += pk_size
@@ -159,7 +178,6 @@ def _get_merge_task_options(
         f"[Merge task {index}]: Params used for calculating merge memory: {debug_memory_params}",
         memory_logs_enabled,
     )
-
     return _get_task_options(0.01, total_memory, ray_custom_resources)
 
 
@@ -255,8 +273,6 @@ def merge_resource_options_provider(
     compacted_delta_manifest: Optional[Manifest] = None,
     ray_custom_resources: Optional[Dict] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
     **kwargs,
 ) -> Dict:
@@ -286,8 +302,6 @@ def merge_resource_options_provider(
         round_completion_info=round_completion_info,
         compacted_delta_manifest=compacted_delta_manifest,
         primary_keys=primary_keys,
-        deltacat_storage=deltacat_storage,
-        deltacat_storage_kwargs=deltacat_storage_kwargs,
         memory_logs_enabled=memory_logs_enabled,
         estimate_resources_params=estimate_resources_params,
     )
@@ -302,7 +316,7 @@ def local_merge_resource_options_provider(
     compacted_delta_manifest: Optional[Manifest] = None,
     ray_custom_resources: Optional[Dict] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
     **kwargs,
@@ -328,8 +342,6 @@ def local_merge_resource_options_provider(
         round_completion_info=round_completion_info,
         compacted_delta_manifest=compacted_delta_manifest,
         primary_keys=primary_keys,
-        deltacat_storage=deltacat_storage,
-        deltacat_storage_kwargs=deltacat_storage_kwargs,
         memory_logs_enabled=memory_logs_enabled,
         estimate_resources_params=estimate_resources_params,
     )

deltacat/compute/converter/converter_session.py

@@ -1,3 +1,4 @@
+from deltacat.constants import DEFAULT_NAMESPACE
 from deltacat.utils.ray_utils.concurrency import (
     invoke_parallel,
     task_resource_options_provider,
@@ -12,8 +13,7 @@ from deltacat import logs
 from deltacat.compute.converter.model.converter_session_params import (
     ConverterSessionParams,
 )
-
-
+from typing import Dict, List, Any, Callable
 from deltacat.compute.converter.constants import DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD
 from deltacat.compute.converter.steps.convert import convert
 from deltacat.compute.converter.model.convert_input import ConvertInput
@@ -31,30 +31,80 @@ from deltacat.compute.converter.pyiceberg.catalog import load_table
 from deltacat.compute.converter.utils.converter_session_utils import (
     group_all_files_to_each_bucket,
 )
+from deltacat.compute.converter.model.convert_result import ConvertResult
+from deltacat.compute.converter.utils.converter_session_utils import (
+    _get_snapshot_action_description,
+    _determine_snapshot_type,
+    SnapshotType,
+)
+
+from pyiceberg.manifest import DataFile
+from pyiceberg.table.metadata import TableMetadata
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-def converter_session(params: ConverterSessionParams, **kwargs):
+def converter_session(params: ConverterSessionParams, **kwargs: Any) -> TableMetadata:
     """
-    Convert equality delete to position delete.
-    Compute and memory heavy work from downloading equality delete table and compute position deletes
-    will be executed on Ray remote tasks.
+    Convert equality deletes to position deletes with option to enforce primary key uniqueness.
+
+    This function processes Iceberg table files to convert equality delete files to position delete files.
+    It can optionally enforce primary key uniqueness by keeping only the latest version of each
+    primary key across all data files.
+
+    **Memory Requirements:**
+    - Minimum 512MB of free memory is required to run the converter
+
+    **Process Overview:**
+    1. Fetches all bucket files (data files, equality deletes, position deletes)
+    2. Groups files by bucket for parallel processing
+    3. Converts equality deletes to position deletes using Ray parallel tasks
+    4. Enforces primary key uniqueness if enabled
+    5. Commits appropriate snapshot (append, replace, or delete) to the Iceberg table
+
+
+    Args:
+        params: ConverterSessionParams containing all configuration parameters
+            - catalog: Iceberg catalog instance
+            - iceberg_table_name: Name of the target Iceberg table
+            - enforce_primary_key_uniqueness: Whether to enforce PK uniqueness
+            - iceberg_warehouse_bucket_name: S3 bucket for Iceberg warehouse
+            - iceberg_namespace: Iceberg namespace
+            - merge_keys: Optional list of merge key fields (uses table identifier fields if not provided)
+            - compact_previous_position_delete_files: Whether to compact existing position delete files
+            - task_max_parallelism: Maximum number of parallel Ray tasks
+            - s3_client_kwargs: Additional S3 client configuration
+            - s3_file_system: S3 file system instance
+            - location_provider_prefix_override: Optional prefix override for file locations
+            - position_delete_for_multiple_data_files: Whether to generate position deletes for multiple data files
+        **kwargs: Additional keyword arguments (currently unused)
+
+    Raises:
+        Exception: If snapshot commitment fails or other critical errors occur
+
     """
 
     catalog = params.catalog
     table_name = params.iceberg_table_name
-    iceberg_table = load_table(catalog, table_name)
+    if "." not in table_name:
+        iceberg_namespace = params.iceberg_namespace or DEFAULT_NAMESPACE
+        table_name = params.iceberg_table_name
+        table_identifier = f"{iceberg_namespace}.{table_name}"
+    else:
+        table_identifier = table_name
+        identifier_parts = table_identifier.split(".")
+        iceberg_namespace = identifier_parts[0]
+        table_name = identifier_parts[1]
+    iceberg_table = load_table(catalog, table_identifier)
     enforce_primary_key_uniqueness = params.enforce_primary_key_uniqueness
     iceberg_warehouse_bucket_name = params.iceberg_warehouse_bucket_name
-    iceberg_namespace = params.iceberg_namespace
     merge_keys = params.merge_keys
     compact_previous_position_delete_files = (
         params.compact_previous_position_delete_files
     )
     task_max_parallelism = params.task_max_parallelism
     s3_client_kwargs = params.s3_client_kwargs
-    s3_file_system = params.s3_file_system
+    s3_file_system = params.filesystem
     location_provider_prefix_override = params.location_provider_prefix_override
     position_delete_for_multiple_data_files = (
         params.position_delete_for_multiple_data_files
@@ -86,7 +136,7 @@ def converter_session(params: ConverterSessionParams, **kwargs):
     else:
         identifier_fields = merge_keys
 
-    convert_options_provider = functools.partial(
+    convert_options_provider: Callable = functools.partial(
         task_resource_options_provider,
         resource_amount_provider=convert_resource_options_provider,
     )
@@ -98,7 +148,8 @@ def converter_session(params: ConverterSessionParams, **kwargs):
     #  Note that approach 2 will ideally require shared object store to avoid download equality delete files * number of child tasks times.
     max_parallel_data_file_download = DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD
 
-    def convert_input_provider(index, item):
+    def convert_input_provider(index: int, item: Any) -> Dict[str, ConvertInput]:
+        task_opts = convert_options_provider(index, item)
         return {
             "convert_input": ConvertInput.of(
                 convert_input_files=item,
@@ -112,7 +163,8 @@ def converter_session(params: ConverterSessionParams, **kwargs):
                 position_delete_for_multiple_data_files=position_delete_for_multiple_data_files,
                 max_parallel_data_file_download=max_parallel_data_file_download,
                 s3_client_kwargs=s3_client_kwargs,
-                s3_file_system=s3_file_system,
+                filesystem=s3_file_system,
+                task_memory=task_opts["memory"],
             )
         }
 
@@ -127,10 +179,10 @@ def converter_session(params: ConverterSessionParams, **kwargs):
         kwargs_provider=convert_input_provider,
     )
 
-    to_be_deleted_files_list = []
+    to_be_deleted_files_list: List[List[DataFile]] = []
     logger.info(f"Finished invoking {len(convert_tasks_pending)} convert tasks.")
 
-    convert_results = ray.get(convert_tasks_pending)
+    convert_results: List[ConvertResult] = ray.get(convert_tasks_pending)
     logger.info(f"Got {len(convert_tasks_pending)} convert tasks.")
 
     total_position_delete_record_count = sum(
@@ -153,8 +205,36 @@ def converter_session(params: ConverterSessionParams, **kwargs):
         convert_result.position_delete_on_disk_sizes
         for convert_result in convert_results
     )
+    total_input_data_files_on_disk_size = sum(
+        convert_result.input_data_files_on_disk_size
+        for convert_result in convert_results
+    )
+
+    # Calculate memory usage statistics
+    max_peak_memory_usage = max(
+        convert_result.peak_memory_usage_bytes for convert_result in convert_results
+    )
+    avg_memory_usage_percentage = sum(
+        convert_result.memory_usage_percentage for convert_result in convert_results
+    ) / len(convert_results)
+    max_memory_usage_percentage = max(
+        convert_result.memory_usage_percentage for convert_result in convert_results
+    )
+
+    logger.info(
+        f"Aggregated stats for {table_identifier}: "
+        f"total position delete record count: {total_position_delete_record_count}, "
+        f"total input data file record count: {total_input_data_file_record_count}, "
+        f"total data file hash columns in memory sizes: {total_data_file_hash_columns_in_memory_sizes}, "
+        f"total position delete file in memory sizes: {total_position_delete_file_in_memory_sizes}, "
+        f"total position delete file on disk sizes: {total_position_delete_on_disk_sizes}, "
+        f"total input data files on disk size: {total_input_data_files_on_disk_size}, "
+        f"max peak memory usage: {max_peak_memory_usage} bytes, "
+        f"average memory usage percentage: {avg_memory_usage_percentage:.2f}%, "
+        f"max memory usage percentage: {max_memory_usage_percentage:.2f}%"
+    )
 
-    to_be_added_files_list = []
+    to_be_added_files_list: List[DataFile] = []
     for convert_result in convert_results:
         to_be_added_files = convert_result.to_be_added_files
         to_be_deleted_files = convert_result.to_be_deleted_files
@@ -162,24 +242,57 @@ def converter_session(params: ConverterSessionParams, **kwargs):
         to_be_deleted_files_list.extend(to_be_deleted_files.values())
         to_be_added_files_list.extend(to_be_added_files)
 
-    if not to_be_deleted_files_list and to_be_added_files_list:
-        commit_append_snapshot(
-            iceberg_table=iceberg_table,
-            new_position_delete_files=to_be_added_files_list,
-        )
-    else:
-        commit_replace_snapshot(
-            iceberg_table=iceberg_table,
-            to_be_deleted_files_list=to_be_deleted_files_list,
-            new_position_delete_files=to_be_added_files_list,
+    logger.info(f"To be deleted files list length: {len(to_be_deleted_files_list)}")
+    logger.info(f"To be added files list length: {len(to_be_added_files_list)}")
+
+    # Determine snapshot type and commit
+    snapshot_type = _determine_snapshot_type(
+        to_be_deleted_files_list, to_be_added_files_list
+    )
+
+    if snapshot_type == SnapshotType.NONE:
+        logger.info(
+            _get_snapshot_action_description(
+                snapshot_type, to_be_deleted_files_list, to_be_added_files_list
+            )
         )
+        return
+
     logger.info(
-        f"Aggregated stats for {table_name}: "
-        f"total position delete record count: {total_position_delete_record_count}, "
-        f"total input data file record_count: {total_input_data_file_record_count}, "
-        f"total data file hash columns in memory sizes: {total_data_file_hash_columns_in_memory_sizes}, "
-        f"total position delete file in memory sizes: {total_position_delete_file_in_memory_sizes}, "
-        f"total position delete file on disk sizes: {total_position_delete_on_disk_sizes}."
+        f"Snapshot action: {_get_snapshot_action_description(snapshot_type, to_be_deleted_files_list, to_be_added_files_list)}"
     )
 
-    logger.info(f"Committed new Iceberg snapshot.")
+    try:
+        if snapshot_type == SnapshotType.APPEND:
+            logger.info(f"Committing append snapshot for {table_identifier}.")
+            updated_table_metadata = commit_append_snapshot(
+                iceberg_table=iceberg_table,
+                new_position_delete_files=to_be_added_files_list,
+            )
+        elif snapshot_type == SnapshotType.REPLACE:
+            logger.info(f"Committing replace snapshot for {table_identifier}.")
+            updated_table_metadata = commit_replace_snapshot(
+                iceberg_table=iceberg_table,
+                to_be_deleted_files=to_be_deleted_files_list,
+                new_position_delete_files=to_be_added_files_list,
+            )
+        elif snapshot_type == SnapshotType.DELETE:
+            logger.info(f"Committing delete snapshot for {table_identifier}.")
+            updated_table_metadata = commit_replace_snapshot(
+                iceberg_table=iceberg_table,
+                to_be_deleted_files=to_be_deleted_files_list,
+                new_position_delete_files=[],  # No new files to add
+            )
+        else:
+            logger.warning(f"Unexpected snapshot type: {snapshot_type}")
+            return
+
+        logger.info(
+            f"Committed new Iceberg snapshot for {table_identifier}: {updated_table_metadata.current_snapshot_id}"
+        )
+
+        # Return the updated table metadata with the new snapshot
+        return updated_table_metadata
+    except Exception as e:
+        logger.error(f"Failed to commit snapshot for {table_identifier}: {str(e)}")
+        raise