deltacat 2.0.0b7__py3-none-any.whl → 2.0.0b10__py3-none-any.whl

deltacat/compute/converter/model/converter_session_params.py

@@ -1,6 +1,10 @@
 from __future__ import annotations
 from typing import Optional, Dict
-from deltacat.compute.converter.constants import DEFAULT_CONVERTER_TASK_MAX_PARALLELISM
+from deltacat.compute.converter.constants import (
+    DEFAULT_CONVERTER_TASK_MAX_PARALLELISM,
+)
+from deltacat.constants import DEFAULT_NAMESPACE
+from fsspec import AbstractFileSystem
 
 
 class ConverterSessionParams(dict):
@@ -18,15 +22,15 @@ class ConverterSessionParams(dict):
         assert (
             params.get("iceberg_warehouse_bucket_name") is not None
         ), "iceberg_warehouse_bucket_name is a required arg"
-        assert (
-            params.get("iceberg_namespace") is not None
-        ), "iceberg_namespace is a required arg"
         result = ConverterSessionParams(params)
 
+        result.iceberg_namespace = params.get("iceberg_namespace", DEFAULT_NAMESPACE)
         result.enforce_primary_key_uniqueness = params.get(
             "enforce_primary_key_uniqueness", False
         )
-        result.compact_small_files = params.get("compact_small_files", False)
+        result.compact_previous_position_delete_files = params.get(
+            "compact_previous_position_delete_files", False
+        )
 
         # For Iceberg v3 spec, option to produce delete vector that can establish 1:1 mapping with data files.
         result.position_delete_for_multiple_data_files = params.get(
@@ -36,6 +40,10 @@ class ConverterSessionParams(dict):
             "task_max_parallelism", DEFAULT_CONVERTER_TASK_MAX_PARALLELISM
         )
         result.merge_keys = params.get("merge_keys", None)
+        result.s3_client_kwargs = params.get("s3_client_kwargs", {})
+        result.s3_file_system = params.get("s3_file_system", None)
+        result.s3_prefix_override = params.get("s3_prefix_override", None)
+
         return result
 
     @property
@@ -54,6 +62,10 @@ class ConverterSessionParams(dict):
     def iceberg_namespace(self) -> str:
         return self["iceberg_namespace"]
 
+    @iceberg_namespace.setter
+    def iceberg_namespace(self, iceberg_namespace) -> None:
+        self["iceberg_namespace"] = iceberg_namespace
+
     @property
     def enforce_primary_key_uniqueness(self) -> bool:
         return self["enforce_primary_key_uniqueness"]
@@ -63,12 +75,16 @@ class ConverterSessionParams(dict):
         self["enforce_primary_key_uniqueness"] = enforce_primary_key_uniqueness
 
     @property
-    def compact_small_files(self) -> bool:
-        return self["compact_small_files"]
+    def compact_previous_position_delete_files(self) -> bool:
+        return self["compact_previous_position_delete_files"]
 
-    @compact_small_files.setter
-    def compact_small_files(self, compact_small_files) -> None:
-        self["compact_small_files"] = compact_small_files
+    @compact_previous_position_delete_files.setter
+    def compact_previous_position_delete_files(
+        self, compact_previous_position_delete_files
+    ) -> None:
+        self[
+            "compact_previous_position_delete_files"
+        ] = compact_previous_position_delete_files
 
     @property
     def position_delete_for_multiple_data_files(self) -> bool:
@@ -97,3 +113,29 @@ class ConverterSessionParams(dict):
     @merge_keys.setter
     def merge_keys(self, merge_keys) -> None:
         self["merge_keys"] = merge_keys
+
+    @property
+    def s3_client_kwargs(self) -> Dict:
+        return self["s3_client_kwargs"]
+
+    @s3_client_kwargs.setter
+    def s3_client_kwargs(self, s3_client_kwargs) -> None:
+        self["s3_client_kwargs"] = s3_client_kwargs
+
+    @property
+    def s3_file_system(self) -> AbstractFileSystem:
+        return self["s3_file_system"]
+
+    @s3_file_system.setter
+    def s3_file_system(self, s3_file_system) -> None:
+        self["s3_file_system"] = s3_file_system
+
+    @property
+    def location_provider_prefix_override(self) -> str:
+        return self["location_provider_prefix_override"]
+
+    @location_provider_prefix_override.setter
+    def location_provider_prefix_override(
+        self, location_provider_prefix_override
+    ) -> None:
+        self["location_provider_prefix_override"] = location_provider_prefix_override

deltacat/compute/converter/pyiceberg/overrides.py

@@ -1,68 +1,200 @@
 from collections import defaultdict
 import logging
 from deltacat import logs
+import pyarrow
 import pyarrow.parquet as pq
+from pyiceberg.io.pyarrow import (
+    parquet_path_to_id_mapping,
+    StatisticsCollector,
+    MetricModeTypes,
+    DataFileStatistics,
+    MetricsMode,
+    StatsAggregator,
+)
+from typing import Dict, List, Set
+from deltacat.compute.converter.utils.iceberg_columns import (
+    ICEBERG_RESERVED_FIELD_ID_FOR_FILE_PATH_COLUMN,
+    ICEBERG_RESERVED_FIELD_ID_FOR_POS_COLUMN,
+)
+from pyiceberg.io.pyarrow import (
+    compute_statistics_plan,
+)
+from pyiceberg.manifest import (
+    DataFile,
+    DataFileContent,
+    FileFormat,
+)
+from pyiceberg.table import _min_sequence_number, _open_manifest
+from pyiceberg.utils.concurrent import ExecutorFactory
+from itertools import chain
+from pyiceberg.typedef import (
+    KeyDefaultDict,
+)
+
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-def parquet_files_dict_to_iceberg_data_files(io, table_metadata, files_dict_list):
-    from pyiceberg.io.pyarrow import (
-        _check_pyarrow_schema_compatible,
-        data_file_statistics_from_parquet_metadata,
-        compute_statistics_plan,
-        parquet_path_to_id_mapping,
-    )
-    from pyiceberg.manifest import (
-        DataFile,
-        DataFileContent,
-        FileFormat,
+def parquet_path_to_id_mapping_override(schema):
+    res = parquet_path_to_id_mapping(schema)
+    # Override here to insert position delete reserved column field IDs
+    res["file_path"] = ICEBERG_RESERVED_FIELD_ID_FOR_FILE_PATH_COLUMN
+    res["pos"] = ICEBERG_RESERVED_FIELD_ID_FOR_POS_COLUMN
+    return res
+
+
+def data_file_statistics_from_parquet_metadata(
+    parquet_metadata: pq.FileMetaData,
+    stats_columns: Dict[int, StatisticsCollector],
+    parquet_column_mapping: Dict[str, int],
+) -> DataFileStatistics:
+    """
+    Overrides original Pyiceberg function: Compute and return DataFileStatistics that includes the following.
+
+    - record_count
+    - column_sizes
+    - value_counts
+    - null_value_counts
+    - nan_value_counts
+    - column_aggregates
+    - split_offsets
+
+    Args:
+        parquet_metadata (pyarrow.parquet.FileMetaData): A pyarrow metadata object.
+        stats_columns (Dict[int, StatisticsCollector]): The statistics gathering plan. It is required to
+            set the mode for column metrics collection
+        parquet_column_mapping (Dict[str, int]): The mapping of the parquet file name to the field ID
+    """
+    column_sizes: Dict[int, int] = {}
+    value_counts: Dict[int, int] = {}
+    split_offsets: List[int] = []
+
+    null_value_counts: Dict[int, int] = {}
+    nan_value_counts: Dict[int, int] = {}
+
+    col_aggs = {}
+
+    invalidate_col: Set[int] = set()
+    for r in range(parquet_metadata.num_row_groups):
+        # References:
+        # https://github.com/apache/iceberg/blob/fc381a81a1fdb8f51a0637ca27cd30673bd7aad3/parquet/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java#L232
+        # https://github.com/apache/parquet-mr/blob/ac29db4611f86a07cc6877b416aa4b183e09b353/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java#L184
+
+        row_group = parquet_metadata.row_group(r)
+
+        data_offset = row_group.column(0).data_page_offset
+        dictionary_offset = row_group.column(0).dictionary_page_offset
+
+        if row_group.column(0).has_dictionary_page and dictionary_offset < data_offset:
+            split_offsets.append(dictionary_offset)
+        else:
+            split_offsets.append(data_offset)
+
+        for pos in range(parquet_metadata.num_columns):
+            column = row_group.column(pos)
+            field_id = parquet_column_mapping[column.path_in_schema]
+            if field_id in stats_columns:
+                stats_col = stats_columns[field_id]
+
+                column_sizes.setdefault(field_id, 0)
+                column_sizes[field_id] += column.total_compressed_size
+
+                if stats_col.mode == MetricsMode(MetricModeTypes.NONE):
+                    continue
+
+                value_counts[field_id] = (
+                    value_counts.get(field_id, 0) + column.num_values
+                )
+
+                if column.is_stats_set:
+                    try:
+                        statistics = column.statistics
+
+                        if statistics.has_null_count:
+                            null_value_counts[field_id] = (
+                                null_value_counts.get(field_id, 0)
+                                + statistics.null_count
+                            )
+
+                        if stats_col.mode == MetricsMode(MetricModeTypes.COUNTS):
+                            continue
+
+                        if field_id not in col_aggs:
+                            col_aggs[field_id] = StatsAggregator(
+                                stats_col.iceberg_type,
+                                statistics.physical_type,
+                                stats_col.mode.length,
+                            )
+
+                        col_aggs[field_id].update_min(statistics.min)
+                        col_aggs[field_id].update_max(statistics.max)
+
+                    except pyarrow.lib.ArrowNotImplementedError as e:
+                        invalidate_col.add(field_id)
+                        logger.warning(e)
+            else:
+                # Note: Removed original adding columns without stats to invalid column logic here
+                logger.warning(
+                    "PyArrow statistics missing for column %d when writing file", pos
+                )
+
+    split_offsets.sort()
+
+    for field_id in invalidate_col:
+        del col_aggs[field_id]
+        del null_value_counts[field_id]
+
+    return DataFileStatistics(
+        record_count=parquet_metadata.num_rows,
+        column_sizes=column_sizes,
+        value_counts=value_counts,
+        null_value_counts=null_value_counts,
+        nan_value_counts=nan_value_counts,
+        column_aggregates=col_aggs,
+        split_offsets=split_offsets,
     )
 
+
+def parquet_files_dict_to_iceberg_data_files(io, table_metadata, files_dict):
     data_file_content_type = DataFileContent.POSITION_DELETES
     iceberg_files = []
     schema = table_metadata.schema()
-    for files_dict in files_dict_list:
-        for partition_value, file_paths in files_dict.items():
-            for file_path in file_paths:
-                input_file = io.new_input(file_path)
-                with input_file.open() as input_stream:
-                    parquet_metadata = pq.read_metadata(input_stream)
-                _check_pyarrow_schema_compatible(
-                    schema, parquet_metadata.schema.to_arrow_schema()
-                )
+    for partition_value, file_paths in files_dict.items():
+        for file_path in file_paths:
+            logger.info(f"DEBUG_file_path:{file_path}")
+            input_file = io.new_input(file_path)
+            with input_file.open() as input_stream:
+                parquet_metadata = pq.read_metadata(input_stream)
 
-                statistics = data_file_statistics_from_parquet_metadata(
-                    parquet_metadata=parquet_metadata,
-                    stats_columns=compute_statistics_plan(
-                        schema, table_metadata.properties
-                    ),
-                    parquet_column_mapping=parquet_path_to_id_mapping(schema),
-                )
+            # Removed _check_pyarrow_schema_compatible() here since reserved columns does not comply to all rules.
 
-                data_file = DataFile(
-                    content=data_file_content_type,
-                    file_path=file_path,
-                    file_format=FileFormat.PARQUET,
-                    partition=partition_value,
-                    # partition=Record(**{"pk": "111", "bucket": 2}),
-                    file_size_in_bytes=len(input_file),
-                    sort_order_id=None,
-                    spec_id=table_metadata.default_spec_id,
-                    equality_ids=None,
-                    key_metadata=None,
-                    **statistics.to_serialized_dict(),
-                )
-                iceberg_files.append(data_file)
+            statistics = data_file_statistics_from_parquet_metadata(
+                parquet_metadata=parquet_metadata,
+                stats_columns=compute_statistics_plan(
+                    schema, table_metadata.properties
+                ),
+                parquet_column_mapping=parquet_path_to_id_mapping_override(schema),
+            )
+
+            data_file = DataFile(
+                content=data_file_content_type,
+                file_path=file_path,
+                file_format=FileFormat.PARQUET,
+                partition=partition_value,
+                file_size_in_bytes=len(input_file),
+                sort_order_id=None,
+                spec_id=table_metadata.default_spec_id,
+                equality_ids=None,
+                key_metadata=None,
+                **statistics.to_serialized_dict(),
+            )
+            iceberg_files.append(data_file)
     return iceberg_files
 
 
 def fetch_all_bucket_files(table):
     # step 1: filter manifests using partition summaries
     # the filter depends on the partition spec used to write the manifest file, so create a cache of filters for each spec id
-    from pyiceberg.typedef import (
-        KeyDefaultDict,
-    )
 
     data_scan = table.scan()
     snapshot = data_scan.snapshot()
@@ -78,23 +210,8 @@ def fetch_all_bucket_files(table):
 
     # step 2: filter the data files in each manifest
     # this filter depends on the partition spec used to write the manifest file
-    from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator
-    from pyiceberg.types import (
-        strtobool,
-    )
-    from pyiceberg.table import _min_sequence_number, _open_manifest
-    from pyiceberg.utils.concurrent import ExecutorFactory
-    from itertools import chain
-    from pyiceberg.manifest import DataFileContent
-
     partition_evaluators = KeyDefaultDict(data_scan._build_partition_evaluator)
-    metrics_evaluator = _InclusiveMetricsEvaluator(
-        data_scan.table_metadata.schema(),
-        data_scan.row_filter,
-        data_scan.case_sensitive,
-        strtobool(data_scan.options.get("include_empty_files", "false")),
-    ).eval
-
+    residual_evaluators = KeyDefaultDict(data_scan._build_residual_evaluator)
     min_sequence_number = _min_sequence_number(manifests)
 
     # {"bucket_index": List[DataFile]}
@@ -111,7 +228,8 @@ def fetch_all_bucket_files(table):
                     data_scan.io,
                     manifest,
                     partition_evaluators[manifest.partition_spec_id],
-                    metrics_evaluator,
+                    residual_evaluators[manifest.partition_spec_id],
+                    data_scan._build_metrics_evaluator(),
                 )
                 for manifest in manifests
                 if data_scan._check_sequence_number(min_sequence_number, manifest)
@@ -122,9 +240,10 @@ def fetch_all_bucket_files(table):
         file_sequence_number = manifest_entry.sequence_number
         data_file_tuple = (file_sequence_number, data_file)
         partition_value = data_file.partition
+
         if data_file.content == DataFileContent.DATA:
             data_entries[partition_value].append(data_file_tuple)
-        if data_file.content == DataFileContent.POSITION_DELETES:
+        elif data_file.content == DataFileContent.POSITION_DELETES:
             positional_delete_entries[partition_value].append(data_file_tuple)
         elif data_file.content == DataFileContent.EQUALITY_DELETES:
             equality_data_entries[partition_value].append(data_file_tuple)

deltacat/compute/converter/steps/convert.py

@@ -15,7 +15,10 @@ from deltacat.compute.converter.utils.io import (
 from deltacat.compute.converter.utils.converter_session_utils import (
     partition_value_record_to_partition_value_string,
 )
-
+from deltacat.compute.converter.pyiceberg.overrides import (
+    parquet_files_dict_to_iceberg_data_files,
+)
+from deltacat.compute.converter.model.convert_result import ConvertResult
 from deltacat import logs
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
@@ -27,17 +30,22 @@ def convert(convert_input: ConvertInput):
     convert_task_index = convert_input.convert_task_index
     iceberg_table_warehouse_prefix = convert_input.iceberg_table_warehouse_prefix
     identifier_fields = convert_input.identifier_fields
-    compact_small_files = convert_input.compact_small_files
+    table_io = convert_input.table_io
+    table_metadata = convert_input.table_metadata
+    compact_previous_position_delete_files = (
+        convert_input.compact_previous_position_delete_files
+    )
     position_delete_for_multiple_data_files = (
         convert_input.position_delete_for_multiple_data_files
     )
     max_parallel_data_file_download = convert_input.max_parallel_data_file_download
     s3_file_system = convert_input.s3_file_system
+    s3_client_kwargs = convert_input.s3_client_kwargs
     if not position_delete_for_multiple_data_files:
         raise NotImplementedError(
             f"Distributed file level position delete compute is not supported yet"
         )
-    if compact_small_files:
+    if compact_previous_position_delete_files:
         raise NotImplementedError(f"Compact previous position delete not supported yet")
 
     logger.info(f"Starting convert task index: {convert_task_index}")
@@ -52,9 +60,15 @@ def convert(convert_input: ConvertInput):
         convert_input_files.partition_value
     )
     partition_value = convert_input_files.partition_value
-    iceberg_table_warehouse_prefix_with_partition = (
-        f"{iceberg_table_warehouse_prefix}/{partition_value_str}"
-    )
+
+    if partition_value_str:
+        iceberg_table_warehouse_prefix_with_partition = (
+            f"{iceberg_table_warehouse_prefix}/{partition_value_str}"
+        )
+    else:
+        iceberg_table_warehouse_prefix_with_partition = (
+            f"{iceberg_table_warehouse_prefix}"
+        )
     enforce_primary_key_uniqueness = convert_input.enforce_primary_key_uniqueness
     total_pos_delete_table = []
     if applicable_equality_delete_files:
@@ -65,8 +79,10 @@ def convert(convert_input: ConvertInput):
             identifier_columns=identifier_fields,
             equality_delete_files_list=applicable_equality_delete_files,
             iceberg_table_warehouse_prefix_with_partition=iceberg_table_warehouse_prefix_with_partition,
+            convert_task_index=convert_task_index,
             max_parallel_data_file_download=max_parallel_data_file_download,
             s3_file_system=s3_file_system,
+            s3_client_kwargs=s3_client_kwargs,
         )
         if pos_delete_after_converting_equality_delete:
             total_pos_delete_table.append(pos_delete_after_converting_equality_delete)
@@ -76,31 +92,71 @@ def convert(convert_input: ConvertInput):
             all_data_files=all_data_files_for_this_bucket,
             data_files_downloaded=applicable_data_files,
         )
-        pos_delete_after_dedupe = dedupe_data_files(
+        logger.info(
+            f"[Convert task {convert_task_index}]: Got {len(data_files_to_dedupe)} files to dedupe."
+        )
+        (
+            pos_delete_after_dedupe,
+            data_file_to_dedupe_record_count,
+            data_file_to_dedupe_size,
+        ) = dedupe_data_files(
             data_file_to_dedupe=data_files_to_dedupe,
-            identify_column_name_concatenated=identifier_fields[0],
             identifier_columns=identifier_fields,
             merge_sort_column=sc._ORDERED_RECORD_IDX_COLUMN_NAME,
+            s3_client_kwargs=s3_client_kwargs,
+        )
+        logger.info(
+            f"[Convert task {convert_task_index}]: Dedupe produced {len(pos_delete_after_dedupe)} position delete records."
         )
         total_pos_delete_table.append(pos_delete_after_dedupe)
 
     total_pos_delete = pa.concat_tables(total_pos_delete_table)
-    to_be_added_files_list = upload_table_with_retry(
-        table=total_pos_delete,
-        s3_url_prefix=iceberg_table_warehouse_prefix_with_partition,
-        s3_table_writer_kwargs={},
-        s3_file_system=s3_file_system,
+
+    logger.info(
+        f"[Convert task {convert_task_index}]: Total position delete produced:{len(total_pos_delete)}"
     )
 
+    to_be_added_files_list = []
+    if total_pos_delete:
+        to_be_added_files_list_parquet = upload_table_with_retry(
+            table=total_pos_delete,
+            s3_url_prefix=iceberg_table_warehouse_prefix_with_partition,
+            s3_table_writer_kwargs={},
+            s3_file_system=s3_file_system,
+        )
+
+        to_be_added_files_dict = defaultdict()
+        to_be_added_files_dict[partition_value] = to_be_added_files_list_parquet
+
+        logger.info(
+            f"[Convert task {convert_task_index}]: Produced {len(to_be_added_files_list_parquet)} position delete files."
+        )
+        to_be_added_files_list = parquet_files_dict_to_iceberg_data_files(
+            io=table_io,
+            table_metadata=table_metadata,
+            files_dict=to_be_added_files_dict,
+        )
+
     to_be_delete_files_dict = defaultdict()
     if applicable_equality_delete_files:
         to_be_delete_files_dict[partition_value] = [
             equality_delete_file[1]
             for equality_delete_file in applicable_equality_delete_files
         ]
-    to_be_added_files_dict = defaultdict()
-    to_be_added_files_dict[partition_value] = to_be_added_files_list
-    return (to_be_delete_files_dict, to_be_added_files_dict)
+
+    convert_res = ConvertResult.of(
+        convert_task_index=convert_task_index,
+        to_be_added_files=to_be_added_files_list,
+        to_be_deleted_files=to_be_delete_files_dict,
+        position_delete_record_count=len(total_pos_delete),
+        input_data_files_record_count=data_file_to_dedupe_record_count,
+        input_data_files_hash_columns_in_memory_sizes=data_file_to_dedupe_size,
+        position_delete_in_memory_sizes=int(total_pos_delete.nbytes),
+        position_delete_on_disk_sizes=sum(
+            file.file_size_in_bytes for file in to_be_added_files_list
+        ),
+    )
+    return convert_res
 
 
 def get_additional_applicable_data_files(all_data_files, data_files_downloaded):
@@ -120,11 +176,6 @@ def filter_rows_to_be_deleted(
             equality_delete_table[identifier_column],
         )
         position_delete_table = data_file_table.filter(equality_deletes)
-        logger.info(f"positional_delete_table:{position_delete_table.to_pydict()}")
-        logger.info(f"data_file_table:{data_file_table.to_pydict()}")
-        logger.info(
-            f"length_pos_delete_table, {len(position_delete_table)}, length_data_table:{len(data_file_table)}"
-        )
     return position_delete_table
 
 
@@ -149,25 +200,15 @@ def compute_pos_delete_converting_equality_deletes(
     return new_position_delete_table
 
 
-def download_bucketed_table(data_files, equality_delete_files):
-    from deltacat.utils.pyarrow import s3_file_to_table
-
-    compacted_table = s3_file_to_table(
-        [data_file.file_path for data_file in data_files]
-    )
-    equality_delete_table = s3_file_to_table(
-        [eq_file.file_path for eq_file in equality_delete_files]
-    )
-    return compacted_table, equality_delete_table
-
-
 def compute_pos_delete_with_limited_parallelism(
     data_files_list,
     identifier_columns,
     equality_delete_files_list,
     iceberg_table_warehouse_prefix_with_partition,
+    convert_task_index,
     max_parallel_data_file_download,
     s3_file_system,
+    s3_client_kwargs,
 ):
     for data_files, equality_delete_files in zip(
         data_files_list, equality_delete_files_list
@@ -182,6 +223,7 @@ def compute_pos_delete_with_limited_parallelism(
                     sc._ORDERED_RECORD_IDX_COLUMN_NAME,
                 ],
                 sequence_number=data_file[0],
+                s3_client_kwargs=s3_client_kwargs,
             )
             data_table_total.append(data_table)
         data_table_total = pa.concat_tables(data_table_total)
@@ -191,6 +233,7 @@ def compute_pos_delete_with_limited_parallelism(
             equality_delete_table = download_data_table_and_append_iceberg_columns(
                 data_files=equality_delete[1],
                 columns_to_download=identifier_columns,
+                s3_client_kwargs=s3_client_kwargs,
             )
             equality_delete_table_total.append(equality_delete_table)
         equality_delete_table_total = pa.concat_tables(equality_delete_table_total)
@@ -201,11 +244,16 @@ def compute_pos_delete_with_limited_parallelism(
         iceberg_table_warehouse_prefix_with_partition=iceberg_table_warehouse_prefix_with_partition,
         identifier_columns=identifier_columns,
         s3_file_system=s3_file_system,
+        s3_client_kwargs=s3_client_kwargs,
     )
-    if not new_pos_delete_table:
-        logger.info("No records deleted based on equality delete converstion")
 
     logger.info(
-        f"Number of records to delete based on equality delete convertion:{len(new_pos_delete_table)}"
+        f"[Convert task {convert_task_index}]: Find deletes got {len(data_table_total)} data table records, "
+        f"{len(equality_delete_table_total)} equality deletes as input, "
+        f"Produced {len(new_pos_delete_table)} position deletes based off find deletes input."
     )
+
+    if not new_pos_delete_table:
+        logger.info("No records deleted based on equality delete convertion")
+
     return new_pos_delete_table

deltacat/compute/converter/steps/dedupe.py

@@ -4,16 +4,21 @@ import deltacat.compute.converter.utils.iceberg_columns as sc
 from deltacat.compute.converter.utils.io import (
     download_data_table_and_append_iceberg_columns,
 )
+import logging
+from deltacat import logs
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 def dedupe_data_files(
     data_file_to_dedupe,
-    identify_column_name_concatenated,
     identifier_columns,
     merge_sort_column,
+    s3_client_kwargs,
 ):
     data_file_table = []
 
+    downloaded_data_file_record_count = 0
     # Sort data files by file sequence number first
     data_file_to_dedupe = sorted(data_file_to_dedupe, key=lambda f: f[0])
     for file_tuple in data_file_to_dedupe:
@@ -27,11 +32,20 @@ def dedupe_data_files(
                 sc._ORDERED_RECORD_IDX_COLUMN_NAME,
             ],
             sequence_number=sequence_number,
+            s3_client_kwargs=s3_client_kwargs,
         )
+        downloaded_data_file_record_count += len(data_file_to_dedupe_table)
         data_file_table.append(data_file_to_dedupe_table)
 
     final_data_to_dedupe = pa.concat_tables(data_file_table)
 
+    assert len(final_data_to_dedupe) == downloaded_data_file_record_count, (
+        f"Mismatch record count while performing table concat, Got {len(final_data_to_dedupe)} in final table, "
+        f"while input table length is: {downloaded_data_file_record_count}"
+    )
+
+    logger.info(f"Length of pyarrow table to dedupe:{len(final_data_to_dedupe)}")
+
     record_idx_iterator = iter(range(len(final_data_to_dedupe)))
 
     # Append global record index to used as aggregate column
@@ -40,7 +54,7 @@ def dedupe_data_files(
     )
 
     final_data_table_indices = final_data_to_dedupe.group_by(
-        identify_column_name_concatenated, use_threads=False
+        sc._IDENTIFIER_COLUMNS_HASH_COLUMN_NAME, use_threads=False
     ).aggregate([(sc._GLOBAL_RECORD_IDX_COLUMN_NAME, "max")])
 
     pos_delete_indices = pc.invert(
@@ -55,6 +69,13 @@ def dedupe_data_files(
     final_data_table_to_delete = final_data_to_dedupe.filter(pos_delete_indices)
 
     final_data_table_to_delete = final_data_table_to_delete.drop(
-        [identify_column_name_concatenated, sc._GLOBAL_RECORD_IDX_COLUMN_NAME]
+        [sc._IDENTIFIER_COLUMNS_HASH_COLUMN_NAME, sc._GLOBAL_RECORD_IDX_COLUMN_NAME]
+    )
+    logger.info(
+        f"Deduped {len(final_data_table_to_delete)} Records based off identifier columns."
+    )
+    return (
+        final_data_table_to_delete,
+        len(final_data_to_dedupe),
+        int(final_data_to_dedupe.nbytes),
     )
-    return final_data_table_to_delete