deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/converter/steps/dedupe.py

@@ -0,0 +1,94 @@
+import pyarrow as pa
+import pyarrow.compute as pc
+import deltacat.compute.converter.utils.iceberg_columns as sc
+from deltacat.compute.converter.utils.io import (
+    download_data_table_and_append_iceberg_columns,
+)
+from deltacat.compute.converter.utils.converter_session_utils import (
+    sort_data_files_maintaining_order,
+)
+import logging
+from deltacat import logs
+from typing import List, Dict, Tuple, Optional, Any
+from pyiceberg.manifest import DataFile
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def dedupe_data_files(
+    data_file_to_dedupe: List[Tuple[int, DataFile]],
+    identifier_columns: List[str],
+    remaining_data_table_after_convert: Optional[pa.Table],
+    merge_sort_column: str,
+    s3_client_kwargs: Optional[Dict[str, Any]],
+) -> Tuple[pa.Table, int, int]:
+    data_file_table = []
+    if remaining_data_table_after_convert:
+        data_file_table.append(remaining_data_table_after_convert)
+
+    data_file_to_dedupe = sort_data_files_maintaining_order(
+        data_files=data_file_to_dedupe
+    )
+    downloaded_data_file_record_count = 0
+    for file_tuple in data_file_to_dedupe:
+        data_file = file_tuple[1]
+        data_file_to_dedupe_table = download_data_table_and_append_iceberg_columns(
+            file=data_file,
+            columns_to_download=identifier_columns,
+            additional_columns_to_append=[
+                sc._FILE_PATH_COLUMN_NAME,
+                sc._ORDERED_RECORD_IDX_COLUMN_NAME,
+            ],
+            s3_client_kwargs=s3_client_kwargs,
+        )
+        logger.info(
+            f"Length of downloaded data file table: {len(data_file_to_dedupe_table)}"
+        )
+        downloaded_data_file_record_count += len(data_file_to_dedupe_table)
+        data_file_table.append(data_file_to_dedupe_table)
+
+    final_data_to_dedupe = pa.concat_tables(data_file_table)
+
+    dedupe_input_record_count = downloaded_data_file_record_count
+    if remaining_data_table_after_convert:
+        dedupe_input_record_count += len(remaining_data_table_after_convert)
+    assert len(final_data_to_dedupe) == dedupe_input_record_count, (
+        f"Mismatch record count while performing table concat, Got {len(final_data_to_dedupe)} in final table, "
+        f"while input table length is: {dedupe_input_record_count}"
+    )
+
+    logger.info(f"Length of pyarrow table to dedupe:{len(final_data_to_dedupe)}")
+
+    record_idx_iterator = iter(range(len(final_data_to_dedupe)))
+
+    # Append global record index to used as aggregate column
+    final_data_to_dedupe = sc.append_global_record_idx_column(
+        final_data_to_dedupe, record_idx_iterator
+    )
+
+    final_data_table_indices = final_data_to_dedupe.group_by(
+        sc._IDENTIFIER_COLUMNS_HASH_COLUMN_NAME, use_threads=False
+    ).aggregate([(sc._GLOBAL_RECORD_IDX_COLUMN_NAME, "max")])
+
+    pos_delete_indices = pc.invert(
+        pc.is_in(
+            final_data_to_dedupe[sc._GLOBAL_RECORD_IDX_COLUMN_NAME],
+            value_set=final_data_table_indices[
+                f"{sc._GLOBAL_RECORD_IDX_COLUMN_NAME}_max"
+            ],
+        )
+    )
+
+    final_data_table_to_delete = final_data_to_dedupe.filter(pos_delete_indices)
+
+    final_data_table_to_delete = final_data_table_to_delete.drop(
+        [sc._IDENTIFIER_COLUMNS_HASH_COLUMN_NAME, sc._GLOBAL_RECORD_IDX_COLUMN_NAME]
+    )
+    logger.info(
+        f"Deduped {len(final_data_table_to_delete)} Records based off identifier columns."
+    )
+    return (
+        final_data_table_to_delete,
+        len(final_data_to_dedupe),
+        int(final_data_to_dedupe.nbytes),
+    )

deltacat/compute/converter/utils/convert_task_options.py

@@ -0,0 +1,132 @@
+from typing import Optional, Dict, List, Tuple, Any
+from deltacat.exceptions import RetryableError
+from pyiceberg.manifest import DataFile
+from deltacat.compute.converter.model.convert_input_files import ConvertInputFiles
+
+AVERAGE_FILE_PATH_COLUMN_SIZE_BYTES = 160
+AVERAGE_POS_COLUMN_SIZE_BYTES = 4
+XXHASH_BYTE_PER_RECORD = 8
+MEMORY_BUFFER_RATE = 2
+# Worst case 2 as no duplicates exists across all pk
+PYARROW_AGGREGATE_MEMORY_MULTIPLIER = 2
+# Observed base memory usage at the beginning of each worker process
+BASE_MEMORY_BUFFER = 0.3 * 1024 * 1024 * 1024
+
+
+def estimate_fixed_hash_columns(
+    hash_value_size_bytes_per_record: int, total_record_count: int
+) -> int:
+    return hash_value_size_bytes_per_record * total_record_count
+
+
+def get_total_record_from_iceberg_files(
+    iceberg_files_list: List[Tuple[int, DataFile]]
+) -> int:
+    total_record_count = 0
+    # file are in form of tuple (sequence_number, DataFile)
+    total_record_count += sum(file[1].record_count for file in iceberg_files_list)
+    return total_record_count
+
+
+def estimate_iceberg_pos_delete_additional_columns(
+    include_columns: List[str], num_of_record_count: int
+) -> int:
+    total_additional_columns_sizes = 0
+    if "file_path" in include_columns:
+        total_additional_columns_sizes += (
+            AVERAGE_FILE_PATH_COLUMN_SIZE_BYTES * num_of_record_count
+        )
+    elif "pos" in include_columns:
+        total_additional_columns_sizes += (
+            AVERAGE_POS_COLUMN_SIZE_BYTES * num_of_record_count
+        )
+    return total_additional_columns_sizes
+
+
+def estimate_convert_remote_option_resources(
+    data_files: List[Tuple[int, DataFile]],
+    equality_delete_files: List[Tuple[int, DataFile]],
+) -> float:
+    data_file_record_count = get_total_record_from_iceberg_files(data_files)
+    equality_delete_record_count = get_total_record_from_iceberg_files(
+        equality_delete_files
+    )
+    hash_column_sizes = estimate_fixed_hash_columns(
+        XXHASH_BYTE_PER_RECORD, data_file_record_count + equality_delete_record_count
+    )
+    pos_delete_sizes = estimate_iceberg_pos_delete_additional_columns(
+        ["file_path", "pos"], data_file_record_count + equality_delete_record_count
+    )
+    total_memory_required = hash_column_sizes + pos_delete_sizes
+    return total_memory_required * MEMORY_BUFFER_RATE
+
+
+def _get_task_options(
+    memory: float,
+    ray_custom_resources: Optional[Dict[str, Any]] = None,
+    scheduling_strategy: str = "SPREAD",
+) -> Dict[str, Any]:
+
+    # NOTE: With DEFAULT scheduling strategy in Ray 2.20.0, autoscaler does
+    # not spin up enough nodes fast and hence we see only approximately
+    # 20 tasks get scheduled out of 100 tasks in queue. Hence, we use SPREAD
+    # which is also ideal for merge and hash bucket tasks.
+    # https://docs.ray.io/en/latest/ray-core/scheduling/index.html
+    task_opts = {
+        "memory": memory,
+        "scheduling_strategy": scheduling_strategy,
+    }
+
+    if ray_custom_resources:
+        task_opts["resources"] = ray_custom_resources
+
+    task_opts["max_retries"] = 3
+    task_opts["num_cpus"] = 1
+    task_opts["resources"] = {"convert_task": 1}
+    # List of possible botocore exceptions are available at
+    # https://github.com/boto/botocore/blob/develop/botocore/exceptions.py
+    task_opts["retry_exceptions"] = [RetryableError]
+
+    return task_opts
+
+
+def estimate_dedupe_memory(
+    all_data_files_for_dedupe: List[Tuple[int, DataFile]]
+) -> float:
+    dedupe_record_count = get_total_record_from_iceberg_files(all_data_files_for_dedupe)
+    produced_pos_memory_required = estimate_iceberg_pos_delete_additional_columns(
+        ["file_path", "pos"], dedupe_record_count
+    )
+    download_pk_memory_required = estimate_fixed_hash_columns(
+        XXHASH_BYTE_PER_RECORD, dedupe_record_count
+    )
+    memory_required_by_dedupe = (
+        produced_pos_memory_required + download_pk_memory_required
+    ) * PYARROW_AGGREGATE_MEMORY_MULTIPLIER
+    memory_with_buffer = memory_required_by_dedupe * MEMORY_BUFFER_RATE
+    return memory_with_buffer
+
+
+def convert_resource_options_provider(
+    index: int, convert_input_files: ConvertInputFiles
+) -> Dict[str, Any]:
+    applicable_data_files = convert_input_files.applicable_data_files
+    applicable_equality_delete_files = (
+        convert_input_files.applicable_equality_delete_files
+    )
+    all_data_files_for_dedupe = convert_input_files.all_data_files_for_dedupe
+    total_memory_required = 0
+    total_memory_required += BASE_MEMORY_BUFFER
+    if applicable_data_files and applicable_equality_delete_files:
+        memory_requirement_for_convert_equality_deletes = (
+            estimate_convert_remote_option_resources(
+                applicable_data_files, applicable_equality_delete_files
+            )
+        )
+        total_memory_required += memory_requirement_for_convert_equality_deletes
+    if all_data_files_for_dedupe:
+        memory_requirement_for_dedupe = estimate_dedupe_memory(
+            all_data_files_for_dedupe
+        )
+        total_memory_required += memory_requirement_for_dedupe
+    return _get_task_options(memory=total_memory_required)

deltacat/compute/converter/utils/converter_session_utils.py

@@ -0,0 +1,175 @@
+from collections import defaultdict
+import logging
+from deltacat import logs
+from deltacat.compute.converter.model.convert_input_files import (
+    ConvertInputFiles,
+    DataFileList,
+    DataFileListGroup,
+)
+from typing import List, Dict, Tuple, Any
+from enum import Enum
+from pyiceberg.manifest import DataFile
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def check_data_files_sequence_number(
+    data_files_list: DataFileList,
+    equality_delete_files_list: DataFileList,
+) -> Tuple[DataFileListGroup, DataFileListGroup]:
+    # Sort by file sequence number
+    data_files_list.sort(key=lambda file_tuple: file_tuple[0])
+    equality_delete_files_list.sort(key=lambda file_tuple: file_tuple[0])
+    data_file_delete_applicable = []
+    result_eq_files_list = []
+
+    # Loop through each value in equality_delete_file
+    for data_file_tuple in data_files_list:
+
+        # Find all values in equality delete file that having a larger sequence number than current data file
+        valid_values_eq = []
+
+        # Pointer for equality delete file
+        eq_file_pointer = 0
+        # Move data_file_pointer to the first value in data_file that is smaller than val_equality
+        while (
+            eq_file_pointer < len(equality_delete_files_list)
+            and equality_delete_files_list[eq_file_pointer][0] > data_file_tuple[0]
+        ):
+            valid_values_eq.append(equality_delete_files_list[eq_file_pointer])
+            eq_file_pointer += 1
+
+        if valid_values_eq:
+            # Append the value for both applicable eq files list and applicable data files list
+            data_file_delete_applicable.append(data_file_tuple)
+            result_eq_files_list.append(valid_values_eq)
+
+    res_data_file_list = []
+    res_equality_delete_file_list = []
+    merged_file_dict = defaultdict(list)
+    for data_file_sublist, eq_delete_sublist in zip(
+        data_file_delete_applicable, result_eq_files_list
+    ):
+        merged_file_dict[tuple(eq_delete_sublist)].append(data_file_sublist)
+    for eq_file_list, data_file_list in merged_file_dict.items():
+        res_data_file_list.append(list(set(data_file_list)))
+        res_equality_delete_file_list.append(list(set(eq_file_list)))
+
+    assert len(res_data_file_list) == len(res_equality_delete_file_list), (
+        f"length of applicable data files list: {len(res_data_file_list)} "
+        f"should equal to length of equality delete files list:{len(res_equality_delete_file_list)}"
+    )
+
+    return res_equality_delete_file_list, res_data_file_list
+
+
+def construct_iceberg_table_prefix(
+    iceberg_warehouse_bucket_name: str, table_name: str, iceberg_namespace: str
+) -> str:
+    return f"{iceberg_warehouse_bucket_name}/{iceberg_namespace}/{table_name}/data"
+
+
+def partition_value_record_to_partition_value_string(partition: Any) -> str:
+    # Get string representation of partition value out of Record[partition_value]
+    partition_value_str = partition.__repr__().split("[", 1)[1].split("]")[0]
+    return partition_value_str
+
+
+def group_all_files_to_each_bucket(
+    data_file_dict: Dict[Any, DataFileList],
+    equality_delete_dict: Dict[Any, DataFileList],
+    pos_delete_dict: Dict[Any, DataFileList],
+) -> List[ConvertInputFiles]:
+    convert_input_files_for_all_buckets = []
+    files_for_each_bucket_for_deletes = defaultdict(tuple)
+    if equality_delete_dict:
+        for partition_value, equality_delete_file_list in equality_delete_dict.items():
+            if partition_value in data_file_dict:
+                (
+                    result_equality_delete_file,
+                    result_data_file,
+                ) = check_data_files_sequence_number(
+                    data_files_list=data_file_dict[partition_value],
+                    equality_delete_files_list=equality_delete_dict[partition_value],
+                )
+                files_for_each_bucket_for_deletes[partition_value] = (
+                    result_data_file,
+                    result_equality_delete_file,
+                    [],
+                )
+
+    for partition_value, all_data_files_for_each_bucket in data_file_dict.items():
+        convert_input_file = ConvertInputFiles.of(
+            partition_value=partition_value,
+            all_data_files_for_dedupe=all_data_files_for_each_bucket,
+        )
+        if partition_value in files_for_each_bucket_for_deletes:
+            convert_input_file.applicable_data_files = (
+                files_for_each_bucket_for_deletes[partition_value][0]
+            )
+            convert_input_file.applicable_equality_delete_files = (
+                files_for_each_bucket_for_deletes[partition_value][1]
+            )
+        convert_input_files_for_all_buckets.append(convert_input_file)
+    return convert_input_files_for_all_buckets
+
+
+def sort_data_files_maintaining_order(data_files: DataFileList) -> DataFileList:
+    """
+    Sort data files deterministically based on two criterias:
+    1. Sequence number: Newly added files will have a higher sequence number
+    2. File path: If file sequence is the same, files are guaranteed to be returned in a deterministic order since file path is unique.
+    """
+    if data_files:
+        data_files = sorted(data_files, key=lambda f: (f[0], f[1].file_path))
+    return data_files
+
+
+class SnapshotType(Enum):
+    """Enumeration of possible snapshot types."""
+
+    NONE = "none"
+    APPEND = "append"
+    REPLACE = "replace"
+    DELETE = "delete"
+
+
+def _get_snapshot_action_description(
+    snapshot_type: SnapshotType,
+    files_to_delete: List[List[DataFile]],
+    files_to_add: List[DataFile],
+) -> str:
+    """Get a human-readable description of the snapshot action."""
+    descriptions = {
+        SnapshotType.NONE: "No changes needed",
+        SnapshotType.APPEND: f"Adding {len(files_to_add)} new files",
+        SnapshotType.REPLACE: f"Replacing {sum(len(files) for files in files_to_delete)} files with {len(files_to_add)} new files",
+        SnapshotType.DELETE: f"Deleting {sum(len(files) for files in files_to_delete)} files",
+    }
+    return descriptions[snapshot_type]
+
+
+def _determine_snapshot_type(
+    to_be_deleted_files: List[List[DataFile]], to_be_added_files: List[DataFile]
+) -> SnapshotType:
+    """
+    Determine the snapshot type based on file changes.
+
+    Args:
+        to_be_deleted_files: List of files to be deleted
+        to_be_added_files: List of files to be added
+
+    Returns:
+        SnapshotType indicating what kind of snapshot to commit
+    """
+    has_files_to_delete = bool(to_be_deleted_files)
+    has_files_to_add = bool(to_be_added_files)
+
+    if not has_files_to_delete and not has_files_to_add:
+        return SnapshotType.NONE
+    elif not has_files_to_delete and has_files_to_add:
+        return SnapshotType.APPEND
+    elif has_files_to_delete and has_files_to_add:
+        return SnapshotType.REPLACE
+    else:  # has_files_to_delete and not has_files_to_add
+        return SnapshotType.DELETE

deltacat/compute/converter/utils/iceberg_columns.py

@@ -0,0 +1,87 @@
+import pyarrow as pa
+from typing import Union, Iterator, Any
+import numpy as np
+
+# Refer to: https://iceberg.apache.org/spec/#reserved-field-ids for reserved field ids
+ICEBERG_RESERVED_FIELD_ID_FOR_FILE_PATH_COLUMN = 2147483546
+
+# Refer to: https://iceberg.apache.org/spec/#reserved-field-ids for reserved field ids
+ICEBERG_RESERVED_FIELD_ID_FOR_POS_COLUMN = 2147483545
+
+
+def _get_iceberg_col_name(suffix: str) -> str:
+    return suffix
+
+
+_ORDERED_RECORD_IDX_COLUMN_NAME = _get_iceberg_col_name("pos")
+_ORDERED_RECORD_IDX_COLUMN_TYPE = pa.int64()
+_ORDERED_RECORD_IDX_FIELD_METADATA = {
+    b"PARQUET:field_id": f"{ICEBERG_RESERVED_FIELD_ID_FOR_POS_COLUMN}"
+}
+_ORDERED_RECORD_IDX_COLUMN_FIELD = pa.field(
+    _ORDERED_RECORD_IDX_COLUMN_NAME,
+    _ORDERED_RECORD_IDX_COLUMN_TYPE,
+    metadata=_ORDERED_RECORD_IDX_FIELD_METADATA,
+    nullable=False,
+)
+
+
+def get_record_index_column_array(obj: Any) -> Union[pa.Array, pa.ChunkedArray]:
+    return pa.array(
+        obj,
+        _ORDERED_RECORD_IDX_COLUMN_TYPE,
+    )
+
+
+def append_record_idx_col(
+    table: pa.Table, ordered_record_indices: Iterator[int]
+) -> pa.Table:
+    table = table.append_column(
+        _ORDERED_RECORD_IDX_COLUMN_FIELD,
+        get_record_index_column_array(ordered_record_indices),
+    )
+    return table
+
+
+_FILE_PATH_COLUMN_NAME = _get_iceberg_col_name("file_path")
+_FILE_PATH_COLUMN_TYPE = pa.string()
+_FILE_PATH_FIELD_METADATA = {
+    b"PARQUET:field_id": f"{ICEBERG_RESERVED_FIELD_ID_FOR_FILE_PATH_COLUMN}"
+}
+_FILE_PATH_COLUMN_FIELD = pa.field(
+    _FILE_PATH_COLUMN_NAME,
+    _FILE_PATH_COLUMN_TYPE,
+    metadata=_FILE_PATH_FIELD_METADATA,
+    nullable=False,
+)
+
+
+def append_file_path_column(table: pa.Table, file_path: str) -> pa.Table:
+    table = table.append_column(
+        _FILE_PATH_COLUMN_FIELD,
+        pa.array(np.repeat(file_path, len(table)), _FILE_PATH_COLUMN_TYPE),
+    )
+    return table
+
+
+_GLOBAL_RECORD_IDX_COLUMN_NAME = _get_iceberg_col_name("global_record_index")
+_GLOBAL_RECORD_IDX_COLUMN_TYPE = pa.int64()
+_GLOBAL_RECORD_IDX_COLUMN_FIELD = pa.field(
+    _GLOBAL_RECORD_IDX_COLUMN_NAME,
+    _GLOBAL_RECORD_IDX_COLUMN_TYPE,
+)
+
+
+def append_global_record_idx_column(
+    table: pa.Table, ordered_record_indices: Iterator[int]
+) -> pa.Table:
+    table = table.append_column(
+        _GLOBAL_RECORD_IDX_COLUMN_NAME,
+        pa.array(ordered_record_indices, _GLOBAL_RECORD_IDX_COLUMN_TYPE),
+    )
+    return table
+
+
+_IDENTIFIER_COLUMNS_HASH_COLUMN_NAME = _get_iceberg_col_name(
+    "identifier_columns_hashed"
+)

deltacat/compute/converter/utils/io.py

@@ -0,0 +1,203 @@
+import logging
+
+from fsspec import AbstractFileSystem
+from deltacat import logs
+import deltacat.compute.converter.utils.iceberg_columns as sc
+import daft
+from deltacat.utils.daft import _get_s3_io_config
+from daft import TimeUnit, DataFrame
+import pyarrow as pa
+from typing import Callable, Optional, List, Dict, Any
+from deltacat.utils.pyarrow import sliced_string_cast
+from deltacat.compute.converter.constants import IDENTIFIER_FIELD_DELIMITER
+from deltacat.compute.converter.utils.s3u import upload_table_with_retry
+from pyiceberg.manifest import DataFile
+import pyarrow.compute as pc
+from deltacat.types.media import ContentType
+from deltacat.types.tables import (
+    get_table_writer,
+    get_table_slicer,
+    write_sliced_table as types_write_sliced_table,
+)
+from deltacat.storage import LocalTable, DistributedDataset
+from typing import Union
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def download_data_table_and_append_iceberg_columns(
+    file: DataFile,
+    columns_to_download: List[str],
+    additional_columns_to_append: Optional[List[str]] = [],
+    s3_client_kwargs: Optional[Dict[str, Any]] = None,
+) -> pa.Table:
+    table = download_parquet_with_daft_hash_applied(
+        identifier_columns=columns_to_download,
+        file=file,
+        s3_client_kwargs=s3_client_kwargs,
+    )
+
+    if sc._FILE_PATH_COLUMN_NAME in additional_columns_to_append:
+        table = sc.append_file_path_column(table, file.file_path)
+    if sc._ORDERED_RECORD_IDX_COLUMN_NAME in additional_columns_to_append:
+        record_idx_iterator = iter(range(len(table)))
+        table = sc.append_record_idx_col(table, record_idx_iterator)
+
+    return table
+
+
+def download_parquet_with_daft_hash_applied(
+    identifier_columns: List[str],
+    file: DataFile,
+    s3_client_kwargs: Optional[Dict[str, Any]],
+    **kwargs: Any,
+) -> pa.Table:
+
+    # TODO: Add correct read kwargs as in:
+    #  https://github.com/ray-project/deltacat/blob/383855a4044e4dfe03cf36d7738359d512a517b4/deltacat/utils/daft.py#L97
+
+    coerce_int96_timestamp_unit = TimeUnit.from_str(
+        kwargs.get("coerce_int96_timestamp_unit", "ms")
+    )
+
+    # TODO: Use Daft SHA1 hash instead to minimize probably of data corruption
+    io_config = _get_s3_io_config(s3_client_kwargs=s3_client_kwargs)
+    df = daft_read_parquet(
+        path=file.file_path,
+        io_config=io_config,
+        coerce_int96_timestamp_unit=coerce_int96_timestamp_unit,
+    )
+
+    hash_column = concatenate_hashed_identifier_columns(
+        df=df, identifier_columns=identifier_columns
+    )
+
+    table = pa.Table.from_arrays(
+        [hash_column], names=[sc._IDENTIFIER_COLUMNS_HASH_COLUMN_NAME]
+    )
+
+    return table
+
+
+def daft_read_parquet(
+    path: str, io_config: Dict[str, Any], coerce_int96_timestamp_unit: TimeUnit
+) -> DataFrame:
+    df = daft.read_parquet(
+        path=path,
+        io_config=io_config,
+        coerce_int96_timestamp_unit=coerce_int96_timestamp_unit,
+    )
+    return df
+
+
+def concatenate_hashed_identifier_columns(
+    df: DataFrame, identifier_columns: List[str]
+) -> pa.Array:
+    pk_hash_columns = []
+    previous_hash_column_length = None
+    for i in range(len(identifier_columns)):
+        pk_hash_column = df.select(daft.col(identifier_columns[i]).hash())
+        pk_hash_column_arrow = pk_hash_column.to_arrow()
+
+        # Assert that each hash column downloaded are same length to ensure we don't create mismatch between columns.
+        if not previous_hash_column_length:
+            previous_hash_column_length = len(pk_hash_column_arrow)
+        else:
+            assert previous_hash_column_length == len(pk_hash_column_arrow), (
+                f"Identifier column Length mismatch: {identifier_columns[i]} has length {len(pk_hash_column_arrow)} "
+                f"but expected {previous_hash_column_length}."
+            )
+            previous_hash_column_length = len(pk_hash_column_arrow)
+
+        # Convert identifier from different datatypes to string here
+        pk_hash_column_str = sliced_string_cast(
+            pk_hash_column_arrow[identifier_columns[i]]
+        )
+        assert len(pk_hash_column_str) == previous_hash_column_length, (
+            f"Casting column Length mismatch: {identifier_columns[i]} has length {len(pk_hash_column_str)} after casting, "
+            f"before casting length: {previous_hash_column_length}."
+        )
+
+        pk_hash_columns.append(pk_hash_column_str)
+
+    pk_hash_columns.append(IDENTIFIER_FIELD_DELIMITER)
+    pk_hash_columns_concatenated = pc.binary_join_element_wise(
+        *pk_hash_columns, null_handling="replace"
+    )
+    assert len(pk_hash_columns_concatenated) == previous_hash_column_length, (
+        f"Concatenated column Length mismatch: Final concatenated identifier column has length {len(pk_hash_columns_concatenated)}, "
+        f"before concatenating length: {previous_hash_column_length}."
+    )
+
+    return pk_hash_columns_concatenated
+
+
+def write_sliced_table(
+    table: Union[LocalTable, DistributedDataset],
+    base_path: str,
+    table_writer_kwargs: Optional[Dict[str, Any]],
+    content_type: ContentType = ContentType.PARQUET,
+    max_records_per_file: Optional[int] = 4000000,
+    filesystem: Optional[Union[AbstractFileSystem, pa.fs.FileSystem]] = None,
+    **kwargs,
+) -> List[str]:
+    """
+    Writes the given table to 1 or more files and return the paths
+    of the files written.
+    """
+    if isinstance(filesystem, pa.fs.FileSystem):
+        table_writer_fn = get_table_writer(table)
+        table_slicer_fn = get_table_slicer(table)
+
+        # Create a wrapper for the table writer that ensures directory creation
+        def table_writer_with_dir_creation(
+            dataframe: Any,
+            base_path: str,
+            filesystem: Optional[Union[AbstractFileSystem, pa.fs.FileSystem]],
+            block_path_provider: Callable,
+            content_type: str = ContentType.PARQUET.value,
+            **kwargs,
+        ):
+            try:
+                # Ensure base path directory exists
+                if isinstance(base_path, str):
+                    # Normalize the base path and ensure it's treated as a directory path
+                    base_dir = base_path.rstrip("/")
+                    filesystem.create_dir(base_dir, recursive=True)
+            except Exception:
+                # Directory might already exist or there might be permission issues
+                # Let the original write attempt proceed
+                pass
+            return table_writer_fn(
+                dataframe,
+                base_path,
+                filesystem,
+                block_path_provider,
+                content_type,
+                **kwargs,
+            )
+
+        # TODO(pdames): Disable redundant file info fetch currently
+        #   used to construct unused manifest entry metadata.
+        manifest_entry_list = types_write_sliced_table(
+            table=table,
+            base_path=base_path,
+            filesystem=filesystem,
+            max_records_per_entry=max_records_per_file,
+            table_writer_fn=table_writer_with_dir_creation,
+            table_slicer_fn=table_slicer_fn,
+            table_writer_kwargs=table_writer_kwargs,
+            content_type=content_type,
+        )
+        paths = [entry.uri for entry in manifest_entry_list]
+        return paths
+    else:
+        return upload_table_with_retry(
+            table=table,
+            s3_url_prefix=base_path,
+            s3_table_writer_kwargs=table_writer_kwargs,
+            content_type=content_type,
+            max_records_per_file=max_records_per_file,
+            s3_file_system=filesystem,
+            **kwargs,
+        )