deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/tests/compute/converter/test_convert_session.py

@@ -0,0 +1,826 @@
+from collections import defaultdict
+import pytest
+import ray
+from typing import List, Dict, Any, Tuple
+from pyiceberg.catalog.rest import RestCatalog
+from pyiceberg.schema import Schema
+from pyiceberg.types import (
+    NestedField,
+    StringType,
+    LongType,
+)
+from pyiceberg.partitioning import PartitionSpec, PartitionField
+from pyiceberg.transforms import IdentityTransform
+import pyarrow as pa
+import daft
+
+from deltacat.compute.converter.steps.convert import convert
+from deltacat.compute.converter.model.convert_input import ConvertInput
+from deltacat.compute.converter.pyiceberg.overrides import (
+    fetch_all_bucket_files,
+)
+from deltacat.compute.converter.utils.converter_session_utils import (
+    group_all_files_to_each_bucket,
+)
+from deltacat.tests.compute.converter.utils import (
+    get_s3_file_system,
+    drop_table_if_exists,
+    commit_equality_delete_to_table,
+)
+from deltacat.compute.converter.pyiceberg.update_snapshot_overrides import (
+    commit_append_snapshot,
+    commit_replace_snapshot,
+)
+
+from pyiceberg.typedef import Record
+from deltacat.compute.converter.utils.convert_task_options import BASE_MEMORY_BUFFER
+from deltacat.tests.test_utils.filesystem import temp_dir_autocleanup
+from deltacat.compute.converter.converter_session import converter_session
+from deltacat.compute.converter.model.converter_session_params import (
+    ConverterSessionParams,
+)
+from pyiceberg.catalog import load_catalog
+import os
+import pyarrow.parquet as pq
+from pyiceberg.manifest import DataFile, DataFileContent, FileFormat
+from pyiceberg.io.pyarrow import (
+    data_file_statistics_from_parquet_metadata,
+    compute_statistics_plan,
+    parquet_path_to_id_mapping,
+)
+from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible
+from pyiceberg.exceptions import NamespaceAlreadyExistsError, NoSuchTableError
+from pyiceberg.io.pyarrow import schema_to_pyarrow
+
+# Task memory in bytes for testing
+TASK_MEMORY_BYTES = BASE_MEMORY_BUFFER
+
+
+# Test data fixtures
+@pytest.fixture
+def base_schema():
+    return Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key", field_type=StringType(), required=False
+        ),
+        NestedField(
+            field_id=2147483546,
+            name="file_path",
+            field_type=StringType(),
+            required=False,
+        ),
+        NestedField(
+            field_id=2147483545, name="pos", field_type=LongType(), required=False
+        ),
+        schema_id=0,
+    )
+
+
+@pytest.fixture
+def base_schema_without_metadata():
+    return Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key", field_type=StringType(), required=False
+        ),
+        schema_id=0,
+    )
+
+
+@pytest.fixture
+def multi_key_schema():
+    return Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key1", field_type=StringType(), required=False
+        ),
+        NestedField(
+            field_id=3, name="primary_key2", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2147483546,
+            name="file_path",
+            field_type=StringType(),
+            required=False,
+        ),
+        NestedField(
+            field_id=2147483545, name="pos", field_type=LongType(), required=False
+        ),
+        schema_id=0,
+    )
+
+
+@pytest.fixture
+def multi_key_schema_without_file_path():
+    return Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key1", field_type=StringType(), required=False
+        ),
+        NestedField(
+            field_id=3, name="primary_key2", field_type=LongType(), required=False
+        ),
+        schema_id=0,
+    )
+
+
+@pytest.fixture
+def base_partition_spec():
+    partition_field_identity = PartitionField(
+        source_id=1,
+        field_id=101,
+        transform=IdentityTransform(),
+        name="number_partitioned",
+    )
+    return PartitionSpec(partition_field_identity)
+
+
+@pytest.fixture
+def table_properties():
+    return {
+        "write.format.default": "parquet",
+        "write.delete.mode": "merge-on-read",
+        "write.update.mode": "merge-on-read",
+        "write.merge.mode": "merge-on-read",
+        "format-version": "2",
+    }
+
+
+def create_test_table(
+    session_catalog: RestCatalog,
+    namespace: str,
+    table_name: str,
+    schema: Schema,
+    partition_spec: PartitionSpec,
+    properties: Dict[str, str],
+) -> str:
+    """Helper function to create a test table"""
+    identifier = f"{namespace}.{table_name}"
+    drop_table_if_exists(identifier, session_catalog)
+    session_catalog.create_table(
+        identifier,
+        schema=schema,
+        partition_spec=partition_spec,
+        properties=properties,
+    )
+    return identifier
+
+
+def create_mock_data_tables(test_case: Dict[str, Any]) -> Tuple[daft.DataFrame, ...]:
+    """Helper function to create mock data tables based on test case"""
+    tables = []
+    for data in test_case["mock_data"]:
+        if "primary_key2" in data:  # Multi-key case
+            names = ["primary_key1", "primary_key2"]
+            table = pa.Table.from_arrays(
+                [pa.array(data["primary_key1"]), pa.array(data["primary_key2"])],
+                names=names,
+            )
+        else:  # Single key case
+            names = ["primary_key"]
+            table = pa.Table.from_arrays([pa.array(data["primary_key"])], names=names)
+        tables.append(daft.from_arrow(table))
+    if "equality_delete_data_mock" in test_case:
+        for data in test_case["equality_delete_data_mock"]:
+            if "primary_key2" in data:  # Multi-key case
+                names = ["primary_key1", "primary_key2"]
+                table = pa.Table.from_arrays(
+                    [pa.array(data["primary_key1"]), pa.array(data["primary_key2"])],
+                    names=names,
+                )
+            else:  # Single key case
+                names = ["primary_key"]
+                table = pa.Table.from_arrays(
+                    [pa.array(data["primary_key"])], names=names
+                )
+            tables.append(daft.from_arrow(table))
+    return tuple(tables)
+
+
+def run_spark_commands(spark, sqls: List[str]) -> None:
+    """Helper function to run Spark SQL commands"""
+    for sql in sqls:
+        spark.sql(sql)
+
+
+def insert_test_data(spark, identifier: str, test_case: Dict[str, Any]) -> None:
+    """Helper function to insert test data into the table"""
+    if "primary_key2" in test_case["mock_data"][0]:
+        # Multi-key case
+        for data in test_case["mock_data"]:
+            values = ", ".join(
+                f"(0, '{pk1}', {pk2})"
+                for pk1, pk2 in zip(data["primary_key1"], data["primary_key2"])
+            )
+            run_spark_commands(spark, [f"INSERT INTO {identifier} VALUES {values}"])
+    else:
+        # Single key case
+        if test_case["schema"] == "base_schema":
+            # For drop duplicates test, use file_path and pos from mock_data
+            for data in test_case["mock_data"]:
+                values = ", ".join(
+                    f"(0, '{pk}', '{path}', {pos})"
+                    for pk, path, pos in zip(
+                        data["primary_key"], data["file_path"], data["pos"]
+                    )
+                )
+                run_spark_commands(spark, [f"INSERT INTO {identifier} VALUES {values}"])
+        else:
+            # For other tests, just include the basic columns
+            for data in test_case["mock_data"]:
+                values = ", ".join(f"(0, '{pk}')" for pk in data["primary_key"])
+                run_spark_commands(spark, [f"INSERT INTO {identifier} VALUES {values}"])
+
+
+def create_convert_input(
+    tbl,
+    convert_input_files_for_all_buckets: List[Any],
+    test_case: Dict[str, Any],
+    s3_file_system: Any,
+) -> List[ConvertInput]:
+    """Helper function to create convert inputs"""
+    convert_inputs = []
+    for i, one_bucket_files in enumerate(convert_input_files_for_all_buckets):
+        convert_input = ConvertInput.of(
+            convert_input_files=one_bucket_files,
+            convert_task_index=i,
+            iceberg_table_warehouse_prefix="warehouse/default",
+            identifier_fields=test_case["identifier_fields"],
+            table_io=tbl.io,
+            table_metadata=tbl.metadata,
+            compact_previous_position_delete_files=False,
+            enforce_primary_key_uniqueness=True,
+            position_delete_for_multiple_data_files=True,
+            max_parallel_data_file_download=10,
+            filesystem=s3_file_system,
+            s3_client_kwargs={},
+            task_memory=TASK_MEMORY_BYTES,
+        )
+        convert_inputs.append(convert_input)
+    return convert_inputs
+
+
+def process_convert_result(convert_result: Any) -> Tuple[List[Any], List[Any]]:
+    """Helper function to process convert results
+
+    Args:
+        convert_result: The result from convert_session
+
+    Returns:
+        Tuple[List[Any], List[Any]]: Lists of files to be deleted and added
+    """
+    to_be_deleted_files_list = []
+    to_be_added_files_list = []
+    if convert_result.to_be_deleted_files:
+        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
+    if convert_result.to_be_added_files:
+        to_be_added_files_list.extend(convert_result.to_be_added_files)
+    return to_be_deleted_files_list, to_be_added_files_list
+
+
+def verify_result(result, expected_result, verify_pos_index=False):
+    """Verify the result matches the expected result.
+
+    Args:
+        result: The result to verify
+        expected_result: The expected result
+        verify_pos_index: Whether to verify position values for primary keys
+    """
+    if "primary_keys" in expected_result and "primary_key" in result:
+        # Single key case
+        assert set(result["primary_key"]) == set(expected_result["primary_keys"])
+        if verify_pos_index and "pk_to_pos" in expected_result:
+            for index in range(len(result["primary_key"])):
+                assert (
+                    result["pos"][index]
+                    == expected_result["pk_to_pos"][result["primary_key"][index]]
+                )
+    elif "pk_tuples" in expected_result:
+        pk_combined_res = []
+        for pk1, pk2 in zip(
+            result["primary_key1"],
+            result["primary_key2"],
+        ):
+            pk_combined_res.append((pk1, pk2))
+
+        # Multi-key case
+        assert set(pk_combined_res) == set(expected_result["pk_tuples"])
+    else:
+        assert set(result) == set(expected_result["primary_keys"])
+
+
+def verify_spark_read_results(spark, identifier, expected_result):
+    spark_read_pos_delete = spark.sql(f"select * from {identifier}").collect()
+    all_pk = [
+        spark_read_pos_delete[row_idx][1]
+        for row_idx in range(len(spark_read_pos_delete))
+    ]
+    verify_result(all_pk, expected_result, verify_pos_index=False)
+
+
+def get_file_prefix(tbl):
+    """Get the file prefix from a table's data files.
+
+    Args:
+        tbl: The table to get the file prefix from
+
+    Returns:
+        str: The file prefix
+    """
+    df = tbl.inspect.entries()
+    data_files = df.to_pydict()["data_file"]
+    file_link = data_files[0]["file_path"]
+    file_prefix = "/".join(file_link.split("/")[:-1])
+    return file_prefix.split("//")[1]
+
+
+# Test cases configuration
+TEST_CASES = [
+    {
+        "name": "single_key_drop_duplicates",
+        "table_name": "table_converter_ray_drop_duplicates_success",
+        "schema": "base_schema",
+        "identifier_fields": ["primary_key"],
+        "mock_data": [
+            {
+                "primary_key": ["pk1", "pk2", "pk3"],
+                "file_path": ["path1", "path2", "path3"],
+                "pos": [1, 2, 3],
+            },
+            {
+                "primary_key": ["pk1", "pk2", "pk3"],
+                "file_path": ["path1", "path2", "path3"],
+                "pos": [4, 5, 6],
+            },
+            {
+                "primary_key": ["pk4", "pk2", "pk3"],
+                "file_path": ["path4", "path2", "path3"],
+                "pos": [7, 8, 9],
+            },
+        ],
+        "expected_result": {
+            "primary_keys": ["pk1", "pk2", "pk3", "pk4"],
+            "pk_to_pos": {"pk1": 4, "pk2": 8, "pk3": 9, "pk4": 7},
+        },
+    },
+    {
+        "name": "multi_key_drop_duplicates",
+        "table_name": "table_converter_ray_pos_delete_multiple_identifier_fields",
+        "schema": "multi_key_schema_without_file_path",
+        "identifier_fields": ["primary_key1", "primary_key2"],
+        "mock_data": [
+            {"primary_key1": ["pk1", "pk2", "pk3"], "primary_key2": [1, 2, 3]},
+            {"primary_key1": ["pk1", "pk2", "pk3"], "primary_key2": [1, 2, 3]},
+            {"primary_key1": ["pk4", "pk2", "pk3"], "primary_key2": [1, 3, 4]},
+        ],
+        "expected_result": {
+            "pk_tuples": [
+                ("pk1", 1),
+                ("pk2", 2),
+                ("pk2", 3),
+                ("pk3", 3),
+                ("pk3", 4),
+                ("pk4", 1),
+            ]
+        },
+    },
+    {
+        "name": "equality_delete",
+        "table_name": "table_converter_ray_equality_delete_success",
+        "schema": "base_schema_without_metadata",
+        "identifier_fields": ["primary_key"],
+        "mock_data": [
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk4", "pk2", "pk3"]},
+        ],
+        "equality_delete_data_mock": [{"primary_key": ["pk1"]}],
+        "equality_delete_data": pa.Table.from_arrays(["pk1"], names=["primary_key"]),
+        "verify_spark_read": True,
+        "expected_result": {"primary_keys": ["pk2", "pk3", "pk4"]},
+    },
+    {
+        "name": "position_delete",
+        "table_name": "table_converter_ray_position_delete_success",
+        "schema": "base_schema_without_metadata",
+        "identifier_fields": ["primary_key"],
+        "mock_data": [
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk4", "pk2", "pk3"]},
+        ],
+        "expected_result": {"primary_keys": ["pk1", "pk2", "pk3", "pk4"]},
+    },
+    {
+        "name": "position_delete_read_by_spark",
+        "table_name": "table_converter_ray_pos_delete_read_by_spark_success",
+        "schema": "base_schema_without_metadata",
+        "identifier_fields": ["primary_key"],
+        "mock_data": [
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk4", "pk2", "pk3"]},
+        ],
+        "expected_result": {"primary_keys": ["pk1", "pk2", "pk3", "pk4"]},
+        "verify_spark_read": True,
+        "expected_spark_count": 4,
+    },
+]
+
+
+@pytest.mark.parametrize("test_case", TEST_CASES)
+@pytest.mark.integration
+def test_converter(
+    test_case: Dict[str, Any],
+    spark,
+    session_catalog: RestCatalog,
+    setup_ray_cluster,
+    mocker,
+    request,
+) -> None:
+    """
+    Parameterized test for converter functionality.
+    Tests drop duplicates, equality delete, and position delete scenarios.
+    """
+    # Get schema fixture based on test case
+    schema = request.getfixturevalue(test_case["schema"])
+
+    # Create test table
+    identifier = create_test_table(
+        session_catalog=session_catalog,
+        namespace="default",
+        table_name=test_case["table_name"],
+        schema=schema,
+        partition_spec=request.getfixturevalue("base_partition_spec"),
+        properties=request.getfixturevalue("table_properties"),
+    )
+
+    # Insert test data
+    insert_test_data(spark, identifier, test_case)
+
+    # Get files and create convert input
+    tbl = session_catalog.load_table(identifier)
+    data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(tbl)
+
+    # Handle equality delete if present
+    if "equality_delete_data" in test_case:
+        tbl = session_catalog.load_table(identifier)
+        file_prefix = get_file_prefix(tbl)
+        partition_value = Record(number_partitioned=0)
+
+        # Note: Just upload to S3 to mock input data here.
+        # NOT committing to Iceberg metadata as equality delete write path not implemented in Pyiceberg/Spark.
+        equality_file_list = commit_equality_delete_to_table(
+            table=tbl,
+            partition_value=partition_value,
+            equality_delete_table=test_case["equality_delete_data"],
+            file_link_prefix=file_prefix,
+        )
+        # Mock equality delete input to converter with latest file sequence, so equality delete can be applied to all data before
+        equality_delete_dict = defaultdict()
+        equality_delete_dict[partition_value] = [(4, equality_file_list[0])]
+
+    convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
+        data_file_dict=data_file_dict,
+        equality_delete_dict=equality_delete_dict,
+        pos_delete_dict=pos_delete_dict,
+    )
+
+    s3_file_system = get_s3_file_system()
+    convert_inputs = create_convert_input(
+        tbl, convert_input_files_for_all_buckets, test_case, s3_file_system
+    )
+
+    # Create and set up mock data
+    mock_data_tables = create_mock_data_tables(test_case)
+    download_data_mock = mocker.patch(
+        "deltacat.compute.converter.utils.io.daft_read_parquet"
+    )
+
+    download_data_mock.side_effect = mock_data_tables
+
+    # Run conversion
+    convert_ref = convert.remote(convert_inputs[0])
+    convert_result = ray.get(convert_ref)
+
+    # Process results
+    to_be_deleted_files_list, to_be_added_files_list = process_convert_result(
+        convert_result
+    )
+
+    if not to_be_deleted_files_list:
+        # Commit changes
+        commit_append_snapshot(
+            iceberg_table=tbl,
+            new_position_delete_files=to_be_added_files_list,
+        )
+    else:
+        commit_replace_snapshot(
+            iceberg_table=tbl,
+            to_be_deleted_files=to_be_deleted_files_list[0],
+            new_position_delete_files=to_be_added_files_list,
+        )
+    tbl.refresh()
+
+    # Verify results
+    pyiceberg_scan_table_rows = tbl.scan().to_arrow().to_pydict()
+
+    # Verify Spark read if required
+    if test_case.get("verify_spark_read", False):
+        verify_spark_read_results(spark, identifier, test_case["expected_result"])
+    else:
+        verify_result(
+            pyiceberg_scan_table_rows,
+            test_case["expected_result"],
+            verify_pos_index=test_case.get("verify_pos_index", False),
+        )
+
+
+def test_converter_session_with_local_filesystem_and_duplicate_ids(
+    setup_ray_cluster,
+) -> None:
+    """
+    Test converter_session functionality with local PyArrow filesystem using duplicate IDs.
+    This test simulates the pattern where duplicate IDs represent updates to existing records.
+    The converter should merge these updates by creating position delete files.
+    """
+    with temp_dir_autocleanup() as temp_catalog_dir:
+        # Create warehouse directory
+        warehouse_path = os.path.join(temp_catalog_dir, "iceberg_warehouse")
+        os.makedirs(warehouse_path, exist_ok=True)
+
+        # Set up local in-memory catalog
+        local_catalog = load_catalog(
+            "local_sql_catalog",
+            **{
+                "type": "in-memory",
+                "warehouse": warehouse_path,
+            },
+        )
+
+        # Create local PyArrow filesystem
+        import pyarrow.fs as pafs
+
+        local_filesystem = pafs.LocalFileSystem()
+
+        # Define schema (id, name, value, version)
+        schema = Schema(
+            NestedField(field_id=1, name="id", field_type=LongType(), required=True),
+            NestedField(
+                field_id=2, name="name", field_type=StringType(), required=False
+            ),
+            NestedField(
+                field_id=3, name="value", field_type=LongType(), required=False
+            ),
+            NestedField(
+                field_id=4, name="version", field_type=LongType(), required=False
+            ),
+            schema_id=0,
+        )
+
+        # Create table properties for merge-on-read
+        properties = {
+            "write.format.default": "parquet",
+            "write.delete.mode": "merge-on-read",
+            "write.update.mode": "merge-on-read",
+            "write.merge.mode": "merge-on-read",
+            "format-version": "2",
+        }
+
+        # Create the table
+        table_identifier = "default.test_duplicate_ids"
+        try:
+            local_catalog.create_namespace("default")
+        except NamespaceAlreadyExistsError:
+            pass  # Namespace may already exist
+        try:
+            local_catalog.drop_table(table_identifier)
+        except NoSuchTableError:
+            pass  # Table may not exist
+
+        local_catalog.create_table(
+            table_identifier,
+            schema=schema,
+            properties=properties,
+        )
+        tbl = local_catalog.load_table(table_identifier)
+
+        # Set the name mapping property so Iceberg can read parquet files without field IDs
+        with tbl.transaction() as tx:
+            tx.set_properties(
+                **{"schema.name-mapping.default": schema.name_mapping.model_dump_json()}
+            )
+
+        # Step 1: Write initial data
+        # Create PyArrow table with explicit schema to match Iceberg schema
+        arrow_schema = schema_to_pyarrow(schema)
+
+        initial_data = pa.table(
+            {
+                "id": [1, 2, 3, 4],
+                "name": ["Alice", "Bob", "Charlie", "David"],
+                "value": [100, 200, 300, 400],
+                "version": [1, 1, 1, 1],
+            },
+            schema=arrow_schema,
+        )
+
+        # Step 2: Write additional data
+        additional_data = pa.table(
+            {
+                "id": [5, 6, 7, 8],
+                "name": ["Eve", "Frank", "Grace", "Henry"],
+                "value": [500, 600, 700, 800],
+                "version": [1, 1, 1, 1],
+            },
+            schema=arrow_schema,
+        )
+
+        # Step 3: Write updates to existing records (this creates duplicates by ID)
+        # These should overwrite the original records with same IDs
+        updated_data = pa.table(
+            {
+                "id": [2, 3, 9],  # IDs 2 and 3 are duplicates, 9 is new
+                "name": [
+                    "Robert",
+                    "Charles",
+                    "Ivan",
+                ],  # Updated names for Bob and Charlie
+                "value": [201, 301, 900],  # Updated values
+                "version": [2, 2, 1],  # Higher version numbers for updates
+            },
+            schema=arrow_schema,
+        )
+
+        # Write all data to separate parquet files to simulate multiple writes
+        data_files_to_commit = []
+
+        for i, data in enumerate([initial_data, additional_data, updated_data]):
+            data_file_path = os.path.join(warehouse_path, f"data_{i}.parquet")
+            pq.write_table(data, data_file_path)
+
+            # Create DataFile objects for Iceberg
+            parquet_metadata = pq.read_metadata(data_file_path)
+            file_size = os.path.getsize(data_file_path)
+
+            # Check schema compatibility
+            _check_pyarrow_schema_compatible(
+                schema, parquet_metadata.schema.to_arrow_schema()
+            )
+
+            # Calculate statistics
+            statistics = data_file_statistics_from_parquet_metadata(
+                parquet_metadata=parquet_metadata,
+                stats_columns=compute_statistics_plan(schema, tbl.metadata.properties),
+                parquet_column_mapping=parquet_path_to_id_mapping(schema),
+            )
+
+            data_file = DataFile(
+                content=DataFileContent.DATA,
+                file_path=data_file_path,
+                file_format=FileFormat.PARQUET,
+                partition={},  # No partitioning
+                file_size_in_bytes=file_size,
+                sort_order_id=None,
+                spec_id=tbl.metadata.default_spec_id,
+                key_metadata=None,
+                equality_ids=None,
+                **statistics.to_serialized_dict(),
+            )
+            data_files_to_commit.append(data_file)
+
+        # Commit all data files to the table
+        with tbl.transaction() as tx:
+            with tx.update_snapshot().fast_append() as update_snapshot:
+                for data_file in data_files_to_commit:
+                    update_snapshot.append_data_file(data_file)
+
+        tbl.refresh()
+
+        # Verify we have duplicate IDs before conversion
+        initial_scan = tbl.scan().to_arrow().to_pydict()
+        print(f"Before conversion - Records with IDs: {sorted(initial_scan['id'])}")
+
+        # There should be duplicates: [1, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9]
+        expected_duplicate_ids = [1, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9]
+        assert (
+            sorted(initial_scan["id"]) == expected_duplicate_ids
+        ), f"Expected duplicate IDs {expected_duplicate_ids}, got {sorted(initial_scan['id'])}"
+
+        # Now call converter_session to convert equality deletes to position deletes
+        converter_params = ConverterSessionParams.of(
+            {
+                "catalog": local_catalog,
+                "iceberg_table_name": table_identifier,
+                "iceberg_warehouse_bucket_name": warehouse_path,  # Local warehouse path
+                "merge_keys": ["id"],  # Use ID as the merge key
+                "enforce_primary_key_uniqueness": True,
+                "task_max_parallelism": 1,  # Single task for local testing
+                "filesystem": local_filesystem,
+                "location_provider_prefix_override": None,  # Use local filesystem
+                "location_provider_prefix_override": None,  # Let the system auto-generate the prefix
+            }
+        )
+
+        print(f"Running converter_session with local filesystem...")
+        print(f"Warehouse path: {warehouse_path}")
+        print(f"Merge keys: ['id']")
+        print(f"Enforce uniqueness: True")
+
+        # Run the converter
+        converter_session(params=converter_params)
+
+        # Refresh table and scan again
+        tbl.refresh()
+        final_scan = tbl.scan().to_arrow().to_pydict()
+
+        print(f"After conversion - Records with IDs: {sorted(final_scan['id'])}")
+        print(f"Final data: {final_scan}")
+
+        # Verify position delete files were created by checking table metadata
+        latest_snapshot = tbl.metadata.current_snapshot()
+        if latest_snapshot:
+            manifests = latest_snapshot.manifests(tbl.io)
+            position_delete_files = []
+
+            for manifest in manifests:
+                entries = manifest.fetch_manifest_entry(tbl.io)
+                for entry in entries:
+                    if entry.data_file.content == DataFileContent.POSITION_DELETES:
+                        position_delete_files.append(entry.data_file.file_path)
+
+            print(f"Position delete files found: {position_delete_files}")
+            assert (
+                len(position_delete_files) > 0
+            ), "No position delete files were created by converter_session"
+
+        # Verify the final result has unique IDs (duplicates should be resolved)
+        # Expected: Latest values for each ID based on the updates
+        expected_unique_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9]  # All unique IDs
+        actual_ids = sorted(final_scan["id"])
+
+        print(f"Expected unique IDs: {expected_unique_ids}")
+        print(f"Actual IDs after conversion: {actual_ids}")
+
+        assert (
+            actual_ids == expected_unique_ids
+        ), f"Expected unique IDs {expected_unique_ids}, got {actual_ids}"
+
+        # Verify the updated values are present (higher version should win)
+        final_data_by_id = {}
+        for i, id_val in enumerate(final_scan["id"]):
+            final_data_by_id[id_val] = {
+                "name": final_scan["name"][i],
+                "value": final_scan["value"][i],
+                "version": final_scan["version"][i],
+            }
+
+        # Check that ID 2 has updated value (Robert, 201, version 2)
+        assert (
+            final_data_by_id[2]["name"] == "Robert"
+        ), f"ID 2 should have updated name 'Robert', got '{final_data_by_id[2]['name']}'"
+        assert (
+            final_data_by_id[2]["value"] == 201
+        ), f"ID 2 should have updated value 201, got {final_data_by_id[2]['value']}"
+        assert (
+            final_data_by_id[2]["version"] == 2
+        ), f"ID 2 should have version 2, got {final_data_by_id[2]['version']}"
+
+        # Check that ID 3 has updated value (Charles, 301, version 2)
+        assert (
+            final_data_by_id[3]["name"] == "Charles"
+        ), f"ID 3 should have updated name 'Charles', got '{final_data_by_id[3]['name']}'"
+        assert (
+            final_data_by_id[3]["value"] == 301
+        ), f"ID 3 should have updated value 301, got {final_data_by_id[3]['value']}"
+        assert (
+            final_data_by_id[3]["version"] == 2
+        ), f"ID 3 should have version 2, got {final_data_by_id[3]['version']}"
+
+        # Check that new ID 9 is present
+        assert (
+            final_data_by_id[9]["name"] == "Ivan"
+        ), f"ID 9 should have name 'Ivan', got '{final_data_by_id[9]['name']}'"
+        assert (
+            final_data_by_id[9]["value"] == 900
+        ), f"ID 9 should have value 900, got {final_data_by_id[9]['value']}"
+
+        print(f"✅ Test completed successfully!")
+        print(
+            f"✅ Position delete files were created: {len(position_delete_files)} files"
+        )
+        print(f"✅ Duplicate IDs were resolved correctly")
+        print(
+            f"✅ Updated values were applied (ID 2: Bob->Robert, ID 3: Charlie->Charles)"
+        )
+        print(f"✅ Final table has {len(actual_ids)} unique records")
+        print(f"✅ Temporary warehouse cleaned up at: {temp_catalog_dir}")