deltacat 2.0.0b11__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/tests/compute/converter/test_convert_session.py

@@ -1,8 +1,8 @@
+from collections import defaultdict
 import pytest
 import ray
-from typing import List
+from typing import List, Dict, Any, Tuple
 from pyiceberg.catalog.rest import RestCatalog
-from pyiceberg.expressions import EqualTo
 from pyiceberg.schema import Schema
 from pyiceberg.types import (
     NestedField,
@@ -25,144 +25,85 @@ from deltacat.compute.converter.utils.converter_session_utils import (
 from deltacat.tests.compute.converter.utils import (
     get_s3_file_system,
     drop_table_if_exists,
+    commit_equality_delete_to_table,
 )
 from deltacat.compute.converter.pyiceberg.update_snapshot_overrides import (
     commit_append_snapshot,
+    commit_replace_snapshot,
 )
 
+from pyiceberg.typedef import Record
+from deltacat.compute.converter.utils.convert_task_options import BASE_MEMORY_BUFFER
+from deltacat.tests.test_utils.filesystem import temp_dir_autocleanup
+from deltacat.compute.converter.converter_session import converter_session
+from deltacat.compute.converter.model.converter_session_params import (
+    ConverterSessionParams,
+)
+from pyiceberg.catalog import load_catalog
+import os
+import pyarrow.parquet as pq
+from pyiceberg.manifest import DataFile, DataFileContent, FileFormat
+from pyiceberg.io.pyarrow import (
+    data_file_statistics_from_parquet_metadata,
+    compute_statistics_plan,
+    parquet_path_to_id_mapping,
+)
+from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible
+from pyiceberg.exceptions import NamespaceAlreadyExistsError, NoSuchTableError
+from pyiceberg.io.pyarrow import schema_to_pyarrow
 
-def run_spark_commands(spark, sqls: List[str]) -> None:
-    for sql in sqls:
-        spark.sql(sql)
+# Task memory in bytes for testing
+TASK_MEMORY_BYTES = BASE_MEMORY_BUFFER
 
 
-@pytest.mark.integration
-def test_pyiceberg_spark_setup_sanity(spark, session_catalog: RestCatalog) -> None:
-    """
-    This Test was copied over from Pyiceberg integ test: https://github.com/apache/iceberg-python/blob/main/tests/integration/test_deletes.py#L62
-    First sanity check to ensure all integration with Pyiceberg and Spark are working as expected.
-    """
-    identifier = "default.table_partitioned_delete"
-
-    run_spark_commands(
-        spark,
-        [
-            f"DROP TABLE IF EXISTS {identifier}",
-            f"""
-            CREATE TABLE {identifier} (
-                number_partitioned  int,
-                number              int
-            )
-            USING iceberg
-            PARTITIONED BY (number_partitioned)
-            TBLPROPERTIES('format-version' = 2)
-        """,
-            f"""
-            INSERT INTO {identifier} VALUES (10, 20), (10, 30)
-        """,
-            f"""
-            INSERT INTO {identifier} VALUES (11, 20), (11, 30)
-        """,
-        ],
+# Test data fixtures
+@pytest.fixture
+def base_schema():
+    return Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key", field_type=StringType(), required=False
+        ),
+        NestedField(
+            field_id=2147483546,
+            name="file_path",
+            field_type=StringType(),
+            required=False,
+        ),
+        NestedField(
+            field_id=2147483545, name="pos", field_type=LongType(), required=False
+        ),
+        schema_id=0,
     )
 
-    tbl = session_catalog.load_table(identifier)
-    tbl.delete(EqualTo("number_partitioned", 10))
-
-    # No overwrite operation
-    assert [snapshot.summary.operation.value for snapshot in tbl.snapshots()] == [
-        "append",
-        "append",
-        "delete",
-    ]
-    assert tbl.scan().to_arrow().to_pydict() == {
-        "number_partitioned": [11, 11],
-        "number": [20, 30],
-    }
-
-
-@pytest.mark.integration
-def test_spark_position_delete_production_sanity(
-    spark, session_catalog: RestCatalog
-) -> None:
-    """
-    Sanity test to ensure Spark position delete production is successful with `merge-on-read` spec V2.
-    Table has two partition levels. 1. BucketTransform on primary key
-    """
-    identifier = "default.table_spark_position_delete_production_sanity"
-
-    run_spark_commands(
-        spark,
-        [
-            f"DROP TABLE IF EXISTS {identifier}",
-            f"""
-            CREATE TABLE {identifier} (
-                number_partitioned INT,
-                primary_key STRING
-            )
-            USING iceberg
-            PARTITIONED BY (bucket(3, primary_key), number_partitioned)
-            TBLPROPERTIES(
-                'format-version' = 2,
-                'write.delete.mode'='merge-on-read',
-                'write.update.mode'='merge-on-read',
-                'write.merge.mode'='merge-on-read'
-            )
-            """,
-            f"""
-            INSERT INTO {identifier} VALUES (0, 'pk1'), (0, 'pk2'), (0, 'pk3')
-            """,
-            f"""
-            INSERT INTO {identifier} VALUES (1, 'pk1'), (1, 'pk2'), (1, 'pk3')
-            """,
-        ],
-    )
 
-    run_spark_commands(
-        spark,
-        [
-            f"""
-                DELETE FROM {identifier} WHERE primary_key in ("pk1")
-            """,
-        ],
+@pytest.fixture
+def base_schema_without_metadata():
+    return Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key", field_type=StringType(), required=False
+        ),
+        schema_id=0,
     )
 
-    tbl = session_catalog.load_table(identifier)
-    tbl.refresh()
-
-    assert [snapshot.summary.operation.value for snapshot in tbl.snapshots()] == [
-        "append",
-        "append",
-        "delete",
-    ]
-
-    assert tbl.scan().to_arrow().to_pydict() == {
-        "number_partitioned": [1, 1, 0, 0],
-        "primary_key": ["pk2", "pk3", "pk2", "pk3"],
-    }
-
-
-@pytest.mark.integration
-def test_converter_drop_duplicates_success(
-    spark, session_catalog: RestCatalog, setup_ray_cluster, mocker
-) -> None:
-    """
-    Test for convert compute remote function happy case. Download file results are mocked.
-    """
-
-    # 1. Create Iceberg table
-    namespace = "default"
-    table_name = "table_converter_ray_pos_delete_drop_duplicates_compute"
-    identifier = f"{namespace}.{table_name}"
 
-    schema = Schema(
+@pytest.fixture
+def multi_key_schema():
+    return Schema(
         NestedField(
             field_id=1, name="number_partitioned", field_type=LongType(), required=False
         ),
         NestedField(
-            field_id=2, name="primary_key", field_type=StringType(), required=False
+            field_id=2, name="primary_key1", field_type=StringType(), required=False
+        ),
+        NestedField(
+            field_id=3, name="primary_key2", field_type=LongType(), required=False
         ),
-        # Explicitly define "file_path" and "pos" for assertion of deterministic record after dedupe
         NestedField(
             field_id=2147483546,
             name="file_path",
@@ -175,21 +116,55 @@ def test_converter_drop_duplicates_success(
         schema_id=0,
     )
 
+
+@pytest.fixture
+def multi_key_schema_without_file_path():
+    return Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key1", field_type=StringType(), required=False
+        ),
+        NestedField(
+            field_id=3, name="primary_key2", field_type=LongType(), required=False
+        ),
+        schema_id=0,
+    )
+
+
+@pytest.fixture
+def base_partition_spec():
     partition_field_identity = PartitionField(
         source_id=1,
         field_id=101,
         transform=IdentityTransform(),
         name="number_partitioned",
     )
-    partition_spec = PartitionSpec(partition_field_identity)
+    return PartitionSpec(partition_field_identity)
 
-    properties = dict()
-    properties["write.format.default"] = "parquet"
-    properties["write.delete.mode"] = "merge-on-read"
-    properties["write.update.mode"] = "merge-on-read"
-    properties["write.merge.mode"] = "merge-on-read"
-    properties["format-version"] = "2"
 
+@pytest.fixture
+def table_properties():
+    return {
+        "write.format.default": "parquet",
+        "write.delete.mode": "merge-on-read",
+        "write.update.mode": "merge-on-read",
+        "write.merge.mode": "merge-on-read",
+        "format-version": "2",
+    }
+
+
+def create_test_table(
+    session_catalog: RestCatalog,
+    namespace: str,
+    table_name: str,
+    schema: Schema,
+    partition_spec: PartitionSpec,
+    properties: Dict[str, str],
+) -> str:
+    """Helper function to create a test table"""
+    identifier = f"{namespace}.{table_name}"
     drop_table_if_exists(identifier, session_catalog)
     session_catalog.create_table(
         identifier,
@@ -197,204 +172,323 @@ def test_converter_drop_duplicates_success(
         partition_spec=partition_spec,
         properties=properties,
     )
+    return identifier
 
-    # 2. Use Spark to generate initial data files
-    tbl = session_catalog.load_table(identifier)
-    tbl.refresh()
-    run_spark_commands(
-        spark,
-        [
-            f"""
-            INSERT INTO {identifier} VALUES (0, "pk1", "path1", 1), (0, "pk2", "path2", 2), (0, "pk3", "path3", 3)
-            """
-        ],
-    )
-    run_spark_commands(
-        spark,
-        [
-            f"""
-                INSERT INTO {identifier} VALUES (0, "pk1", "path1", 4), (0, "pk2", "path2", 5), (0, "pk3", "path3", 6)
-                """
-        ],
-    )
-    run_spark_commands(
-        spark,
-        [
-            f"""
-                INSERT INTO {identifier} VALUES (0, "pk4", "path4", 7), (0, "pk2", "path2", 8), (0, "pk3", "path3", 9)
-                """
-        ],
-    )
 
-    tbl = session_catalog.load_table(identifier)
-    # 3. Use convert.remote() function to compute position deletes
-    data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(tbl)
+def create_mock_data_tables(test_case: Dict[str, Any]) -> Tuple[daft.DataFrame, ...]:
+    """Helper function to create mock data tables based on test case"""
+    tables = []
+    for data in test_case["mock_data"]:
+        if "primary_key2" in data:  # Multi-key case
+            names = ["primary_key1", "primary_key2"]
+            table = pa.Table.from_arrays(
+                [pa.array(data["primary_key1"]), pa.array(data["primary_key2"])],
+                names=names,
+            )
+        else:  # Single key case
+            names = ["primary_key"]
+            table = pa.Table.from_arrays([pa.array(data["primary_key"])], names=names)
+        tables.append(daft.from_arrow(table))
+    if "equality_delete_data_mock" in test_case:
+        for data in test_case["equality_delete_data_mock"]:
+            if "primary_key2" in data:  # Multi-key case
+                names = ["primary_key1", "primary_key2"]
+                table = pa.Table.from_arrays(
+                    [pa.array(data["primary_key1"]), pa.array(data["primary_key2"])],
+                    names=names,
+                )
+            else:  # Single key case
+                names = ["primary_key"]
+                table = pa.Table.from_arrays(
+                    [pa.array(data["primary_key"])], names=names
+                )
+            tables.append(daft.from_arrow(table))
+    return tuple(tables)
 
-    convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
-        data_file_dict=data_file_dict,
-        equality_delete_dict=equality_delete_dict,
-        pos_delete_dict=pos_delete_dict,
-    )
 
-    s3_file_system = get_s3_file_system()
+def run_spark_commands(spark, sqls: List[str]) -> None:
+    """Helper function to run Spark SQL commands"""
+    for sql in sqls:
+        spark.sql(sql)
+
+
+def insert_test_data(spark, identifier: str, test_case: Dict[str, Any]) -> None:
+    """Helper function to insert test data into the table"""
+    if "primary_key2" in test_case["mock_data"][0]:
+        # Multi-key case
+        for data in test_case["mock_data"]:
+            values = ", ".join(
+                f"(0, '{pk1}', {pk2})"
+                for pk1, pk2 in zip(data["primary_key1"], data["primary_key2"])
+            )
+            run_spark_commands(spark, [f"INSERT INTO {identifier} VALUES {values}"])
+    else:
+        # Single key case
+        if test_case["schema"] == "base_schema":
+            # For drop duplicates test, use file_path and pos from mock_data
+            for data in test_case["mock_data"]:
+                values = ", ".join(
+                    f"(0, '{pk}', '{path}', {pos})"
+                    for pk, path, pos in zip(
+                        data["primary_key"], data["file_path"], data["pos"]
+                    )
+                )
+                run_spark_commands(spark, [f"INSERT INTO {identifier} VALUES {values}"])
+        else:
+            # For other tests, just include the basic columns
+            for data in test_case["mock_data"]:
+                values = ", ".join(f"(0, '{pk}')" for pk in data["primary_key"])
+                run_spark_commands(spark, [f"INSERT INTO {identifier} VALUES {values}"])
+
+
+def create_convert_input(
+    tbl,
+    convert_input_files_for_all_buckets: List[Any],
+    test_case: Dict[str, Any],
+    s3_file_system: Any,
+) -> List[ConvertInput]:
+    """Helper function to create convert inputs"""
+    convert_inputs = []
     for i, one_bucket_files in enumerate(convert_input_files_for_all_buckets):
         convert_input = ConvertInput.of(
             convert_input_files=one_bucket_files,
             convert_task_index=i,
             iceberg_table_warehouse_prefix="warehouse/default",
-            identifier_fields=["primary_key"],
+            identifier_fields=test_case["identifier_fields"],
             table_io=tbl.io,
             table_metadata=tbl.metadata,
             compact_previous_position_delete_files=False,
             enforce_primary_key_uniqueness=True,
             position_delete_for_multiple_data_files=True,
             max_parallel_data_file_download=10,
-            s3_file_system=s3_file_system,
+            filesystem=s3_file_system,
             s3_client_kwargs={},
+            task_memory=TASK_MEMORY_BYTES,
         )
+        convert_inputs.append(convert_input)
+    return convert_inputs
 
-    number_partitioned_array_1 = pa.array([0, 0, 0], type=pa.int32())
-    primary_key_array_1 = pa.array(["pk1", "pk2", "pk3"])
-    names = ["number_partitioned", "primary_key"]
-    data_table_1 = pa.Table.from_arrays(
-        [number_partitioned_array_1, primary_key_array_1], names=names
-    )
-
-    number_partitioned_array_2 = pa.array([0, 0, 0], type=pa.int32())
-    primary_key_array_2 = pa.array(["pk1", "pk2", "pk3"])
-    names = ["number_partitioned", "primary_key"]
-    data_table_2 = pa.Table.from_arrays(
-        [number_partitioned_array_2, primary_key_array_2], names=names
-    )
-
-    number_partitioned_array_3 = pa.array([0, 0, 0], type=pa.int32())
-    primary_key_array_3 = pa.array(["pk4", "pk2", "pk3"])
-    names = ["number_partitioned", "primary_key"]
-    data_table_3 = pa.Table.from_arrays(
-        [number_partitioned_array_3, primary_key_array_3], names=names
-    )
-
-    daft_df_1 = daft.from_arrow(data_table_1)
-    daft_df_2 = daft.from_arrow(data_table_2)
-    daft_df_3 = daft.from_arrow(data_table_3)
 
-    download_data_mock = mocker.patch(
-        "deltacat.compute.converter.utils.io.daft_read_parquet"
-    )
-    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
+def process_convert_result(convert_result: Any) -> Tuple[List[Any], List[Any]]:
+    """Helper function to process convert results
 
-    convert_ref = convert.remote(convert_input)
+    Args:
+        convert_result: The result from convert_session
 
+    Returns:
+        Tuple[List[Any], List[Any]]: Lists of files to be deleted and added
+    """
     to_be_deleted_files_list = []
-
-    convert_result = ray.get(convert_ref)
-
     to_be_added_files_list = []
-    # Check if there're files to delete
     if convert_result.to_be_deleted_files:
         to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
     if convert_result.to_be_added_files:
         to_be_added_files_list.extend(convert_result.to_be_added_files)
+    return to_be_deleted_files_list, to_be_added_files_list
 
-    commit_append_snapshot(
-        iceberg_table=tbl,
-        new_position_delete_files=to_be_added_files_list,
-    )
-    tbl.refresh()
 
-    # 5. Only primary key 2 and 3 should exist in table, as primary key 1 is deleted.
-    pyiceberg_scan_table_rows = tbl.scan().to_arrow().to_pydict()
+def verify_result(result, expected_result, verify_pos_index=False):
+    """Verify the result matches the expected result.
+
+    Args:
+        result: The result to verify
+        expected_result: The expected result
+        verify_pos_index: Whether to verify position values for primary keys
+    """
+    if "primary_keys" in expected_result and "primary_key" in result:
+        # Single key case
+        assert set(result["primary_key"]) == set(expected_result["primary_keys"])
+        if verify_pos_index and "pk_to_pos" in expected_result:
+            for index in range(len(result["primary_key"])):
+                assert (
+                    result["pos"][index]
+                    == expected_result["pk_to_pos"][result["primary_key"][index]]
+                )
+    elif "pk_tuples" in expected_result:
+        pk_combined_res = []
+        for pk1, pk2 in zip(
+            result["primary_key1"],
+            result["primary_key2"],
+        ):
+            pk_combined_res.append((pk1, pk2))
+
+        # Multi-key case
+        assert set(pk_combined_res) == set(expected_result["pk_tuples"])
+    else:
+        assert set(result) == set(expected_result["primary_keys"])
+
+
+def verify_spark_read_results(spark, identifier, expected_result):
+    spark_read_pos_delete = spark.sql(f"select * from {identifier}").collect()
+    all_pk = [
+        spark_read_pos_delete[row_idx][1]
+        for row_idx in range(len(spark_read_pos_delete))
+    ]
+    verify_result(all_pk, expected_result, verify_pos_index=False)
+
+
+def get_file_prefix(tbl):
+    """Get the file prefix from a table's data files.
 
-    # Only one unique record for each pk exists
-    all_pk = sorted(pyiceberg_scan_table_rows["primary_key"])
-    assert all_pk == ["pk1", "pk2", "pk3", "pk4"]
+    Args:
+        tbl: The table to get the file prefix from
 
-    # Expected unique record to keep for each pk
-    expected_pk_to_pos_mapping = {"pk1": 4, "pk2": 8, "pk3": 9, "pk4": 7}
-    for pk, pos in zip(
-        pyiceberg_scan_table_rows["primary_key"], pyiceberg_scan_table_rows["pos"]
-    ):
-        assert pos == expected_pk_to_pos_mapping[pk]
+    Returns:
+        str: The file prefix
+    """
+    df = tbl.inspect.entries()
+    data_files = df.to_pydict()["data_file"]
+    file_link = data_files[0]["file_path"]
+    file_prefix = "/".join(file_link.split("/")[:-1])
+    return file_prefix.split("//")[1]
+
+
+# Test cases configuration
+TEST_CASES = [
+    {
+        "name": "single_key_drop_duplicates",
+        "table_name": "table_converter_ray_drop_duplicates_success",
+        "schema": "base_schema",
+        "identifier_fields": ["primary_key"],
+        "mock_data": [
+            {
+                "primary_key": ["pk1", "pk2", "pk3"],
+                "file_path": ["path1", "path2", "path3"],
+                "pos": [1, 2, 3],
+            },
+            {
+                "primary_key": ["pk1", "pk2", "pk3"],
+                "file_path": ["path1", "path2", "path3"],
+                "pos": [4, 5, 6],
+            },
+            {
+                "primary_key": ["pk4", "pk2", "pk3"],
+                "file_path": ["path4", "path2", "path3"],
+                "pos": [7, 8, 9],
+            },
+        ],
+        "expected_result": {
+            "primary_keys": ["pk1", "pk2", "pk3", "pk4"],
+            "pk_to_pos": {"pk1": 4, "pk2": 8, "pk3": 9, "pk4": 7},
+        },
+    },
+    {
+        "name": "multi_key_drop_duplicates",
+        "table_name": "table_converter_ray_pos_delete_multiple_identifier_fields",
+        "schema": "multi_key_schema_without_file_path",
+        "identifier_fields": ["primary_key1", "primary_key2"],
+        "mock_data": [
+            {"primary_key1": ["pk1", "pk2", "pk3"], "primary_key2": [1, 2, 3]},
+            {"primary_key1": ["pk1", "pk2", "pk3"], "primary_key2": [1, 2, 3]},
+            {"primary_key1": ["pk4", "pk2", "pk3"], "primary_key2": [1, 3, 4]},
+        ],
+        "expected_result": {
+            "pk_tuples": [
+                ("pk1", 1),
+                ("pk2", 2),
+                ("pk2", 3),
+                ("pk3", 3),
+                ("pk3", 4),
+                ("pk4", 1),
+            ]
+        },
+    },
+    {
+        "name": "equality_delete",
+        "table_name": "table_converter_ray_equality_delete_success",
+        "schema": "base_schema_without_metadata",
+        "identifier_fields": ["primary_key"],
+        "mock_data": [
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk4", "pk2", "pk3"]},
+        ],
+        "equality_delete_data_mock": [{"primary_key": ["pk1"]}],
+        "equality_delete_data": pa.Table.from_arrays(["pk1"], names=["primary_key"]),
+        "verify_spark_read": True,
+        "expected_result": {"primary_keys": ["pk2", "pk3", "pk4"]},
+    },
+    {
+        "name": "position_delete",
+        "table_name": "table_converter_ray_position_delete_success",
+        "schema": "base_schema_without_metadata",
+        "identifier_fields": ["primary_key"],
+        "mock_data": [
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk4", "pk2", "pk3"]},
+        ],
+        "expected_result": {"primary_keys": ["pk1", "pk2", "pk3", "pk4"]},
+    },
+    {
+        "name": "position_delete_read_by_spark",
+        "table_name": "table_converter_ray_pos_delete_read_by_spark_success",
+        "schema": "base_schema_without_metadata",
+        "identifier_fields": ["primary_key"],
+        "mock_data": [
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk1", "pk2", "pk3"]},
+            {"primary_key": ["pk4", "pk2", "pk3"]},
+        ],
+        "expected_result": {"primary_keys": ["pk1", "pk2", "pk3", "pk4"]},
+        "verify_spark_read": True,
+        "expected_spark_count": 4,
+    },
+]
 
 
+@pytest.mark.parametrize("test_case", TEST_CASES)
 @pytest.mark.integration
-def test_converter_pos_delete_read_by_spark_success(
-    spark, session_catalog: RestCatalog, setup_ray_cluster, mocker
+def test_converter(
+    test_case: Dict[str, Any],
+    spark,
+    session_catalog: RestCatalog,
+    setup_ray_cluster,
+    mocker,
+    request,
 ) -> None:
     """
-    Test for convert compute remote function happy case. Download file results are mocked.
+    Parameterized test for converter functionality.
+    Tests drop duplicates, equality delete, and position delete scenarios.
     """
-
-    # 1. Create Iceberg table
-    namespace = "default"
-    table_name = "table_converter_ray_pos_delete_read_by_spark_success"
-    identifier = f"{namespace}.{table_name}"
-
-    schema = Schema(
-        NestedField(
-            field_id=1, name="number_partitioned", field_type=LongType(), required=False
-        ),
-        NestedField(
-            field_id=2, name="primary_key", field_type=StringType(), required=False
-        ),
-        schema_id=0,
-    )
-
-    partition_field_identity = PartitionField(
-        source_id=1,
-        field_id=101,
-        transform=IdentityTransform(),
-        name="number_partitioned",
-    )
-    partition_spec = PartitionSpec(partition_field_identity)
-
-    properties = dict()
-    properties["write.format.default"] = "parquet"
-    properties["write.delete.mode"] = "merge-on-read"
-    properties["write.update.mode"] = "merge-on-read"
-    properties["write.merge.mode"] = "merge-on-read"
-    properties["format-version"] = "2"
-
-    drop_table_if_exists(identifier, session_catalog)
-    session_catalog.create_table(
-        identifier,
+    # Get schema fixture based on test case
+    schema = request.getfixturevalue(test_case["schema"])
+
+    # Create test table
+    identifier = create_test_table(
+        session_catalog=session_catalog,
+        namespace="default",
+        table_name=test_case["table_name"],
         schema=schema,
-        partition_spec=partition_spec,
-        properties=properties,
+        partition_spec=request.getfixturevalue("base_partition_spec"),
+        properties=request.getfixturevalue("table_properties"),
     )
 
-    # 2. Use Spark to generate initial data files
-    tbl = session_catalog.load_table(identifier)
-
-    run_spark_commands(
-        spark,
-        [
-            f"""
-               INSERT INTO {identifier} VALUES (0, "pk1"), (0, "pk2"), (0, "pk3")
-               """
-        ],
-    )
-    run_spark_commands(
-        spark,
-        [
-            f"""
-                   INSERT INTO {identifier} VALUES (0, "pk1"), (0, "pk2"), (0, "pk3")
-                   """
-        ],
-    )
-    run_spark_commands(
-        spark,
-        [
-            f"""
-                   INSERT INTO {identifier} VALUES (0, "pk4"), (0, "pk2"), (0, "pk3")
-                   """
-        ],
-    )
-    tbl.refresh()
+    # Insert test data
+    insert_test_data(spark, identifier, test_case)
 
-    # 3. Use convert.remote() function to compute position deletes
+    # Get files and create convert input
+    tbl = session_catalog.load_table(identifier)
     data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(tbl)
 
+    # Handle equality delete if present
+    if "equality_delete_data" in test_case:
+        tbl = session_catalog.load_table(identifier)
+        file_prefix = get_file_prefix(tbl)
+        partition_value = Record(number_partitioned=0)
+
+        # Note: Just upload to S3 to mock input data here.
+        # NOT committing to Iceberg metadata as equality delete write path not implemented in Pyiceberg/Spark.
+        equality_file_list = commit_equality_delete_to_table(
+            table=tbl,
+            partition_value=partition_value,
+            equality_delete_table=test_case["equality_delete_data"],
+            file_link_prefix=file_prefix,
+        )
+        # Mock equality delete input to converter with latest file sequence, so equality delete can be applied to all data before
+        equality_delete_dict = defaultdict()
+        equality_delete_dict[partition_value] = [(4, equality_file_list[0])]
+
     convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
         data_file_dict=data_file_dict,
         equality_delete_dict=equality_delete_dict,
@@ -402,240 +496,331 @@ def test_converter_pos_delete_read_by_spark_success(
     )
 
     s3_file_system = get_s3_file_system()
-    for i, one_bucket_files in enumerate(convert_input_files_for_all_buckets):
-        convert_input = ConvertInput.of(
-            convert_input_files=one_bucket_files,
-            convert_task_index=i,
-            iceberg_table_warehouse_prefix="warehouse/default",
-            identifier_fields=["primary_key"],
-            table_io=tbl.io,
-            table_metadata=tbl.metadata,
-            compact_previous_position_delete_files=False,
-            enforce_primary_key_uniqueness=True,
-            position_delete_for_multiple_data_files=True,
-            max_parallel_data_file_download=10,
-            s3_file_system=s3_file_system,
-            s3_client_kwargs={},
-        )
-
-    primary_key_array_1 = pa.array(["pk1", "pk2", "pk3"])
-    names = ["primary_key"]
-    data_table_1 = pa.Table.from_arrays([primary_key_array_1], names=names)
-
-    primary_key_array_2 = pa.array(["pk1", "pk2", "pk3"])
-    names = ["primary_key"]
-    data_table_2 = pa.Table.from_arrays([primary_key_array_2], names=names)
-
-    primary_key_array_3 = pa.array(["pk4", "pk2", "pk3"])
-    names = ["primary_key"]
-    data_table_3 = pa.Table.from_arrays([primary_key_array_3], names=names)
-
-    daft_df_1 = daft.from_arrow(data_table_1)
-    daft_df_2 = daft.from_arrow(data_table_2)
-    daft_df_3 = daft.from_arrow(data_table_3)
+    convert_inputs = create_convert_input(
+        tbl, convert_input_files_for_all_buckets, test_case, s3_file_system
+    )
 
+    # Create and set up mock data
+    mock_data_tables = create_mock_data_tables(test_case)
     download_data_mock = mocker.patch(
         "deltacat.compute.converter.utils.io.daft_read_parquet"
     )
-    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
 
-    convert_ref = convert.remote(convert_input)
+    download_data_mock.side_effect = mock_data_tables
 
-    to_be_deleted_files_list = []
-    to_be_added_files_list = []
+    # Run conversion
+    convert_ref = convert.remote(convert_inputs[0])
     convert_result = ray.get(convert_ref)
 
-    if convert_result.to_be_deleted_files:
-        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
-    if convert_result.to_be_added_files:
-        to_be_added_files_list.extend(convert_result.to_be_added_files)
-
-    # 4. Commit position delete, delete equality deletes from table
-    commit_append_snapshot(
-        iceberg_table=tbl,
-        new_position_delete_files=to_be_added_files_list,
+    # Process results
+    to_be_deleted_files_list, to_be_added_files_list = process_convert_result(
+        convert_result
     )
+
+    if not to_be_deleted_files_list:
+        # Commit changes
+        commit_append_snapshot(
+            iceberg_table=tbl,
+            new_position_delete_files=to_be_added_files_list,
+        )
+    else:
+        commit_replace_snapshot(
+            iceberg_table=tbl,
+            to_be_deleted_files=to_be_deleted_files_list[0],
+            new_position_delete_files=to_be_added_files_list,
+        )
     tbl.refresh()
 
-    # 5. Result assertion: Spark read table contains unique primary key
-    spark_read_pos_delete = spark.sql(f"select * from {identifier}").collect()
-    all_pk = [
-        spark_read_pos_delete[row_idx][1]
-        for row_idx in range(len(spark_read_pos_delete))
-    ]
-    all_pk_sorted = sorted(all_pk)
-    assert all_pk_sorted == ["pk1", "pk2", "pk3", "pk4"]
+    # Verify results
+    pyiceberg_scan_table_rows = tbl.scan().to_arrow().to_pydict()
 
+    # Verify Spark read if required
+    if test_case.get("verify_spark_read", False):
+        verify_spark_read_results(spark, identifier, test_case["expected_result"])
+    else:
+        verify_result(
+            pyiceberg_scan_table_rows,
+            test_case["expected_result"],
+            verify_pos_index=test_case.get("verify_pos_index", False),
+        )
 
-@pytest.mark.integration
-def test_converter_pos_delete_multiple_identifier_fields_success(
-    spark, session_catalog: RestCatalog, setup_ray_cluster, mocker
+
+def test_converter_session_with_local_filesystem_and_duplicate_ids(
+    setup_ray_cluster,
 ) -> None:
     """
-    Test for convert compute remote function happy case. Download file results are mocked.
+    Test converter_session functionality with local PyArrow filesystem using duplicate IDs.
+    This test simulates the pattern where duplicate IDs represent updates to existing records.
+    The converter should merge these updates by creating position delete files.
     """
+    with temp_dir_autocleanup() as temp_catalog_dir:
+        # Create warehouse directory
+        warehouse_path = os.path.join(temp_catalog_dir, "iceberg_warehouse")
+        os.makedirs(warehouse_path, exist_ok=True)
+
+        # Set up local in-memory catalog
+        local_catalog = load_catalog(
+            "local_sql_catalog",
+            **{
+                "type": "in-memory",
+                "warehouse": warehouse_path,
+            },
+        )
 
-    # 1. Create Iceberg table
-    namespace = "default"
-    table_name = "table_converter_ray_pos_delete_multiple_identifier_fields"
-
-    identifier = f"{namespace}.{table_name}"
-
-    schema = Schema(
-        NestedField(
-            field_id=1, name="number_partitioned", field_type=LongType(), required=False
-        ),
-        NestedField(
-            field_id=2, name="primary_key1", field_type=StringType(), required=False
-        ),
-        NestedField(
-            field_id=3, name="primary_key2", field_type=LongType(), required=False
-        ),
-        schema_id=0,
-    )
-
-    partition_field_identity = PartitionField(
-        source_id=1,
-        field_id=101,
-        transform=IdentityTransform(),
-        name="number_partitioned",
-    )
-    partition_spec = PartitionSpec(partition_field_identity)
-
-    properties = dict()
-    properties["write.format.default"] = "parquet"
-    properties["write.delete.mode"] = "merge-on-read"
-    properties["write.update.mode"] = "merge-on-read"
-    properties["write.merge.mode"] = "merge-on-read"
-    properties["format-version"] = "2"
-
-    drop_table_if_exists(identifier, session_catalog)
-    session_catalog.create_table(
-        identifier,
-        schema=schema,
-        partition_spec=partition_spec,
-        properties=properties,
-    )
-
-    # 2. Use Spark to generate initial data files
-    tbl = session_catalog.load_table(identifier)
-
-    run_spark_commands(
-        spark,
-        [
-            f"""
-               INSERT INTO {identifier} VALUES (0, "pk1", 1), (0, "pk2", 2), (0, "pk3", 3)
-               """
-        ],
-    )
-    run_spark_commands(
-        spark,
-        [
-            f"""
-               INSERT INTO {identifier} VALUES (0, "pk1", 1), (0, "pk2", 2), (0, "pk3", 3)
-               """
-        ],
-    )
-    run_spark_commands(
-        spark,
-        [
-            f"""
-               INSERT INTO {identifier} VALUES (0, "pk4", 1), (0, "pk2", 3), (0, "pk3", 4)
-               """
-        ],
-    )
-    tbl.refresh()
-
-    # 3. Use convert.remote() function to compute position deletes
-    data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(tbl)
-
-    convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
-        data_file_dict=data_file_dict,
-        equality_delete_dict=equality_delete_dict,
-        pos_delete_dict=pos_delete_dict,
-    )
-
-    s3_file_system = get_s3_file_system()
-    for i, one_bucket_files in enumerate(convert_input_files_for_all_buckets):
-        convert_input = ConvertInput.of(
-            convert_input_files=one_bucket_files,
-            convert_task_index=i,
-            iceberg_table_warehouse_prefix="warehouse/default",
-            identifier_fields=["primary_key1", "primary_key2"],
-            table_io=tbl.io,
-            table_metadata=tbl.metadata,
-            compact_previous_position_delete_files=False,
-            enforce_primary_key_uniqueness=True,
-            position_delete_for_multiple_data_files=True,
-            max_parallel_data_file_download=10,
-            s3_file_system=s3_file_system,
-            s3_client_kwargs={},
+        # Create local PyArrow filesystem
+        import pyarrow.fs as pafs
+
+        local_filesystem = pafs.LocalFileSystem()
+
+        # Define schema (id, name, value, version)
+        schema = Schema(
+            NestedField(field_id=1, name="id", field_type=LongType(), required=True),
+            NestedField(
+                field_id=2, name="name", field_type=StringType(), required=False
+            ),
+            NestedField(
+                field_id=3, name="value", field_type=LongType(), required=False
+            ),
+            NestedField(
+                field_id=4, name="version", field_type=LongType(), required=False
+            ),
+            schema_id=0,
         )
 
-    names = ["primary_key1", "primary_key2"]
+        # Create table properties for merge-on-read
+        properties = {
+            "write.format.default": "parquet",
+            "write.delete.mode": "merge-on-read",
+            "write.update.mode": "merge-on-read",
+            "write.merge.mode": "merge-on-read",
+            "format-version": "2",
+        }
+
+        # Create the table
+        table_identifier = "default.test_duplicate_ids"
+        try:
+            local_catalog.create_namespace("default")
+        except NamespaceAlreadyExistsError:
+            pass  # Namespace may already exist
+        try:
+            local_catalog.drop_table(table_identifier)
+        except NoSuchTableError:
+            pass  # Table may not exist
+
+        local_catalog.create_table(
+            table_identifier,
+            schema=schema,
+            properties=properties,
+        )
+        tbl = local_catalog.load_table(table_identifier)
 
-    primary_key1_array_1 = pa.array(["pk1", "pk2", "pk3"])
-    primary_key2_array_1 = pa.array([1, 2, 3])
-    data_table_1 = pa.Table.from_arrays(
-        [primary_key1_array_1, primary_key2_array_1], names=names
-    )
+        # Set the name mapping property so Iceberg can read parquet files without field IDs
+        with tbl.transaction() as tx:
+            tx.set_properties(
+                **{"schema.name-mapping.default": schema.name_mapping.model_dump_json()}
+            )
 
-    primary_key1_array_2 = pa.array(["pk1", "pk2", "pk3"])
-    primary_key2_array_2 = pa.array([1, 2, 3])
-    data_table_2 = pa.Table.from_arrays(
-        [primary_key1_array_2, primary_key2_array_2], names=names
-    )
+        # Step 1: Write initial data
+        # Create PyArrow table with explicit schema to match Iceberg schema
+        arrow_schema = schema_to_pyarrow(schema)
+
+        initial_data = pa.table(
+            {
+                "id": [1, 2, 3, 4],
+                "name": ["Alice", "Bob", "Charlie", "David"],
+                "value": [100, 200, 300, 400],
+                "version": [1, 1, 1, 1],
+            },
+            schema=arrow_schema,
+        )
 
-    primary_key1_array_3 = pa.array(["pk4", "pk2", "pk3"])
-    primary_key2_array_3 = pa.array([1, 3, 4])
-    data_table_3 = pa.Table.from_arrays(
-        [primary_key1_array_3, primary_key2_array_3], names=names
-    )
+        # Step 2: Write additional data
+        additional_data = pa.table(
+            {
+                "id": [5, 6, 7, 8],
+                "name": ["Eve", "Frank", "Grace", "Henry"],
+                "value": [500, 600, 700, 800],
+                "version": [1, 1, 1, 1],
+            },
+            schema=arrow_schema,
+        )
 
-    daft_df_1 = daft.from_arrow(data_table_1)
-    daft_df_2 = daft.from_arrow(data_table_2)
-    daft_df_3 = daft.from_arrow(data_table_3)
+        # Step 3: Write updates to existing records (this creates duplicates by ID)
+        # These should overwrite the original records with same IDs
+        updated_data = pa.table(
+            {
+                "id": [2, 3, 9],  # IDs 2 and 3 are duplicates, 9 is new
+                "name": [
+                    "Robert",
+                    "Charles",
+                    "Ivan",
+                ],  # Updated names for Bob and Charlie
+                "value": [201, 301, 900],  # Updated values
+                "version": [2, 2, 1],  # Higher version numbers for updates
+            },
+            schema=arrow_schema,
+        )
 
-    download_data_mock = mocker.patch(
-        "deltacat.compute.converter.utils.io.daft_read_parquet"
-    )
-    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
+        # Write all data to separate parquet files to simulate multiple writes
+        data_files_to_commit = []
 
-    convert_ref = convert.remote(convert_input)
+        for i, data in enumerate([initial_data, additional_data, updated_data]):
+            data_file_path = os.path.join(warehouse_path, f"data_{i}.parquet")
+            pq.write_table(data, data_file_path)
 
-    to_be_deleted_files_list = []
-    to_be_added_files_list = []
-    convert_result = ray.get(convert_ref)
+            # Create DataFile objects for Iceberg
+            parquet_metadata = pq.read_metadata(data_file_path)
+            file_size = os.path.getsize(data_file_path)
 
-    if convert_result.to_be_deleted_files:
-        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
-    if convert_result.to_be_added_files:
-        to_be_added_files_list.extend(convert_result.to_be_added_files)
+            # Check schema compatibility
+            _check_pyarrow_schema_compatible(
+                schema, parquet_metadata.schema.to_arrow_schema()
+            )
 
-    # 4. Commit position delete, delete equality deletes from table
+            # Calculate statistics
+            statistics = data_file_statistics_from_parquet_metadata(
+                parquet_metadata=parquet_metadata,
+                stats_columns=compute_statistics_plan(schema, tbl.metadata.properties),
+                parquet_column_mapping=parquet_path_to_id_mapping(schema),
+            )
 
-    commit_append_snapshot(
-        iceberg_table=tbl,
-        new_position_delete_files=to_be_added_files_list,
-    )
-    tbl.refresh()
+            data_file = DataFile(
+                content=DataFileContent.DATA,
+                file_path=data_file_path,
+                file_format=FileFormat.PARQUET,
+                partition={},  # No partitioning
+                file_size_in_bytes=file_size,
+                sort_order_id=None,
+                spec_id=tbl.metadata.default_spec_id,
+                key_metadata=None,
+                equality_ids=None,
+                **statistics.to_serialized_dict(),
+            )
+            data_files_to_commit.append(data_file)
+
+        # Commit all data files to the table
+        with tbl.transaction() as tx:
+            with tx.update_snapshot().fast_append() as update_snapshot:
+                for data_file in data_files_to_commit:
+                    update_snapshot.append_data_file(data_file)
+
+        tbl.refresh()
+
+        # Verify we have duplicate IDs before conversion
+        initial_scan = tbl.scan().to_arrow().to_pydict()
+        print(f"Before conversion - Records with IDs: {sorted(initial_scan['id'])}")
+
+        # There should be duplicates: [1, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9]
+        expected_duplicate_ids = [1, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9]
+        assert (
+            sorted(initial_scan["id"]) == expected_duplicate_ids
+        ), f"Expected duplicate IDs {expected_duplicate_ids}, got {sorted(initial_scan['id'])}"
+
+        # Now call converter_session to convert equality deletes to position deletes
+        converter_params = ConverterSessionParams.of(
+            {
+                "catalog": local_catalog,
+                "iceberg_table_name": table_identifier,
+                "iceberg_warehouse_bucket_name": warehouse_path,  # Local warehouse path
+                "merge_keys": ["id"],  # Use ID as the merge key
+                "enforce_primary_key_uniqueness": True,
+                "task_max_parallelism": 1,  # Single task for local testing
+                "filesystem": local_filesystem,
+                "location_provider_prefix_override": None,  # Use local filesystem
+                "location_provider_prefix_override": None,  # Let the system auto-generate the prefix
+            }
+        )
 
-    # 5. Result assertion: Expected unique primary keys to be kept
-    pyiceberg_scan_table_rows = tbl.scan().to_arrow().to_pydict()
-    expected_result_tuple_list = [
-        ("pk1", 1),
-        ("pk2", 2),
-        ("pk2", 3),
-        ("pk3", 3),
-        ("pk3", 4),
-        ("pk4", 1),
-    ]
-    pk_combined_res = []
-    for pk1, pk2 in zip(
-        pyiceberg_scan_table_rows["primary_key1"],
-        pyiceberg_scan_table_rows["primary_key2"],
-    ):
-        pk_combined_res.append((pk1, pk2))
-
-    # Assert elements are same disregard ordering in list
-    assert sorted(pk_combined_res) == sorted(expected_result_tuple_list)
+        print(f"Running converter_session with local filesystem...")
+        print(f"Warehouse path: {warehouse_path}")
+        print(f"Merge keys: ['id']")
+        print(f"Enforce uniqueness: True")
+
+        # Run the converter
+        converter_session(params=converter_params)
+
+        # Refresh table and scan again
+        tbl.refresh()
+        final_scan = tbl.scan().to_arrow().to_pydict()
+
+        print(f"After conversion - Records with IDs: {sorted(final_scan['id'])}")
+        print(f"Final data: {final_scan}")
+
+        # Verify position delete files were created by checking table metadata
+        latest_snapshot = tbl.metadata.current_snapshot()
+        if latest_snapshot:
+            manifests = latest_snapshot.manifests(tbl.io)
+            position_delete_files = []
+
+            for manifest in manifests:
+                entries = manifest.fetch_manifest_entry(tbl.io)
+                for entry in entries:
+                    if entry.data_file.content == DataFileContent.POSITION_DELETES:
+                        position_delete_files.append(entry.data_file.file_path)
+
+            print(f"Position delete files found: {position_delete_files}")
+            assert (
+                len(position_delete_files) > 0
+            ), "No position delete files were created by converter_session"
+
+        # Verify the final result has unique IDs (duplicates should be resolved)
+        # Expected: Latest values for each ID based on the updates
+        expected_unique_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9]  # All unique IDs
+        actual_ids = sorted(final_scan["id"])
+
+        print(f"Expected unique IDs: {expected_unique_ids}")
+        print(f"Actual IDs after conversion: {actual_ids}")
+
+        assert (
+            actual_ids == expected_unique_ids
+        ), f"Expected unique IDs {expected_unique_ids}, got {actual_ids}"
+
+        # Verify the updated values are present (higher version should win)
+        final_data_by_id = {}
+        for i, id_val in enumerate(final_scan["id"]):
+            final_data_by_id[id_val] = {
+                "name": final_scan["name"][i],
+                "value": final_scan["value"][i],
+                "version": final_scan["version"][i],
+            }
+
+        # Check that ID 2 has updated value (Robert, 201, version 2)
+        assert (
+            final_data_by_id[2]["name"] == "Robert"
+        ), f"ID 2 should have updated name 'Robert', got '{final_data_by_id[2]['name']}'"
+        assert (
+            final_data_by_id[2]["value"] == 201
+        ), f"ID 2 should have updated value 201, got {final_data_by_id[2]['value']}"
+        assert (
+            final_data_by_id[2]["version"] == 2
+        ), f"ID 2 should have version 2, got {final_data_by_id[2]['version']}"
+
+        # Check that ID 3 has updated value (Charles, 301, version 2)
+        assert (
+            final_data_by_id[3]["name"] == "Charles"
+        ), f"ID 3 should have updated name 'Charles', got '{final_data_by_id[3]['name']}'"
+        assert (
+            final_data_by_id[3]["value"] == 301
+        ), f"ID 3 should have updated value 301, got {final_data_by_id[3]['value']}"
+        assert (
+            final_data_by_id[3]["version"] == 2
+        ), f"ID 3 should have version 2, got {final_data_by_id[3]['version']}"
+
+        # Check that new ID 9 is present
+        assert (
+            final_data_by_id[9]["name"] == "Ivan"
+        ), f"ID 9 should have name 'Ivan', got '{final_data_by_id[9]['name']}'"
+        assert (
+            final_data_by_id[9]["value"] == 900
+        ), f"ID 9 should have value 900, got {final_data_by_id[9]['value']}"
+
+        print(f"✅ Test completed successfully!")
+        print(
+            f"✅ Position delete files were created: {len(position_delete_files)} files"
+        )
+        print(f"✅ Duplicate IDs were resolved correctly")
+        print(
+            f"✅ Updated values were applied (ID 2: Bob->Robert, ID 3: Charlie->Charles)"
+        )
+        print(f"✅ Final table has {len(actual_ids)} unique records")
+        print(f"✅ Temporary warehouse cleaned up at: {temp_catalog_dir}")