deltacat 2.0.0b9__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

deltacat/tests/catalog/test_catalogs.py

@@ -1,23 +1,24 @@
 import unittest
 import pytest
-import ray
 import tempfile
 import shutil
 import uuid
 from unittest import mock
 import os
 
-from deltacat.catalog import CatalogProperties
-from pyiceberg.catalog import Catalog as IcebergCatalog
-
-from deltacat.catalog.model.catalog import (
+from deltacat.catalog import (
+    CatalogProperties,
     Catalog,
-    init,
+    clear_catalogs,
     get_catalog,
-    put_catalog,
+    init,
     is_initialized,
+    put_catalog,
 )
-from deltacat.catalog.iceberg.iceberg_catalog_config import IcebergCatalogConfig
+from deltacat.experimental.catalog.iceberg import impl as IcebergCatalog
+from pyiceberg.catalog import Catalog as PyIcebergCatalog
+
+from deltacat.experimental.catalog.iceberg import IcebergCatalogConfig
 
 from pyiceberg.catalog import CatalogType
 
@@ -25,41 +26,19 @@ from pyiceberg.catalog import CatalogType
 # Test module to mock a catalog implementation
 class MockCatalogImpl:
     @staticmethod
-    def initialize(*args, **kwargs):
+    def initialize(config, *args, **kwargs):
         # Return some state that the catalog would normally maintain
-        return {"initialized": True, "args": args, "kwargs": kwargs}
+        return {
+            "initialized": True,
+            "config": config,
+            "args": args,
+            "kwargs": kwargs,
+        }
 
 
 @pytest.fixture(scope="function")
-def reset_catalogs_ray_actor():
-    """
-    Setup and teardown for Ray environment for tests.
-
-    This will kill the actor all_catalogs, essentially wiping global state for catalogs
-
-    NOTE: tests using this fixture must be run serially. As of April 7 2025, the unit test suite had various
-      failures if run in parallel, in part because the state of all_catalogs in ray is shared across tests.
-
-    NOTE: when using this fixture, ensure you pass ray_init_args={"ignore_reinit_error": True} into all
-        functions which may re-initialize ray. This is because the production code checks the all_catalogs actor
-        in order to determine whether it needs to initialize Ray
-    """
-    # Reset the global catalog_actor state before each test
-    import deltacat.catalog.model.catalog as catalog_module
-
-    # Initialize Ray if not already initialized
-    if not ray.is_initialized():
-        ray.init(ignore_reinit_error=True)
-    yield
-
-    # Clean up the actor if it exists
-    if catalog_module.all_catalogs is not None:
-        try:
-            ray.kill(catalog_module.all_catalogs)
-        except Exception:
-            pass
-        finally:
-            catalog_module.all_catalogs = None
+def reset_catalogs():
+    clear_catalogs()
 
 
 class TestCatalog(unittest.TestCase):
@@ -74,6 +53,7 @@ class TestCatalog(unittest.TestCase):
         # Check that inner state was correctly initialized
         # This just asserts that kwargs were plumbed through from Catalog constructor
         self.assertTrue(catalog.inner["initialized"])
+        self.assertIsNone(catalog.inner["config"])
         self.assertEqual(catalog.inner["args"], ())
         self.assertEqual(catalog.inner["kwargs"], {})
 
@@ -81,16 +61,18 @@ class TestCatalog(unittest.TestCase):
         """Test the iceberg factory method correctly creates an Iceberg catalog."""
         # Create a mock for the Iceberg catalog module
         with mock.patch(
-            "deltacat.catalog.model.catalog.IcebergCatalog"
+            "deltacat.experimental.catalog.iceberg.impl.IcebergCatalog"
         ) as mock_iceberg_catalog:
             # Configure the mock to return a known value when initialize is called
             mock_iceberg_catalog.initialize.return_value = {"iceberg": True}
 
             # Create an Iceberg catalog config and invoke iceberg factory method
             config = IcebergCatalogConfig(type=CatalogType.IN_MEMORY, properties={})
-            catalog = Catalog.iceberg(config)
+            catalog = IcebergCatalog.from_config(config)
 
             # Check that the implementation is set to iceberg_catalog
+            print(f"catalog.impl: {catalog.impl}")
+            print(f"mock_iceberg_catalog: {mock_iceberg_catalog}")
             self.assertEqual(catalog.impl, mock_iceberg_catalog)
             # Check that the inner state is set to the output of initialize
             self.assertEqual(catalog.inner, {"iceberg": True})
@@ -109,8 +91,7 @@ class TestCatalogsIntegration:
         catalog = Catalog(impl=MockCatalogImpl)
         init(
             catalog,
-            ray_init_args={"ignore_reinit_error": True},
-            **{"force_reinitialize": True},
+            force=True,
         )
 
     @classmethod
@@ -118,17 +99,13 @@ class TestCatalogsIntegration:
         if cls.temp_dir and os.path.exists(cls.temp_dir):
             shutil.rmtree(cls.temp_dir)
 
-    def test_init_single_catalog(self, reset_catalogs_ray_actor):
+    def test_init_single_catalog(self, reset_catalogs):
         """Test initializing a single catalog."""
 
         catalog = Catalog(impl=MockCatalogImpl)
 
         # Initialize with a single catalog and Ray init args including the namespace
-        init(
-            catalog,
-            ray_init_args={"ignore_reinit_error": True},
-            **{"force_reinitialize": True},
-        )
+        init(catalog, force=True)
 
         assert is_initialized()
 
@@ -137,7 +114,7 @@ class TestCatalogsIntegration:
         assert retrieved_catalog.impl == MockCatalogImpl
         assert retrieved_catalog.inner["initialized"]
 
-    def test_init_multiple_catalogs(self, reset_catalogs_ray_actor):
+    def test_init_multiple_catalogs(self, reset_catalogs):
         """Test initializing multiple catalogs."""
         # Create catalogs
         catalog1 = Catalog(impl=MockCatalogImpl, id=1)
@@ -145,11 +122,7 @@ class TestCatalogsIntegration:
 
         # Initialize with multiple catalogs and Ray init args including the namespace
         catalogs_dict = {"catalog1": catalog1, "catalog2": catalog2}
-        init(
-            catalogs_dict,
-            ray_init_args={"ignore_reinit_error": True},
-            **{"force_reinitialize": True},
-        )
+        init(catalogs_dict, force=True)
 
         assert is_initialized()
 
@@ -162,7 +135,7 @@ class TestCatalogsIntegration:
         assert retrieved_catalog2.impl == MockCatalogImpl
         assert retrieved_catalog2.inner["kwargs"]["id"] == 2
 
-    def test_init_with_default_catalog_name(self, reset_catalogs_ray_actor):
+    def test_init_with_default_catalog_name(self, reset_catalogs):
         """Test initializing with a specified default catalog name."""
         # Create catalogs
         catalog1 = Catalog(impl=MockCatalogImpl, id=1)
@@ -173,8 +146,7 @@ class TestCatalogsIntegration:
         init(
             catalogs_dict,
             default="catalog2",
-            ray_init_args={"ignore_reinit_error": True},
-            **{"force_reinitialize": True},
+            force=True,
         )
 
         # Get the default catalog and check it's catalog2
@@ -182,16 +154,12 @@ class TestCatalogsIntegration:
         assert default_catalog.impl == MockCatalogImpl
         assert default_catalog.inner["kwargs"]["id"] == 2
 
-    def test_put_catalog(self, reset_catalogs_ray_actor):
+    def test_put_catalog(self, reset_catalogs):
         """Test adding a catalog after initialization."""
         # Initialize with a single catalog
         catalog1 = Catalog(impl=MockCatalogImpl, id=1)
         catalog2 = Catalog(impl=MockCatalogImpl, id=2)
-        init(
-            {"catalog1": catalog1},
-            ray_init_args={"ignore_reinit_error": True},
-            **{"force_reinitialize": True},
-        )
+        init({"catalog1": catalog1}, force=True)
 
         # Add a second catalog
         put_catalog("catalog2", catalog2)
@@ -203,21 +171,19 @@ class TestCatalogsIntegration:
         retrieved_catalog2 = get_catalog("catalog2")
         assert retrieved_catalog2.inner["kwargs"]["id"] == 2
 
-    def test_put_catalog_that_already_exists(self, reset_catalogs_ray_actor):
+    def test_put_catalog_that_already_exists(self, reset_catalogs):
         catalog = Catalog(impl=MockCatalogImpl, id=1)
         catalog2 = Catalog(impl=MockCatalogImpl, id=2)
         put_catalog(
             "test_catalog",
             catalog,
             id=1,
-            ray_init_args={"ignore_reinit_error": True},
         )
 
         # Try to add another catalog with the same name. Should not error
         put_catalog(
             "test_catalog",
             catalog2,
-            ray_init_args={"ignore_reinit_error": True},
         )
 
         retrieved_catalog = get_catalog("test_catalog")
@@ -228,40 +194,31 @@ class TestCatalogsIntegration:
             put_catalog(
                 "test_catalog",
                 catalog,
-                ray_init_args={"ignore_reinit_error": True},
                 fail_if_exists=True,
             )
 
-    def test_get_catalog_nonexistent(self, reset_catalogs_ray_actor):
+    def test_get_catalog_nonexistent(self, reset_catalogs):
         """Test that trying to get a nonexistent catalog raises an error."""
         # Initialize with a catalog
         catalog = Catalog(impl=MockCatalogImpl)
-        init(
-            {"test_catalog": catalog},
-            ray_init_args={"ignore_reinit_error": True},
-            **{"force_reinitialize": True},
-        )
+        init({"test_catalog": catalog}, force=True)
 
         # Try to get a nonexistent catalog
         with pytest.raises(ValueError):
             get_catalog("nonexistent")
 
-    def test_get_catalog_no_default(self, reset_catalogs_ray_actor):
+    def test_get_catalog_no_default(self, reset_catalogs):
         """Test that trying to get the default catalog when none is set raises an error."""
         # Initialize with multiple catalogs but no default
         catalog1 = Catalog(impl=MockCatalogImpl, id=1)
         catalog2 = Catalog(impl=MockCatalogImpl, id=2)
-        init(
-            {"catalog1": catalog1, "catalog2": catalog2},
-            ray_init_args={"ignore_reinit_error": True},
-            **{"force_reinitialize": True},
-        )
+        init({"catalog1": catalog1, "catalog2": catalog2}, force=True)
 
         # Try to get the default catalog
         with pytest.raises(ValueError):
             get_catalog()
 
-    def test_default_catalog_initialization(self, reset_catalogs_ray_actor):
+    def test_default_catalog_initialization(self, reset_catalogs):
         """Test that a Default catalog can be initialized and accessed using the factory method."""
         from deltacat.catalog.model.properties import CatalogProperties
 
@@ -270,15 +227,11 @@ class TestCatalogsIntegration:
         # Create the catalog properties
         config = CatalogProperties(root=self.temp_dir)
 
-        # Create the catalog using the factory method
-        catalog = Catalog.default(config)
+        # Create the catalog
+        catalog = Catalog(config)
 
         # Initialize DeltaCAT with this catalog
-        init(
-            {catalog_name: catalog},
-            ray_init_args={"ignore_reinit_error": True},
-            **{"force_reinitialize": True},
-        )
+        init({catalog_name: catalog}, force=True)
 
         # Retrieve the catalog and verify it's the same one
         retrieved_catalog = get_catalog(catalog_name)
@@ -286,16 +239,14 @@ class TestCatalogsIntegration:
         assert isinstance(retrieved_catalog.inner, CatalogProperties)
         assert retrieved_catalog.inner.root == self.temp_dir
 
-    def test_default_catalog_initialization_from_kwargs(self, reset_catalogs_ray_actor):
+    def test_default_catalog_initialization_from_kwargs(self, reset_catalogs):
 
         catalog_name = str(uuid.uuid4())
-        # Initialize DeltaCAT with this catalog
-        from deltacat.catalog.main import impl as DeltacatCatalog
 
+        # Initialize DeltaCAT with this catalog
         put_catalog(
             catalog_name,
-            Catalog(DeltacatCatalog, **{"root": "test_root"}),
-            ray_init_args={"ignore_reinit_error": True},
+            Catalog(root="test_root"),
         )
 
         # Retrieve the catalog and verify it's the same one
@@ -304,7 +255,7 @@ class TestCatalogsIntegration:
         assert isinstance(retrieved_catalog.inner, CatalogProperties)
         assert retrieved_catalog.inner.root == "test_root"
 
-    def test_iceberg_catalog_initialization(self, reset_catalogs_ray_actor):
+    def test_iceberg_catalog_initialization(self, reset_catalogs):
         """Test that an Iceberg catalog can be initialized and accessed."""
         catalog_name = str(uuid.uuid4())
 
@@ -314,11 +265,14 @@ class TestCatalogsIntegration:
         )
 
         # Create the catalog using the factory method
-        catalog = Catalog.iceberg(config)
+        catalog = IcebergCatalog.from_config(config)
 
-        put_catalog(catalog_name, catalog, ray_init_args={"ignore_reinit_error": True})
+        put_catalog(catalog_name, catalog)
 
         # Retrieve the catalog and verify it's the same one
         retrieved_catalog = get_catalog(catalog_name)
-        assert retrieved_catalog.impl.__name__ == "deltacat.catalog.iceberg.impl"
-        assert isinstance(retrieved_catalog.inner, IcebergCatalog)
+        assert (
+            retrieved_catalog.impl.__name__
+            == "deltacat.experimental.catalog.iceberg.impl"
+        )
+        assert isinstance(retrieved_catalog.inner, PyIcebergCatalog)

deltacat/tests/catalog/test_default_catalog_impl.py

@@ -38,8 +38,7 @@ class TestReadTable(unittest.TestCase):
         catalog_config = CatalogProperties(storage=ds)
         dc.put_catalog(
             cls.catalog_name,
-            catalog=Catalog.default(config=catalog_config),
-            ray_init_args={"ignore_reinit_error": True},
+            catalog=Catalog(catalog_config),
         )
         super().setUpClass()
 

deltacat/tests/compute/converter/test_convert_session.py

@@ -12,14 +12,13 @@ from pyiceberg.types import (
 from pyiceberg.partitioning import PartitionSpec, PartitionField
 from pyiceberg.transforms import IdentityTransform
 import pyarrow as pa
+import daft
 
 from deltacat.compute.converter.steps.convert import convert
 from deltacat.compute.converter.model.convert_input import ConvertInput
 from deltacat.compute.converter.pyiceberg.overrides import (
     fetch_all_bucket_files,
-    parquet_files_dict_to_iceberg_data_files,
 )
-from collections import defaultdict
 from deltacat.compute.converter.utils.converter_session_utils import (
     group_all_files_to_each_bucket,
 )
@@ -244,11 +243,14 @@ def test_converter_drop_duplicates_success(
             convert_task_index=i,
             iceberg_table_warehouse_prefix="warehouse/default",
             identifier_fields=["primary_key"],
-            compact_small_files=False,
+            table_io=tbl.io,
+            table_metadata=tbl.metadata,
+            compact_previous_position_delete_files=False,
             enforce_primary_key_uniqueness=True,
             position_delete_for_multiple_data_files=True,
             max_parallel_data_file_download=10,
             s3_file_system=s3_file_system,
+            s3_client_kwargs={},
         )
 
     number_partitioned_array_1 = pa.array([0, 0, 0], type=pa.int32())
@@ -272,38 +274,31 @@ def test_converter_drop_duplicates_success(
         [number_partitioned_array_3, primary_key_array_3], names=names
     )
 
+    daft_df_1 = daft.from_arrow(data_table_1)
+    daft_df_2 = daft.from_arrow(data_table_2)
+    daft_df_3 = daft.from_arrow(data_table_3)
+
     download_data_mock = mocker.patch(
-        "deltacat.compute.converter.utils.io.download_parquet_with_daft_hash_applied"
+        "deltacat.compute.converter.utils.io.daft_read_parquet"
     )
-    download_data_mock.side_effect = (data_table_1, data_table_2, data_table_3)
+    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
 
     convert_ref = convert.remote(convert_input)
 
     to_be_deleted_files_list = []
-    to_be_added_files_dict_list = []
-    convert_result = ray.get(convert_ref)
-
-    partition_value = convert_input.convert_input_files.partition_value
 
-    if convert_result[0]:
-        to_be_deleted_files_list.extend(convert_result[0].values())
-
-    file_location = convert_result[1][partition_value][0]
-    to_be_added_files = f"s3://{file_location}"
+    convert_result = ray.get(convert_ref)
 
-    to_be_added_files_dict = defaultdict()
-    to_be_added_files_dict[partition_value] = [to_be_added_files]
-    to_be_added_files_dict_list.append(to_be_added_files_dict)
+    to_be_added_files_list = []
+    # Check if there're files to delete
+    if convert_result.to_be_deleted_files:
+        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
+    if convert_result.to_be_added_files:
+        to_be_added_files_list.extend(convert_result.to_be_added_files)
 
-    # 4. Commit position delete, delete equality deletes from table
-    new_position_delete_files = parquet_files_dict_to_iceberg_data_files(
-        io=tbl.io,
-        table_metadata=tbl.metadata,
-        files_dict_list=to_be_added_files_dict_list,
-    )
     commit_append_snapshot(
         iceberg_table=tbl,
-        new_position_delete_files=new_position_delete_files,
+        new_position_delete_files=to_be_added_files_list,
     )
     tbl.refresh()
 
@@ -413,11 +408,14 @@ def test_converter_pos_delete_read_by_spark_success(
             convert_task_index=i,
             iceberg_table_warehouse_prefix="warehouse/default",
             identifier_fields=["primary_key"],
-            compact_small_files=False,
+            table_io=tbl.io,
+            table_metadata=tbl.metadata,
+            compact_previous_position_delete_files=False,
             enforce_primary_key_uniqueness=True,
             position_delete_for_multiple_data_files=True,
             max_parallel_data_file_download=10,
             s3_file_system=s3_file_system,
+            s3_client_kwargs={},
         )
 
     primary_key_array_1 = pa.array(["pk1", "pk2", "pk3"])
@@ -432,39 +430,30 @@ def test_converter_pos_delete_read_by_spark_success(
     names = ["primary_key"]
     data_table_3 = pa.Table.from_arrays([primary_key_array_3], names=names)
 
+    daft_df_1 = daft.from_arrow(data_table_1)
+    daft_df_2 = daft.from_arrow(data_table_2)
+    daft_df_3 = daft.from_arrow(data_table_3)
+
     download_data_mock = mocker.patch(
-        "deltacat.compute.converter.utils.io.download_parquet_with_daft_hash_applied"
+        "deltacat.compute.converter.utils.io.daft_read_parquet"
     )
-    download_data_mock.side_effect = (data_table_1, data_table_2, data_table_3)
+    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
 
     convert_ref = convert.remote(convert_input)
 
     to_be_deleted_files_list = []
-    to_be_added_files_dict_list = []
+    to_be_added_files_list = []
     convert_result = ray.get(convert_ref)
 
-    partition_value = convert_input.convert_input_files.partition_value
-
-    if convert_result[0]:
-        to_be_deleted_files_list.extend(convert_result[0].values())
-
-    file_location = convert_result[1][partition_value][0]
-    to_be_added_files = f"s3://{file_location}"
-
-    to_be_added_files_dict = defaultdict()
-    to_be_added_files_dict[partition_value] = [to_be_added_files]
-    to_be_added_files_dict_list.append(to_be_added_files_dict)
+    if convert_result.to_be_deleted_files:
+        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
+    if convert_result.to_be_added_files:
+        to_be_added_files_list.extend(convert_result.to_be_added_files)
 
     # 4. Commit position delete, delete equality deletes from table
-    new_position_delete_files = parquet_files_dict_to_iceberg_data_files(
-        io=tbl.io,
-        table_metadata=tbl.metadata,
-        files_dict_list=to_be_added_files_dict_list,
-    )
-
     commit_append_snapshot(
         iceberg_table=tbl,
-        new_position_delete_files=new_position_delete_files,
+        new_position_delete_files=to_be_added_files_list,
     )
     tbl.refresh()
 
@@ -476,3 +465,177 @@ def test_converter_pos_delete_read_by_spark_success(
     ]
     all_pk_sorted = sorted(all_pk)
     assert all_pk_sorted == ["pk1", "pk2", "pk3", "pk4"]
+
+
+@pytest.mark.integration
+def test_converter_pos_delete_multiple_identifier_fields_success(
+    spark, session_catalog: RestCatalog, setup_ray_cluster, mocker
+) -> None:
+    """
+    Test for convert compute remote function happy case. Download file results are mocked.
+    """
+
+    # 1. Create Iceberg table
+    namespace = "default"
+    table_name = "table_converter_ray_pos_delete_multiple_identifier_fields"
+
+    identifier = f"{namespace}.{table_name}"
+
+    schema = Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key1", field_type=StringType(), required=False
+        ),
+        NestedField(
+            field_id=3, name="primary_key2", field_type=LongType(), required=False
+        ),
+        schema_id=0,
+    )
+
+    partition_field_identity = PartitionField(
+        source_id=1,
+        field_id=101,
+        transform=IdentityTransform(),
+        name="number_partitioned",
+    )
+    partition_spec = PartitionSpec(partition_field_identity)
+
+    properties = dict()
+    properties["write.format.default"] = "parquet"
+    properties["write.delete.mode"] = "merge-on-read"
+    properties["write.update.mode"] = "merge-on-read"
+    properties["write.merge.mode"] = "merge-on-read"
+    properties["format-version"] = "2"
+
+    drop_table_if_exists(identifier, session_catalog)
+    session_catalog.create_table(
+        identifier,
+        schema=schema,
+        partition_spec=partition_spec,
+        properties=properties,
+    )
+
+    # 2. Use Spark to generate initial data files
+    tbl = session_catalog.load_table(identifier)
+
+    run_spark_commands(
+        spark,
+        [
+            f"""
+               INSERT INTO {identifier} VALUES (0, "pk1", 1), (0, "pk2", 2), (0, "pk3", 3)
+               """
+        ],
+    )
+    run_spark_commands(
+        spark,
+        [
+            f"""
+               INSERT INTO {identifier} VALUES (0, "pk1", 1), (0, "pk2", 2), (0, "pk3", 3)
+               """
+        ],
+    )
+    run_spark_commands(
+        spark,
+        [
+            f"""
+               INSERT INTO {identifier} VALUES (0, "pk4", 1), (0, "pk2", 3), (0, "pk3", 4)
+               """
+        ],
+    )
+    tbl.refresh()
+
+    # 3. Use convert.remote() function to compute position deletes
+    data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(tbl)
+
+    convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
+        data_file_dict=data_file_dict,
+        equality_delete_dict=equality_delete_dict,
+        pos_delete_dict=pos_delete_dict,
+    )
+
+    s3_file_system = get_s3_file_system()
+    for i, one_bucket_files in enumerate(convert_input_files_for_all_buckets):
+        convert_input = ConvertInput.of(
+            convert_input_files=one_bucket_files,
+            convert_task_index=i,
+            iceberg_table_warehouse_prefix="warehouse/default",
+            identifier_fields=["primary_key1", "primary_key2"],
+            table_io=tbl.io,
+            table_metadata=tbl.metadata,
+            compact_previous_position_delete_files=False,
+            enforce_primary_key_uniqueness=True,
+            position_delete_for_multiple_data_files=True,
+            max_parallel_data_file_download=10,
+            s3_file_system=s3_file_system,
+            s3_client_kwargs={},
+        )
+
+    names = ["primary_key1", "primary_key2"]
+
+    primary_key1_array_1 = pa.array(["pk1", "pk2", "pk3"])
+    primary_key2_array_1 = pa.array([1, 2, 3])
+    data_table_1 = pa.Table.from_arrays(
+        [primary_key1_array_1, primary_key2_array_1], names=names
+    )
+
+    primary_key1_array_2 = pa.array(["pk1", "pk2", "pk3"])
+    primary_key2_array_2 = pa.array([1, 2, 3])
+    data_table_2 = pa.Table.from_arrays(
+        [primary_key1_array_2, primary_key2_array_2], names=names
+    )
+
+    primary_key1_array_3 = pa.array(["pk4", "pk2", "pk3"])
+    primary_key2_array_3 = pa.array([1, 3, 4])
+    data_table_3 = pa.Table.from_arrays(
+        [primary_key1_array_3, primary_key2_array_3], names=names
+    )
+
+    daft_df_1 = daft.from_arrow(data_table_1)
+    daft_df_2 = daft.from_arrow(data_table_2)
+    daft_df_3 = daft.from_arrow(data_table_3)
+
+    download_data_mock = mocker.patch(
+        "deltacat.compute.converter.utils.io.daft_read_parquet"
+    )
+    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
+
+    convert_ref = convert.remote(convert_input)
+
+    to_be_deleted_files_list = []
+    to_be_added_files_list = []
+    convert_result = ray.get(convert_ref)
+
+    if convert_result.to_be_deleted_files:
+        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
+    if convert_result.to_be_added_files:
+        to_be_added_files_list.extend(convert_result.to_be_added_files)
+
+    # 4. Commit position delete, delete equality deletes from table
+
+    commit_append_snapshot(
+        iceberg_table=tbl,
+        new_position_delete_files=to_be_added_files_list,
+    )
+    tbl.refresh()
+
+    # 5. Result assertion: Expected unique primary keys to be kept
+    pyiceberg_scan_table_rows = tbl.scan().to_arrow().to_pydict()
+    expected_result_tuple_list = [
+        ("pk1", 1),
+        ("pk2", 2),
+        ("pk2", 3),
+        ("pk3", 3),
+        ("pk3", 4),
+        ("pk4", 1),
+    ]
+    pk_combined_res = []
+    for pk1, pk2 in zip(
+        pyiceberg_scan_table_rows["primary_key1"],
+        pyiceberg_scan_table_rows["primary_key2"],
+    ):
+        pk_combined_res.append((pk1, pk2))
+
+    # Assert elements are same disregard ordering in list
+    assert sorted(pk_combined_res) == sorted(expected_result_tuple_list)