deltacat 2.0.0b9__py3-none-any.whl → 2.0.0b10__py3-none-any.whl

deltacat/tests/compute/converter/test_convert_session.py

@@ -12,14 +12,13 @@ from pyiceberg.types import (
 from pyiceberg.partitioning import PartitionSpec, PartitionField
 from pyiceberg.transforms import IdentityTransform
 import pyarrow as pa
+import daft
 
 from deltacat.compute.converter.steps.convert import convert
 from deltacat.compute.converter.model.convert_input import ConvertInput
 from deltacat.compute.converter.pyiceberg.overrides import (
     fetch_all_bucket_files,
-    parquet_files_dict_to_iceberg_data_files,
 )
-from collections import defaultdict
 from deltacat.compute.converter.utils.converter_session_utils import (
     group_all_files_to_each_bucket,
 )
@@ -244,11 +243,14 @@ def test_converter_drop_duplicates_success(
             convert_task_index=i,
             iceberg_table_warehouse_prefix="warehouse/default",
             identifier_fields=["primary_key"],
-            compact_small_files=False,
+            table_io=tbl.io,
+            table_metadata=tbl.metadata,
+            compact_previous_position_delete_files=False,
             enforce_primary_key_uniqueness=True,
             position_delete_for_multiple_data_files=True,
             max_parallel_data_file_download=10,
             s3_file_system=s3_file_system,
+            s3_client_kwargs={},
         )
 
     number_partitioned_array_1 = pa.array([0, 0, 0], type=pa.int32())
@@ -272,38 +274,31 @@ def test_converter_drop_duplicates_success(
         [number_partitioned_array_3, primary_key_array_3], names=names
     )
 
+    daft_df_1 = daft.from_arrow(data_table_1)
+    daft_df_2 = daft.from_arrow(data_table_2)
+    daft_df_3 = daft.from_arrow(data_table_3)
+
     download_data_mock = mocker.patch(
-        "deltacat.compute.converter.utils.io.download_parquet_with_daft_hash_applied"
+        "deltacat.compute.converter.utils.io.daft_read_parquet"
     )
-    download_data_mock.side_effect = (data_table_1, data_table_2, data_table_3)
+    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
 
     convert_ref = convert.remote(convert_input)
 
     to_be_deleted_files_list = []
-    to_be_added_files_dict_list = []
-    convert_result = ray.get(convert_ref)
-
-    partition_value = convert_input.convert_input_files.partition_value
 
-    if convert_result[0]:
-        to_be_deleted_files_list.extend(convert_result[0].values())
-
-    file_location = convert_result[1][partition_value][0]
-    to_be_added_files = f"s3://{file_location}"
+    convert_result = ray.get(convert_ref)
 
-    to_be_added_files_dict = defaultdict()
-    to_be_added_files_dict[partition_value] = [to_be_added_files]
-    to_be_added_files_dict_list.append(to_be_added_files_dict)
+    to_be_added_files_list = []
+    # Check if there're files to delete
+    if convert_result.to_be_deleted_files:
+        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
+    if convert_result.to_be_added_files:
+        to_be_added_files_list.extend(convert_result.to_be_added_files)
 
-    # 4. Commit position delete, delete equality deletes from table
-    new_position_delete_files = parquet_files_dict_to_iceberg_data_files(
-        io=tbl.io,
-        table_metadata=tbl.metadata,
-        files_dict_list=to_be_added_files_dict_list,
-    )
     commit_append_snapshot(
         iceberg_table=tbl,
-        new_position_delete_files=new_position_delete_files,
+        new_position_delete_files=to_be_added_files_list,
     )
     tbl.refresh()
 
@@ -413,11 +408,14 @@ def test_converter_pos_delete_read_by_spark_success(
             convert_task_index=i,
             iceberg_table_warehouse_prefix="warehouse/default",
             identifier_fields=["primary_key"],
-            compact_small_files=False,
+            table_io=tbl.io,
+            table_metadata=tbl.metadata,
+            compact_previous_position_delete_files=False,
             enforce_primary_key_uniqueness=True,
             position_delete_for_multiple_data_files=True,
             max_parallel_data_file_download=10,
             s3_file_system=s3_file_system,
+            s3_client_kwargs={},
         )
 
     primary_key_array_1 = pa.array(["pk1", "pk2", "pk3"])
@@ -432,39 +430,30 @@ def test_converter_pos_delete_read_by_spark_success(
     names = ["primary_key"]
     data_table_3 = pa.Table.from_arrays([primary_key_array_3], names=names)
 
+    daft_df_1 = daft.from_arrow(data_table_1)
+    daft_df_2 = daft.from_arrow(data_table_2)
+    daft_df_3 = daft.from_arrow(data_table_3)
+
     download_data_mock = mocker.patch(
-        "deltacat.compute.converter.utils.io.download_parquet_with_daft_hash_applied"
+        "deltacat.compute.converter.utils.io.daft_read_parquet"
     )
-    download_data_mock.side_effect = (data_table_1, data_table_2, data_table_3)
+    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
 
     convert_ref = convert.remote(convert_input)
 
     to_be_deleted_files_list = []
-    to_be_added_files_dict_list = []
+    to_be_added_files_list = []
     convert_result = ray.get(convert_ref)
 
-    partition_value = convert_input.convert_input_files.partition_value
-
-    if convert_result[0]:
-        to_be_deleted_files_list.extend(convert_result[0].values())
-
-    file_location = convert_result[1][partition_value][0]
-    to_be_added_files = f"s3://{file_location}"
-
-    to_be_added_files_dict = defaultdict()
-    to_be_added_files_dict[partition_value] = [to_be_added_files]
-    to_be_added_files_dict_list.append(to_be_added_files_dict)
+    if convert_result.to_be_deleted_files:
+        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
+    if convert_result.to_be_added_files:
+        to_be_added_files_list.extend(convert_result.to_be_added_files)
 
     # 4. Commit position delete, delete equality deletes from table
-    new_position_delete_files = parquet_files_dict_to_iceberg_data_files(
-        io=tbl.io,
-        table_metadata=tbl.metadata,
-        files_dict_list=to_be_added_files_dict_list,
-    )
-
     commit_append_snapshot(
         iceberg_table=tbl,
-        new_position_delete_files=new_position_delete_files,
+        new_position_delete_files=to_be_added_files_list,
     )
     tbl.refresh()
 
@@ -476,3 +465,177 @@ def test_converter_pos_delete_read_by_spark_success(
     ]
     all_pk_sorted = sorted(all_pk)
     assert all_pk_sorted == ["pk1", "pk2", "pk3", "pk4"]
+
+
+@pytest.mark.integration
+def test_converter_pos_delete_multiple_identifier_fields_success(
+    spark, session_catalog: RestCatalog, setup_ray_cluster, mocker
+) -> None:
+    """
+    Test for convert compute remote function happy case. Download file results are mocked.
+    """
+
+    # 1. Create Iceberg table
+    namespace = "default"
+    table_name = "table_converter_ray_pos_delete_multiple_identifier_fields"
+
+    identifier = f"{namespace}.{table_name}"
+
+    schema = Schema(
+        NestedField(
+            field_id=1, name="number_partitioned", field_type=LongType(), required=False
+        ),
+        NestedField(
+            field_id=2, name="primary_key1", field_type=StringType(), required=False
+        ),
+        NestedField(
+            field_id=3, name="primary_key2", field_type=LongType(), required=False
+        ),
+        schema_id=0,
+    )
+
+    partition_field_identity = PartitionField(
+        source_id=1,
+        field_id=101,
+        transform=IdentityTransform(),
+        name="number_partitioned",
+    )
+    partition_spec = PartitionSpec(partition_field_identity)
+
+    properties = dict()
+    properties["write.format.default"] = "parquet"
+    properties["write.delete.mode"] = "merge-on-read"
+    properties["write.update.mode"] = "merge-on-read"
+    properties["write.merge.mode"] = "merge-on-read"
+    properties["format-version"] = "2"
+
+    drop_table_if_exists(identifier, session_catalog)
+    session_catalog.create_table(
+        identifier,
+        schema=schema,
+        partition_spec=partition_spec,
+        properties=properties,
+    )
+
+    # 2. Use Spark to generate initial data files
+    tbl = session_catalog.load_table(identifier)
+
+    run_spark_commands(
+        spark,
+        [
+            f"""
+               INSERT INTO {identifier} VALUES (0, "pk1", 1), (0, "pk2", 2), (0, "pk3", 3)
+               """
+        ],
+    )
+    run_spark_commands(
+        spark,
+        [
+            f"""
+               INSERT INTO {identifier} VALUES (0, "pk1", 1), (0, "pk2", 2), (0, "pk3", 3)
+               """
+        ],
+    )
+    run_spark_commands(
+        spark,
+        [
+            f"""
+               INSERT INTO {identifier} VALUES (0, "pk4", 1), (0, "pk2", 3), (0, "pk3", 4)
+               """
+        ],
+    )
+    tbl.refresh()
+
+    # 3. Use convert.remote() function to compute position deletes
+    data_file_dict, equality_delete_dict, pos_delete_dict = fetch_all_bucket_files(tbl)
+
+    convert_input_files_for_all_buckets = group_all_files_to_each_bucket(
+        data_file_dict=data_file_dict,
+        equality_delete_dict=equality_delete_dict,
+        pos_delete_dict=pos_delete_dict,
+    )
+
+    s3_file_system = get_s3_file_system()
+    for i, one_bucket_files in enumerate(convert_input_files_for_all_buckets):
+        convert_input = ConvertInput.of(
+            convert_input_files=one_bucket_files,
+            convert_task_index=i,
+            iceberg_table_warehouse_prefix="warehouse/default",
+            identifier_fields=["primary_key1", "primary_key2"],
+            table_io=tbl.io,
+            table_metadata=tbl.metadata,
+            compact_previous_position_delete_files=False,
+            enforce_primary_key_uniqueness=True,
+            position_delete_for_multiple_data_files=True,
+            max_parallel_data_file_download=10,
+            s3_file_system=s3_file_system,
+            s3_client_kwargs={},
+        )
+
+    names = ["primary_key1", "primary_key2"]
+
+    primary_key1_array_1 = pa.array(["pk1", "pk2", "pk3"])
+    primary_key2_array_1 = pa.array([1, 2, 3])
+    data_table_1 = pa.Table.from_arrays(
+        [primary_key1_array_1, primary_key2_array_1], names=names
+    )
+
+    primary_key1_array_2 = pa.array(["pk1", "pk2", "pk3"])
+    primary_key2_array_2 = pa.array([1, 2, 3])
+    data_table_2 = pa.Table.from_arrays(
+        [primary_key1_array_2, primary_key2_array_2], names=names
+    )
+
+    primary_key1_array_3 = pa.array(["pk4", "pk2", "pk3"])
+    primary_key2_array_3 = pa.array([1, 3, 4])
+    data_table_3 = pa.Table.from_arrays(
+        [primary_key1_array_3, primary_key2_array_3], names=names
+    )
+
+    daft_df_1 = daft.from_arrow(data_table_1)
+    daft_df_2 = daft.from_arrow(data_table_2)
+    daft_df_3 = daft.from_arrow(data_table_3)
+
+    download_data_mock = mocker.patch(
+        "deltacat.compute.converter.utils.io.daft_read_parquet"
+    )
+    download_data_mock.side_effect = (daft_df_1, daft_df_2, daft_df_3)
+
+    convert_ref = convert.remote(convert_input)
+
+    to_be_deleted_files_list = []
+    to_be_added_files_list = []
+    convert_result = ray.get(convert_ref)
+
+    if convert_result.to_be_deleted_files:
+        to_be_deleted_files_list.extend(convert_result.to_be_deleted_files.values())
+    if convert_result.to_be_added_files:
+        to_be_added_files_list.extend(convert_result.to_be_added_files)
+
+    # 4. Commit position delete, delete equality deletes from table
+
+    commit_append_snapshot(
+        iceberg_table=tbl,
+        new_position_delete_files=to_be_added_files_list,
+    )
+    tbl.refresh()
+
+    # 5. Result assertion: Expected unique primary keys to be kept
+    pyiceberg_scan_table_rows = tbl.scan().to_arrow().to_pydict()
+    expected_result_tuple_list = [
+        ("pk1", 1),
+        ("pk2", 2),
+        ("pk2", 3),
+        ("pk3", 3),
+        ("pk3", 4),
+        ("pk4", 1),
+    ]
+    pk_combined_res = []
+    for pk1, pk2 in zip(
+        pyiceberg_scan_table_rows["primary_key1"],
+        pyiceberg_scan_table_rows["primary_key2"],
+    ):
+        pk_combined_res.append((pk1, pk2))
+
+    # Assert elements are same disregard ordering in list
+    assert sorted(pk_combined_res) == sorted(expected_result_tuple_list)

deltacat/tests/local_deltacat_storage/__init__.py

@@ -7,6 +7,7 @@ import sqlite3
 from sqlite3 import Cursor, Connection
 import uuid
 import ray
+
 import io
 
 from deltacat.tests.test_utils.storage import create_empty_delta

deltacat/tests/storage/model/test_expression.py

@@ -0,0 +1,327 @@
+import pytest
+import pyarrow as pa
+
+from deltacat.storage.model.expression import (
+    Reference,
+    Literal,
+    Equal,
+    NotEqual,
+    GreaterThan,
+    LessThan,
+    GreaterThanEqual,
+    LessThanEqual,
+    And,
+    Or,
+    Not,
+    In,
+    Between,
+    Like,
+    IsNull,
+)
+from deltacat.storage.model.expression.visitor import DisplayVisitor, ExpressionVisitor
+
+
+@pytest.fixture
+def field_ref():
+    return Reference("field1")
+
+
+@pytest.fixture
+def field_ref2():
+    return Reference("field2")
+
+
+@pytest.fixture
+def literal_int():
+    return Literal(pa.scalar(42))
+
+
+@pytest.fixture
+def literal_str():
+    return Literal(pa.scalar("test"))
+
+
+@pytest.fixture
+def display_visitor():
+    return DisplayVisitor()
+
+
+class TestExpressionLibrary:
+    """Test suite for the Deltacat expression library."""
+
+    def test_reference_creation(self):
+        ref = Reference("field1")
+        assert ref.field == "field1"
+        assert ref.index is None
+
+    def test_reference_with_index(self):
+        ref = Reference("field1", 0)
+        assert ref.field == "field1"
+        assert ref.index == 0
+
+    def test_literal_creation(self):
+        lit = Literal(pa.scalar(42))
+        assert lit.value.as_py() == 42
+
+    # Test the factory methods (.of)
+    def test_factory_methods(self):
+        # Reference.of
+        ref = Reference.of("field1")
+        assert ref.field == "field1"
+
+        # Literal.of
+        lit = Literal.of(42)
+        assert lit.value.as_py() == 42
+
+        # Equal.of with mixed types
+        eq = Equal.of("field1", 42)
+        assert isinstance(eq.left, Literal)
+        assert isinstance(eq.right, Literal)
+        assert eq.left.value.as_py() == "field1"
+        assert eq.right.value.as_py() == 42
+
+        # Not.of
+        not_expr = Not.of(Equal.of("field1", 42))
+        assert isinstance(not_expr.operand, Equal)
+
+        # In.of
+        in_expr = In.of("field1", [1, 2, 3])
+        assert isinstance(in_expr.value, Literal)
+        assert len(in_expr.values) == 3
+        assert all(isinstance(v, Literal) for v in in_expr.values)
+
+        # Between.of
+        between_expr = Between.of("field1", 10, 20)
+        assert isinstance(between_expr.value, Literal)
+        assert between_expr.lower.value.as_py() == 10
+        assert between_expr.upper.value.as_py() == 20
+
+        # Like.of
+        like_expr = Like.of("field1", "%test%")
+        assert isinstance(like_expr.value, Literal)
+        assert like_expr.pattern.value.as_py() == "%test%"
+
+    # Test reference comparison helper methods
+    def test_reference_comparison_helpers(self, field_ref):
+        # Test eq, ne, gt, lt, ge, le methods
+        eq_expr = field_ref.eq(42)
+        assert isinstance(eq_expr, Equal)
+        assert eq_expr.left == field_ref
+        assert eq_expr.right.value.as_py() == 42
+
+        ne_expr = field_ref.ne(42)
+        assert isinstance(ne_expr, NotEqual)
+
+        gt_expr = field_ref.gt(42)
+        assert isinstance(gt_expr, GreaterThan)
+
+        lt_expr = field_ref.lt(42)
+        assert isinstance(lt_expr, LessThan)
+
+        ge_expr = field_ref.ge(42)
+        assert isinstance(ge_expr, GreaterThanEqual)
+
+        le_expr = field_ref.le(42)
+        assert isinstance(le_expr, LessThanEqual)
+
+    # Test reference special operation helpers
+    def test_reference_special_helpers(self, field_ref):
+        # Test is_null, in_, between, like methods
+        is_null_expr = field_ref.is_null()
+        assert isinstance(is_null_expr, IsNull)
+        assert is_null_expr.operand == field_ref
+
+        in_expr = field_ref.in_([1, 2, 3])
+        assert isinstance(in_expr, In)
+        assert in_expr.value == field_ref
+        assert len(in_expr.values) == 3
+        assert in_expr.values[0].value.as_py() == 1
+
+        between_expr = field_ref.between(10, 20)
+        assert isinstance(between_expr, Between)
+        assert between_expr.value == field_ref
+        assert between_expr.lower.value.as_py() == 10
+        assert between_expr.upper.value.as_py() == 20
+
+        like_expr = field_ref.like("%test%")
+        assert isinstance(like_expr, Like)
+        assert like_expr.value == field_ref
+        assert like_expr.pattern.value.as_py() == "%test%"
+
+    # Test boolean expression helper methods
+    def test_boolean_expression_helpers(self, field_ref):
+        # Test and_, or_, not_ methods
+        expr1 = field_ref.eq(42)
+        expr2 = field_ref.gt(10)
+
+        and_expr = expr1.and_(expr2)
+        assert isinstance(and_expr, And)
+        assert and_expr.left == expr1
+        assert and_expr.right == expr2
+
+        or_expr = expr1.or_(expr2)
+        assert isinstance(or_expr, Or)
+        assert or_expr.left == expr1
+        assert or_expr.right == expr2
+
+        not_expr = expr1.not_()
+        assert isinstance(not_expr, Not)
+        assert not_expr.operand == expr1
+
+    # Test building complex expressions
+    def test_complex_expression_building(self, field_ref, field_ref2):
+        # Test building more complex expressions using method chaining
+        expr = field_ref.eq(42).and_(field_ref2.gt(10)).or_(field_ref.is_null()).not_()
+
+        assert isinstance(expr, Not)
+        assert isinstance(expr.operand, Or)
+        assert isinstance(expr.operand.left, And)
+        assert isinstance(expr.operand.right, IsNull)
+
+    # Test DisplayVisitor for different expression types
+    def test_reference_display(self, field_ref, display_visitor):
+        assert display_visitor.visit(field_ref) == "field1"
+
+    def test_literal_display(self, literal_int, literal_str, display_visitor):
+        assert display_visitor.visit(literal_int) == "42"
+        assert display_visitor.visit(literal_str) == "test"
+
+    def test_comparison_display(self, field_ref, literal_int, display_visitor):
+        assert display_visitor.visit(Equal(field_ref, literal_int)) == "field1 = 42"
+        assert display_visitor.visit(NotEqual(field_ref, literal_int)) == "field1 <> 42"
+        assert (
+            display_visitor.visit(GreaterThan(field_ref, literal_int)) == "field1 > 42"
+        )
+        assert display_visitor.visit(LessThan(field_ref, literal_int)) == "field1 < 42"
+        assert (
+            display_visitor.visit(GreaterThanEqual(field_ref, literal_int))
+            == "field1 >= 42"
+        )
+        assert (
+            display_visitor.visit(LessThanEqual(field_ref, literal_int))
+            == "field1 <= 42"
+        )
+
+    def test_logical_operator_display(self, field_ref, literal_int, display_visitor):
+        eq_expr = Equal(field_ref, literal_int)
+        gt_expr = GreaterThan(field_ref, literal_int)
+
+        assert (
+            display_visitor.visit(And(eq_expr, gt_expr))
+            == "(field1 = 42 AND field1 > 42)"
+        )
+        assert (
+            display_visitor.visit(Or(eq_expr, gt_expr))
+            == "(field1 = 42 OR field1 > 42)"
+        )
+        assert display_visitor.visit(Not(eq_expr)) == "NOT (field1 = 42)"
+
+    def test_special_operator_display(self, field_ref, display_visitor):
+        assert display_visitor.visit(IsNull(field_ref)) == "(field1) IS NULL"
+
+        values = [Literal(pa.scalar(1)), Literal(pa.scalar(2)), Literal(pa.scalar(3))]
+        assert display_visitor.visit(In(field_ref, values)) == "field1 IN (1, 2, 3)"
+
+        lower = Literal(pa.scalar(10))
+        upper = Literal(pa.scalar(20))
+        assert (
+            display_visitor.visit(Between(field_ref, lower, upper))
+            == "field1 BETWEEN 10 AND 20"
+        )
+
+        pattern = Literal(pa.scalar("%test%"))
+        assert display_visitor.visit(Like(field_ref, pattern)) == "field1 LIKE %test%"
+
+    def test_complex_expression_display(self, field_ref, field_ref2, display_visitor):
+        expr = field_ref.eq(42).and_(field_ref2.gt(10)).or_(field_ref.is_null()).not_()
+
+        # Check that the DisplayVisitor correctly formats the complex expression
+        assert (
+            display_visitor.visit(expr)
+            == "NOT (((field1 = 42 AND field2 > 10) OR (field1) IS NULL))"
+        )
+
+    # Test BinaryExpression with_ methods
+    def test_binary_expression_with_methods(self, field_ref, field_ref2, literal_int):
+        eq_expr = Equal(field_ref, literal_int)
+
+        # Test with_left
+        new_expr = eq_expr.with_left(field_ref2)
+        assert isinstance(new_expr, Equal)
+        assert new_expr.left == field_ref2
+        assert new_expr.right == literal_int
+
+        # Test with_right
+        new_lit = Literal(pa.scalar(100))
+        new_expr = eq_expr.with_right(new_lit)
+        assert new_expr.left == field_ref
+        assert new_expr.right == new_lit
+
+    # Test __str__ method which uses DisplayVisitor
+    def test_expression_str_method(self, field_ref, literal_int):
+        eq_expr = Equal(field_ref, literal_int)
+        assert str(eq_expr) == "field1 = 42"
+
+    # Test proper parenthesization in complex expressions
+    def test_nested_parentheses(self, field_ref, field_ref2, display_visitor):
+        # Create a complex expression: (field1 = 1 AND field2 = 2) OR field2 = 3
+        expr1 = Equal(field_ref, Literal(pa.scalar(1)))
+        expr2 = Equal(field_ref2, Literal(pa.scalar(2)))
+        expr3 = Equal(field_ref2, Literal(pa.scalar(3)))
+
+        and_expr = And(expr1, expr2)
+        or_expr = Or(and_expr, expr3)
+
+        assert (
+            display_visitor.visit(or_expr)
+            == "((field1 = 1 AND field2 = 2) OR field2 = 3)"
+        )
+
+    # Test Literal comparison methods
+    def test_literal_comparison_methods(self, literal_int):
+        eq_expr = literal_int.eq("test")
+        assert isinstance(eq_expr, Equal)
+        assert eq_expr.left == literal_int
+        assert eq_expr.right.value.as_py() == "test"
+
+        ne_expr = literal_int.ne("test")
+        assert isinstance(ne_expr, NotEqual)
+        assert ne_expr.left == literal_int
+        assert ne_expr.right.value.as_py() == "test"
+
+    # Test a custom ExpressionVisitor implementation
+    def test_custom_visitor(self, field_ref, literal_int):
+        class CountingVisitor(ExpressionVisitor[None, int]):
+            """Simple visitor that counts expression nodes"""
+
+            def visit_reference(self, expr, context=None):
+                return 1
+
+            def visit_literal(self, expr, context=None):
+                return 1
+
+            def visit_binary_expression(self, expr, left, right, context=None):
+                return left + right + 1
+
+            def visit_unary_expression(self, expr, operand, context=None):
+                return operand + 1
+
+            def visit_in(self, expr, context=None):
+                return 1 + len(expr.values) + 1  # value + all values + In operator
+
+            def visit_between(self, expr, context=None):
+                return 3  # value + lower + upper
+
+            def visit_like(self, expr, context=None):
+                return 2  # value + pattern
+
+        visitor = CountingVisitor()
+
+        # Count nodes in simple expressions
+        assert visitor.visit(field_ref) == 1
+        assert visitor.visit(literal_int) == 1
+        assert visitor.visit(Equal(field_ref, literal_int)) == 3  # left + right + Equal
+
+        # Count nodes in a more complex expression
+        expr = field_ref.eq(42).and_(field_ref.gt(10))
+        assert visitor.visit(expr) == 7  # (1+1+1) + (1+1+1) + 1

deltacat/tests/storage/rivulet/fs/test_file_location_provider.py

@@ -2,7 +2,8 @@ import pytest
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-from deltacat import Datatype, Dataset
+from deltacat.storage.rivulet.schema.datatype import Datatype
+from deltacat.storage.rivulet.dataset import Dataset
 from deltacat.storage.rivulet import Schema, Field
 from deltacat.utils.metafile_locator import _find_partition_path
 

deltacat/tests/storage/rivulet/test_dataset.py

@@ -57,7 +57,7 @@ def test_dataset_creation_metadata_structure(tmp_path):
     dataset = Dataset(dataset_name="test_dataset", metadata_uri=str(tmp_path))
 
     assert dataset._metadata_folder.startswith(".riv-meta")
-    assert dataset._namespace == "DEFAULT"
+    assert dataset._namespace == "default"
     assert dataset.dataset_name == "test_dataset"
     assert dataset._metadata_path == str(tmp_path / ".riv-meta-test_dataset")
 

deltacat/tests/storage/rivulet/test_manifest.py

@@ -2,7 +2,7 @@ import os
 
 import pytest
 
-from deltacat import Dataset
+from deltacat.storage.rivulet.dataset import Dataset
 from deltacat.storage.rivulet.fs.file_store import FileStore
 from deltacat.storage.rivulet.schema.datatype import Datatype
 from deltacat.storage.rivulet.metastore.delta import DeltacatManifestIO