deltacat 1.1.35__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/tests/storage/model/test_schema.py

@@ -0,0 +1,308 @@
+import pytest
+import pyarrow as pa
+
+from deltacat.storage.model.schema import (
+    Schema,
+    Field,
+    BASE_SCHEMA_NAME,
+)
+
+
+@pytest.fixture
+def schema_a():
+    return Schema.of(
+        [
+            Field.of(
+                field=pa.field("col1", pa.int32(), nullable=False),
+                field_id=1,
+                is_merge_key=True,
+            )
+        ]
+    )
+
+
+@pytest.fixture
+def schema_b():
+    return Schema.of(
+        [
+            Field.of(
+                field=pa.field("col2", pa.string(), nullable=True),
+                field_id=2,
+                is_merge_key=False,
+            )
+        ]
+    )
+
+
+@pytest.fixture
+def schema_c():
+    return Schema.of(
+        [
+            Field.of(
+                field=pa.field("col3", pa.float64(), nullable=False),
+                field_id=3,
+                is_merge_key=False,
+            )
+        ]
+    )
+
+
+@pytest.fixture
+def schema_d():
+    schema = Schema.of(
+        [
+            Field.of(
+                field=pa.field("col_named", pa.int64(), nullable=True),
+                field_id=4,
+                is_merge_key=False,
+            )
+        ]
+    )
+    return schema
+
+
+def test_of_with_dict(schema_a, schema_b):
+    input_dict = {"schema_a": schema_a.arrow, "schema_b": schema_b.arrow}
+    schema = Schema.of(input_dict)
+
+    assert isinstance(schema, Schema)
+    assert list(schema.subschemas.keys()) == ["schema_a", "schema_b"]
+    assert list(schema.subschemas_to_field_ids.keys()) == ["schema_a", "schema_b"]
+    assert schema.subschemas["schema_a"].equivalent_to(schema_a, True)
+    assert schema.subschemas["schema_b"].equivalent_to(schema_b, True)
+
+
+def test_of_invalid_input():
+    with pytest.raises(ValueError):
+        Schema.of(42)
+    with pytest.raises(ValueError):
+        Schema.of(["foo"])
+    with pytest.raises(ValueError):
+        Schema.of({"foo": [42]})
+
+
+def test_insert_explicit_name(schema_a, schema_b):
+    schema = Schema.of(schema_a.arrow)
+    assert not schema.subschemas
+    new_schema = schema.add_subschema("explicit", schema_b.arrow)
+    assert not schema.subschemas
+    assert "explicit" in new_schema.subschemas
+    assert "explicit" in new_schema.subschemas_to_field_ids
+    assert BASE_SCHEMA_NAME in new_schema.subschemas
+    assert BASE_SCHEMA_NAME in new_schema.subschemas_to_field_ids
+    assert new_schema.subschemas[BASE_SCHEMA_NAME].equivalent_to(schema_a)
+    assert new_schema.subschemas["explicit"].equivalent_to(schema_b)
+
+
+def test_insert_reserved_name_fails(schema_a, schema_b):
+    schema = Schema.of(schema_a.arrow)
+    with pytest.raises(ValueError):
+        schema.add_subschema(BASE_SCHEMA_NAME, schema_b.arrow)
+
+
+def test_insert_duplicate_schema_name_fails(schema_a, schema_b):
+    schema = Schema.of({"dupe": schema_a.arrow})
+    with pytest.raises(ValueError):
+        schema.add_subschema("dupe", schema_b.arrow)
+
+
+def test_insert_duplicate_field_name_fails(schema_a):
+    schema = Schema.of(schema_a.arrow)
+    with pytest.raises(ValueError):
+        schema.add_subschema("dupe_field_name", schema_a.arrow)
+
+
+def test_insert_autofill_field_id():
+    schema1 = [
+        Field.of(
+            field=pa.field("col1", pa.int32(), nullable=False),
+            is_merge_key=True,
+        )
+    ]
+    schema2 = [
+        Field.of(
+            field=pa.field("col2", pa.int32(), nullable=False),
+            is_merge_key=True,
+        )
+    ]
+    schema = Schema.of(schema1)
+    new_schema = schema.add_subschema("explicit", schema2)
+    assert not schema.subschemas
+    assert "explicit" in new_schema.subschemas
+    assert "explicit" in new_schema.subschemas_to_field_ids
+    assert BASE_SCHEMA_NAME in new_schema.subschemas
+    assert BASE_SCHEMA_NAME in new_schema.subschemas_to_field_ids
+    assert len(new_schema.subschemas[BASE_SCHEMA_NAME].fields) == 1
+    assert len(new_schema.subschemas["explicit"].fields) == 1
+    assert new_schema.subschemas[BASE_SCHEMA_NAME].fields[0].id == 0
+    assert new_schema.subschemas["explicit"].fields[0].id == 1
+    assert new_schema.subschemas[BASE_SCHEMA_NAME].equivalent_to(Schema.of(schema1))
+    schema2_with_expected_field_id = [
+        Field.of(
+            field=pa.field("col2", pa.int32(), nullable=False),
+            field_id=1,
+            is_merge_key=True,
+        )
+    ]
+    assert new_schema.subschemas["explicit"].equivalent_to(
+        Schema.of(schema2_with_expected_field_id)
+    )
+
+
+def test_insert_duplicate_field_name_case_insensitive_fails():
+    schema1 = Schema.of(
+        [
+            Field.of(
+                field=pa.field("col1", pa.int32(), nullable=False),
+                field_id=1,
+                is_merge_key=True,
+            )
+        ]
+    )
+    schema2 = Schema.of(
+        [
+            Field.of(
+                field=pa.field("COL1", pa.int32(), nullable=False),
+                field_id=2,
+                is_merge_key=True,
+            )
+        ]
+    )
+    schema = Schema.of(schema1.arrow)
+    with pytest.raises(ValueError):
+        schema.add_subschema("dupe_field_name_case_insensitive", schema2.arrow)
+
+
+def test_insert_duplicate_field_id_fails():
+    schema1 = Schema.of(
+        [
+            Field.of(
+                field=pa.field("col1", pa.int32(), nullable=False),
+                field_id=1,
+                is_merge_key=True,
+            )
+        ]
+    )
+    schema2 = Schema.of(
+        [
+            Field.of(
+                field=pa.field("col2", pa.int32(), nullable=False),
+                field_id=1,
+                is_merge_key=True,
+            )
+        ]
+    )
+    schema = Schema.of(schema1.arrow)
+    with pytest.raises(ValueError):
+        schema.add_subschema("dupe_field_id", schema2.arrow)
+
+
+def test_update_success(schema_a, schema_b):
+    schema = Schema.of({"key": schema_a.arrow})
+    new_schema = schema.replace_subschema("key", schema_b.arrow)
+    assert schema.subschemas["key"].equivalent_to(schema_a)
+    assert new_schema.subschemas["key"].equivalent_to(schema_b)
+
+
+def test_update_not_exist(schema_a):
+    schema = Schema.of({"key": schema_a.arrow})
+    with pytest.raises(ValueError):
+        schema.replace_subschema("nonexistent", schema_a.arrow)
+
+
+def test_delete_schema_success(schema_a, schema_b):
+    schema = Schema.of({"key1": schema_a.arrow, "key2": schema_b.arrow})
+    new_schema = schema.delete_subschema("key1")
+    assert "key1" in schema.subschemas
+    assert "key2" in schema.subschemas
+    assert "key1" not in new_schema.subschemas
+    assert "key2" in new_schema.subschemas
+    assert "key1" in schema.subschemas_to_field_ids
+    assert "key2" in schema.subschemas_to_field_ids
+    assert "key1" not in new_schema.subschemas_to_field_ids
+    assert "key2" in new_schema.subschemas_to_field_ids
+
+
+def test_delete_only_schema_fails(schema_a):
+    schema = Schema.of({"key": schema_a.arrow})
+    with pytest.raises(ValueError):
+        schema.delete_subschema("key")
+
+
+def test_delete_schema_not_exist(schema_a):
+    schema = Schema.of({"key": schema_a.arrow})
+    with pytest.raises(ValueError):
+        schema.delete_subschema("nonexistent")
+
+
+def test_get_schemas_order(schema_a, schema_b, schema_c):
+    schema = Schema.of({"a": schema_a.arrow, "b": schema_b.arrow, "c": schema_c.arrow})
+    assert list(schema.subschemas.keys()) == ["a", "b", "c"]
+    assert list(schema.subschemas.values()) == [schema_a, schema_b, schema_c]
+    schema = Schema.of({"a": schema_a.arrow})
+    schema = schema.add_subschema("b", schema_b.arrow)
+    assert list(schema.subschemas.keys()) == ["a", "b"]
+    assert list(schema.subschemas.values()) == [schema_a, schema_b]
+    schema = schema.add_subschema("c", schema_c.arrow)
+    assert list(schema.subschemas.keys()) == ["a", "b", "c"]
+    assert list(schema.subschemas.values()) == [schema_a, schema_b, schema_c]
+
+
+def test_equivalent_to_same(schema_a, schema_b):
+    schema1 = Schema.of({"a": schema_a.arrow, "b": schema_b.arrow})
+    schema2 = Schema.of({"a": schema_a.arrow, "b": schema_b.arrow})
+    assert schema1.equivalent_to(schema2)
+    assert schema2.equivalent_to(schema1)
+
+
+def test_equivalent_to_different_subschema_names(schema_a, schema_b):
+    schema1 = Schema.of({"a": schema_a.arrow, "b": schema_b.arrow})
+    schema2 = Schema.of({"x": schema_a.arrow, "b": schema_b.arrow})
+    assert schema1.equivalent_to(schema2)
+    assert not schema1.equivalent_to(schema2, True)
+
+
+def test_not_equivalent_to_different_subschema_order(schema_a, schema_b):
+    schema1 = Schema.of({"a": schema_a.arrow, "b": schema_b.arrow})
+    schema2 = Schema.of({"b": schema_b.arrow, "a": schema_a.arrow})
+    assert not schema1.equivalent_to(schema2)
+
+
+def test_equivalent_to_non_schema(schema_a):
+    schema = Schema.of({"a": schema_a.arrow})
+    assert not schema.equivalent_to("not a schema")
+
+
+def test_equivalent_schemas_different_instances():
+    """
+    Edge case: ensure equivalent schemas with different instances
+    are considered equivalent.
+    """
+    schema1 = Schema.of(
+        [
+            Field.of(
+                pa.field("col1", pa.int32(), nullable=False),
+                field_id=1,
+                is_merge_key=True,
+            )
+        ]
+    )
+    schema2 = Schema.of(
+        [
+            Field.of(
+                pa.field("col1", pa.int32(), nullable=False),
+                field_id=1,
+                is_merge_key=True,
+            )
+        ]
+    )
+    schema1 = Schema.of({"key": schema1.arrow})
+    schema2 = Schema.of({"key": schema2.arrow})
+    assert schema1.equivalent_to(schema2)
+
+
+def test_empty_schema_fails():
+    with pytest.raises(ValueError):
+        Schema.of({})
+    with pytest.raises(ValueError):
+        Schema.of([])

deltacat/tests/storage/model/test_shard.py

@@ -0,0 +1,22 @@
+import pytest
+
+from deltacat.storage.model.shard import ShardingStrategy
+
+
+def test_sharding_strategy_from_string_range():
+    """
+    Tests that from_string('range') returns an instance of RangeShardingStrategy.
+    """
+    from deltacat.storage.rivulet.shard.range_shard import RangeShardingStrategy
+
+    strategy = ShardingStrategy.from_string("range")
+    assert isinstance(strategy, RangeShardingStrategy)
+
+
+def test_sharding_strategy_from_string_invalid():
+    """
+    Tests that from_string(...) raises ValueError for an unknown strategy string.
+    """
+    with pytest.raises(ValueError) as exc_info:
+        ShardingStrategy.from_string("unknown_strategy")
+    assert "Unsupported sharding strategy type: unknown_strategy" in str(exc_info.value)

deltacat/tests/storage/model/test_table_version.py

@@ -0,0 +1,110 @@
+import pytest
+
+from deltacat.constants import BYTES_PER_KIBIBYTE
+from deltacat.storage.model.table_version import TableVersion, TableVersionLocator
+
+
+@pytest.mark.parametrize(
+    "previous_version, expected_next_version",
+    [
+        (None, "1"),
+        ("v.1", "v.2"),
+        ("1", "2"),
+        ("1.0", "1.1"),
+        ("v.999", "v.1000"),
+    ],
+)
+def test_next_version(previous_version, expected_next_version):
+    new_version = TableVersion.next_version(previous_version)
+    assert isinstance(new_version, str)
+    assert new_version == expected_next_version
+
+
+@pytest.mark.parametrize(
+    "table_version, expected_parsed_version",
+    [
+        ("v.1", ("v.", 1)),
+        ("1", (None, 1)),
+        ("1.0", ("1.", 0)),
+        ("v.999", ("v.", 999)),
+    ],
+)
+def test_parse_version(table_version, expected_parsed_version):
+    prefix, version_number = TableVersion.parse_table_version(table_version)
+    if prefix is not None:
+        assert isinstance(prefix, str)
+    assert isinstance(version_number, int)
+    assert (prefix, version_number) == expected_parsed_version
+
+
+def test_version_validation_invalid():
+    with pytest.raises(ValueError):
+        TableVersionLocator.at(
+            namespace="test_namespace",
+            table_name="test_table",
+            table_version="invalid_version",
+        )
+
+
+def test_version_validation_valid_to_invalid():
+    valid_tv_locator = TableVersionLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+        table_version="1",
+    )
+    assert valid_tv_locator.table_version == "1"
+    tv = TableVersion.of(
+        locator=valid_tv_locator,
+        schema=None,
+    )
+    assert tv.current_version_number() == 1
+    with pytest.raises(ValueError):
+        valid_tv_locator.table_version = "invalid_version"
+
+
+def test_version_validation_numeric_name():
+    valid_tv_locator = TableVersionLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+        table_version="1.0",
+    )
+    assert valid_tv_locator.table_version == "1.0"
+    tv = TableVersion.of(
+        locator=valid_tv_locator,
+        schema=None,
+    )
+    assert tv.current_version_number() == 0
+
+
+def test_version_validation_truncate_leading_zeros():
+    valid_tv_locator = TableVersionLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+        table_version="1.00002",
+    )
+    # ensure that leading 0's are truncated
+    assert valid_tv_locator.table_version == "1.2"
+    tv = TableVersion.of(
+        locator=valid_tv_locator,
+        schema=None,
+    )
+    assert tv.current_version_number() == 2
+
+
+def test_version_validation_version_id_length_limits():
+    # ensure that long version identifiers are accepted
+    long_tv_id = "a" * (BYTES_PER_KIBIBYTE - 2) + ".1"
+    valid_tv_locator = TableVersionLocator.at(
+        namespace="test_namespace",
+        table_name="test_table",
+        table_version=long_tv_id,
+    )
+    assert valid_tv_locator.table_version == long_tv_id
+    tv = TableVersion.of(
+        locator=valid_tv_locator,
+        schema=None,
+    )
+    assert tv.current_version_number() == 1
+    # ensure that an excessively long version identifier is rejected
+    with pytest.raises(ValueError):
+        valid_tv_locator.table_version = "0" * (BYTES_PER_KIBIBYTE + 1)