deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/tests/experimental/daft/test_deltacat_daft_integration.py

@@ -0,0 +1,136 @@
+import daft
+from daft import Table, Identifier
+import pytest
+import uuid
+
+from deltacat.catalog import Catalog as DeltaCATCatalog
+from deltacat.catalog import CatalogProperties
+from deltacat.experimental.daft.daft_catalog import DaftCatalog
+import shutil
+import tempfile
+
+from deltacat.experimental.catalog.iceberg import impl as IcebergCatalog
+from deltacat.experimental.catalog.iceberg import IcebergCatalogConfig
+
+from pyiceberg.catalog import CatalogType
+
+
+class TestCatalogIntegration:
+    @classmethod
+    def setup_method(cls):
+        cls.tmpdir = tempfile.mkdtemp()
+
+    @classmethod
+    def teardown_method(cls):
+        shutil.rmtree(cls.tmpdir)
+
+    def test_create_table(self):
+        """Demonstrate DeltaCAT-Daft integration."""
+        # Create a DeltaCAT catalog
+        catalog_props = CatalogProperties(root=self.tmpdir)
+        dc_catalog = DeltaCATCatalog(catalog_props)
+
+        # Use a random catalog name to prevent namespacing conflicts with other tests
+        # Convert the DeltaCAT catalog to a Daft catalog
+        catalog_name = f"deltacat_{uuid.uuid4().hex[:8]}"
+
+        daft_catalog = DaftCatalog(catalog=dc_catalog, name=catalog_name)
+
+        # Register the catalog with Daft's catalog system
+        daft.attach_catalog(daft_catalog, catalog_name)
+
+        # Create a sample DataFrame
+        df = daft.from_pydict({"id": [1, 2, 3], "value": ["a", "b", "c"]})
+        # Create then get table
+        daft_catalog.create_table(Identifier("example_table"), df)
+        table: Table = daft_catalog.get_table(Identifier("example_table"))
+        assert table.name == "example_table"
+
+    def test_get_table(self):
+        """Test getting a table from the DeltaCAT-Daft catalog."""
+        # Create a DeltaCAT catalog using the existing tmpdir
+        catalog_props = CatalogProperties(root=self.tmpdir)
+        dc_catalog = DeltaCATCatalog(catalog_props)
+
+        # Convert to DaftCatalog and attach to Daft
+        catalog_name = f"deltacat_{uuid.uuid4().hex[:8]}"
+        daft_catalog = DaftCatalog(dc_catalog, catalog_name)
+        daft.attach_catalog(daft_catalog, catalog_name)
+
+        # Create a sample DataFrame and table
+        df = daft.from_pydict({"id": [1, 2, 3], "value": ["a", "b", "c"]})
+        table_name = "test_get_table"
+        daft_catalog.create_table(Identifier(table_name), df)
+
+        # Get the table using different forms of identifiers
+        table2 = daft_catalog.get_table(Identifier(table_name))
+        assert table2 is not None
+        assert table2.name == table_name
+
+        # 3. With namespace. DeltaCAT used the default namespace since it was not provided
+        table3 = daft_catalog.get_table(Identifier("default", table_name))
+        assert table3 is not None
+        assert table3.name == table_name
+
+        # Test non-existent table raises an appropriate error
+        with pytest.raises(ValueError, match="Table nonexistent_table not found"):
+            daft_catalog.get_table(Identifier("nonexistent_table"))
+
+
+class TestIcebergCatalogIntegration:
+    @classmethod
+    def setup_method(cls):
+        cls.tmpdir = tempfile.mkdtemp()
+
+    @classmethod
+    def teardown_method(cls):
+        shutil.rmtree(cls.tmpdir)
+
+    def test_iceberg_catalog_integration(self):
+        # Create a unique warehouse path for this test
+        warehouse_path = self.tmpdir
+
+        # Configure an Iceberg catalog with the warehouse path
+        config = IcebergCatalogConfig(
+            type=CatalogType.SQL,
+            properties={
+                "warehouse": warehouse_path,
+                "uri": f"sqlite:////{warehouse_path}/sql-catalog.db",
+            },
+        )
+        dc_catalog = IcebergCatalog.from_config(config)
+
+        # Convert the DeltaCAT catalog to a Daft catalog
+        catalog_name = f"deltacat_iceberg_{uuid.uuid4().hex[:8]}"
+        daft_catalog = DaftCatalog(dc_catalog, catalog_name)
+        daft.attach_catalog(daft_catalog, catalog_name)
+
+        # Create a sample DataFrame
+        df = daft.from_pydict({"id": [1, 2, 3], "value": ["a", "b", "c"]})
+
+        # Create a table with the Daft catalog
+        table_name = "example_table"
+        namespace = "example_namespace"
+        daft_catalog.create_table(Identifier(namespace, table_name), df)
+
+        # Query that Iceberg table exists using PyIceberg
+        iceberg_catalog = dc_catalog.inner
+
+        # Verify the table exists in the Iceberg catalog
+        tables = iceberg_catalog.list_tables(namespace)
+
+        assert any(
+            t[0] == namespace and t[1] == table_name for t in tables
+        ), f"Table {table_name} not found in Iceberg catalog"
+
+        # Load the table from Iceberg catalog and verify its properties
+        iceberg_table = iceberg_catalog.load_table(f"{namespace}.{table_name}")
+
+        # Check that the schema matches our DataFrame
+        schema = iceberg_table.schema()
+        assert (
+            schema.find_field("id") is not None
+        ), "Field 'id' not fcound in table schema"
+        assert (
+            schema.find_field("value") is not None
+        ), "Field 'value' not found in table schema"

deltacat/tests/{storage → experimental/storage}/rivulet/conftest.py

@@ -3,9 +3,9 @@ import io
 import pytest
 from faker import Faker
 
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet.mvp.Table import MvpTable
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 import random
 import string
 from PIL import Image

deltacat/tests/{storage → experimental/storage}/rivulet/fs/test_file_location_provider.py

@@ -2,9 +2,9 @@ import pytest
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet import Schema, Field
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet import Schema, Field
 from deltacat.utils.metafile_locator import _find_partition_path
 
 

deltacat/tests/experimental/storage/rivulet/reader/query_expression.py

@@ -0,0 +1,80 @@
+import pytest
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet.shard.range_shard import RangeShard
+
+
+@pytest.fixture
+def sample_range_shard():
+    return RangeShard(min_key=5, max_key=15)
+
+
+@pytest.fixture
+def sample_string_shard():
+    return RangeShard(min_key="apple", max_key="zebra")
+
+
+def test_with_key():
+    query = QueryExpression[int]()
+    query.with_key(5)
+    assert query.min_key == 5
+    assert query.max_key == 5
+    with pytest.raises(ValueError):
+        query.with_key(10)
+
+
+def test_with_range():
+    query = QueryExpression[int]()
+    query.with_range(10, 5)
+    assert query.min_key == 5
+    assert query.max_key == 10
+    with pytest.raises(ValueError):
+        query.with_range(20, 25)
+
+
+def test_matches_query():
+    query = QueryExpression[int]()
+    assert query.matches_query(5)
+    assert query.matches_query(-999)
+    query.with_range(10, 20)
+    assert query.matches_query(15)
+    assert not query.matches_query(25)
+    assert not query.matches_query(5)
+
+
+def test_below_query_range():
+    query = QueryExpression[int]()
+    assert not query.below_query_range(5)
+    query.with_range(10, 20)
+    assert query.below_query_range(5)
+    assert not query.below_query_range(15)
+    assert not query.below_query_range(25)
+
+
+def test_with_shard_existing_query(sample_range_shard):
+    query = QueryExpression[int]().with_range(10, 20)
+    new_query = QueryExpression.with_shard(query, sample_range_shard)
+    assert new_query.min_key == 5
+    assert new_query.max_key == 20
+
+
+def test_with_shard_none_shard():
+    query = QueryExpression[int]().with_range(10, 20)
+    result = QueryExpression.with_shard(query, None)
+    assert result.min_key == 10
+    assert result.max_key == 20
+
+
+def test_with_shard_existing_query_string(sample_string_shard):
+    query = QueryExpression[str]().with_range("banana", "yellow")
+    new_query = QueryExpression.with_shard(query, sample_string_shard)
+    assert new_query.min_key == "apple"
+    assert new_query.max_key == "zebra"
+
+
+def test_query_expression_string_matches():
+    query = QueryExpression[str]().with_range("apple", "cat")
+    assert query.matches_query("apple")
+    assert query.matches_query("banana")
+    assert not query.matches_query("dog")

deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py

@@ -0,0 +1,119 @@
+import pytest
+from deltacat.tests.experimental.storage.rivulet.test_utils import verify_pyarrow_scan
+import pyarrow as pa
+from deltacat.experimental.storage.rivulet import Schema, Field, Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+
+
+@pytest.fixture
+def combined_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int64(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+            Field("height", Datatype.int64()),
+            Field("gender", Datatype.string()),
+        ]
+    )
+
+
+@pytest.fixture
+def initial_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+        ]
+    )
+
+
+@pytest.fixture
+def extended_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int64(), is_merge_key=True),
+            Field("height", Datatype.int64()),
+            Field("gender", Datatype.string()),
+        ]
+    )
+
+
+@pytest.fixture
+def sample_data():
+    return {
+        "id": [1, 2, 3],
+        "name": ["Alice", "Bob", "Charlie"],
+        "age": [25, 30, 35],
+    }
+
+
+@pytest.fixture
+def extended_data():
+    return {
+        "id": [1, 2, 3],
+        "height": [150, 160, 159],
+        "gender": ["male", "female", "male"],
+    }
+
+
+@pytest.fixture
+def combined_data(sample_data, extended_data):
+    data = sample_data.copy()
+    data.update(extended_data)
+    return data
+
+
+@pytest.fixture
+def parquet_data(tmp_path, sample_data):
+    parquet_path = tmp_path / "test.parquet"
+    table = pa.Table.from_pydict(sample_data)
+    pa.parquet.write_table(table, parquet_path)
+    return parquet_path
+
+
+@pytest.fixture
+def sample_dataset(parquet_data, tmp_path):
+    return Dataset.from_parquet(
+        name="test_dataset",
+        file_uri=str(parquet_data),
+        metadata_uri=str(tmp_path),
+        merge_keys="id",
+    )
+
+
+def test_end_to_end_scan_with_multiple_schemas(
+    sample_dataset,
+    initial_schema,
+    extended_schema,
+    combined_schema,
+    sample_data,
+    extended_data,
+    combined_data,
+):
+    # Verify initial scan.
+    verify_pyarrow_scan(sample_dataset.scan().to_arrow(), initial_schema, sample_data)
+
+    # Add a new schema to the dataset
+    sample_dataset.add_schema(schema=extended_schema, schema_name="schema2")
+    new_data = [
+        {"id": 1, "height": 150, "gender": "male"},
+        {"id": 2, "height": 160, "gender": "female"},
+        {"id": 3, "height": 159, "gender": "male"},
+    ]
+    writer = sample_dataset.writer(schema_name="schema2")
+    writer.write(new_data)
+    writer.flush()
+
+    # Verify scan with the extended schema retrieves only extended datfa
+    verify_pyarrow_scan(
+        sample_dataset.scan(schema_name="schema2").to_arrow(),
+        extended_schema,
+        extended_data,
+    )
+
+    # Verify a combined scan retrieves data matching the combined schema
+    verify_pyarrow_scan(
+        sample_dataset.scan().to_arrow(), combined_schema, combined_data
+    )

deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py

@@ -0,0 +1,71 @@
+import pytest
+import os
+
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltacatManifestIO
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet import Schema
+
+
+@pytest.fixture
+def sample_schema():
+    return Schema(
+        {("id", Datatype.int32()), ("name", Datatype.string())},
+        "id",
+    )
+
+
+@pytest.fixture
+def sample_pydict():
+    return {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]}
+
+
+def test_dataset_metastore_e2e(sample_schema, tmp_path):
+    # Setup
+    dataset = Dataset(metadata_uri=tmp_path, dataset_name="dataset")
+    file_provider = dataset._file_provider
+    manifest_io = DeltacatManifestIO(file_provider.uri, dataset._locator)
+
+    # Create multiple manifests
+    manifests_data = [
+        {"sst_files": ["sst1.sst", "sst2.sst"], "level": 1},
+        {"sst_files": ["sst3.sst", "sst4.sst"], "level": 2},
+    ]
+
+    # Create SST files and manifests
+    manifest_paths = []
+    for manifest_data in manifests_data:
+        sst_files = manifest_data["sst_files"]
+        for sst in sst_files:
+            with open(os.path.join(file_provider.uri, sst), "w") as f:
+                f.write("test data")
+
+        manifest_path = manifest_io.write(
+            sst_files, sample_schema, manifest_data["level"]
+        )
+        manifest_paths.append(manifest_path)
+
+    # Initialize DatasetMetastore
+    metastore = DatasetMetastore(
+        file_provider.uri,
+        file_provider,
+        file_provider._locator,
+        manifest_io=manifest_io,
+    )
+
+    # Test manifest generation
+    manifest_accessors = list(metastore.generate_manifests())
+    assert len(manifest_accessors) == len(manifests_data)
+
+    # Verify each manifest accessor
+    for accessor in manifest_accessors:
+        assert accessor.context.schema == sample_schema
+        manifests_data_index = 0 if accessor.context.level == 1 else 1
+        assert accessor.context.level == manifests_data[manifests_data_index]["level"]
+        assert (
+            accessor.manifest.sst_files
+            == manifests_data[manifests_data_index]["sst_files"]
+        )

deltacat/tests/{storage → experimental/storage}/rivulet/schema/test_schema.py

@@ -1,6 +1,6 @@
 import pytest
 import pyarrow as pa
-from deltacat.storage.rivulet import Schema, Field, Datatype
+from deltacat.experimental.storage.rivulet import Schema, Field, Datatype
 
 
 def test_field_initialization():

deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py

@@ -0,0 +1,162 @@
+import pytest
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.shard.range_shard import (
+    RangeShard,
+    RangeShardingStrategy,
+)
+
+
+@pytest.fixture
+def sample_numeric_dataset(tmp_path):
+    """
+    Creates a small Parquet file with integer-based min/max keys and
+    initializes a Dataset from it. Merge key is 'id' with values [1,2,3].
+    So min_key=1, max_key=3.
+    """
+    data = {
+        "id": [1, 2, 3],
+        "name": ["Alice", "Bob", "Charlie"],
+        "age": [25, 30, 35],
+    }
+    table = pa.Table.from_pydict(data)
+    parquet_file = tmp_path / "numeric_data.parquet"
+    pq.write_table(table, parquet_file)
+
+    ds = Dataset.from_parquet(
+        name="numeric_dataset",
+        file_uri=str(parquet_file),
+        metadata_uri=tmp_path,
+        merge_keys="id",
+    )
+    return ds
+
+
+@pytest.fixture
+def sample_string_dataset(tmp_path):
+    """
+    Creates a small Parquet file with a string-based merge key ('name')
+    and initializes a Dataset from it. Merge key has values
+    ['Alice', 'Bob', 'Charlie'] => min_key='Alice', max_key='Charlie'.
+    """
+    data = {
+        "name": ["Alice", "Charlie", "Bob"],  # random order
+        "value": [100, 200, 150],
+    }
+    table = pa.Table.from_pydict(data)
+    parquet_file = tmp_path / "string_data.parquet"
+    pq.write_table(table, parquet_file)
+
+    ds = Dataset.from_parquet(
+        name="string_dataset",
+        file_uri=str(parquet_file),
+        metadata_uri=tmp_path,
+        merge_keys="name",
+    )
+    return ds
+
+
+def test_shards(sample_numeric_dataset, sample_string_dataset):
+    shards = sample_numeric_dataset.shards(num_shards=2)
+
+    num_shards = len(list(shards))
+    assert num_shards == 2
+
+    shard = shards[0]
+    records = list(sample_numeric_dataset.scan(shard=shard).to_pydict())
+    num_records = len(records)
+    assert num_records == 2
+
+    assert records[0]["id"] == 1
+    assert records[0]["name"] == "Alice"
+
+    assert records[1]["id"] == 2
+    assert records[1]["name"] == "Bob"
+
+
+def test_range_shard_repr():
+    shard = RangeShard(min_key=5, max_key=15)
+    assert repr(shard) == "Shard(type=range, min_key=5, max_key=15)"
+
+
+def test_range_shard_split_integers():
+    shards = RangeShard.split(global_min=1, global_max=10, num_shards=2)
+    assert len(shards) == 2
+
+    assert shards[0].min_key == 1
+    assert shards[0].max_key == 5
+    assert shards[1].min_key == 6
+    assert shards[1].max_key == 10
+
+
+def test_range_shard_split_integers_single_shard():
+    shards = RangeShard.split(global_min=1, global_max=10, num_shards=1)
+    assert len(shards) == 1
+    assert shards[0].min_key == 1
+    assert shards[0].max_key == 10
+
+
+def test_range_shard_split_integers_same_value():
+    shards = RangeShard.split(global_min=5, global_max=5, num_shards=3)
+    assert len(shards) == 1
+
+
+def test_range_sharding_strategy_integers(sample_numeric_dataset):
+    strategy = RangeShardingStrategy()
+    shards = list(
+        strategy.shards(num_shards=2, metastore=sample_numeric_dataset._metastore)
+    )
+
+    assert len(shards) == 2, "Expected 2 shards for dataset with keys [1,2,3]"
+
+    shard1, shard2 = shards
+    assert isinstance(shard1, RangeShard)
+    assert isinstance(shard2, RangeShard)
+    assert shard1.min_key == 1
+    assert shard1.max_key == 2
+    assert shard2.min_key == 3
+    assert shard2.max_key == 3
+
+
+def test_range_sharding_strategy_integers_single_shard(sample_numeric_dataset):
+    strategy = RangeShardingStrategy()
+    shards = list(
+        strategy.shards(num_shards=1, metastore=sample_numeric_dataset._metastore)
+    )
+    assert len(shards) == 1
+    shard = shards[0]
+    assert shard.min_key == 1
+    assert shard.max_key == 3
+
+
+def test_range_sharding_strategy_strings(sample_string_dataset):
+    strategy = RangeShardingStrategy()
+    shards = list(
+        strategy.shards(num_shards=2, metastore=sample_string_dataset._metastore)
+    )
+
+    assert len(shards) == 2, "Expected 2 shards for string-based dataset"
+    shard1, shard2 = shards
+    assert isinstance(shard1, RangeShard)
+    assert isinstance(shard2, RangeShard)
+
+    assert shard1.min_key == "Alice"
+    assert shard1.max_key < "Charlie"
+
+    assert shard2.min_key == shard1.max_key
+    assert shard2.max_key == "Charlie"
+
+
+def test_range_sharding_strategy_strings_single_shard(sample_string_dataset):
+    strategy = RangeShardingStrategy()
+    shards = list(
+        strategy.shards(num_shards=1, metastore=sample_string_dataset._metastore)
+    )
+
+    assert len(shards) == 1
+
+    shard = shards[0]
+    assert shard.min_key == "Alice"
+    assert shard.max_key == "Charlie"

deltacat/tests/{storage → experimental/storage}/rivulet/test_dataset.py

@@ -3,9 +3,11 @@ from deltacat.utils.metafile_locator import _find_partition_path
 import pytest
 
 import pyarrow as pa
-from deltacat.storage.rivulet import Schema, Field, Datatype
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.experimental.storage.rivulet import Schema, Field, Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
 
 
 @pytest.fixture

deltacat/tests/{storage → experimental/storage}/rivulet/test_manifest.py

@@ -2,11 +2,11 @@ import os
 
 import pytest
 
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet.metastore.delta import DeltacatManifestIO
-from deltacat.storage.rivulet import Schema, Field
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltacatManifestIO
+from deltacat.experimental.storage.rivulet import Schema, Field
 import pyarrow as pa
 import pyarrow.parquet
 

deltacat/tests/{storage → experimental/storage}/rivulet/test_sst_interval_tree.py

@@ -2,16 +2,16 @@ from typing import List, FrozenSet, Dict
 
 import pytest
 
-from deltacat.storage.rivulet.metastore.delta import DeltaContext
-from deltacat.storage.rivulet.metastore.sst import SSTable, SSTableRow
-from deltacat.storage.rivulet.metastore.sst_interval_tree import (
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTable, SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
     BlockIntervalTree,
     BlockGroup,
     OrderedBlockGroups,
     Block,
 )
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet import Schema
 
 
 @pytest.fixture