deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/tests/experimental/daft/test_deltacat_daft_integration.py

@@ -0,0 +1,136 @@
+import daft
+from daft import Table, Identifier
+import pytest
+import uuid
+
+from deltacat.catalog import Catalog as DeltaCATCatalog
+from deltacat.catalog import CatalogProperties
+from deltacat.experimental.daft.daft_catalog import DaftCatalog
+import shutil
+import tempfile
+
+from deltacat.experimental.catalog.iceberg import impl as IcebergCatalog
+from deltacat.experimental.catalog.iceberg import IcebergCatalogConfig
+
+from pyiceberg.catalog import CatalogType
+
+
+class TestCatalogIntegration:
+    @classmethod
+    def setup_method(cls):
+        cls.tmpdir = tempfile.mkdtemp()
+
+    @classmethod
+    def teardown_method(cls):
+        shutil.rmtree(cls.tmpdir)
+
+    def test_create_table(self):
+        """Demonstrate DeltaCAT-Daft integration."""
+        # Create a DeltaCAT catalog
+        catalog_props = CatalogProperties(root=self.tmpdir)
+        dc_catalog = DeltaCATCatalog(catalog_props)
+
+        # Use a random catalog name to prevent namespacing conflicts with other tests
+        # Convert the DeltaCAT catalog to a Daft catalog
+        catalog_name = f"deltacat_{uuid.uuid4().hex[:8]}"
+
+        daft_catalog = DaftCatalog(catalog=dc_catalog, name=catalog_name)
+
+        # Register the catalog with Daft's catalog system
+        daft.attach_catalog(daft_catalog, catalog_name)
+
+        # Create a sample DataFrame
+        df = daft.from_pydict({"id": [1, 2, 3], "value": ["a", "b", "c"]})
+        # Create then get table
+        daft_catalog.create_table(Identifier("example_table"), df)
+        table: Table = daft_catalog.get_table(Identifier("example_table"))
+        assert table.name == "example_table"
+
+    def test_get_table(self):
+        """Test getting a table from the DeltaCAT-Daft catalog."""
+        # Create a DeltaCAT catalog using the existing tmpdir
+        catalog_props = CatalogProperties(root=self.tmpdir)
+        dc_catalog = DeltaCATCatalog(catalog_props)
+
+        # Convert to DaftCatalog and attach to Daft
+        catalog_name = f"deltacat_{uuid.uuid4().hex[:8]}"
+        daft_catalog = DaftCatalog(dc_catalog, catalog_name)
+        daft.attach_catalog(daft_catalog, catalog_name)
+
+        # Create a sample DataFrame and table
+        df = daft.from_pydict({"id": [1, 2, 3], "value": ["a", "b", "c"]})
+        table_name = "test_get_table"
+        daft_catalog.create_table(Identifier(table_name), df)
+
+        # Get the table using different forms of identifiers
+        table2 = daft_catalog.get_table(Identifier(table_name))
+        assert table2 is not None
+        assert table2.name == table_name
+
+        # 3. With namespace. DeltaCAT used the default namespace since it was not provided
+        table3 = daft_catalog.get_table(Identifier("default", table_name))
+        assert table3 is not None
+        assert table3.name == table_name
+
+        # Test non-existent table raises an appropriate error
+        with pytest.raises(ValueError, match="Table nonexistent_table not found"):
+            daft_catalog.get_table(Identifier("nonexistent_table"))
+
+
+class TestIcebergCatalogIntegration:
+    @classmethod
+    def setup_method(cls):
+        cls.tmpdir = tempfile.mkdtemp()
+
+    @classmethod
+    def teardown_method(cls):
+        shutil.rmtree(cls.tmpdir)
+
+    def test_iceberg_catalog_integration(self):
+        # Create a unique warehouse path for this test
+        warehouse_path = self.tmpdir
+
+        # Configure an Iceberg catalog with the warehouse path
+        config = IcebergCatalogConfig(
+            type=CatalogType.SQL,
+            properties={
+                "warehouse": warehouse_path,
+                "uri": f"sqlite:////{warehouse_path}/sql-catalog.db",
+            },
+        )
+        dc_catalog = IcebergCatalog.from_config(config)
+
+        # Convert the DeltaCAT catalog to a Daft catalog
+        catalog_name = f"deltacat_iceberg_{uuid.uuid4().hex[:8]}"
+        daft_catalog = DaftCatalog(dc_catalog, catalog_name)
+        daft.attach_catalog(daft_catalog, catalog_name)
+
+        # Create a sample DataFrame
+        df = daft.from_pydict({"id": [1, 2, 3], "value": ["a", "b", "c"]})
+
+        # Create a table with the Daft catalog
+        table_name = "example_table"
+        namespace = "example_namespace"
+        daft_catalog.create_table(Identifier(namespace, table_name), df)
+
+        # Query that Iceberg table exists using PyIceberg
+        iceberg_catalog = dc_catalog.inner
+
+        # Verify the table exists in the Iceberg catalog
+        tables = iceberg_catalog.list_tables(namespace)
+
+        assert any(
+            t[0] == namespace and t[1] == table_name for t in tables
+        ), f"Table {table_name} not found in Iceberg catalog"
+
+        # Load the table from Iceberg catalog and verify its properties
+        iceberg_table = iceberg_catalog.load_table(f"{namespace}.{table_name}")
+
+        # Check that the schema matches our DataFrame
+        schema = iceberg_table.schema()
+        assert (
+            schema.find_field("id") is not None
+        ), "Field 'id' not fcound in table schema"
+        assert (
+            schema.find_field("value") is not None
+        ), "Field 'value' not found in table schema"

deltacat/tests/experimental/storage/rivulet/conftest.py

@@ -0,0 +1,149 @@
+import io
+
+import pytest
+from faker import Faker
+
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
+import random
+import string
+from PIL import Image
+
+FIXTURE_ROW_COUNT = 10000
+
+
+@pytest.fixture
+def ds1_dataset() -> MvpTable:
+    """
+    dataset with one million rows
+    primary key is integer between 1 and 1,000,000
+
+    TODO change to user Faker instead of int ranges
+    """
+
+    # Function to generate random names
+    def generate_random_name():
+        return "".join(
+            random.choices(
+                string.ascii_uppercase + string.ascii_lowercase, k=random.randint(3, 10)
+            )
+        )
+
+    # Create a list of numbers from 1 to TEST_ROW_COUNT
+    ids = list(range(1, FIXTURE_ROW_COUNT + 1))
+    random.shuffle(ids)
+
+    # Generate one million rows
+    return MvpTable(
+        {
+            "id": ids,
+            "name": [generate_random_name() for _ in range(FIXTURE_ROW_COUNT)],
+            "age": [random.randint(18, 100) for _ in range(FIXTURE_ROW_COUNT)],
+        }
+    )
+
+
+@pytest.fixture
+def ds1_schema():
+    return Schema(
+        {
+            ("id", Datatype.int32()),
+            ("name", Datatype.string()),
+            ("age", Datatype.int32()),
+        },
+        "id",
+    )
+
+
+@pytest.fixture
+def ds2_dataset():
+    """
+    dataset2 with one million rows that can be joined to ds1
+    primary key is integer between 1 and 1,000,000
+    """
+    # Create a list of numbers from 1 to 1,000,000
+    ids = list(range(1, FIXTURE_ROW_COUNT + 1))
+    random.shuffle(ids)
+
+    fake = Faker()
+
+    # Generate one million rows
+    return MvpTable(
+        {
+            "id": ids,
+            "address": [fake.address() for _ in range(FIXTURE_ROW_COUNT)],
+            "zip": [fake.zipcode() for _ in range(FIXTURE_ROW_COUNT)],
+        }
+    )
+
+
+@pytest.fixture
+def ds2_schema():
+    return Schema(
+        {
+            ("id", Datatype.int32()),
+            ("address", Datatype.string()),
+            ("zip", Datatype.string()),
+        },
+        "id",
+    )
+
+
+@pytest.fixture
+def combined_schema(ds1_schema, ds2_schema):
+    return Schema(
+        {
+            ("id", Datatype.int32()),
+            ("address", Datatype.string()),
+            ("zip", Datatype.string()),
+            ("name", Datatype.string()),
+            ("age", Datatype.int32()),
+        },
+        "id",
+    )
+
+
+@pytest.fixture
+def dataset_images_with_label() -> (MvpTable, Schema):
+    """
+    dataset with one thousand images and labels, generated dynamically
+    primary key is integer between 1 and 1,000
+    """
+    ROW_COUNT = 1000
+    fake = Faker()
+    schema = Schema(
+        {
+            ("id", Datatype.int32()),
+            ("image", Datatype.image("jpg")),
+            ("label", Datatype.string()),
+        },
+        "id",
+    )
+
+    # Create a list of numbers from 1 to ROW_COUNT
+    ids = list(range(1, ROW_COUNT + 1))
+    random.shuffle(ids)
+
+    fake_image = Image.new(
+        "RGB",
+        (512, 512),
+        color=(random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)),
+    )
+    # get bytes from image encoded as png
+    buffer = io.BytesIO()
+    fake_image.save(buffer, format="PNG")
+    # seek to start of buffer since we just wrote to it
+    buffer.seek(0)
+    image_bytes = buffer.read()
+    # Generate one million rows
+    return (
+        MvpTable(
+            {
+                "id": ids,
+                "image": [image_bytes for _ in range(ROW_COUNT)],
+                "label": [fake.name() for _ in range(ROW_COUNT)],
+            }
+        ),
+        schema,
+    )

deltacat/tests/experimental/storage/rivulet/fs/test_file_location_provider.py

@@ -0,0 +1,94 @@
+import pytest
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet import Schema, Field
+from deltacat.utils.metafile_locator import _find_partition_path
+
+
+@pytest.fixture
+def sample_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+        ]
+    )
+
+
+@pytest.fixture
+def sample_pydict():
+    return {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"], "age": [25, 30, 35]}
+
+
+@pytest.fixture
+def temp_storage_path(tmp_path):
+    return tmp_path
+
+
+@pytest.fixture
+def sample_parquet_data(temp_storage_path, sample_pydict):
+    parquet_path = temp_storage_path / "test.parquet"
+    table = pa.Table.from_pydict(sample_pydict)
+    pq.write_table(table, parquet_path)
+    return parquet_path
+
+
+@pytest.fixture
+def dataset(sample_parquet_data):
+    return Dataset.from_parquet(
+        file_uri=sample_parquet_data, name="dataset", merge_keys="id"
+    )
+
+
+@pytest.fixture
+def file_provider(dataset):
+    return dataset._file_provider
+
+
+def test_provide_data_file(file_provider):
+    output_file = file_provider.provide_data_file("parquet")
+    assert "data" in output_file.location
+    assert output_file.location.endswith(".parquet")
+
+    output_file2 = file_provider.provide_data_file("parquet")
+    assert "data" in output_file2.location
+    assert output_file2.location.endswith(".parquet")
+
+    assert (
+        output_file.location != output_file2.location
+    ), "Two output files should have different locations."
+
+
+def test_provide_manifest_file(file_provider):
+    output_file = file_provider.provide_manifest_file()
+    assert "metadata/manifests" in output_file.location
+    assert output_file.location.endswith(".json")
+
+
+def test_provide_l0_sst_file(file_provider):
+    output_file = file_provider.provide_l0_sst_file()
+    assert "metadata/ssts/0" in output_file.location
+    assert output_file.location.endswith(".json")
+
+
+def test_provide_input_file(file_provider, sample_parquet_data):
+    input_file = file_provider.provide_input_file(str(sample_parquet_data))
+    assert input_file.location == str(sample_parquet_data)
+
+
+def test_generate_sst_uris(file_provider):
+    generated_files = list(file_provider.generate_sst_uris())
+    for file in generated_files:
+        assert "metadata/ssts/0" in file.location
+        assert file.location.endswith(".json")
+
+
+def test_get_scan_directories(file_provider):
+    partition_path = _find_partition_path(file_provider.uri, file_provider._locator)
+    assert file_provider.get_sst_scan_directories() == [
+        f"{partition_path}/metadata/ssts/0/"
+    ]

deltacat/tests/experimental/storage/rivulet/reader/query_expression.py

@@ -0,0 +1,80 @@
+import pytest
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet.shard.range_shard import RangeShard
+
+
+@pytest.fixture
+def sample_range_shard():
+    return RangeShard(min_key=5, max_key=15)
+
+
+@pytest.fixture
+def sample_string_shard():
+    return RangeShard(min_key="apple", max_key="zebra")
+
+
+def test_with_key():
+    query = QueryExpression[int]()
+    query.with_key(5)
+    assert query.min_key == 5
+    assert query.max_key == 5
+    with pytest.raises(ValueError):
+        query.with_key(10)
+
+
+def test_with_range():
+    query = QueryExpression[int]()
+    query.with_range(10, 5)
+    assert query.min_key == 5
+    assert query.max_key == 10
+    with pytest.raises(ValueError):
+        query.with_range(20, 25)
+
+
+def test_matches_query():
+    query = QueryExpression[int]()
+    assert query.matches_query(5)
+    assert query.matches_query(-999)
+    query.with_range(10, 20)
+    assert query.matches_query(15)
+    assert not query.matches_query(25)
+    assert not query.matches_query(5)
+
+
+def test_below_query_range():
+    query = QueryExpression[int]()
+    assert not query.below_query_range(5)
+    query.with_range(10, 20)
+    assert query.below_query_range(5)
+    assert not query.below_query_range(15)
+    assert not query.below_query_range(25)
+
+
+def test_with_shard_existing_query(sample_range_shard):
+    query = QueryExpression[int]().with_range(10, 20)
+    new_query = QueryExpression.with_shard(query, sample_range_shard)
+    assert new_query.min_key == 5
+    assert new_query.max_key == 20
+
+
+def test_with_shard_none_shard():
+    query = QueryExpression[int]().with_range(10, 20)
+    result = QueryExpression.with_shard(query, None)
+    assert result.min_key == 10
+    assert result.max_key == 20
+
+
+def test_with_shard_existing_query_string(sample_string_shard):
+    query = QueryExpression[str]().with_range("banana", "yellow")
+    new_query = QueryExpression.with_shard(query, sample_string_shard)
+    assert new_query.min_key == "apple"
+    assert new_query.max_key == "zebra"
+
+
+def test_query_expression_string_matches():
+    query = QueryExpression[str]().with_range("apple", "cat")
+    assert query.matches_query("apple")
+    assert query.matches_query("banana")
+    assert not query.matches_query("dog")

deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py

@@ -0,0 +1,119 @@
+import pytest
+from deltacat.tests.experimental.storage.rivulet.test_utils import verify_pyarrow_scan
+import pyarrow as pa
+from deltacat.experimental.storage.rivulet import Schema, Field, Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+
+
+@pytest.fixture
+def combined_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int64(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+            Field("height", Datatype.int64()),
+            Field("gender", Datatype.string()),
+        ]
+    )
+
+
+@pytest.fixture
+def initial_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+        ]
+    )
+
+
+@pytest.fixture
+def extended_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int64(), is_merge_key=True),
+            Field("height", Datatype.int64()),
+            Field("gender", Datatype.string()),
+        ]
+    )
+
+
+@pytest.fixture
+def sample_data():
+    return {
+        "id": [1, 2, 3],
+        "name": ["Alice", "Bob", "Charlie"],
+        "age": [25, 30, 35],
+    }
+
+
+@pytest.fixture
+def extended_data():
+    return {
+        "id": [1, 2, 3],
+        "height": [150, 160, 159],
+        "gender": ["male", "female", "male"],
+    }
+
+
+@pytest.fixture
+def combined_data(sample_data, extended_data):
+    data = sample_data.copy()
+    data.update(extended_data)
+    return data
+
+
+@pytest.fixture
+def parquet_data(tmp_path, sample_data):
+    parquet_path = tmp_path / "test.parquet"
+    table = pa.Table.from_pydict(sample_data)
+    pa.parquet.write_table(table, parquet_path)
+    return parquet_path
+
+
+@pytest.fixture
+def sample_dataset(parquet_data, tmp_path):
+    return Dataset.from_parquet(
+        name="test_dataset",
+        file_uri=str(parquet_data),
+        metadata_uri=str(tmp_path),
+        merge_keys="id",
+    )
+
+
+def test_end_to_end_scan_with_multiple_schemas(
+    sample_dataset,
+    initial_schema,
+    extended_schema,
+    combined_schema,
+    sample_data,
+    extended_data,
+    combined_data,
+):
+    # Verify initial scan.
+    verify_pyarrow_scan(sample_dataset.scan().to_arrow(), initial_schema, sample_data)
+
+    # Add a new schema to the dataset
+    sample_dataset.add_schema(schema=extended_schema, schema_name="schema2")
+    new_data = [
+        {"id": 1, "height": 150, "gender": "male"},
+        {"id": 2, "height": 160, "gender": "female"},
+        {"id": 3, "height": 159, "gender": "male"},
+    ]
+    writer = sample_dataset.writer(schema_name="schema2")
+    writer.write(new_data)
+    writer.flush()
+
+    # Verify scan with the extended schema retrieves only extended datfa
+    verify_pyarrow_scan(
+        sample_dataset.scan(schema_name="schema2").to_arrow(),
+        extended_schema,
+        extended_data,
+    )
+
+    # Verify a combined scan retrieves data matching the combined schema
+    verify_pyarrow_scan(
+        sample_dataset.scan().to_arrow(), combined_schema, combined_data
+    )

deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py

@@ -0,0 +1,71 @@
+import pytest
+import os
+
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltacatManifestIO
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet import Schema
+
+
+@pytest.fixture
+def sample_schema():
+    return Schema(
+        {("id", Datatype.int32()), ("name", Datatype.string())},
+        "id",
+    )
+
+
+@pytest.fixture
+def sample_pydict():
+    return {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]}
+
+
+def test_dataset_metastore_e2e(sample_schema, tmp_path):
+    # Setup
+    dataset = Dataset(metadata_uri=tmp_path, dataset_name="dataset")
+    file_provider = dataset._file_provider
+    manifest_io = DeltacatManifestIO(file_provider.uri, dataset._locator)
+
+    # Create multiple manifests
+    manifests_data = [
+        {"sst_files": ["sst1.sst", "sst2.sst"], "level": 1},
+        {"sst_files": ["sst3.sst", "sst4.sst"], "level": 2},
+    ]
+
+    # Create SST files and manifests
+    manifest_paths = []
+    for manifest_data in manifests_data:
+        sst_files = manifest_data["sst_files"]
+        for sst in sst_files:
+            with open(os.path.join(file_provider.uri, sst), "w") as f:
+                f.write("test data")
+
+        manifest_path = manifest_io.write(
+            sst_files, sample_schema, manifest_data["level"]
+        )
+        manifest_paths.append(manifest_path)
+
+    # Initialize DatasetMetastore
+    metastore = DatasetMetastore(
+        file_provider.uri,
+        file_provider,
+        file_provider._locator,
+        manifest_io=manifest_io,
+    )
+
+    # Test manifest generation
+    manifest_accessors = list(metastore.generate_manifests())
+    assert len(manifest_accessors) == len(manifests_data)
+
+    # Verify each manifest accessor
+    for accessor in manifest_accessors:
+        assert accessor.context.schema == sample_schema
+        manifests_data_index = 0 if accessor.context.level == 1 else 1
+        assert accessor.context.level == manifests_data[manifests_data_index]["level"]
+        assert (
+            accessor.manifest.sst_files
+            == manifests_data[manifests_data_index]["sst_files"]
+        )