deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

deltacat/tests/{storage → experimental/storage}/rivulet/fs/test_file_location_provider.py

@@ -2,9 +2,9 @@ import pytest
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet import Schema, Field
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet import Schema, Field
 from deltacat.utils.metafile_locator import _find_partition_path
 
 

deltacat/tests/experimental/storage/rivulet/reader/query_expression.py

@@ -0,0 +1,80 @@
+import pytest
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet.shard.range_shard import RangeShard
+
+
+@pytest.fixture
+def sample_range_shard():
+    return RangeShard(min_key=5, max_key=15)
+
+
+@pytest.fixture
+def sample_string_shard():
+    return RangeShard(min_key="apple", max_key="zebra")
+
+
+def test_with_key():
+    query = QueryExpression[int]()
+    query.with_key(5)
+    assert query.min_key == 5
+    assert query.max_key == 5
+    with pytest.raises(ValueError):
+        query.with_key(10)
+
+
+def test_with_range():
+    query = QueryExpression[int]()
+    query.with_range(10, 5)
+    assert query.min_key == 5
+    assert query.max_key == 10
+    with pytest.raises(ValueError):
+        query.with_range(20, 25)
+
+
+def test_matches_query():
+    query = QueryExpression[int]()
+    assert query.matches_query(5)
+    assert query.matches_query(-999)
+    query.with_range(10, 20)
+    assert query.matches_query(15)
+    assert not query.matches_query(25)
+    assert not query.matches_query(5)
+
+
+def test_below_query_range():
+    query = QueryExpression[int]()
+    assert not query.below_query_range(5)
+    query.with_range(10, 20)
+    assert query.below_query_range(5)
+    assert not query.below_query_range(15)
+    assert not query.below_query_range(25)
+
+
+def test_with_shard_existing_query(sample_range_shard):
+    query = QueryExpression[int]().with_range(10, 20)
+    new_query = QueryExpression.with_shard(query, sample_range_shard)
+    assert new_query.min_key == 5
+    assert new_query.max_key == 20
+
+
+def test_with_shard_none_shard():
+    query = QueryExpression[int]().with_range(10, 20)
+    result = QueryExpression.with_shard(query, None)
+    assert result.min_key == 10
+    assert result.max_key == 20
+
+
+def test_with_shard_existing_query_string(sample_string_shard):
+    query = QueryExpression[str]().with_range("banana", "yellow")
+    new_query = QueryExpression.with_shard(query, sample_string_shard)
+    assert new_query.min_key == "apple"
+    assert new_query.max_key == "zebra"
+
+
+def test_query_expression_string_matches():
+    query = QueryExpression[str]().with_range("apple", "cat")
+    assert query.matches_query("apple")
+    assert query.matches_query("banana")
+    assert not query.matches_query("dog")

deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py

@@ -0,0 +1,119 @@
+import pytest
+from deltacat.tests.experimental.storage.rivulet.test_utils import verify_pyarrow_scan
+import pyarrow as pa
+from deltacat.experimental.storage.rivulet import Schema, Field, Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+
+
+@pytest.fixture
+def combined_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int64(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+            Field("height", Datatype.int64()),
+            Field("gender", Datatype.string()),
+        ]
+    )
+
+
+@pytest.fixture
+def initial_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+        ]
+    )
+
+
+@pytest.fixture
+def extended_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int64(), is_merge_key=True),
+            Field("height", Datatype.int64()),
+            Field("gender", Datatype.string()),
+        ]
+    )
+
+
+@pytest.fixture
+def sample_data():
+    return {
+        "id": [1, 2, 3],
+        "name": ["Alice", "Bob", "Charlie"],
+        "age": [25, 30, 35],
+    }
+
+
+@pytest.fixture
+def extended_data():
+    return {
+        "id": [1, 2, 3],
+        "height": [150, 160, 159],
+        "gender": ["male", "female", "male"],
+    }
+
+
+@pytest.fixture
+def combined_data(sample_data, extended_data):
+    data = sample_data.copy()
+    data.update(extended_data)
+    return data
+
+
+@pytest.fixture
+def parquet_data(tmp_path, sample_data):
+    parquet_path = tmp_path / "test.parquet"
+    table = pa.Table.from_pydict(sample_data)
+    pa.parquet.write_table(table, parquet_path)
+    return parquet_path
+
+
+@pytest.fixture
+def sample_dataset(parquet_data, tmp_path):
+    return Dataset.from_parquet(
+        name="test_dataset",
+        file_uri=str(parquet_data),
+        metadata_uri=str(tmp_path),
+        merge_keys="id",
+    )
+
+
+def test_end_to_end_scan_with_multiple_schemas(
+    sample_dataset,
+    initial_schema,
+    extended_schema,
+    combined_schema,
+    sample_data,
+    extended_data,
+    combined_data,
+):
+    # Verify initial scan.
+    verify_pyarrow_scan(sample_dataset.scan().to_arrow(), initial_schema, sample_data)
+
+    # Add a new schema to the dataset
+    sample_dataset.add_schema(schema=extended_schema, schema_name="schema2")
+    new_data = [
+        {"id": 1, "height": 150, "gender": "male"},
+        {"id": 2, "height": 160, "gender": "female"},
+        {"id": 3, "height": 159, "gender": "male"},
+    ]
+    writer = sample_dataset.writer(schema_name="schema2")
+    writer.write(new_data)
+    writer.flush()
+
+    # Verify scan with the extended schema retrieves only extended datfa
+    verify_pyarrow_scan(
+        sample_dataset.scan(schema_name="schema2").to_arrow(),
+        extended_schema,
+        extended_data,
+    )
+
+    # Verify a combined scan retrieves data matching the combined schema
+    verify_pyarrow_scan(
+        sample_dataset.scan().to_arrow(), combined_schema, combined_data
+    )

deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py

@@ -0,0 +1,71 @@
+import pytest
+import os
+
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltacatManifestIO
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet import Schema
+
+
+@pytest.fixture
+def sample_schema():
+    return Schema(
+        {("id", Datatype.int32()), ("name", Datatype.string())},
+        "id",
+    )
+
+
+@pytest.fixture
+def sample_pydict():
+    return {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]}
+
+
+def test_dataset_metastore_e2e(sample_schema, tmp_path):
+    # Setup
+    dataset = Dataset(metadata_uri=tmp_path, dataset_name="dataset")
+    file_provider = dataset._file_provider
+    manifest_io = DeltacatManifestIO(file_provider.uri, dataset._locator)
+
+    # Create multiple manifests
+    manifests_data = [
+        {"sst_files": ["sst1.sst", "sst2.sst"], "level": 1},
+        {"sst_files": ["sst3.sst", "sst4.sst"], "level": 2},
+    ]
+
+    # Create SST files and manifests
+    manifest_paths = []
+    for manifest_data in manifests_data:
+        sst_files = manifest_data["sst_files"]
+        for sst in sst_files:
+            with open(os.path.join(file_provider.uri, sst), "w") as f:
+                f.write("test data")
+
+        manifest_path = manifest_io.write(
+            sst_files, sample_schema, manifest_data["level"]
+        )
+        manifest_paths.append(manifest_path)
+
+    # Initialize DatasetMetastore
+    metastore = DatasetMetastore(
+        file_provider.uri,
+        file_provider,
+        file_provider._locator,
+        manifest_io=manifest_io,
+    )
+
+    # Test manifest generation
+    manifest_accessors = list(metastore.generate_manifests())
+    assert len(manifest_accessors) == len(manifests_data)
+
+    # Verify each manifest accessor
+    for accessor in manifest_accessors:
+        assert accessor.context.schema == sample_schema
+        manifests_data_index = 0 if accessor.context.level == 1 else 1
+        assert accessor.context.level == manifests_data[manifests_data_index]["level"]
+        assert (
+            accessor.manifest.sst_files
+            == manifests_data[manifests_data_index]["sst_files"]
+        )

deltacat/tests/{storage → experimental/storage}/rivulet/schema/test_schema.py

@@ -1,6 +1,6 @@
 import pytest
 import pyarrow as pa
-from deltacat.storage.rivulet import Schema, Field, Datatype
+from deltacat.experimental.storage.rivulet import Schema, Field, Datatype
 
 
 def test_field_initialization():

deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py

@@ -0,0 +1,162 @@
+import pytest
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.shard.range_shard import (
+    RangeShard,
+    RangeShardingStrategy,
+)
+
+
+@pytest.fixture
+def sample_numeric_dataset(tmp_path):
+    """
+    Creates a small Parquet file with integer-based min/max keys and
+    initializes a Dataset from it. Merge key is 'id' with values [1,2,3].
+    So min_key=1, max_key=3.
+    """
+    data = {
+        "id": [1, 2, 3],
+        "name": ["Alice", "Bob", "Charlie"],
+        "age": [25, 30, 35],
+    }
+    table = pa.Table.from_pydict(data)
+    parquet_file = tmp_path / "numeric_data.parquet"
+    pq.write_table(table, parquet_file)
+
+    ds = Dataset.from_parquet(
+        name="numeric_dataset",
+        file_uri=str(parquet_file),
+        metadata_uri=tmp_path,
+        merge_keys="id",
+    )
+    return ds
+
+
+@pytest.fixture
+def sample_string_dataset(tmp_path):
+    """
+    Creates a small Parquet file with a string-based merge key ('name')
+    and initializes a Dataset from it. Merge key has values
+    ['Alice', 'Bob', 'Charlie'] => min_key='Alice', max_key='Charlie'.
+    """
+    data = {
+        "name": ["Alice", "Charlie", "Bob"],  # random order
+        "value": [100, 200, 150],
+    }
+    table = pa.Table.from_pydict(data)
+    parquet_file = tmp_path / "string_data.parquet"
+    pq.write_table(table, parquet_file)
+
+    ds = Dataset.from_parquet(
+        name="string_dataset",
+        file_uri=str(parquet_file),
+        metadata_uri=tmp_path,
+        merge_keys="name",
+    )
+    return ds
+
+
+def test_shards(sample_numeric_dataset, sample_string_dataset):
+    shards = sample_numeric_dataset.shards(num_shards=2)
+
+    num_shards = len(list(shards))
+    assert num_shards == 2
+
+    shard = shards[0]
+    records = list(sample_numeric_dataset.scan(shard=shard).to_pydict())
+    num_records = len(records)
+    assert num_records == 2
+
+    assert records[0]["id"] == 1
+    assert records[0]["name"] == "Alice"
+
+    assert records[1]["id"] == 2
+    assert records[1]["name"] == "Bob"
+
+
+def test_range_shard_repr():
+    shard = RangeShard(min_key=5, max_key=15)
+    assert repr(shard) == "Shard(type=range, min_key=5, max_key=15)"
+
+
+def test_range_shard_split_integers():
+    shards = RangeShard.split(global_min=1, global_max=10, num_shards=2)
+    assert len(shards) == 2
+
+    assert shards[0].min_key == 1
+    assert shards[0].max_key == 5
+    assert shards[1].min_key == 6
+    assert shards[1].max_key == 10
+
+
+def test_range_shard_split_integers_single_shard():
+    shards = RangeShard.split(global_min=1, global_max=10, num_shards=1)
+    assert len(shards) == 1
+    assert shards[0].min_key == 1
+    assert shards[0].max_key == 10
+
+
+def test_range_shard_split_integers_same_value():
+    shards = RangeShard.split(global_min=5, global_max=5, num_shards=3)
+    assert len(shards) == 1
+
+
+def test_range_sharding_strategy_integers(sample_numeric_dataset):
+    strategy = RangeShardingStrategy()
+    shards = list(
+        strategy.shards(num_shards=2, metastore=sample_numeric_dataset._metastore)
+    )
+
+    assert len(shards) == 2, "Expected 2 shards for dataset with keys [1,2,3]"
+
+    shard1, shard2 = shards
+    assert isinstance(shard1, RangeShard)
+    assert isinstance(shard2, RangeShard)
+    assert shard1.min_key == 1
+    assert shard1.max_key == 2
+    assert shard2.min_key == 3
+    assert shard2.max_key == 3
+
+
+def test_range_sharding_strategy_integers_single_shard(sample_numeric_dataset):
+    strategy = RangeShardingStrategy()
+    shards = list(
+        strategy.shards(num_shards=1, metastore=sample_numeric_dataset._metastore)
+    )
+    assert len(shards) == 1
+    shard = shards[0]
+    assert shard.min_key == 1
+    assert shard.max_key == 3
+
+
+def test_range_sharding_strategy_strings(sample_string_dataset):
+    strategy = RangeShardingStrategy()
+    shards = list(
+        strategy.shards(num_shards=2, metastore=sample_string_dataset._metastore)
+    )
+
+    assert len(shards) == 2, "Expected 2 shards for string-based dataset"
+    shard1, shard2 = shards
+    assert isinstance(shard1, RangeShard)
+    assert isinstance(shard2, RangeShard)
+
+    assert shard1.min_key == "Alice"
+    assert shard1.max_key < "Charlie"
+
+    assert shard2.min_key == shard1.max_key
+    assert shard2.max_key == "Charlie"
+
+
+def test_range_sharding_strategy_strings_single_shard(sample_string_dataset):
+    strategy = RangeShardingStrategy()
+    shards = list(
+        strategy.shards(num_shards=1, metastore=sample_string_dataset._metastore)
+    )
+
+    assert len(shards) == 1
+
+    shard = shards[0]
+    assert shard.min_key == "Alice"
+    assert shard.max_key == "Charlie"

deltacat/tests/{storage → experimental/storage}/rivulet/test_dataset.py

@@ -3,9 +3,11 @@ from deltacat.utils.metafile_locator import _find_partition_path
 import pytest
 
 import pyarrow as pa
-from deltacat.storage.rivulet import Schema, Field, Datatype
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.experimental.storage.rivulet import Schema, Field, Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
 
 
 @pytest.fixture

deltacat/tests/{storage → experimental/storage}/rivulet/test_manifest.py

@@ -2,11 +2,11 @@ import os
 
 import pytest
 
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet.metastore.delta import DeltacatManifestIO
-from deltacat.storage.rivulet import Schema, Field
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltacatManifestIO
+from deltacat.experimental.storage.rivulet import Schema, Field
 import pyarrow as pa
 import pyarrow.parquet
 

deltacat/tests/{storage → experimental/storage}/rivulet/test_sst_interval_tree.py

@@ -2,16 +2,16 @@ from typing import List, FrozenSet, Dict
 
 import pytest
 
-from deltacat.storage.rivulet.metastore.delta import DeltaContext
-from deltacat.storage.rivulet.metastore.sst import SSTable, SSTableRow
-from deltacat.storage.rivulet.metastore.sst_interval_tree import (
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTable, SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
     BlockIntervalTree,
     BlockGroup,
     OrderedBlockGroups,
     Block,
 )
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet import Schema
 
 
 @pytest.fixture

deltacat/tests/{storage → experimental/storage}/rivulet/test_utils.py

@@ -3,12 +3,14 @@ import os
 
 from pyarrow import RecordBatch, Table
 
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet.writer.dataset_writer import DatasetWriter
-
-from deltacat.storage.rivulet.mvp.Table import MvpTable, MvpRow
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet.writer.dataset_writer import DatasetWriter
+
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable, MvpRow
+from deltacat.experimental.storage.rivulet import Schema
 from typing import Dict, List, Generator, Set
 
 FIXTURE_ROW_COUNT = 10000

deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_write_then_read.py

@@ -7,23 +7,25 @@ import msgpack
 import pytest
 from pyarrow import RecordBatch
 
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.metastore.delta import (
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.metastore.delta import (
     ManifestIO,
     TreeLevel,
     DeltacatManifestIO,
 )
 
-from deltacat.storage.rivulet.mvp.Table import MvpTable, MvpRow
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.writer.memtable_dataset_writer import (
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable, MvpRow
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.writer.memtable_dataset_writer import (
     MemtableDatasetWriter,
 )
 
-from deltacat.tests.storage.rivulet.test_utils import FIXTURE_ROW_COUNT
-from deltacat.tests.storage.rivulet.test_utils import (
+from deltacat.tests.experimental.storage.rivulet.test_utils import FIXTURE_ROW_COUNT
+from deltacat.tests.experimental.storage.rivulet.test_utils import (
     write_mvp_table,
     compare_mvp_table_to_scan_results,
     mvp_table_to_record_batches,

deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_writer.py

@@ -2,8 +2,8 @@ import pytest
 import shutil
 import tempfile
 
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.writer.memtable_dataset_writer import (
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.writer.memtable_dataset_writer import (
     MemtableDatasetWriter,
 )
 from ..test_utils import (

deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_memtable_dataset_writer.py

@@ -1,12 +1,12 @@
 import pytest
 
-from deltacat.storage.rivulet.dataset import Dataset
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.metastore.delta import DeltacatManifestIO
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.schema.datatype import Datatype
-from deltacat.storage.rivulet.writer.memtable_dataset_writer import (
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltacatManifestIO
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.writer.memtable_dataset_writer import (
     MemtableDatasetWriter,
 )
 

deltacat/tests/storage/model/test_shard.py

@@ -7,7 +7,9 @@ def test_sharding_strategy_from_string_range():
     """
     Tests that from_string('range') returns an instance of RangeShardingStrategy.
     """
-    from deltacat.storage.rivulet.shard.range_shard import RangeShardingStrategy
+    from deltacat.experimental.storage.rivulet.shard.range_shard import (
+        RangeShardingStrategy,
+    )
 
     strategy = ShardingStrategy.from_string("range")
     assert isinstance(strategy, RangeShardingStrategy)

deltacat/types/media.py

@@ -148,9 +148,9 @@ class DatastoreType(str, Enum):
     writer for that data store. Note that, although some overlap exists between
     enum values here and in :class:`deltacat.types.media.ContentType`, each
     enum serve a different purpose. The purpose of
-    :class:`deltacat.types.media.ContentType` is to resolve the MIME type for
-    specific types of files, and may be used together with multi-content-type
-    datastore types to describe the specific file types read/written to that
+    :class:`deltacat.types.media.ContentType` is to resolve a file's MIME type,
+    and may be used together with datastores that support storing different
+    file types to describe the specific file type read/written from/to that
     datastore (e.g., Iceberg, Hudi, Delta Lake, Audio, Images, Video, etc.)
     """