deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py

@@ -13,9 +13,9 @@ from typing import (
     Optional,
 )
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 
 FILE_FORMAT = TypeVar("FILE_FORMAT")
 MEMORY_FORMAT = TypeVar("MEMORY_FORMAT")

deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py

@@ -3,9 +3,11 @@ from typing import Generator, Dict, Optional
 import pyarrow as pa
 
 from deltacat.storage.model.shard import Shard
-from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet import Schema
 
 
 class DataScan:

deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py

@@ -7,16 +7,16 @@ import pyarrow.fs
 
 from deltacat.storage import Delta
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 from deltacat.utils.filesystem import resolve_path_and_filesystem
-from deltacat.storage.rivulet.metastore.json_sst import JsonSstReader
-from deltacat.storage.rivulet.metastore.delta import (
+from deltacat.experimental.storage.rivulet.metastore.json_sst import JsonSstReader
+from deltacat.experimental.storage.rivulet.metastore.delta import (
     ManifestIO,
     DeltaContext,
     RivuletDelta,
     DeltacatManifestIO,
 )
-from deltacat.storage.rivulet.metastore.sst import SSTReader, SSTable
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTReader, SSTable
 from deltacat.utils.metafile_locator import _find_table_path
 from deltacat import logs
 

deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py

@@ -2,18 +2,20 @@ import logging
 from typing import Generator, Optional, Set, Type, TypeVar, Any
 
 from deltacat.storage.model.shard import Shard
-from deltacat.storage.rivulet.metastore.sst import SSTableRow, SSTable
-from deltacat.storage.rivulet.metastore.sst_interval_tree import (
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow, SSTable
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
     BlockIntervalTree,
     OrderedBlockGroups,
 )
-from deltacat.storage.rivulet.reader.block_scanner import BlockScanner
-from deltacat.storage.rivulet.reader.dataset_metastore import (
+from deltacat.experimental.storage.rivulet.reader.block_scanner import BlockScanner
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
     DatasetMetastore,
     ManifestAccessor,
 )
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet import Schema
 
 # The type of data returned to reader
 T = TypeVar("T")

deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py

@@ -4,7 +4,10 @@ from typing import Generator, Dict, Type, NamedTuple, List
 
 from pyarrow import RecordBatch
 
-from deltacat.storage.rivulet.reader.data_reader import DataReader, MEMORY_FORMAT
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
+    DataReader,
+    MEMORY_FORMAT,
+)
 import pyarrow as pa
 
 

deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py

@@ -1,9 +1,9 @@
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.reader.data_reader import FileReader
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.reader.data_reader import FileReader
 from typing import Type, Dict
 
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 
 
 class FileReaderRegistrar:

deltacat/{storage → experimental/storage}/rivulet/schema/schema.py

@@ -5,7 +5,7 @@ from typing import MutableMapping, Dict, Iterable, Tuple, Optional
 
 import pyarrow as pa
 
-from deltacat.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
 
 
 @dataclass(frozen=True)

deltacat/{storage → experimental/storage}/rivulet/serializer.py

@@ -1,6 +1,6 @@
 from typing import Protocol, Iterable, List, Union, Any, Dict
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
 import pyarrow as pa
 
 MEMTABLE_DATA = Union[Iterable[Dict[str, Any]], pa.Table]

deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
-from deltacat.storage.rivulet.parquet.serializer import ParquetDataSerializer
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.serializer import DataSerializer
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.parquet.serializer import (
+    ParquetDataSerializer,
+)
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.serializer import DataSerializer
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
-from deltacat.storage.rivulet.feather.serializer import FeatherDataSerializer
+from deltacat.experimental.storage.rivulet.feather.serializer import (
+    FeatherDataSerializer,
+)
 
 
 class DataSerializerFactory:

deltacat/experimental/storage/rivulet/shard/range_shard.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+from typing import Generic, List, Union, Iterable
+from deltacat.storage.model.shard import T, Shard, ShardingStrategy
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+
+
+class RangeShard(Shard, Generic[T]):
+    """
+    Represents a range-based shard with minimum and maximum keys.
+
+    param: min_key: The minimum key for the shard.
+    param: max_key: The maximum key for the shard.
+    """
+
+    def __init__(self, min_key: T, max_key: T):
+        self.min_key = min_key
+        self.max_key = max_key
+
+    def __repr__(self) -> str:
+        return f"Shard(type=range, min_key={self.min_key}, max_key={self.max_key})"
+
+    @staticmethod
+    def split(
+        global_min: Union[int, str], global_max: Union[int, str], num_shards: int
+    ) -> List[RangeShard]:
+        """
+        Splits a range into `num_shards` shards.
+        Currently supports splitting ranges of integers and strings.
+
+        Note: If global_min == global_max or num_shards <= 1, a single shard is returned,
+              num_shards is ignored.
+
+        :param global_min: The minimum key for the entire range (int or str).
+        :param global_max: The maximum key for the entire range (int or str).
+        :param num_shards: The number of shards to create.
+        :return: A list of RangeShard objects.
+        """
+        if global_min == global_max or num_shards <= 1:
+            return [RangeShard(global_min, global_max)]
+
+        # Determine which interpolation function to use based on the type of min/max
+        if isinstance(global_min, int) and isinstance(global_max, int):
+            interpolate = RangeShard._interpolate_numeric
+        elif isinstance(global_min, str) and isinstance(global_max, str):
+            interpolate = RangeShard._interpolate_str
+        else:
+            raise ValueError(
+                "Unsupported combination of types for global_min and global_max."
+            )
+
+        shards: List[RangeShard] = []
+        for i in range(num_shards):
+            start = interpolate(global_min, global_max, i, num_shards)
+            end = interpolate(global_min, global_max, i + 1, num_shards)
+
+            if i > 0:
+                if isinstance(start, int):
+                    start = shards[-1].max_key + 1
+                elif isinstance(start, int):
+                    char_list = list(start)
+                    char_list[-1] = chr(ord(char_list[-1]) + 1)
+                    start = "".join(char_list)
+
+            shards.append(RangeShard(start, end))
+
+        return shards
+
+    @staticmethod
+    def _interpolate_numeric(start: int, end: int, step: int, total_steps: int) -> int:
+        """
+        Integer interpolation using integer (floor) division.
+
+        param: start (int): The starting number.
+        param: end (int): The ending number.
+        param: step (int): The current step in the interpolation (0-based).
+        param: total_steps (int): The total number of interpolation steps.
+
+        returns: int: The interpolated integer.
+        """
+        return start + (end - start) * step // total_steps
+
+    @staticmethod
+    def _interpolate_str(start: str, end: str, step: int, total_steps: int) -> str:
+        """
+        Interpolates between two strings lexicographically.
+
+        param: start (str): The starting string.
+        param: end (str): The ending string.
+        param: step (int): The current step in the interpolation (0-based).
+        param: total_steps (int): The total number of interpolation steps.
+
+        returns: str: The interpolated string.
+        """
+        max_len = max(len(start), len(end))
+
+        # Pad strings to the same length with spaces (smallest lexicographical character).
+        start = start.ljust(max_len, " ")
+        end = end.ljust(max_len, " ")
+
+        # Interpolate character by character based on ordinal values.
+        interpolated_chars = [
+            chr(round(ord(s) + (ord(e) - ord(s)) * step / total_steps))
+            for s, e in zip(start, end)
+        ]
+
+        return "".join(interpolated_chars).rstrip()
+
+
+class RangeShardingStrategy(ShardingStrategy, Generic[T]):
+    """
+    Implements a sharding strategy to divide a range of keys into shards.
+
+    method: shards: Generates a list of RangeShard objects based on the global range.
+    """
+
+    def shards(
+        self, num_shards: int, metastore: DatasetMetastore
+    ) -> Iterable[RangeShard[T]]:
+        """
+        Divides the global range of keys into evenly sized shards.
+
+        param: num_shards: The number of shards to divide the range into.
+        param: metastore: The dataset metastore providing access to manifests.
+        returns: A list of RangeShard objects representing the divided range.
+        """
+        min, max = metastore.get_min_max_keys()
+        return RangeShard.split(min, max, num_shards)

deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py

@@ -6,15 +6,26 @@ from typing import Any, List, Set, Protocol, TypeVar, Dict, Iterable
 
 from pyarrow import RecordBatch, Table
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.metastore.delta import ManifestIO, DeltacatManifestIO
-
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.metastore.json_sst import JsonSstWriter
-from deltacat.storage.rivulet.serializer import MEMTABLE_DATA, DataSerializer
-from deltacat.storage.rivulet.serializer_factory import DataSerializerFactory
-from deltacat.storage.rivulet.writer.dataset_writer import DatasetWriter, DATA
-from deltacat.storage.rivulet.metastore.sst import SSTWriter
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.delta import (
+    ManifestIO,
+    DeltacatManifestIO,
+)
+
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.metastore.json_sst import JsonSstWriter
+from deltacat.experimental.storage.rivulet.serializer import (
+    MEMTABLE_DATA,
+    DataSerializer,
+)
+from deltacat.experimental.storage.rivulet.serializer_factory import (
+    DataSerializerFactory,
+)
+from deltacat.experimental.storage.rivulet.writer.dataset_writer import (
+    DatasetWriter,
+    DATA,
+)
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTWriter
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 INPUT_ROW = TypeVar("INPUT_ROW")
 

deltacat/io/reader/deltacat_read_api.py

@@ -97,7 +97,7 @@ def read_deltacat(
         >>> # Read the Iceberg stream of the latest active DeltaCAT table version,
         >>> import deltacat as dc
         >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table/default/iceberg")
-        >>> # Or, if `my_catalog is the default catalog, this is equivalent to:
+        >>> # Or, if `my_catalog` is the default catalog, this is equivalent to:
         >>> dc.io.read_deltacat("namespace://my_namespace/my_table/default/iceberg")
         >>> # Or, if `my_namespace` is the default namespace, this is equivalent to:
         >>> dc.io.read_deltacat("table://my_table/default/iceberg")

deltacat/storage/model/shard.py

@@ -1,7 +1,9 @@
 from abc import abstractmethod
 from typing import Iterable, Optional, Protocol, TypeVar, Union
 
-from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
 
 # TODO: Add type validation in dataset/schema classes
 T = TypeVar("T", bound=Union[int, str])
@@ -34,7 +36,9 @@ class ShardingStrategy(Protocol):
         return: ShardingStrategy class.
         """
         if strategy == "range":
-            from deltacat.storage.rivulet.shard.range_shard import RangeShardingStrategy
+            from deltacat.experimental.storage.rivulet.shard.range_shard import (
+                RangeShardingStrategy,
+            )
 
             return RangeShardingStrategy()
         else:

deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py

@@ -0,0 +1,130 @@
+import shutil
+
+from deltacat.catalog import get_catalog_properties
+from deltacat.exceptions import NamespaceAlreadyExistsError
+import pytest
+import tempfile
+import deltacat.catalog.main.impl as catalog
+
+
+class TestCatalogNamespaceOperations:
+    temp_dir = None
+    property_catalog = None
+    catalog = None
+
+    @classmethod
+    def setup_class(cls):
+        cls.temp_dir = tempfile.mkdtemp()
+        cls.catalog_properties = get_catalog_properties(root=cls.temp_dir)
+
+    @classmethod
+    def teardown_class(cls):
+        shutil.rmtree(cls.temp_dir)
+
+    def test_create_namespace(self):
+        """Test creating a namespace with properties"""
+        namespace = "test_create_namespace"
+        properties = {"description": "Test Namespace", "owner": "test-user"}
+
+        # Create namespace
+        catalog.create_namespace(
+            namespace=namespace, properties=properties, inner=self.catalog_properties
+        )
+
+        # Verify namespace exists
+        assert catalog.namespace_exists(namespace, inner=self.catalog_properties)
+
+        # Get namespace and verify properties
+        namespace = catalog.get_namespace(namespace, inner=self.catalog_properties)
+        assert namespace.namespace == "test_create_namespace"
+        assert namespace.properties["description"] == "Test Namespace"
+
+    def test_get_namespace(self):
+        """Test getting namespace properties"""
+        namespace = "test_get_namespace"
+        properties = {"description": "foo", "created_by": "bar"}
+
+        # Create namespace
+        catalog.create_namespace(
+            namespace=namespace, properties=properties, inner=self.catalog_properties
+        )
+
+        # Get namespace properties
+        namespace = catalog.get_namespace(namespace, inner=self.catalog_properties)
+
+        # Verify properties
+        assert namespace.namespace == "test_get_namespace"
+        assert namespace.properties["created_by"] == "bar"
+
+    def test_namespace_exists(self):
+        """Test checking if a namespace exists"""
+        existing_namespace = "test_namespace_exists"
+        non_existing_namespace = "non_existing_namespace"
+
+        # Create namespace
+        catalog.create_namespace(
+            namespace=existing_namespace, properties={}, inner=self.catalog_properties
+        )
+
+        # Check existing namespace
+        assert catalog.namespace_exists(
+            existing_namespace, inner=self.catalog_properties
+        )
+
+        # Check non-existing namespace
+        assert not catalog.namespace_exists(
+            non_existing_namespace, inner=self.catalog_properties
+        )
+
+    def test_create_namespace_already_exists(self):
+        """Test creating a namespace that already exists should fail"""
+        namespace = "test_create_namespace_already_exists"
+        properties = {"description": "Test namespace", "owner": "test-user"}
+
+        # Create namespace first time
+        catalog.create_namespace(
+            namespace=namespace,
+            properties=properties,
+            inner=self.catalog_properties,
+        )
+
+        # Verify namespace exists
+        assert catalog.namespace_exists(namespace, inner=self.catalog_properties)
+
+        # Try to create the same namespace again, should raise ValueError
+        with pytest.raises(NamespaceAlreadyExistsError, match=namespace):
+            catalog.create_namespace(
+                namespace=namespace,
+                properties=properties,
+                inner=self.catalog_properties,
+            )
+
+    def test_drop_namespace(self):
+        """Test dropping a namespace"""
+        namespace = "test_drop_namespace"
+        properties = {"description": "Test Namespace", "owner": "test-user"}
+
+        # Create namespace
+        catalog.create_namespace(
+            namespace=namespace,
+            properties=properties,
+            inner=self.catalog_properties,
+        )
+
+        # Verify namespace exists
+        assert catalog.namespace_exists(
+            namespace,
+            inner=self.catalog_properties,
+        )
+
+        # Drop namespace
+        catalog.drop_namespace(
+            namespace,
+            inner=self.catalog_properties,
+        )
+
+        # Verify namespace does not exist
+        assert not catalog.namespace_exists(
+            namespace,
+            inner=self.catalog_properties,
+        )