deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py

@@ -1,9 +1,9 @@
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.reader.data_reader import FileReader
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.reader.data_reader import FileReader
 from typing import Type, Dict
 
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 
 
 class FileReaderRegistrar:

deltacat/{storage → experimental/storage}/rivulet/schema/schema.py

@@ -5,7 +5,7 @@ from typing import MutableMapping, Dict, Iterable, Tuple, Optional
 
 import pyarrow as pa
 
-from deltacat.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
 
 
 @dataclass(frozen=True)

deltacat/{storage → experimental/storage}/rivulet/serializer.py

@@ -1,6 +1,6 @@
 from typing import Protocol, Iterable, List, Union, Any, Dict
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
 import pyarrow as pa
 
 MEMTABLE_DATA = Union[Iterable[Dict[str, Any]], pa.Table]

deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
-from deltacat.storage.rivulet.parquet.serializer import ParquetDataSerializer
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.serializer import DataSerializer
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.parquet.serializer import (
+    ParquetDataSerializer,
+)
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.serializer import DataSerializer
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
-from deltacat.storage.rivulet.feather.serializer import FeatherDataSerializer
+from deltacat.experimental.storage.rivulet.feather.serializer import (
+    FeatherDataSerializer,
+)
 
 
 class DataSerializerFactory:

deltacat/experimental/storage/rivulet/shard/range_shard.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+from typing import Generic, List, Union, Iterable
+from deltacat.storage.model.shard import T, Shard, ShardingStrategy
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+
+
+class RangeShard(Shard, Generic[T]):
+    """
+    Represents a range-based shard with minimum and maximum keys.
+
+    param: min_key: The minimum key for the shard.
+    param: max_key: The maximum key for the shard.
+    """
+
+    def __init__(self, min_key: T, max_key: T):
+        self.min_key = min_key
+        self.max_key = max_key
+
+    def __repr__(self) -> str:
+        return f"Shard(type=range, min_key={self.min_key}, max_key={self.max_key})"
+
+    @staticmethod
+    def split(
+        global_min: Union[int, str], global_max: Union[int, str], num_shards: int
+    ) -> List[RangeShard]:
+        """
+        Splits a range into `num_shards` shards.
+        Currently supports splitting ranges of integers and strings.
+
+        Note: If global_min == global_max or num_shards <= 1, a single shard is returned,
+              num_shards is ignored.
+
+        :param global_min: The minimum key for the entire range (int or str).
+        :param global_max: The maximum key for the entire range (int or str).
+        :param num_shards: The number of shards to create.
+        :return: A list of RangeShard objects.
+        """
+        if global_min == global_max or num_shards <= 1:
+            return [RangeShard(global_min, global_max)]
+
+        # Determine which interpolation function to use based on the type of min/max
+        if isinstance(global_min, int) and isinstance(global_max, int):
+            interpolate = RangeShard._interpolate_numeric
+        elif isinstance(global_min, str) and isinstance(global_max, str):
+            interpolate = RangeShard._interpolate_str
+        else:
+            raise ValueError(
+                "Unsupported combination of types for global_min and global_max."
+            )
+
+        shards: List[RangeShard] = []
+        for i in range(num_shards):
+            start = interpolate(global_min, global_max, i, num_shards)
+            end = interpolate(global_min, global_max, i + 1, num_shards)
+
+            if i > 0:
+                if isinstance(start, int):
+                    start = shards[-1].max_key + 1
+                elif isinstance(start, int):
+                    char_list = list(start)
+                    char_list[-1] = chr(ord(char_list[-1]) + 1)
+                    start = "".join(char_list)
+
+            shards.append(RangeShard(start, end))
+
+        return shards
+
+    @staticmethod
+    def _interpolate_numeric(start: int, end: int, step: int, total_steps: int) -> int:
+        """
+        Integer interpolation using integer (floor) division.
+
+        param: start (int): The starting number.
+        param: end (int): The ending number.
+        param: step (int): The current step in the interpolation (0-based).
+        param: total_steps (int): The total number of interpolation steps.
+
+        returns: int: The interpolated integer.
+        """
+        return start + (end - start) * step // total_steps
+
+    @staticmethod
+    def _interpolate_str(start: str, end: str, step: int, total_steps: int) -> str:
+        """
+        Interpolates between two strings lexicographically.
+
+        param: start (str): The starting string.
+        param: end (str): The ending string.
+        param: step (int): The current step in the interpolation (0-based).
+        param: total_steps (int): The total number of interpolation steps.
+
+        returns: str: The interpolated string.
+        """
+        max_len = max(len(start), len(end))
+
+        # Pad strings to the same length with spaces (smallest lexicographical character).
+        start = start.ljust(max_len, " ")
+        end = end.ljust(max_len, " ")
+
+        # Interpolate character by character based on ordinal values.
+        interpolated_chars = [
+            chr(round(ord(s) + (ord(e) - ord(s)) * step / total_steps))
+            for s, e in zip(start, end)
+        ]
+
+        return "".join(interpolated_chars).rstrip()
+
+
+class RangeShardingStrategy(ShardingStrategy, Generic[T]):
+    """
+    Implements a sharding strategy to divide a range of keys into shards.
+
+    method: shards: Generates a list of RangeShard objects based on the global range.
+    """
+
+    def shards(
+        self, num_shards: int, metastore: DatasetMetastore
+    ) -> Iterable[RangeShard[T]]:
+        """
+        Divides the global range of keys into evenly sized shards.
+
+        param: num_shards: The number of shards to divide the range into.
+        param: metastore: The dataset metastore providing access to manifests.
+        returns: A list of RangeShard objects representing the divided range.
+        """
+        min, max = metastore.get_min_max_keys()
+        return RangeShard.split(min, max, num_shards)

deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py

@@ -6,15 +6,26 @@ from typing import Any, List, Set, Protocol, TypeVar, Dict, Iterable
 
 from pyarrow import RecordBatch, Table
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.metastore.delta import ManifestIO, DeltacatManifestIO
-
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.metastore.json_sst import JsonSstWriter
-from deltacat.storage.rivulet.serializer import MEMTABLE_DATA, DataSerializer
-from deltacat.storage.rivulet.serializer_factory import DataSerializerFactory
-from deltacat.storage.rivulet.writer.dataset_writer import DatasetWriter, DATA
-from deltacat.storage.rivulet.metastore.sst import SSTWriter
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.delta import (
+    ManifestIO,
+    DeltacatManifestIO,
+)
+
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.metastore.json_sst import JsonSstWriter
+from deltacat.experimental.storage.rivulet.serializer import (
+    MEMTABLE_DATA,
+    DataSerializer,
+)
+from deltacat.experimental.storage.rivulet.serializer_factory import (
+    DataSerializerFactory,
+)
+from deltacat.experimental.storage.rivulet.writer.dataset_writer import (
+    DatasetWriter,
+    DATA,
+)
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTWriter
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 INPUT_ROW = TypeVar("INPUT_ROW")
 

deltacat/io/datasource/deltacat_datasource.py

@@ -350,7 +350,6 @@ class DeltaCatDatasource(Datasource):
             ]
         elif self._deltacat_read_type == DeltacatReadType.METADATA_LIST:
             # do a shallow read of the top-level DeltaCAT metadata
-            print(f"listers: {self._reader.listers}")
             listers = copy.deepcopy(self._reader.listers)
             listers = [listers[0]]
             read_tasks = self._list_all_metafiles_read_tasks(

deltacat/io/reader/deltacat_read_api.py

@@ -97,7 +97,7 @@ def read_deltacat(
         >>> # Read the Iceberg stream of the latest active DeltaCAT table version,
         >>> import deltacat as dc
         >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table/default/iceberg")
-        >>> # Or, if `my_catalog is the default catalog, this is equivalent to:
+        >>> # Or, if `my_catalog` is the default catalog, this is equivalent to:
         >>> dc.io.read_deltacat("namespace://my_namespace/my_table/default/iceberg")
         >>> # Or, if `my_namespace` is the default namespace, this is equivalent to:
         >>> dc.io.read_deltacat("table://my_table/default/iceberg")

deltacat/storage/__init__.py

@@ -20,6 +20,9 @@ from deltacat.storage.model.metafile import (
 from deltacat.storage.model.transaction import (
     TransactionOperation,
     Transaction,
+    read_transaction,
+    transactions,
+    transaction,
 )
 from deltacat.storage.model.namespace import (
     Namespace,
@@ -31,6 +34,7 @@ from deltacat.storage.model.partition import (
     PartitionLocator,
     PartitionLocatorAlias,
     PartitionKey,
+    PartitionKeyList,
     PartitionScheme,
     PartitionSchemeList,
     PartitionValues,
@@ -43,6 +47,9 @@ from deltacat.storage.model.schema import (
     NestedFieldName,
     Schema,
     SchemaList,
+    SchemaUpdate,
+    SchemaUpdateOperation,
+    SchemaUpdateOperations,
 )
 from deltacat.storage.model.stream import (
     Stream,
@@ -75,6 +82,7 @@ from deltacat.storage.model.transform import (
     MonthTransform,
     YearTransform,
     TruncateTransform,
+    TruncateStrategy,
 )
 from deltacat.storage.model.types import (
     CommitState,
@@ -88,11 +96,12 @@ from deltacat.storage.model.types import (
     SchemaConsistencyType,
     StreamFormat,
     SortOrder,
-    TransactionType,
     TransactionOperationType,
+    TransactionStatus,
 )
 from deltacat.storage.model.sort_key import (
     SortKey,
+    SortKeyList,
     SortScheme,
     SortSchemeList,
 )
@@ -138,6 +147,7 @@ __all__ = [
     "NullOrder",
     "Partition",
     "PartitionKey",
+    "PartitionKeyList",
     "PartitionLocator",
     "PartitionLocatorAlias",
     "PartitionScheme",
@@ -145,8 +155,12 @@ __all__ = [
     "PartitionValues",
     "Schema",
     "SchemaList",
+    "SchemaUpdate",
+    "SchemaUpdateOperation",
+    "SchemaUpdateOperations",
     "SchemaConsistencyType",
     "SortKey",
+    "SortKeyList",
     "SortOrder",
     "SortScheme",
     "SortSchemeList",
@@ -163,13 +177,17 @@ __all__ = [
     "Transaction",
     "TransactionOperation",
     "TransactionOperationType",
-    "TransactionType",
+    "TransactionStatus",
     "Transform",
     "TransformName",
     "TransformParameters",
     "TruncateTransform",
     "TruncateTransformParameters",
+    "TruncateStrategy",
     "UnknownTransform",
     "VoidTransform",
     "YearTransform",
+    "read_transaction",
+    "transactions",
+    "transaction",
 ]

deltacat/storage/interface.py

@@ -2,6 +2,7 @@ from typing import Any, Callable, Dict, List, Optional, Union, Tuple
 
 from deltacat.storage import (
     EntryParams,
+    EntryType,
     Delta,
     DeltaLocator,
     DeltaProperties,
@@ -30,11 +31,12 @@ from deltacat.storage import (
     TableVersionProperties,
 )
 from deltacat.storage.model.manifest import Manifest
+from deltacat.storage.model.partition import UNKNOWN_PARTITION_ID
 from deltacat.types.media import (
     ContentType,
     DistributedDatasetType,
     StorageType,
-    TableType,
+    DatasetType,
 )
 from deltacat.utils.common import ReadKwargsProvider
 
@@ -205,7 +207,7 @@ def get_latest_delta(
 
 def download_delta(
     delta_like: Union[Delta, DeltaLocator],
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     storage_type: StorageType = StorageType.DISTRIBUTED,
     max_parallelism: Optional[int] = None,
     columns: Optional[List[str]] = None,
@@ -216,7 +218,7 @@ def download_delta(
     **kwargs,
 ) -> Union[LocalDataset, DistributedDataset]:  # type: ignore
     """
-    Download the given delta or delta locator into either a list of
+    Reads the given delta or delta locator into either a list of
     tables resident in the local node's memory, or into a dataset distributed
     across this Ray cluster's object store memory. Ordered table N of a local
     table list, or ordered block N of a distributed dataset, always contain
@@ -228,19 +230,19 @@ def download_delta(
 def download_delta_manifest_entry(
     delta_like: Union[Delta, DeltaLocator],
     entry_index: int,
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     *args,
     **kwargs,
 ) -> LocalTable:
     """
-    Downloads a single manifest entry into the specified table type for the
+    Reads a single manifest entry into the specified table type for the
     given delta or delta locator. If a delta is provided with a non-empty
-    manifest, then the entry is downloaded from this manifest. Otherwise, the
-    manifest is first retrieved then the given entry index downloaded.
+    manifest, then the entry is read from this manifest. Otherwise, the
+    manifest is first retrieved then the given entry index read.
 
-    NOTE: The entry will be downloaded in the current node's memory.
+    NOTE: The entry will be read in the current node's memory.
     """
     raise NotImplementedError("download_delta_manifest_entry not implemented")
 
@@ -288,9 +290,9 @@ def create_table_version(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
+    lifecycle_state: Optional[LifecycleState] = LifecycleState.CREATED,
     schema: Optional[Schema] = None,
     partition_scheme: Optional[PartitionScheme] = None,
-    # TODO(pdames): rename to `sort_scheme`
     sort_keys: Optional[SortScheme] = None,
     table_version_description: Optional[str] = None,
     table_version_properties: Optional[TableVersionProperties] = None,
@@ -299,9 +301,9 @@ def create_table_version(
     supported_content_types: Optional[List[ContentType]] = None,
     *args,
     **kwargs,
-) -> Tuple[Optional[Table], TableVersion, Stream]:
+) -> Tuple[Table, TableVersion, Stream]:
     """
-    Create a table version with an unreleased lifecycle state and an empty delta
+    Create a table version with the given or CREATED lifecycle state and an empty delta
     stream. Table versions may be schemaless and unpartitioned to improve write
     performance, or have their writes governed by a schema and partition scheme
     to improve data consistency and read performance.
@@ -314,6 +316,20 @@ def create_table_version(
     raise NotImplementedError("create_table_version not implemented")
 
 
+def create_table(
+    namespace: str,
+    table_name: str,
+    description: Optional[str] = None,
+    properties: Optional[TableProperties] = None,
+    *args,
+    **kwargs,
+) -> Table:
+    """
+    Create a new table. Raises an error if the given table already exists.
+    """
+    raise NotImplementedError("create_table not implemented")
+
+
 def update_table(
     namespace: str,
     table_name: str,
@@ -322,7 +338,7 @@ def update_table(
     new_table_name: Optional[str] = None,
     *args,
     **kwargs,
-) -> None:
+) -> Table:
     """
     Update table metadata describing the table versions it contains. By default,
     a table's properties are empty, and its description is equal to that given
@@ -345,7 +361,7 @@ def update_table_version(
     sort_keys: Optional[SortScheme] = None,
     *args,
     **kwargs,
-) -> None:
+) -> Tuple[Optional[Table], TableVersion, Optional[Stream]]:
     """
     Update a table version. Notably, updating an unreleased table version's
     lifecycle state to 'active' telegraphs that it is ready for external
@@ -410,15 +426,15 @@ def delete_stream(
 
 def delete_table(
     namespace: str,
-    name: str,
+    table_name: str,
     purge: bool = False,
     *args,
     **kwargs,
 ) -> None:
     """
-    Drops the given table and all its contents (table versions, streams, partitions,
-    and deltas). If purge is True, also removes all data files associated with the table.
-    Raises an error if the given table does not exist.
+    Drops the given table from the catalog. If purge is True, also removes
+    all data files associated with the table. Raises an error if the given table
+    does not exist.
     """
     raise NotImplementedError("delete_table not implemented")
 
@@ -430,10 +446,9 @@ def delete_namespace(
     **kwargs,
 ) -> None:
     """
-    Drops a table namespace and all its contents. If purge is True, then all
-    tables, table versions, and deltas will be deleted. Otherwise, the namespace
-    will be dropped only if it is empty. Raises an error if the given namespace
-    does not exist.
+    Drops the given namespace from the catalog. If purge is True, also removes
+    all data files associated with the namespace. Raises an error if the given
+    namespace does not exist.
     """
     raise NotImplementedError("drop_namespace not implemented")
 
@@ -509,6 +524,7 @@ def stage_partition(
 def commit_partition(
     partition: Partition,
     previous_partition: Optional[Partition] = None,
+    expected_previous_partition_id: Optional[str] = UNKNOWN_PARTITION_ID,
     *args,
     **kwargs,
 ) -> Partition:
@@ -586,23 +602,19 @@ def stage_delta(
     max_records_per_entry: Optional[int] = None,
     author: Optional[ManifestAuthor] = None,
     properties: Optional[DeltaProperties] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
     entry_params: Optional[EntryParams] = None,
+    entry_type: Optional[EntryType] = EntryType.DATA,
+    schema: Optional[Schema] = None,
+    sort_scheme_id: Optional[str] = None,
     *args,
     **kwargs,
 ) -> Delta:
     """
-    Writes the given table to 1 or more S3 files. Returns an unregistered
+    Writes the given dataset to 1 or more files. Returns an unregistered
     delta whose manifest entries point to the uploaded files. Applies any
     schema consistency policies configured for the parent table version.
-
-    The partition spec will be used to split the input table into
-    multiple files. Optionally, partition_values can be provided to avoid
-    this method to recompute partition_values from the provided data.
-
-    Raises an error if the provided data does not conform to a unique ordered
-    list of partition_values
     """
     raise NotImplementedError("stage_delta not implemented")
 
@@ -723,13 +735,23 @@ def table_version_exists(
 
 def can_categorize(e: BaseException, *args, **kwargs) -> bool:
     """
-    Return whether input error is from storage implementation layer.
+    True if the input error originated from the storage
+    implementation layer and can be categorized under an
+    existing DeltaCatError. The "categorize_errors" decorator
+    uses this to determine if an unknown error from the storage
+    implementation can be categorized prior to casting it to
+    the equivalent DeltaCatError via `raise_categorized_error`
     """
     raise NotImplementedError
 
 
 def raise_categorized_error(e: BaseException, *args, **kwargs):
     """
-    Raise and handle storage implementation layer specific errors.
+    Casts a categorizable error that originaed from the storage
+    implementation layer to its equivalent DeltaCatError
+    for uniform handling (e.g., determining whether an error
+    is retryable or not) via the "categorize_errors" decorator.
+    Raises an UnclassifiedDeltaCatError from the input exception
+    if the error cannot be categorized.
     """
     raise NotImplementedError