deltacat 2.0.0b11__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/storage/interface.py

@@ -2,6 +2,7 @@ from typing import Any, Callable, Dict, List, Optional, Union, Tuple
 
 from deltacat.storage import (
     EntryParams,
+    EntryType,
     Delta,
     DeltaLocator,
     DeltaProperties,
@@ -30,11 +31,12 @@ from deltacat.storage import (
     TableVersionProperties,
 )
 from deltacat.storage.model.manifest import Manifest
+from deltacat.storage.model.partition import UNKNOWN_PARTITION_ID
 from deltacat.types.media import (
     ContentType,
     DistributedDatasetType,
     StorageType,
-    TableType,
+    DatasetType,
 )
 from deltacat.utils.common import ReadKwargsProvider
 
@@ -205,7 +207,7 @@ def get_latest_delta(
 
 def download_delta(
     delta_like: Union[Delta, DeltaLocator],
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     storage_type: StorageType = StorageType.DISTRIBUTED,
     max_parallelism: Optional[int] = None,
     columns: Optional[List[str]] = None,
@@ -216,7 +218,7 @@ def download_delta(
     **kwargs,
 ) -> Union[LocalDataset, DistributedDataset]:  # type: ignore
     """
-    Download the given delta or delta locator into either a list of
+    Reads the given delta or delta locator into either a list of
     tables resident in the local node's memory, or into a dataset distributed
     across this Ray cluster's object store memory. Ordered table N of a local
     table list, or ordered block N of a distributed dataset, always contain
@@ -228,19 +230,19 @@ def download_delta(
 def download_delta_manifest_entry(
     delta_like: Union[Delta, DeltaLocator],
     entry_index: int,
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     *args,
     **kwargs,
 ) -> LocalTable:
     """
-    Downloads a single manifest entry into the specified table type for the
+    Reads a single manifest entry into the specified table type for the
     given delta or delta locator. If a delta is provided with a non-empty
-    manifest, then the entry is downloaded from this manifest. Otherwise, the
-    manifest is first retrieved then the given entry index downloaded.
+    manifest, then the entry is read from this manifest. Otherwise, the
+    manifest is first retrieved then the given entry index read.
 
-    NOTE: The entry will be downloaded in the current node's memory.
+    NOTE: The entry will be read in the current node's memory.
     """
     raise NotImplementedError("download_delta_manifest_entry not implemented")
 
@@ -288,9 +290,9 @@ def create_table_version(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
+    lifecycle_state: Optional[LifecycleState] = LifecycleState.CREATED,
     schema: Optional[Schema] = None,
     partition_scheme: Optional[PartitionScheme] = None,
-    # TODO(pdames): rename to `sort_scheme`
     sort_keys: Optional[SortScheme] = None,
     table_version_description: Optional[str] = None,
     table_version_properties: Optional[TableVersionProperties] = None,
@@ -299,9 +301,9 @@ def create_table_version(
     supported_content_types: Optional[List[ContentType]] = None,
     *args,
     **kwargs,
-) -> Tuple[Optional[Table], TableVersion, Stream]:
+) -> Tuple[Table, TableVersion, Stream]:
     """
-    Create a table version with an unreleased lifecycle state and an empty delta
+    Create a table version with the given or CREATED lifecycle state and an empty delta
     stream. Table versions may be schemaless and unpartitioned to improve write
     performance, or have their writes governed by a schema and partition scheme
     to improve data consistency and read performance.
@@ -314,6 +316,20 @@ def create_table_version(
     raise NotImplementedError("create_table_version not implemented")
 
 
+def create_table(
+    namespace: str,
+    table_name: str,
+    description: Optional[str] = None,
+    properties: Optional[TableProperties] = None,
+    *args,
+    **kwargs,
+) -> Table:
+    """
+    Create a new table. Raises an error if the given table already exists.
+    """
+    raise NotImplementedError("create_table not implemented")
+
+
 def update_table(
     namespace: str,
     table_name: str,
@@ -322,7 +338,7 @@ def update_table(
     new_table_name: Optional[str] = None,
     *args,
     **kwargs,
-) -> None:
+) -> Table:
     """
     Update table metadata describing the table versions it contains. By default,
     a table's properties are empty, and its description is equal to that given
@@ -345,7 +361,7 @@ def update_table_version(
     sort_keys: Optional[SortScheme] = None,
     *args,
     **kwargs,
-) -> None:
+) -> Tuple[Optional[Table], TableVersion, Optional[Stream]]:
     """
     Update a table version. Notably, updating an unreleased table version's
     lifecycle state to 'active' telegraphs that it is ready for external
@@ -410,15 +426,15 @@ def delete_stream(
 
 def delete_table(
     namespace: str,
-    name: str,
+    table_name: str,
     purge: bool = False,
     *args,
     **kwargs,
 ) -> None:
     """
-    Drops the given table and all its contents (table versions, streams, partitions,
-    and deltas). If purge is True, also removes all data files associated with the table.
-    Raises an error if the given table does not exist.
+    Drops the given table from the catalog. If purge is True, also removes
+    all data files associated with the table. Raises an error if the given table
+    does not exist.
     """
     raise NotImplementedError("delete_table not implemented")
 
@@ -430,10 +446,9 @@ def delete_namespace(
     **kwargs,
 ) -> None:
     """
-    Drops a table namespace and all its contents. If purge is True, then all
-    tables, table versions, and deltas will be deleted. Otherwise, the namespace
-    will be dropped only if it is empty. Raises an error if the given namespace
-    does not exist.
+    Drops the given namespace from the catalog. If purge is True, also removes
+    all data files associated with the namespace. Raises an error if the given
+    namespace does not exist.
     """
     raise NotImplementedError("drop_namespace not implemented")
 
@@ -509,6 +524,7 @@ def stage_partition(
 def commit_partition(
     partition: Partition,
     previous_partition: Optional[Partition] = None,
+    expected_previous_partition_id: Optional[str] = UNKNOWN_PARTITION_ID,
     *args,
     **kwargs,
 ) -> Partition:
@@ -586,23 +602,19 @@ def stage_delta(
     max_records_per_entry: Optional[int] = None,
     author: Optional[ManifestAuthor] = None,
     properties: Optional[DeltaProperties] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
     entry_params: Optional[EntryParams] = None,
+    entry_type: Optional[EntryType] = EntryType.DATA,
+    schema: Optional[Schema] = None,
+    sort_scheme_id: Optional[str] = None,
     *args,
     **kwargs,
 ) -> Delta:
     """
-    Writes the given table to 1 or more S3 files. Returns an unregistered
+    Writes the given dataset to 1 or more files. Returns an unregistered
     delta whose manifest entries point to the uploaded files. Applies any
     schema consistency policies configured for the parent table version.
-
-    The partition spec will be used to split the input table into
-    multiple files. Optionally, partition_values can be provided to avoid
-    this method to recompute partition_values from the provided data.
-
-    Raises an error if the provided data does not conform to a unique ordered
-    list of partition_values
     """
     raise NotImplementedError("stage_delta not implemented")
 
@@ -723,13 +735,23 @@ def table_version_exists(
 
 def can_categorize(e: BaseException, *args, **kwargs) -> bool:
     """
-    Return whether input error is from storage implementation layer.
+    True if the input error originated from the storage
+    implementation layer and can be categorized under an
+    existing DeltaCatError. The "categorize_errors" decorator
+    uses this to determine if an unknown error from the storage
+    implementation can be categorized prior to casting it to
+    the equivalent DeltaCatError via `raise_categorized_error`
     """
     raise NotImplementedError
 
 
 def raise_categorized_error(e: BaseException, *args, **kwargs):
     """
-    Raise and handle storage implementation layer specific errors.
+    Casts a categorizable error that originaed from the storage
+    implementation layer to its equivalent DeltaCatError
+    for uniform handling (e.g., determining whether an error
+    is retryable or not) via the "categorize_errors" decorator.
+    Raises an UnclassifiedDeltaCatError from the input exception
+    if the error cannot be categorized.
     """
     raise NotImplementedError