deltacat 1.1.36__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

deltacat/storage/interface.py

@@ -1,38 +1,40 @@
-from typing import Any, Callable, Dict, List, Optional, Set, Union
-
-import pyarrow as pa
+from typing import Any, Callable, Dict, List, Optional, Union, Tuple
 
 from deltacat.storage import (
-    DeleteParameters,
+    EntryParams,
     Delta,
     DeltaLocator,
+    DeltaProperties,
     DeltaType,
     DistributedDataset,
     LifecycleState,
     ListResult,
     LocalDataset,
     LocalTable,
-    Manifest,
     ManifestAuthor,
     Namespace,
+    NamespaceProperties,
     Partition,
-    SchemaConsistencyType,
+    PartitionLocator,
+    PartitionScheme,
+    PartitionValues,
+    Schema,
+    SortScheme,
     Stream,
+    StreamFormat,
     StreamLocator,
     Table,
+    TableProperties,
     TableVersion,
-    SortKey,
-    PartitionLocator,
-    PartitionFilter,
-    PartitionValues,
-    DeltaPartitionSpec,
-    StreamPartitionSpec,
+    TableVersionLocator,
+    TableVersionProperties,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.types.media import (
     ContentType,
+    DistributedDatasetType,
     StorageType,
     TableType,
-    DistributedDatasetType,
 )
 from deltacat.utils.common import ReadKwargsProvider
 
@@ -64,12 +66,26 @@ def list_table_versions(
     raise NotImplementedError("list_table_versions not implemented")
 
 
+def list_streams(
+    namespace: str,
+    table_name: str,
+    table_version: str,
+    *args,
+    **kwargs,
+) -> ListResult[Stream]:
+    """
+    Lists a page of streams for the given table version.
+    Raises an error if the table version does not exist.
+    """
+    raise NotImplementedError("list_streams not implemented")
+
+
 def list_partitions(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> ListResult[Partition]:
     """
     Lists a page of partitions for the given table version. Partitions are
@@ -96,9 +112,9 @@ def list_deltas(
     last_stream_position: Optional[int] = None,
     ascending_order: Optional[bool] = None,
     include_manifest: bool = False,
-    partition_filter: Optional[PartitionFilter] = None,
+    partition_scheme_id: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> ListResult[Delta]:
     """
     Lists a page of deltas for the given table version and committed partition.
@@ -106,15 +122,13 @@ def list_deltas(
     limited to inclusive first and last stream positions. Deltas are returned by
     descending stream position by default. Table version resolves to the latest
     active table version if not specified. Partition values should not be
-    specified for unpartitioned tables. Raises an error if the given table
-    version or partition does not exist.
+    specified for unpartitioned tables. Partition scheme ID resolves to the
+    table version's current partition scheme by default. Raises an error if the
+    given table version or partition does not exist.
 
     To conserve memory, the deltas returned do not include manifests by
     default. The manifests can either be optionally retrieved as part of this
     call or lazily loaded via subsequent calls to `get_delta_manifest`.
-
-    Note: partition_values is deprecated and will be removed in future releases.
-    Use partition_filter instead.
     """
     raise NotImplementedError("list_deltas not implemented")
 
@@ -126,7 +140,7 @@ def list_partition_deltas(
     ascending_order: bool = False,
     include_manifest: bool = False,
     *args,
-    **kwargs
+    **kwargs,
 ) -> ListResult[Delta]:
     """
     Lists a page of deltas committed to the given partition.
@@ -145,22 +159,21 @@ def get_delta(
     partition_values: Optional[PartitionValues] = None,
     table_version: Optional[str] = None,
     include_manifest: bool = False,
-    partition_filter: Optional[PartitionFilter] = None,
+    partition_scheme_id: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Optional[Delta]:
     """
     Gets the delta for the given table version, partition, and stream position.
     Table version resolves to the latest active table version if not specified.
-    Partition values should not be specified for unpartitioned tables. Raises
-    an error if the given table version or partition does not exist.
+    Partition values should not be specified for unpartitioned tables. Partition
+    scheme ID resolves to the table version's current partition scheme by
+    default. Raises an error if the given table version or partition does not
+    exist.
 
     To conserve memory, the delta returned does not include a manifest by
     default. The manifest can either be optionally retrieved as part of this
     call or lazily loaded via a subsequent call to `get_delta_manifest`.
-
-    Note: partition_values is deprecated and will be removed in future releases.
-    Use partition_filter instead.
     """
     raise NotImplementedError("get_delta not implemented")
 
@@ -171,23 +184,21 @@ def get_latest_delta(
     partition_values: Optional[PartitionValues] = None,
     table_version: Optional[str] = None,
     include_manifest: bool = False,
-    partition_filter: Optional[PartitionFilter] = None,
+    partition_scheme_id: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Optional[Delta]:
     """
     Gets the latest delta (i.e. the delta with the greatest stream position) for
     the given table version and partition. Table version resolves to the latest
     active table version if not specified. Partition values should not be
-    specified for unpartitioned tables. Raises an error if the given table
-    version or partition does not exist.
+    specified for unpartitioned tables. Partition scheme ID resolves to the
+    table version's current partition scheme by default. Raises an error if the
+    given table version or partition does not exist.
 
     To conserve memory, the delta returned does not include a manifest by
     default. The manifest can either be optionally retrieved as part of this
     call or lazily loaded via a subsequent call to `get_delta_manifest`.
-
-    Note: partition_values is deprecated and will be removed in future releases.
-    Use partition_filter instead.
     """
     raise NotImplementedError("get_latest_delta not implemented")
 
@@ -201,9 +212,8 @@ def download_delta(
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     ray_options_provider: Callable[[int, Any], Dict[str, Any]] = None,
     distributed_dataset_type: DistributedDatasetType = DistributedDatasetType.RAY_DATASET,
-    partition_filter: Optional[PartitionFilter] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Union[LocalDataset, DistributedDataset]:  # type: ignore
     """
     Download the given delta or delta locator into either a list of
@@ -211,10 +221,6 @@ def download_delta(
     across this Ray cluster's object store memory. Ordered table N of a local
     table list, or ordered block N of a distributed dataset, always contain
     the contents of ordered delta manifest entry N.
-
-    partition_filter is an optional parameter which determines which files to
-    download from the delta manifest. A delta manifest contains all the data files
-    for a given delta.
     """
     raise NotImplementedError("download_delta not implemented")
 
@@ -226,7 +232,7 @@ def download_delta_manifest_entry(
     columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> LocalTable:
     """
     Downloads a single manifest entry into the specified table type for the
@@ -244,17 +250,21 @@ def get_delta_manifest(
 ) -> Manifest:
     """
     Get the manifest associated with the given delta or delta locator. This
-    always retrieves the authoritative remote copy of the delta manifest, and
-    never the local manifest defined for any input delta.
+    always retrieves the authoritative durable copy of the delta manifest, and
+    never the local manifest defined for any input delta. Raises an error if
+    the delta can't be found, or if it doesn't contain a manifest.
     """
     raise NotImplementedError("get_delta_manifest not implemented")
 
 
 def create_namespace(
-    namespace: str, permissions: Dict[str, Any], *args, **kwargs
+    namespace: str,
+    properties: Optional[NamespaceProperties] = None,
+    *args,
+    **kwargs,
 ) -> Namespace:
     """
-    Creates a table namespace with the given name and permissions. Returns
+    Creates a table namespace with the given name and properties. Returns
     the created namespace.
     """
     raise NotImplementedError("create_namespace not implemented")
@@ -262,13 +272,13 @@ def create_namespace(
 
 def update_namespace(
     namespace: str,
-    permissions: Optional[Dict[str, Any]] = None,
+    properties: Optional[NamespaceProperties] = None,
     new_namespace: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> None:
     """
-    Updates a table namespace's name and/or permissions. Raises an error if the
+    Updates a table namespace's name and/or properties. Raises an error if the
     given namespace does not exist.
     """
     raise NotImplementedError("update_namespace not implemented")
@@ -278,52 +288,28 @@ def create_table_version(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
-    schema: Optional[Union[pa.Schema, str, bytes]] = None,
-    schema_consistency: Optional[Dict[str, SchemaConsistencyType]] = None,
-    partition_keys: Optional[List[Dict[str, Any]]] = None,
-    primary_key_column_names: Optional[Set[str]] = None,
-    sort_keys: Optional[List[SortKey]] = None,
+    schema: Optional[Schema] = None,
+    partition_scheme: Optional[PartitionScheme] = None,
+    # TODO(pdames): rename to `sort_scheme`
+    sort_keys: Optional[SortScheme] = None,
     table_version_description: Optional[str] = None,
-    table_version_properties: Optional[Dict[str, str]] = None,
-    table_permissions: Optional[Dict[str, Any]] = None,
+    table_version_properties: Optional[TableVersionProperties] = None,
     table_description: Optional[str] = None,
-    table_properties: Optional[Dict[str, str]] = None,
+    table_properties: Optional[TableProperties] = None,
     supported_content_types: Optional[List[ContentType]] = None,
-    partition_spec: Optional[StreamPartitionSpec] = None,
     *args,
-    **kwargs
-) -> Stream:
+    **kwargs,
+) -> Tuple[Optional[Table], TableVersion, Stream]:
     """
     Create a table version with an unreleased lifecycle state and an empty delta
-    stream. Table versions may be schemaless and unpartitioned, or partitioned
-    according to a list of partition key names and types. Note that partition
-    keys are not required to exist in the table's schema, and can thus still be
-    used with schemaless tables. This can be useful for creating logical shards
-    of a delta stream where partition keys are known but not projected onto each
-    row of the table (e.g. all rows of a customer orders table are known to
-    correspond to a given order day, even if this column doesn't exist in the
-    table). Primary and sort keys must exist within the table's schema.
-    Permissions specified at the table level override any conflicting
-    permissions specified at the table namespace level. Returns the stream
-    for the created table version. Raises an error if the given namespace does
-    not exist.
-
-    Schemas are optional for DeltaCAT tables and can be used to inform the data
-    consistency checks run for each field. If a schema is present, it can be
-    used to enforce the following column-level data consistency policies at
-    table load time:
-
-    None: No consistency checks are run. May be mixed with the below two
-    policies by specifying column names to pass through together with
-    column names to coerce/validate.
+    stream. Table versions may be schemaless and unpartitioned to improve write
+    performance, or have their writes governed by a schema and partition scheme
+    to improve data consistency and read performance.
 
-    Coerce: Coerce fields to fit the schema whenever possible. An explicit
-    subset of column names to coerce may optionally be specified.
+    Returns a tuple containing the created/updated table, table version, and
+    stream (respectively).
 
-    Validate: Raise an error for any fields that don't fit the schema. An
-    explicit subset of column names to validate may optionally be specified.
-
-    Either partition_keys or partition_spec must be specified but not both.
+    Raises an error if the given namespace does not exist.
     """
     raise NotImplementedError("create_table_version not implemented")
 
@@ -331,18 +317,17 @@ def create_table_version(
 def update_table(
     namespace: str,
     table_name: str,
-    permissions: Optional[Dict[str, Any]] = None,
     description: Optional[str] = None,
-    properties: Optional[Dict[str, str]] = None,
+    properties: Optional[TableProperties] = None,
     new_table_name: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> None:
     """
     Update table metadata describing the table versions it contains. By default,
-    a table's properties are empty, and its description and permissions are
-    equal to those given when its first table version was created. Raises an
-    error if the given table does not exist.
+    a table's properties are empty, and its description is equal to that given
+    when its first table version was created. Raises an error if the given
+    table does not exist.
     """
     raise NotImplementedError("update_table not implemented")
 
@@ -352,12 +337,14 @@ def update_table_version(
     table_name: str,
     table_version: str,
     lifecycle_state: Optional[LifecycleState] = None,
-    schema: Optional[Union[pa.Schema, str, bytes]] = None,
-    schema_consistency: Optional[Dict[str, SchemaConsistencyType]] = None,
+    schema: Optional[Schema] = None,
     description: Optional[str] = None,
-    properties: Optional[Dict[str, str]] = None,
+    properties: Optional[TableVersionProperties] = None,
+    partition_scheme: Optional[PartitionScheme] = None,
+    # TODO(pdames): rename to `sort_scheme`
+    sort_keys: Optional[SortScheme] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> None:
     """
     Update a table version. Notably, updating an unreleased table version's
@@ -375,18 +362,27 @@ def stage_stream(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
+    stream_format: StreamFormat = StreamFormat.DELTACAT,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Stream:
     """
     Stages a new delta stream for the given table version. Resolves to the
-    latest active table version if no table version is given. Returns the
-    staged stream. Raises an error if the table version does not exist.
+    latest active table version if no table version is given. Resolves to the
+    DeltaCAT stream format if no stream format is given. If this stream
+    will replace another stream with the same format and scheme, then it will
+    have its previous stream ID set to the ID of the stream being replaced.
+    Returns the staged stream. Raises an error if the table version does not
+    exist.
     """
     raise NotImplementedError("stage_stream not implemented")
 
 
-def commit_stream(stream: Stream, *args, **kwargs) -> Stream:
+def commit_stream(
+    stream: Stream,
+    *args,
+    **kwargs,
+) -> Stream:
     """
     Registers a delta stream with a target table version, replacing any
     previous stream registered for the same table version. Returns the
@@ -399,43 +395,112 @@ def delete_stream(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
+    stream_format: StreamFormat = StreamFormat.DELTACAT,
     *args,
-    **kwargs
+    **kwargs,
 ) -> None:
     """
     Deletes the delta stream currently registered with the given table version.
     Resolves to the latest active table version if no table version is given.
-    Raises an error if the table version does not exist.
+    Resolves to the deltacat stream format if no stream format is given.
+    Raises an error if the stream does not exist.
     """
     raise NotImplementedError("delete_stream not implemented")
 
 
+def delete_table(
+    namespace: str,
+    name: str,
+    purge: bool = False,
+    *args,
+    **kwargs,
+) -> None:
+    """
+    Drops the given table and all its contents (table versions, streams, partitions,
+    and deltas). If purge is True, also removes all data files associated with the table.
+    Raises an error if the given table does not exist.
+    """
+    raise NotImplementedError("delete_table not implemented")
+
+
+def delete_namespace(
+    namespace: str,
+    purge: bool = False,
+    *args,
+    **kwargs,
+) -> None:
+    """
+    Drops a table namespace and all its contents. If purge is True, then all
+    tables, table versions, and deltas will be deleted. Otherwise, the namespace
+    will be dropped only if it is empty. Raises an error if the given namespace
+    does not exist.
+    """
+    raise NotImplementedError("drop_namespace not implemented")
+
+
+def get_stream_by_id(
+    table_version_locator: TableVersionLocator,
+    stream_id: str,
+    *args,
+    **kwargs,
+) -> Optional[Partition]:
+    """
+    Gets the stream for the given table version locator and stream ID.
+    Returns None if the stream does not exist. Raises an error if the given
+    table version locator does not exist.
+    """
+    raise NotImplementedError("get_stream_by_id not implemented")
+
+
 def get_stream(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
+    stream_format: StreamFormat = StreamFormat.DELTACAT,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Optional[Stream]:
     """
-    Gets the most recently committed stream for the given table version and
-    partition key values. Resolves to the latest active table version if no
-    table version is given. Returns None if the table version does not exist.
+    Gets the most recently committed stream for the given table version.
+    Resolves to the latest active table version if no table version is given.
+    Resolves to the deltacat stream format if no stream format is given.
+    Returns None if the table version or stream format does not exist.
     """
     raise NotImplementedError("get_stream not implemented")
 
 
+def stream_exists(
+    namespace: str,
+    table_name: str,
+    table_version: Optional[str] = None,
+    stream_format: StreamFormat = StreamFormat.DELTACAT,
+    *args,
+    **kwargs,
+) -> bool:
+    """
+    Returns True if the given Stream exists, False if not.
+    Resolves to the latest active table version if no table version is given.
+    Resolves to the DeltaCAT stream format if no stream format is given.
+    Returns None if the table version or stream format does not exist.
+    """
+    raise NotImplementedError("stream_exists not implemented")
+
+
 def stage_partition(
-    stream: Stream, partition_values: Optional[PartitionValues] = None, *args, **kwargs
+    stream: Stream,
+    partition_values: Optional[PartitionValues] = None,
+    partition_scheme_id: Optional[str] = None,
+    *args,
+    **kwargs,
 ) -> Partition:
     """
     Stages a new partition for the given stream and partition values. Returns
     the staged partition. If this partition will replace another partition
-    with the same partition values, then it will have its previous partition ID
-    set to the ID of the partition being replaced. Partition keys should not be
-    specified for unpartitioned tables.
+    with the same partition values and scheme, then it will have its previous
+    partition ID set to the ID of the partition being replaced. Partition values
+    should not be specified for unpartitioned tables.
 
-    The partition_values must represents the results of transforms in a partition
+    The partition_values must represent the results of transforms in a partition
     spec specified in the stream.
     """
     raise NotImplementedError("stage_partition not implemented")
@@ -445,13 +510,18 @@ def commit_partition(
     partition: Partition,
     previous_partition: Optional[Partition] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Partition:
     """
-    Commits the given partition to its associated table version stream,
-    replacing any previous partition (i.e., "partition being replaced") registered for the same stream and
+    Commits the staged partition to its associated table version stream,
+    replacing any previous partition registered for the same stream and
     partition values.
-    If the previous_partition is passed as an argument, the specified previous_partition will be the partition being replaced, otherwise it will be retrieved.
+
+    If previous partition is given then it will be replaced with its deltas
+    prepended to the new partition being committed. Otherwise the latest
+    committed partition with the same keys and partition scheme ID will be
+    retrieved.
+
     Returns the registered partition. If the partition's
     previous delta stream position is specified, then the commit will
     be rejected if it does not match the actual previous stream position of
@@ -463,33 +533,48 @@ def commit_partition(
 
 
 def delete_partition(
-    namespace: str,
-    table_name: str,
-    table_version: Optional[str] = None,
+    stream_locator: StreamLocator,
     partition_values: Optional[PartitionValues] = None,
+    partition_scheme_id: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> None:
     """
-    Deletes the given partition from the specified table version. Resolves to
-    the latest active table version if no table version is given. Partition
+    Deletes the given partition from the specified stream. Partition
     values should not be specified for unpartitioned tables. Raises an error
-    if the table version or partition does not exist.
+    if the partition does not exist.
     """
     raise NotImplementedError("delete_partition not implemented")
 
 
+def get_partition_by_id(
+    stream_locator: StreamLocator,
+    partition_id: str,
+    *args,
+    **kwargs,
+) -> Optional[Partition]:
+    """
+    Gets the partition for the given stream locator and partition ID.
+    Returns None if the partition does not exist. Raises an error if the
+    given stream locator does not exist.
+    """
+    raise NotImplementedError("get_partition_by_id not implemented")
+
+
 def get_partition(
     stream_locator: StreamLocator,
     partition_values: Optional[PartitionValues] = None,
+    partition_scheme_id: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Optional[Partition]:
     """
     Gets the most recently committed partition for the given stream locator and
     partition key values. Returns None if no partition has been committed for
     the given table version and/or partition key values. Partition values
-    should not be specified for unpartitioned tables.
+    should not be specified for unpartitioned tables. Partition scheme ID
+    resolves to the table version's current partition scheme by default.
+    Raises an error if the given stream locator does not exist.
     """
     raise NotImplementedError("get_partition not implemented")
 
@@ -500,14 +585,12 @@ def stage_delta(
     delta_type: DeltaType = DeltaType.UPSERT,
     max_records_per_entry: Optional[int] = None,
     author: Optional[ManifestAuthor] = None,
-    properties: Optional[Dict[str, str]] = None,
+    properties: Optional[DeltaProperties] = None,
     s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
-    delete_parameters: Optional[DeleteParameters] = None,
-    partition_spec: Optional[DeltaPartitionSpec] = None,
-    partition_values: Optional[PartitionValues] = None,
+    entry_params: Optional[EntryParams] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Delta:
     """
     Writes the given table to 1 or more S3 files. Returns an unregistered
@@ -601,7 +684,7 @@ def get_table_version_column_names(
     table_name: str,
     table_version: Optional[str] = None,
     *args,
-    **kwargs
+    **kwargs,
 ) -> Optional[List[str]]:
     """
     Gets a list of column names for the specified table version, or for the
@@ -619,8 +702,8 @@ def get_table_version_schema(
     table_name: str,
     table_version: Optional[str] = None,
     *args,
-    **kwargs
-) -> Optional[Union[pa.Schema, str, bytes]]:
+    **kwargs,
+) -> Optional[Schema]:
     """
     Gets the schema for the specified table version, or for the latest active
     table version if none is specified. Returns None if the table version is