deltacat 2.0.0b11__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/catalog/interface.py

@@ -1,21 +1,31 @@
 from typing import Any, Dict, List, Optional, Union
 
-from deltacat.storage.model.partition import PartitionScheme
+from deltacat.storage.model.partition import (
+    Partition,
+    PartitionLocator,
+    PartitionScheme,
+)
 from deltacat.catalog.model.table_definition import TableDefinition
 from deltacat.storage.model.sort_key import SortScheme
 from deltacat.storage.model.list_result import ListResult
 from deltacat.storage.model.namespace import Namespace, NamespaceProperties
-from deltacat.storage.model.schema import Schema
+from deltacat.storage.model.schema import (
+    Schema,
+    SchemaUpdateOperations,
+)
 from deltacat.storage.model.table import TableProperties
+from deltacat.storage.model.table_version import TableVersionProperties
 from deltacat.storage.model.types import (
-    DistributedDataset,
+    Dataset,
     LifecycleState,
-    LocalDataset,
-    LocalTable,
     StreamFormat,
 )
+from deltacat.storage.model.transaction import Transaction
 from deltacat.types.media import ContentType
-from deltacat.types.tables import TableWriteMode
+from deltacat.types.tables import (
+    DatasetType,
+    TableWriteMode,
+)
 
 
 # catalog functions
@@ -34,40 +44,73 @@ def initialize(*args, **kwargs) -> Optional[Any]:
 
 # table functions
 def write_to_table(
-    data: Union[LocalTable, LocalDataset, DistributedDataset],
+    data: Dataset,
     table: str,
     *args,
     namespace: Optional[str] = None,
+    table_version: Optional[str] = None,
     mode: TableWriteMode = TableWriteMode.AUTO,
     content_type: ContentType = ContentType.PARQUET,
+    transaction: Optional[Transaction] = None,
     **kwargs,
 ) -> None:
-    """Write data to a DeltaCat table.
+    """Write local or distributed data to a table. Raises an error if the
+    table does not exist and the table write mode is not CREATE or AUTO.
+
+    When creating a table, all `create_table` parameters may be optionally
+    specified as additional keyword arguments. When appending to, or replacing,
+    an existing table, all `alter_table` parameters may be optionally specified
+    as additional keyword arguments.
 
     Args:
-        data: Data to write to the table. Can be a LocalTable, LocalDataset, or DistributedDataset.
+        data: Local or distributed data to write to the table.
         table: Name of the table to write to.
-        namespace: Optional namespace of the table. Uses default namespace if not specified.
-        mode: Write mode to use when writing to the table.
-        content_type: Content type of the data being written.
-
-    Returns:
-        None
+        namespace: Optional namespace for the table. Uses default if not specified.
+        table_version: Optional version of the table to write to. If specified,
+            will create this version if it doesn't exist (in CREATE mode) or
+            get this version if it exists (in other modes). If not specified,
+            uses the latest version.
+        mode: Write mode (AUTO, CREATE, APPEND, REPLACE, MERGE, DELETE).
+        content_type: Content type used to write the data files. Defaults to PARQUET.
+        transaction: Optional transaction to append write operations to instead of
+            creating and committing a new transaction.
+        **kwargs: Additional keyword arguments.
     """
     raise NotImplementedError("write_to_table not implemented")
 
 
 def read_table(
-    table: str, *args, namespace: Optional[str] = None, **kwargs
-) -> DistributedDataset:
-    """Read data from a DeltaCat table.
+    table: str,
+    *args,
+    namespace: Optional[str] = None,
+    table_version: Optional[str] = None,
+    read_as: DatasetType = DatasetType.DAFT,
+    partition_filter: Optional[List[Union[Partition, PartitionLocator]]] = None,
+    max_parallelism: Optional[int] = None,
+    columns: Optional[List[str]] = None,
+    file_path_column: Optional[str] = None,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
+) -> Dataset:
+    """Read a table into a dataset.
 
     Args:
-        table: Name of the table to read from.
-        namespace: Optional namespace of the table. Uses default namespace if not specified.
+        table: Name of the table to read.
+        namespace: Optional namespace of the table. Uses default if not specified.
+        table_version: Optional specific version of the table to read.
+        read_as: Dataset type to use for reading table files. Defaults to DatasetType.DAFT.
+        partition_filter: Optional list of partitions to read from.
+        max_parallelism: Optional maximum parallelism for data download. Defaults to the number of
+            available CPU cores for local dataset type reads (i.e., members of DatasetType.local())
+            and 100 for distributed dataset type reads (i.e., members of DatasetType.distributed()).
+        columns: Optional list of columns to include in the result.
+        file_path_column: Optional column name to add file paths to the result.
+        transaction: Optional transaction to chain this read operation to. If provided, uncommitted
+            changes from the transaction will be visible to this read operation.
+        **kwargs: Additional keyword arguments.
 
     Returns:
-        A Deltacat DistributedDataset containing the table data.
+        Dataset containing the table data.
     """
     raise NotImplementedError("read_table not implemented")
 
@@ -76,12 +119,16 @@ def alter_table(
     table: str,
     *args,
     namespace: Optional[str] = None,
+    table_version: Optional[str] = None,
     lifecycle_state: Optional[LifecycleState] = None,
-    schema_updates: Optional[Dict[str, Any]] = None,
+    schema_updates: Optional[SchemaUpdateOperations] = None,
     partition_updates: Optional[Dict[str, Any]] = None,
-    sort_keys: Optional[SortScheme] = None,
-    description: Optional[str] = None,
-    properties: Optional[TableProperties] = None,
+    sort_scheme: Optional[SortScheme] = None,
+    table_description: Optional[str] = None,
+    table_version_description: Optional[str] = None,
+    table_properties: Optional[TableProperties] = None,
+    table_version_properties: Optional[TableVersionProperties] = None,
+    transaction: Optional[Transaction] = None,
     **kwargs,
 ) -> None:
     """Alter deltacat table/table_version definition.
@@ -92,36 +139,44 @@ def alter_table(
     Args:
         table: Name of the table to alter.
         namespace: Optional namespace of the table. Uses default namespace if not specified.
+        table_version: Optional specific version of the table to alter. Defaults to the latest active version.
         lifecycle_state: New lifecycle state for the table.
-        schema_updates: Map of schema updates to apply.
-        partition_updates: Map of partition scheme updates to apply.
-        sort_keys: New sort keys scheme.
-        description: New description for the table.
-        properties: New table properties.
+        schema_updates: Schema updates to apply.
+        partition_updates: Partition scheme updates to apply.
+        sort_scheme: New sort scheme.
+        table_description: New description for the table.
+        table_version_description: New description for the table version. Defaults to `table_description` if not  specified.
+        table_properties: New table properties.
+        table_version_properties: New table version properties. Defaults to the current parent table properties if not specified.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         None
 
     Raises:
         TableNotFoundError: If the table does not already exist.
+        TableVersionNotFoundError: If the specified table version or active table version does not exist.
     """
     raise NotImplementedError("alter_table not implemented")
 
 
 def create_table(
-    name: str,
+    table: str,
     *args,
     namespace: Optional[str] = None,
-    version: Optional[str] = None,
+    table_version: Optional[str] = None,
     lifecycle_state: Optional[LifecycleState] = LifecycleState.ACTIVE,
     schema: Optional[Schema] = None,
     partition_scheme: Optional[PartitionScheme] = None,
     sort_keys: Optional[SortScheme] = None,
-    description: Optional[str] = None,
+    table_description: Optional[str] = None,
+    table_version_description: Optional[str] = None,
     table_properties: Optional[TableProperties] = None,
+    table_version_properties: Optional[TableVersionProperties] = None,
     namespace_properties: Optional[NamespaceProperties] = None,
     content_types: Optional[List[ContentType]] = None,
     fail_if_exists: bool = True,
+    transaction: Optional[Transaction] = None,
     **kwargs,
 ) -> TableDefinition:
     """Create an empty table in the catalog.
@@ -130,18 +185,21 @@ def create_table(
     Additionally if the provided namespace does not exist, it will be created for you.
 
     Args:
-        name: Name of the table to create.
+        table: Name of the table to create.
         namespace: Optional namespace for the table. Uses default namespace if not specified.
         version: Optional version identifier for the table.
         lifecycle_state: Lifecycle state of the new table. Defaults to ACTIVE.
         schema: Schema definition for the table.
         partition_scheme: Optional partitioning scheme for the table.
         sort_keys: Optional sort keys for the table.
-        description: Optional description of the table.
+        table_description: Optional description of the table.
+        table_version_description: Optional description for the table version.
         table_properties: Optional properties for the table.
+        table_version_properties: Optional properties for the table version. Defaults to the current parent table properties if not specified.
         namespace_properties: Optional properties for the namespace if it needs to be created.
         content_types: Optional list of allowed content types for the table.
         fail_if_exists: If True, raises an error if table already exists. If False, returns existing table.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         TableDefinition object for the created or existing table.
@@ -150,40 +208,53 @@ def create_table(
         TableAlreadyExistsError: If the table already exists and fail_if_exists is True.
         NamespaceNotFoundError: If the provided namespace does not exist.
     """
+
     raise NotImplementedError("create_table not implemented")
 
 
 def drop_table(
-    name: str,
+    table: str,
     *args,
     namespace: Optional[str] = None,
     table_version: Optional[str] = None,
     purge: bool = False,
+    transaction: Optional[Transaction] = None,
     **kwargs,
 ) -> None:
     """Drop a table from the catalog and optionally purges underlying data.
 
     Args:
-        name: Name of the table to drop.
+        table: Name of the table to drop.
         namespace: Optional namespace of the table. Uses default namespace if not specified.
-        table_version: Optional specific version of the table to drop.
+        table_version: Optional specific version of the table to drop. Defaults to the latest active version.
         purge: If True, permanently delete the table data. If False, only remove from catalog.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         None
 
     Raises:
         TableNotFoundError: If the table does not exist.
+        TableVersionNotFoundError: If the table version does not exist.
     """
     raise NotImplementedError("drop_table not implemented")
 
 
-def refresh_table(table: str, *args, namespace: Optional[str] = None, **kwargs) -> None:
+def refresh_table(
+    table: str,
+    *args,
+    namespace: Optional[str] = None,
+    table_version: Optional[str] = None,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
+) -> None:
     """Refresh metadata cached on the Ray cluster for the given table.
 
     Args:
         table: Name of the table to refresh.
         namespace: Optional namespace of the table. Uses default namespace if not specified.
+        table_version: Optional specific version of the table to refresh. Defaults to the latest active version.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         None
@@ -192,12 +263,18 @@ def refresh_table(table: str, *args, namespace: Optional[str] = None, **kwargs)
 
 
 def list_tables(
-    *args, namespace: Optional[str] = None, **kwargs
+    *args,
+    namespace: Optional[str] = None,
+    table: Optional[str] = None,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
 ) -> ListResult[TableDefinition]:
     """List a page of table definitions.
 
     Args:
         namespace: Optional namespace to list tables from. Uses default namespace if not specified.
+        table: Optional table to list its table versions. If not specified, lists the latest active version of each table in the namespace.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         ListResult containing TableDefinition objects for tables in the namespace.
@@ -206,11 +283,12 @@ def list_tables(
 
 
 def get_table(
-    name: str,
+    table: str,
     *args,
     namespace: Optional[str] = None,
     table_version: Optional[str] = None,
     stream_format: StreamFormat = StreamFormat.DELTACAT,
+    transaction: Optional[Transaction] = None,
     **kwargs,
 ) -> Optional[TableDefinition]:
     """Get table definition metadata.
@@ -218,29 +296,33 @@ def get_table(
     Args:
         name: Name of the table to retrieve.
         namespace: Optional namespace of the table. Uses default namespace if not specified.
-        table_version: Optional specific version of the table to retrieve.
-            If not specified, the latest version is used.
-        stream_format: Optional stream format to retrieve. Uses the default Deltacat stream
-            format if not specified.
+        table_version: Optional specific version of the table to retrieve. Defaults to the latest active version.
+        stream_format: Optional stream format to retrieve. Defaults to DELTACAT.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
-        Deltacat TableDefinition if the table exists, None otherwise.
-
-    Raises:
-        TableVersionNotFoundError: If the table version does not exist.
-        StreamNotFoundError: If the stream does not exist.
+        Deltacat TableDefinition if the table exists, None otherwise. The table definition's table version will be
+        None if the requested version is not found. The table definition's stream will be None if the requested stream
+        format is not found.
     """
     raise NotImplementedError("get_table not implemented")
 
 
 def truncate_table(
-    table: str, *args, namespace: Optional[str] = None, **kwargs
+    table: str,
+    *args,
+    namespace: Optional[str] = None,
+    table_version: Optional[str] = None,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
 ) -> None:
     """Truncate table data.
 
     Args:
         table: Name of the table to truncate.
         namespace: Optional namespace of the table. Uses default namespace if not specified.
+        table_version: Optional specific version of the table to truncate. Defaults to the latest active version.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         None
@@ -249,7 +331,12 @@ def truncate_table(
 
 
 def rename_table(
-    table: str, new_name: str, *args, namespace: Optional[str] = None, **kwargs
+    table: str,
+    new_name: str,
+    *args,
+    namespace: Optional[str] = None,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
 ) -> None:
     """Rename an existing table.
 
@@ -257,6 +344,7 @@ def rename_table(
         table: Current name of the table.
         new_name: New name for the table.
         namespace: Optional namespace of the table. Uses default namespace if not specified.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         None
@@ -267,12 +355,23 @@ def rename_table(
     raise NotImplementedError("rename_table not implemented")
 
 
-def table_exists(table: str, *args, namespace: Optional[str] = None, **kwargs) -> bool:
+def table_exists(
+    table: str,
+    *args,
+    namespace: Optional[str] = None,
+    table_version: Optional[str] = None,
+    stream_format: StreamFormat = StreamFormat.DELTACAT,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
+) -> bool:
     """Check if a table exists in the catalog.
 
     Args:
         table: Name of the table to check.
         namespace: Optional namespace of the table. Uses default namespace if not specified.
+        table_version: Optional specific version of the table to check. Defaults to the latest active version.
+        stream_format: Optional stream format to check. Defaults to DELTACAT.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         True if the table exists, False otherwise.
@@ -281,11 +380,15 @@ def table_exists(table: str, *args, namespace: Optional[str] = None, **kwargs) -
 
 
 # namespace functions
-def list_namespaces(*args, **kwargs) -> ListResult[Namespace]:
+def list_namespaces(
+    *args,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
+) -> ListResult[Namespace]:
     """List a page of table namespaces.
 
     Args:
-        catalog: Catalog properties instance.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         ListResult containing Namespace objects.
@@ -293,11 +396,17 @@ def list_namespaces(*args, **kwargs) -> ListResult[Namespace]:
     raise NotImplementedError("list_namespaces not implemented")
 
 
-def get_namespace(namespace: str, *args, **kwargs) -> Optional[Namespace]:
+def get_namespace(
+    namespace: str,
+    *args,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
+) -> Optional[Namespace]:
     """Get metadata for a specific table namespace.
 
     Args:
         namespace: Name of the namespace to retrieve.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         Namespace object if the namespace exists, None otherwise.
@@ -305,11 +414,17 @@ def get_namespace(namespace: str, *args, **kwargs) -> Optional[Namespace]:
     raise NotImplementedError("get_namespace not implemented")
 
 
-def namespace_exists(namespace: str, *args, **kwargs) -> bool:
+def namespace_exists(
+    namespace: str,
+    *args,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
+) -> bool:
     """Check if a namespace exists.
 
     Args:
         namespace: Name of the namespace to check.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         True if the namespace exists, False otherwise.
@@ -319,8 +434,9 @@ def namespace_exists(namespace: str, *args, **kwargs) -> bool:
 
 def create_namespace(
     namespace: str,
-    properties: Optional[NamespaceProperties] = None,
     *args,
+    properties: Optional[NamespaceProperties] = None,
+    transaction: Optional[Transaction] = None,
     **kwargs,
 ) -> Namespace:
     """Create a new namespace.
@@ -328,6 +444,7 @@ def create_namespace(
     Args:
         namespace: Name of the namespace to create.
         properties: Optional properties for the namespace.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         Created Namespace object.
@@ -343,6 +460,7 @@ def alter_namespace(
     *args,
     properties: Optional[NamespaceProperties] = None,
     new_namespace: Optional[str] = None,
+    transaction: Optional[Transaction] = None,
     **kwargs,
 ) -> None:
     """Alter a namespace definition.
@@ -351,6 +469,7 @@ def alter_namespace(
         namespace: Name of the namespace to alter.
         properties: Optional new properties for the namespace.
         new_namespace: Optional new name for the namespace.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         None
@@ -358,13 +477,20 @@ def alter_namespace(
     raise NotImplementedError("alter_namespace not implemented")
 
 
-def drop_namespace(namespace: str, *args, purge: bool = False, **kwargs) -> None:
+def drop_namespace(
+    namespace: str,
+    *args,
+    purge: bool = False,
+    transaction: Optional[Transaction] = None,
+    **kwargs,
+) -> None:
     """Drop a namespace and all of its tables from the catalog.
 
     Args:
         namespace: Name of the namespace to drop.
-        purge: If True, permanently delete all tables in the namespace.
-            If False, only remove from catalog.
+        purge: If True, permanently delete all table data in the namespace.
+            If False, only removes the namespace from the catalog.
+        transaction: Optional transaction to use. If None, creates a new transaction.
 
     Returns:
         None
@@ -376,6 +502,6 @@ def default_namespace(*args, **kwargs) -> str:
     """Return the default namespace for the catalog.
 
     Returns:
-        String name of the default namespace.
+        Name of the default namespace.
     """
     raise NotImplementedError("default_namespace not implemented")