deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/io/reader/deltacat_read_api.py

@@ -0,0 +1,172 @@
+from typing import List, Optional, Union, Dict, Any
+
+from ray.data import Dataset as RayDataset
+from ray.data import read_datasource
+
+from deltacat.io.datasource.deltacat_datasource import DeltaCatDatasource
+from deltacat.io.dataset.deltacat_dataset import DeltaCatDataset
+from deltacat.utils.common import ReadKwargsProvider
+from deltacat.utils.url import DeltaCatUrl, DeltaCatUrlReader
+from deltacat.io.datasource.deltacat_datasource import DeltacatReadType
+
+
+class EmptyReadKwargsProvider(ReadKwargsProvider):
+    def _get_kwargs(
+        self,
+        datasource_type: str,
+        kwargs: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        return {}
+
+
+def read_deltacat(
+    urls: Union[DeltaCatUrl, List[DeltaCatUrl]],
+    *,
+    deltacat_read_type: DeltacatReadType = DeltacatReadType.DATA,
+    timestamp_as_of: Optional[int] = None,
+    merge_on_read: Optional[bool] = False,
+    read_kwargs_provider: Optional[ReadKwargsProvider] = EmptyReadKwargsProvider(),
+) -> DeltaCatDataset:
+    """Reads the given DeltaCAT URLs into a Ray Dataset. DeltaCAT URLs can
+    either reference objects registered in a DeltaCAT catalog, or unregistered
+    external objects that are readable into a Ray Dataset.
+
+    Unless `metadata_only` is `True`, all reads of registered DeltaCAT catalog
+    object data must resolve to a single table version.
+
+    When reading unregistered external objects, all additional keyword
+    arguments specified are passed into the Ray Datasource resolved for the
+    given DeltaCAT URLs.
+
+    Examples:
+        >>> # Read the latest active DeltaCAT table version:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table")
+        >>> # If `my_catalog is the default catalog, this is equivalent to:
+        >>> dc.io.read_deltacat("namespace://my_namespace/my_table")
+        >>> # If `my_namespace` is the default namespace, this is equivalent to:
+        >>> dc.io.read_deltacat("table://my_table")
+
+        >>> # Read metadata from all partitions and deltas of the latest active
+        >>> # DeltaCAT table version:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table", metadata_only=True)
+        >>> # Since "default" always resolves to the latest active table version.
+        >>> # This is equivalent to:
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table/default", metadata_only=True)
+
+        >>> # Read only the latest active table version's top-level metadata:
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table/default", metadata_only=True, recursive=False)
+
+        >>> # Read only top-level metadata from a DeltaCAT table:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table", metadata_only=True, recursive=False)
+
+        >>> # Read top-level table metadata from all table versions:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table/*", metadata_only=True, recursive=False)
+
+        >>> # Read metadata from all partitions and deltas of all table versions:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table/*", metadata_only=True)
+
+        >>> # Read metadata from all tables and table versions of the namespace:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/*", metadata_only=True)
+
+        >>> # Read metadata from the latest active table version for each
+        >>> # table in the namespace:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace", metadata_only=True)
+
+        >>> # Read metadata from the latest active table version for each
+        >>> # table in the namespace:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace", metadata_only=True)
+
+        >>> # Read metadata from the latest active table version for each
+        >>> # table in the catalog's default namespace:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog", metadata_only=True)
+
+        >>> # Read metadata from all table versions for each table in each
+        >>> # catalog namespace:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/*", metadata_only=True)
+
+        >>> # Read the Iceberg stream of the latest active DeltaCAT table version,
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("dc://my_catalog/my_namespace/my_table/default/iceberg")
+        >>> # Or, if `my_catalog` is the default catalog, this is equivalent to:
+        >>> dc.io.read_deltacat("namespace://my_namespace/my_table/default/iceberg")
+        >>> # Or, if `my_namespace` is the default namespace, this is equivalent to:
+        >>> dc.io.read_deltacat("table://my_table/default/iceberg")
+
+        >>> # Read an external unregistered Iceberg table `my_db.my_table`:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("iceberg://my_db.my_table")
+
+        >>> # Read an external unregistered audio file from /my/audio.mp4:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("audio+file:///my/audio.mp4")
+
+        >>> # Read an external unregistered audio file from s3://my/audio.mp4:
+        >>> import deltacat as dc
+        >>> dc.io.read_deltacat("audio+s3://my/audio.mp4")
+
+    Args:
+        urls: The DeltaCAT URLs to read.
+        deltacat_read_type: If METADATA, reads only DeltaCAT metadata for the
+            given URL and skips both recursive metadata expansion and reads
+            of the underlying data files. If METADATA_RECURSIVE then recursively
+            expands child metadata but does not read underlying data files. If
+            DATA then recursively expands child metadata to discover and read
+            all underlying data files.
+        timestamp_as_of: Reads a historic snapshot of the given paths as-of the
+            given millisecond-precision epoch timestamp (only used when reading
+            registered DeltaCAT catalog objects).
+        merge_on_read: If True, merges all unmaterialized inserts, updates,
+            and deletes in the registered DeltaCAT table version being read. Only
+            applicable if `metadata_only` is False.
+            ray_remote_args: kwargs passed to `ray.remote` in the read tasks.
+        read_kwargs_provider: Resolves
+            :class:`~deltacat.types.media.DatasourceType` string keys to
+            kwarg dictionaries to pass to the resolved
+            :class:`~ray.data.Datasource` implementation for each distinct
+            DeltaCAT URL type.
+
+    Returns:
+        DeltacatDataset holding Arrow records read from the specified URL.
+    """
+    # TODO(pdames): The below implementation serializes reads of each URL and
+    #   then unions their respective datasets together. While this was an easy
+    #   starting point to implement, a more efficient implementation should push
+    #   all URLs down into `DeltacatDatasource` to parallelize all reads
+    #   (i.e., by returning the `ReadTask` for all datasources in
+    #   `get_read_tasks()` and estimating the corresponding memory size across
+    #   all datasources in `estimate_inmemory_data_size()`.
+    dataset: RayDataset = None
+    for url in urls:
+        if not url.is_deltacat_catalog_url():
+            # this URL points to an external unregistered Ray Datasource
+            # TODO(pdames): Honor metadata only reads of external datasources
+            #  by registering only file paths & metadata in delta manifests.
+            reader = DeltaCatUrlReader(url)
+            next_ds = reader.read(read_kwargs_provider(url.datastore_type, {}))
+        else:
+            # this URL points to a registered DeltaCAT object
+            next_ds = read_datasource(
+                DeltaCatDatasource(
+                    url=url,
+                    deltacat_read_type=deltacat_read_type,
+                    timestamp_as_of=timestamp_as_of,
+                    merge_on_read=merge_on_read,
+                    read_kwargs_provider=read_kwargs_provider,
+                )
+            )
+        # union the last dataset read into the result set
+        if not dataset:
+            dataset = next_ds
+        else:
+            dataset.union(next_ds)
+    return DeltaCatDataset.from_dataset(dataset)

deltacat/storage/__init__.py

@@ -20,6 +20,9 @@ from deltacat.storage.model.metafile import (
 from deltacat.storage.model.transaction import (
     TransactionOperation,
     Transaction,
+    read_transaction,
+    transactions,
+    transaction,
 )
 from deltacat.storage.model.namespace import (
     Namespace,
@@ -31,6 +34,7 @@ from deltacat.storage.model.partition import (
     PartitionLocator,
     PartitionLocatorAlias,
     PartitionKey,
+    PartitionKeyList,
     PartitionScheme,
     PartitionSchemeList,
     PartitionValues,
@@ -43,6 +47,9 @@ from deltacat.storage.model.schema import (
     NestedFieldName,
     Schema,
     SchemaList,
+    SchemaUpdate,
+    SchemaUpdateOperation,
+    SchemaUpdateOperations,
 )
 from deltacat.storage.model.stream import (
     Stream,
@@ -75,9 +82,11 @@ from deltacat.storage.model.transform import (
     MonthTransform,
     YearTransform,
     TruncateTransform,
+    TruncateStrategy,
 )
 from deltacat.storage.model.types import (
     CommitState,
+    Dataset,
     DeltaType,
     DistributedDataset,
     LifecycleState,
@@ -87,11 +96,12 @@ from deltacat.storage.model.types import (
     SchemaConsistencyType,
     StreamFormat,
     SortOrder,
-    TransactionType,
     TransactionOperationType,
+    TransactionStatus,
 )
 from deltacat.storage.model.sort_key import (
     SortKey,
+    SortKeyList,
     SortScheme,
     SortSchemeList,
 )
@@ -102,6 +112,7 @@ __all__ = [
     "BucketTransform",
     "BucketTransformParameters",
     "CommitState",
+    "Dataset",
     "DayTransform",
     "Delta",
     "DeltaLocator",
@@ -136,6 +147,7 @@ __all__ = [
     "NullOrder",
     "Partition",
     "PartitionKey",
+    "PartitionKeyList",
     "PartitionLocator",
     "PartitionLocatorAlias",
     "PartitionScheme",
@@ -143,8 +155,12 @@ __all__ = [
     "PartitionValues",
     "Schema",
     "SchemaList",
+    "SchemaUpdate",
+    "SchemaUpdateOperation",
+    "SchemaUpdateOperations",
     "SchemaConsistencyType",
     "SortKey",
+    "SortKeyList",
     "SortOrder",
     "SortScheme",
     "SortSchemeList",
@@ -161,13 +177,17 @@ __all__ = [
     "Transaction",
     "TransactionOperation",
     "TransactionOperationType",
-    "TransactionType",
+    "TransactionStatus",
     "Transform",
     "TransformName",
     "TransformParameters",
     "TruncateTransform",
     "TruncateTransformParameters",
+    "TruncateStrategy",
     "UnknownTransform",
     "VoidTransform",
     "YearTransform",
+    "read_transaction",
+    "transactions",
+    "transaction",
 ]

deltacat/storage/interface.py

@@ -2,6 +2,7 @@ from typing import Any, Callable, Dict, List, Optional, Union, Tuple
 
 from deltacat.storage import (
     EntryParams,
+    EntryType,
     Delta,
     DeltaLocator,
     DeltaProperties,
@@ -30,11 +31,12 @@ from deltacat.storage import (
     TableVersionProperties,
 )
 from deltacat.storage.model.manifest import Manifest
+from deltacat.storage.model.partition import UNKNOWN_PARTITION_ID
 from deltacat.types.media import (
     ContentType,
     DistributedDatasetType,
     StorageType,
-    TableType,
+    DatasetType,
 )
 from deltacat.utils.common import ReadKwargsProvider
 
@@ -205,7 +207,7 @@ def get_latest_delta(
 
 def download_delta(
     delta_like: Union[Delta, DeltaLocator],
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     storage_type: StorageType = StorageType.DISTRIBUTED,
     max_parallelism: Optional[int] = None,
     columns: Optional[List[str]] = None,
@@ -216,7 +218,7 @@ def download_delta(
     **kwargs,
 ) -> Union[LocalDataset, DistributedDataset]:  # type: ignore
     """
-    Download the given delta or delta locator into either a list of
+    Reads the given delta or delta locator into either a list of
     tables resident in the local node's memory, or into a dataset distributed
     across this Ray cluster's object store memory. Ordered table N of a local
     table list, or ordered block N of a distributed dataset, always contain
@@ -228,19 +230,19 @@ def download_delta(
 def download_delta_manifest_entry(
     delta_like: Union[Delta, DeltaLocator],
     entry_index: int,
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     *args,
     **kwargs,
 ) -> LocalTable:
     """
-    Downloads a single manifest entry into the specified table type for the
+    Reads a single manifest entry into the specified table type for the
     given delta or delta locator. If a delta is provided with a non-empty
-    manifest, then the entry is downloaded from this manifest. Otherwise, the
-    manifest is first retrieved then the given entry index downloaded.
+    manifest, then the entry is read from this manifest. Otherwise, the
+    manifest is first retrieved then the given entry index read.
 
-    NOTE: The entry will be downloaded in the current node's memory.
+    NOTE: The entry will be read in the current node's memory.
     """
     raise NotImplementedError("download_delta_manifest_entry not implemented")
 
@@ -288,9 +290,9 @@ def create_table_version(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
+    lifecycle_state: Optional[LifecycleState] = LifecycleState.CREATED,
     schema: Optional[Schema] = None,
     partition_scheme: Optional[PartitionScheme] = None,
-    # TODO(pdames): rename to `sort_scheme`
     sort_keys: Optional[SortScheme] = None,
     table_version_description: Optional[str] = None,
     table_version_properties: Optional[TableVersionProperties] = None,
@@ -299,9 +301,9 @@ def create_table_version(
     supported_content_types: Optional[List[ContentType]] = None,
     *args,
     **kwargs,
-) -> Tuple[Optional[Table], TableVersion, Stream]:
+) -> Tuple[Table, TableVersion, Stream]:
     """
-    Create a table version with an unreleased lifecycle state and an empty delta
+    Create a table version with the given or CREATED lifecycle state and an empty delta
     stream. Table versions may be schemaless and unpartitioned to improve write
     performance, or have their writes governed by a schema and partition scheme
     to improve data consistency and read performance.
@@ -314,6 +316,20 @@ def create_table_version(
     raise NotImplementedError("create_table_version not implemented")
 
 
+def create_table(
+    namespace: str,
+    table_name: str,
+    description: Optional[str] = None,
+    properties: Optional[TableProperties] = None,
+    *args,
+    **kwargs,
+) -> Table:
+    """
+    Create a new table. Raises an error if the given table already exists.
+    """
+    raise NotImplementedError("create_table not implemented")
+
+
 def update_table(
     namespace: str,
     table_name: str,
@@ -322,7 +338,7 @@ def update_table(
     new_table_name: Optional[str] = None,
     *args,
     **kwargs,
-) -> None:
+) -> Table:
     """
     Update table metadata describing the table versions it contains. By default,
     a table's properties are empty, and its description is equal to that given
@@ -345,7 +361,7 @@ def update_table_version(
     sort_keys: Optional[SortScheme] = None,
     *args,
     **kwargs,
-) -> None:
+) -> Tuple[Optional[Table], TableVersion, Optional[Stream]]:
     """
     Update a table version. Notably, updating an unreleased table version's
     lifecycle state to 'active' telegraphs that it is ready for external
@@ -410,15 +426,15 @@ def delete_stream(
 
 def delete_table(
     namespace: str,
-    name: str,
+    table_name: str,
     purge: bool = False,
     *args,
     **kwargs,
 ) -> None:
     """
-    Drops the given table and all its contents (table versions, streams, partitions,
-    and deltas). If purge is True, also removes all data files associated with the table.
-    Raises an error if the given table does not exist.
+    Drops the given table from the catalog. If purge is True, also removes
+    all data files associated with the table. Raises an error if the given table
+    does not exist.
     """
     raise NotImplementedError("delete_table not implemented")
 
@@ -430,10 +446,9 @@ def delete_namespace(
     **kwargs,
 ) -> None:
     """
-    Drops a table namespace and all its contents. If purge is True, then all
-    tables, table versions, and deltas will be deleted. Otherwise, the namespace
-    will be dropped only if it is empty. Raises an error if the given namespace
-    does not exist.
+    Drops the given namespace from the catalog. If purge is True, also removes
+    all data files associated with the namespace. Raises an error if the given
+    namespace does not exist.
     """
     raise NotImplementedError("drop_namespace not implemented")
 
@@ -509,6 +524,7 @@ def stage_partition(
 def commit_partition(
     partition: Partition,
     previous_partition: Optional[Partition] = None,
+    expected_previous_partition_id: Optional[str] = UNKNOWN_PARTITION_ID,
     *args,
     **kwargs,
 ) -> Partition:
@@ -586,23 +602,19 @@ def stage_delta(
     max_records_per_entry: Optional[int] = None,
     author: Optional[ManifestAuthor] = None,
     properties: Optional[DeltaProperties] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
     entry_params: Optional[EntryParams] = None,
+    entry_type: Optional[EntryType] = EntryType.DATA,
+    schema: Optional[Schema] = None,
+    sort_scheme_id: Optional[str] = None,
     *args,
     **kwargs,
 ) -> Delta:
     """
-    Writes the given table to 1 or more S3 files. Returns an unregistered
+    Writes the given dataset to 1 or more files. Returns an unregistered
     delta whose manifest entries point to the uploaded files. Applies any
     schema consistency policies configured for the parent table version.
-
-    The partition spec will be used to split the input table into
-    multiple files. Optionally, partition_values can be provided to avoid
-    this method to recompute partition_values from the provided data.
-
-    Raises an error if the provided data does not conform to a unique ordered
-    list of partition_values
     """
     raise NotImplementedError("stage_delta not implemented")
 
@@ -723,13 +735,23 @@ def table_version_exists(
 
 def can_categorize(e: BaseException, *args, **kwargs) -> bool:
     """
-    Return whether input error is from storage implementation layer.
+    True if the input error originated from the storage
+    implementation layer and can be categorized under an
+    existing DeltaCatError. The "categorize_errors" decorator
+    uses this to determine if an unknown error from the storage
+    implementation can be categorized prior to casting it to
+    the equivalent DeltaCatError via `raise_categorized_error`
     """
     raise NotImplementedError
 
 
 def raise_categorized_error(e: BaseException, *args, **kwargs):
     """
-    Raise and handle storage implementation layer specific errors.
+    Casts a categorizable error that originaed from the storage
+    implementation layer to its equivalent DeltaCatError
+    for uniform handling (e.g., determining whether an error
+    is retryable or not) via the "categorize_errors" decorator.
+    Raises an UnclassifiedDeltaCatError from the input exception
+    if the error cannot be categorized.
     """
     raise NotImplementedError