deltacat 1.1.8__py3-none-any.whl → 1.1.10__py3-none-any.whl

deltacat/storage/interface.py

@@ -23,6 +23,10 @@ from deltacat.storage import (
     TableVersion,
     SortKey,
     PartitionLocator,
+    PartitionFilter,
+    PartitionValues,
+    DeltaPartitionSpec,
+    StreamPartitionSpec,
 )
 from deltacat.types.media import (
     ContentType,
@@ -86,12 +90,13 @@ def list_stream_partitions(stream: Stream, *args, **kwargs) -> ListResult[Partit
 def list_deltas(
     namespace: str,
     table_name: str,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     table_version: Optional[str] = None,
     first_stream_position: Optional[int] = None,
     last_stream_position: Optional[int] = None,
     ascending_order: Optional[bool] = None,
     include_manifest: bool = False,
+    partition_filter: Optional[PartitionFilter] = None,
     *args,
     **kwargs
 ) -> ListResult[Delta]:
@@ -107,6 +112,9 @@ def list_deltas(
     To conserve memory, the deltas returned do not include manifests by
     default. The manifests can either be optionally retrieved as part of this
     call or lazily loaded via subsequent calls to `get_delta_manifest`.
+
+    Note: partition_values is deprecated and will be removed in future releases.
+    Use partition_filter instead.
     """
     raise NotImplementedError("list_deltas not implemented")
 
@@ -134,9 +142,10 @@ def get_delta(
     namespace: str,
     table_name: str,
     stream_position: int,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     table_version: Optional[str] = None,
     include_manifest: bool = False,
+    partition_filter: Optional[PartitionFilter] = None,
     *args,
     **kwargs
 ) -> Optional[Delta]:
@@ -149,6 +158,9 @@ def get_delta(
     To conserve memory, the delta returned does not include a manifest by
     default. The manifest can either be optionally retrieved as part of this
     call or lazily loaded via a subsequent call to `get_delta_manifest`.
+
+    Note: partition_values is deprecated and will be removed in future releases.
+    Use partition_filter instead.
     """
     raise NotImplementedError("get_delta not implemented")
 
@@ -156,9 +168,10 @@ def get_delta(
 def get_latest_delta(
     namespace: str,
     table_name: str,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     table_version: Optional[str] = None,
     include_manifest: bool = False,
+    partition_filter: Optional[PartitionFilter] = None,
     *args,
     **kwargs
 ) -> Optional[Delta]:
@@ -172,6 +185,9 @@ def get_latest_delta(
     To conserve memory, the delta returned does not include a manifest by
     default. The manifest can either be optionally retrieved as part of this
     call or lazily loaded via a subsequent call to `get_delta_manifest`.
+
+    Note: partition_values is deprecated and will be removed in future releases.
+    Use partition_filter instead.
     """
     raise NotImplementedError("get_latest_delta not implemented")
 
@@ -185,6 +201,7 @@ def download_delta(
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     ray_options_provider: Callable[[int, Any], Dict[str, Any]] = None,
     distributed_dataset_type: DistributedDatasetType = DistributedDatasetType.RAY_DATASET,
+    partition_filter: Optional[PartitionFilter] = None,
     *args,
     **kwargs
 ) -> Union[LocalDataset, DistributedDataset]:  # type: ignore
@@ -194,6 +211,10 @@ def download_delta(
     across this Ray cluster's object store memory. Ordered table N of a local
     table list, or ordered block N of a distributed dataset, always contain
     the contents of ordered delta manifest entry N.
+
+    partition_filter is an optional parameter which determines which files to
+    download from the delta manifest. A delta manifest contains all the data files
+    for a given delta.
     """
     raise NotImplementedError("download_delta not implemented")
 
@@ -268,6 +289,7 @@ def create_table_version(
     table_description: Optional[str] = None,
     table_properties: Optional[Dict[str, str]] = None,
     supported_content_types: Optional[List[ContentType]] = None,
+    partition_spec: Optional[StreamPartitionSpec] = None,
     *args,
     **kwargs
 ) -> Stream:
@@ -300,6 +322,8 @@ def create_table_version(
 
     Validate: Raise an error for any fields that don't fit the schema. An
     explicit subset of column names to validate may optionally be specified.
+
+    Either partition_keys or partition_spec must be specified but not both.
     """
     raise NotImplementedError("create_table_version not implemented")
 
@@ -402,7 +426,7 @@ def get_stream(
 
 
 def stage_partition(
-    stream: Stream, partition_values: Optional[List[Any]] = None, *args, **kwargs
+    stream: Stream, partition_values: Optional[PartitionValues] = None, *args, **kwargs
 ) -> Partition:
     """
     Stages a new partition for the given stream and partition values. Returns
@@ -410,6 +434,9 @@ def stage_partition(
     with the same partition values, then it will have its previous partition ID
     set to the ID of the partition being replaced. Partition keys should not be
     specified for unpartitioned tables.
+
+    The partition_values must represents the results of transforms in a partition
+    spec specified in the stream.
     """
     raise NotImplementedError("stage_partition not implemented")
 
@@ -439,7 +466,7 @@ def delete_partition(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     *args,
     **kwargs
 ) -> None:
@@ -454,7 +481,7 @@ def delete_partition(
 
 def get_partition(
     stream_locator: StreamLocator,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     *args,
     **kwargs
 ) -> Optional[Partition]:
@@ -477,6 +504,8 @@ def stage_delta(
     s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
     delete_parameters: Optional[DeleteParameters] = None,
+    partition_spec: Optional[DeltaPartitionSpec] = None,
+    partition_values: Optional[PartitionValues] = None,
     *args,
     **kwargs
 ) -> Delta:
@@ -484,6 +513,13 @@ def stage_delta(
     Writes the given table to 1 or more S3 files. Returns an unregistered
     delta whose manifest entries point to the uploaded files. Applies any
     schema consistency policies configured for the parent table version.
+
+    The partition spec will be used to split the input table into
+    multiple files. Optionally, partition_values can be provided to avoid
+    this method to recompute partition_values from the provided data.
+
+    Raises an error if the provided data does not conform to a unique ordered
+    list of partition_values
     """
     raise NotImplementedError("stage_delta not implemented")
 
@@ -600,3 +636,17 @@ def table_version_exists(
     Returns True if the given table version exists, False if not.
     """
     raise NotImplementedError("table_version_exists not implemented")
+
+
+def can_categorize(e: BaseException, *args, **kwargs) -> bool:
+    """
+    Return whether input error is from storage implementation layer.
+    """
+    raise NotImplementedError
+
+
+def raise_categorized_error(e: BaseException, *args, **kwargs):
+    """
+    Raise and handle storage implementation layer specific errors.
+    """
+    raise NotImplementedError

deltacat/storage/model/delta.py

@@ -12,6 +12,7 @@ from deltacat.storage.model.stream import StreamLocator
 from deltacat.storage.model.table import TableLocator
 from deltacat.storage.model.table_version import TableVersionLocator
 from deltacat.storage.model.types import DeltaType
+from deltacat.storage.model.partition_spec import DeltaPartitionSpec, PartitionValues
 
 
 class Delta(dict):
@@ -24,6 +25,7 @@ class Delta(dict):
         manifest: Optional[Manifest],
         previous_stream_position: Optional[int] = None,
         delete_parameters: Optional[DeleteParameters] = None,
+        partition_spec: Optional[DeltaPartitionSpec] = None,
     ) -> Delta:
         """
         Creates a Delta metadata model with the given Delta Locator, Delta Type,
@@ -38,6 +40,7 @@ class Delta(dict):
         delta.manifest = manifest
         delta.previous_stream_position = previous_stream_position
         delta.delete_parameters = delete_parameters
+        delta.partition_spec = partition_spec
         return delta
 
     @staticmethod
@@ -90,6 +93,12 @@ class Delta(dict):
                 f"Deltas to merge must all share the same delta type "
                 f"(found {len(distinct_delta_types)} delta types)."
             )
+        distinct_partition_spec = set([d.partition_spec for d in deltas])
+        if len(distinct_partition_spec) > 1:
+            raise ValueError(
+                f"Deltas to merge must all share the same partition spec "
+                f"(found {len(distinct_partition_spec)} partition specs)."
+            )
         merged_manifest = Manifest.merge_manifests(
             manifests,
             manifest_author,
@@ -252,7 +261,7 @@ class Delta(dict):
         return None
 
     @property
-    def partition_values(self) -> Optional[List[Any]]:
+    def partition_values(self) -> Optional[PartitionValues]:
         delta_locator = self.locator
         if delta_locator:
             return delta_locator.partition_values
@@ -276,6 +285,17 @@ class Delta(dict):
     def delete_parameters(self, delete_parameters: Optional[DeleteParameters]) -> None:
         self["delete_parameters"] = delete_parameters
 
+    @property
+    def partition_spec(self) -> Optional[DeltaPartitionSpec]:
+        val: Dict[str, Any] = self.get("partitionSpec")
+        if val is not None and not isinstance(val, DeltaPartitionSpec):
+            self.partition_spec = val = DeltaPartitionSpec(val)
+        return val
+
+    @partition_spec.setter
+    def partition_spec(self, value: Optional[DeltaPartitionSpec]) -> None:
+        self["partitionSpec"] = value
+
 
 class DeltaLocator(Locator, dict):
     @staticmethod
@@ -299,7 +319,7 @@ class DeltaLocator(Locator, dict):
         table_version: Optional[str],
         stream_id: Optional[str],
         storage_type: Optional[str],
-        partition_values: Optional[List[Any]],
+        partition_values: Optional[PartitionValues],
         partition_id: Optional[str],
         stream_position: Optional[int],
     ) -> DeltaLocator:
@@ -365,7 +385,7 @@ class DeltaLocator(Locator, dict):
         return None
 
     @property
-    def partition_values(self) -> Optional[List[Any]]:
+    def partition_values(self) -> Optional[PartitionValues]:
         partition_locator = self.partition_locator
         if partition_locator:
             return partition_locator.partition_values

deltacat/storage/model/partition.py

@@ -1,10 +1,9 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
-
 from typing import Any, Dict, List, Optional, Union
 
 import pyarrow as pa
-
+from deltacat.storage.model.partition_spec import PartitionValues
 from deltacat.storage.model.locator import Locator
 from deltacat.storage.model.namespace import NamespaceLocator
 from deltacat.storage.model.stream import StreamLocator
@@ -127,7 +126,7 @@ class Partition(dict):
         return None
 
     @property
-    def partition_values(self) -> Optional[List[Any]]:
+    def partition_values(self) -> Optional[PartitionValues]:
         partition_locator = self.locator
         if partition_locator:
             return partition_locator.partition_values
@@ -199,7 +198,7 @@ class PartitionLocator(Locator, dict):
     @staticmethod
     def of(
         stream_locator: Optional[StreamLocator],
-        partition_values: Optional[List[Any]],
+        partition_values: Optional[PartitionValues],
         partition_id: Optional[str],
     ) -> PartitionLocator:
         """
@@ -225,7 +224,7 @@ class PartitionLocator(Locator, dict):
         table_version: Optional[str],
         stream_id: Optional[str],
         storage_type: Optional[str],
-        partition_values: Optional[List[Any]],
+        partition_values: Optional[PartitionValues],
         partition_id: Optional[str],
     ) -> PartitionLocator:
         stream_locator = StreamLocator.at(
@@ -253,11 +252,11 @@ class PartitionLocator(Locator, dict):
         self["streamLocator"] = stream_locator
 
     @property
-    def partition_values(self) -> Optional[List[Any]]:
+    def partition_values(self) -> Optional[PartitionValues]:
         return self.get("partitionValues")
 
     @partition_values.setter
-    def partition_values(self, partition_values: Optional[List[Any]]) -> None:
+    def partition_values(self, partition_values: Optional[PartitionValues]) -> None:
         self["partitionValues"] = partition_values
 
     @property

deltacat/storage/model/partition_spec.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+from typing import List, Optional, Any
+from deltacat.storage.model.transform import Transform
+
+"""
+An ordered list of partition values determining the values of
+ordered transforms specified in the partition spec.
+"""
+PartitionValues = List[Any]
+
+
+class PartitionFilter(dict):
+    """
+    This class represents a filter for partitions.
+    It is used to filter partitions based on certain criteria.
+    """
+
+    @staticmethod
+    def of(
+        partition_values: Optional[PartitionValues] = None,
+    ) -> PartitionFilter:
+        """
+        Creates a new PartitionFilter instance with the specified partition key and value.
+        """
+        partition_filter = PartitionFilter()
+        partition_filter["partitionValues"] = partition_values
+        return partition_filter
+
+    @property
+    def partition_values(self) -> Optional[PartitionValues]:
+        return self.get("partitionValues")
+
+
+class PartitionSpec(dict):
+    """
+    This class determines how the underlying entities in the
+    hierarchy are partitioned. Stream partitions deltas and
+    delta partitions files.
+    """
+
+    @staticmethod
+    def of(ordered_transforms: List[Transform] = None) -> PartitionSpec:
+        partition_spec = PartitionSpec()
+        partition_spec.ordered_transforms = ordered_transforms
+        return partition_spec
+
+    @property
+    def ordered_transforms(self) -> List[Transform]:
+        return self.get("orderedTransforms")
+
+    @ordered_transforms.setter
+    def ordered_transforms(self, value: List[Transform]) -> None:
+        self["orderedTransforms"] = value
+
+
+class StreamPartitionSpec(PartitionSpec):
+    """
+    A class representing a stream partition specification.
+    A stream partitions deltas into multiple different Partition
+    """
+
+    pass
+
+
+class DeltaPartitionSpec(PartitionSpec):
+    """
+    A class representing delta partition specification.
+    The manifest entries in delta are partitioned based on this spec.
+    """
+
+    pass

deltacat/storage/model/stream.py

@@ -8,6 +8,7 @@ from deltacat.storage.model.namespace import NamespaceLocator
 from deltacat.storage.model.table import TableLocator
 from deltacat.storage.model.table_version import TableVersionLocator
 from deltacat.storage.model.types import CommitState
+from deltacat.storage.model.partition_spec import StreamPartitionSpec, PartitionValues
 
 
 class Stream(dict):
@@ -17,12 +18,14 @@ class Stream(dict):
         partition_keys: Optional[List[Dict[str, Any]]],
         state: Optional[CommitState] = None,
         previous_stream_digest: Optional[bytes] = None,
+        partition_spec: Optional[StreamPartitionSpec] = None,
     ) -> Stream:
         stream = Stream()
         stream.locator = locator
         stream.partition_keys = partition_keys
         stream.state = state
         stream.previous_stream_digest = previous_stream_digest
+        stream.partition_spec = partition_spec
         return stream
 
     @property
@@ -38,6 +41,14 @@ class Stream(dict):
 
     @property
     def partition_keys(self) -> Optional[List[Dict[str, Any]]]:
+        """
+        Ordered list of unique column names in the table schema on
+        which the underlying data is partitioned. Either partition_spec
+        or partition_keys must be specified but not both.
+
+        (Deprecated): Partition keys will be deprecated in the favor
+        of partition_spec in future releases.
+        """
         return self.get("partitionKeys")
 
     @partition_keys.setter
@@ -46,6 +57,9 @@ class Stream(dict):
 
     @property
     def previous_stream_digest(self) -> Optional[str]:
+        """
+        Previous stream digest
+        """
         return self.get("previousStreamDigest")
 
     @previous_stream_digest.setter
@@ -54,6 +68,9 @@ class Stream(dict):
 
     @property
     def state(self) -> Optional[CommitState]:
+        """
+        The commit state of a stream.
+        """
         state = self.get("state")
         return None if state is None else CommitState(state)
 
@@ -61,6 +78,26 @@ class Stream(dict):
     def state(self, state: Optional[CommitState]) -> None:
         self["state"] = state
 
+    @property
+    def partition_spec(self) -> Optional[StreamPartitionSpec]:
+        """
+        If a table uses complex partitioning instead of identity,
+        partition spec can be specified to define that strategy.
+        For example, a partition spec can define a bucketing strategy
+        on composite column values or can define iceberg compliant
+        bucketing.
+
+        Either partition_spec or partition_keys must be specified but not both.
+        """
+        val: Dict[str, Any] = self.get("partitionSpec")
+        if val is not None and not isinstance(val, StreamPartitionSpec):
+            self.partition_spec = val = StreamPartitionSpec(val)
+        return val
+
+    @partition_spec.setter
+    def partition_spec(self, spec: StreamPartitionSpec) -> None:
+        self["partitionSpec"] = spec
+
     @property
     def namespace_locator(self) -> Optional[NamespaceLocator]:
         stream_locator = self.locator
@@ -110,7 +147,7 @@ class Stream(dict):
             return stream_locator.table_version
         return None
 
-    def validate_partition_values(self, partition_values: Optional[List[Any]]):
+    def validate_partition_values(self, partition_values: Optional[PartitionValues]):
         # TODO (pdames): ensure value data types match key data types
         partition_keys = self.partition_keys
         num_keys = len(partition_keys) if partition_keys else 0

deltacat/storage/model/transform.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+from typing import List
+from enum import Enum
+
+
+class TransformName(str, Enum):
+    IDENTITY = "identity"
+    BUCKET = "bucket"
+
+
+class TransformParameters(dict):
+    """
+    This is a parent class that contains properties
+    to be passed to the corresponding transform
+    """
+
+    pass
+
+
+class IdentityTransformParameters(TransformParameters):
+    """
+    This class is used to pass parameters to the identity transform
+    """
+
+    @staticmethod
+    def of(column_name: str) -> IdentityTransformParameters:
+        identify_transform_parameters = IdentityTransformParameters()
+        identify_transform_parameters["columnName"] = column_name
+        return identify_transform_parameters
+
+    @property
+    def column_name(self) -> str:
+        """
+        The name of the column to use for identity transform
+        """
+        return self["columnName"]
+
+    @column_name.setter
+    def column_name(self, value: str) -> None:
+        self["columnName"] = value
+
+
+class BucketingStrategy(str, Enum):
+    """
+    A bucketing strategy for the transform
+    """
+
+    # Uses default deltacat bucketing strategy.
+    # This strategy supports hashing on composite keys
+    # and uses SHA1 hashing for determining the bucket.
+    # If no columns passed, it will use a random UUID
+    # for determining the bucket.
+    DEFAULT = "default"
+
+    # Uses iceberg compliant bucketing strategy.
+    # As indicated in the iceberg spec, it does not support
+    # composite keys and uses murmur3 hash for determining
+    # the bucket.
+    # See https://iceberg.apache.org/spec/#partitioning
+    ICEBERG = "iceberg"
+
+
+class BucketTransformParameters(TransformParameters):
+    """
+    Encapsulates parameters for the bucket transform.
+    """
+
+    def of(
+        self,
+        num_buckets: int,
+        column_names: List[str],
+        bucketing_strategy: BucketingStrategy,
+    ) -> BucketTransformParameters:
+        bucket_transform_parameters = BucketTransformParameters()
+        bucket_transform_parameters["numBuckets"] = num_buckets
+        bucket_transform_parameters["columnNames"] = column_names
+        bucket_transform_parameters["bucketingStrategy"] = bucketing_strategy
+
+        return bucket_transform_parameters
+
+    @property
+    def num_buckets(self) -> int:
+        """
+        The total number of buckets to create for values of the column
+        """
+        return self["numBuckets"]
+
+    @property
+    def column_names(self) -> List[str]:
+        """
+        An ordered list of unique column names from the table schema
+        to use for bucketings.
+        """
+        return self["columnNames"]
+
+    @property
+    def bucketing_strategy(self) -> BucketingStrategy:
+        """
+        The bucketing strategy to used.
+        """
+        return self["bucketingStrategy"]
+
+
+class Transform(dict):
+    """
+    A transform is represents how a particular column value can be
+    transformed into a new value. This is mostly used in the context
+    of partitioning the data files in a table.
+    """
+
+    @staticmethod
+    def of(
+        name: TransformName,
+        parameters: TransformParameters,
+    ) -> Transform:
+        partition_transform = Transform()
+        partition_transform["name"] = name
+        partition_transform["parameters"] = parameters
+        return partition_transform
+
+    @property
+    def name(self) -> TransformName:
+        return self["name"]
+
+    @property
+    def parameters(self) -> TransformParameters:
+        return self["parameters"]

deltacat/tests/aws/test_s3u.py

@@ -20,6 +20,7 @@ from botocore.exceptions import (
     ConnectTimeoutError,
     HTTPClientError,
 )
+from ray.data.datasource import FilenameProvider
 from deltacat.exceptions import NonRetryableError
 from moto import mock_s3
 from tenacity import RetryError
@@ -34,6 +35,7 @@ class TestUuidBlockWritePathProvider(unittest.TestCase):
 
         result = provider("base_path")
 
+        self.assertTrue(isinstance(provider, FilenameProvider))
         self.assertRegex(result, r"^base_path/[\w-]{36}$")