deltacat 0.1.10.dev0__py3-none-any.whl → 0.1.12__py3-none-any.whl

deltacat/storage/model/delta.py

@@ -1,26 +1,28 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from deltacat.storage.model.types import DeltaType
+from typing import Any, Dict, List, Optional
+
+from deltacat.aws.redshift import Manifest, ManifestAuthor, ManifestMeta
+from deltacat.storage.model.locator import Locator
 from deltacat.storage.model.namespace import NamespaceLocator
 from deltacat.storage.model.partition import PartitionLocator
 from deltacat.storage.model.stream import StreamLocator
 from deltacat.storage.model.table import TableLocator
 from deltacat.storage.model.table_version import TableVersionLocator
-from deltacat.storage.model.locator import Locator
-from deltacat.aws.redshift import Manifest, ManifestMeta, ManifestAuthor
-
-from typing import Any, Dict, List, Optional
+from deltacat.storage.model.types import DeltaType
 
 
 class Delta(dict):
     @staticmethod
-    def of(locator: Optional[DeltaLocator],
-           delta_type: Optional[DeltaType],
-           meta: Optional[ManifestMeta],
-           properties: Optional[Dict[str, str]],
-           manifest: Optional[Manifest],
-           previous_stream_position: Optional[int] = None) -> Delta:
+    def of(
+        locator: Optional[DeltaLocator],
+        delta_type: Optional[DeltaType],
+        meta: Optional[ManifestMeta],
+        properties: Optional[Dict[str, str]],
+        manifest: Optional[Manifest],
+        previous_stream_position: Optional[int] = None,
+    ) -> Delta:
         """
         Creates a Delta metadata model with the given Delta Locator, Delta Type,
         manifest metadata, properties, manifest, and previous delta stream
@@ -37,10 +39,11 @@ class Delta(dict):
 
     @staticmethod
     def merge_deltas(
-            deltas: List[Delta],
-            manifest_author: Optional[ManifestAuthor] = None,
-            stream_position: Optional[int] = None,
-            properties: Optional[Dict[str, str]] = None) -> Delta:
+        deltas: List[Delta],
+        manifest_author: Optional[ManifestAuthor] = None,
+        stream_position: Optional[int] = None,
+        properties: Optional[Dict[str, str]] = None,
+    ) -> Delta:
         """
         Merges the input list of deltas into a single delta. All input deltas to
         merge must belong to the same partition, share the same delta type, and
@@ -70,18 +73,20 @@ class Delta(dict):
         if len(distinct_storage_types) > 1:
             raise NotImplementedError(
                 f"Deltas to merge must all share the same storage type "
-                f"(found {len(distinct_storage_types)} storage types.")
-        pl_digest_set = set([d.partition_locator.digest()
-                             for d in deltas])
+                f"(found {len(distinct_storage_types)} storage types."
+            )
+        pl_digest_set = set([d.partition_locator.digest() for d in deltas])
         if len(pl_digest_set) > 1:
             raise ValueError(
                 f"Deltas to merge must all belong to the same partition "
-                f"(found {len(pl_digest_set)} partitions).")
+                f"(found {len(pl_digest_set)} partitions)."
+            )
         distinct_delta_types = set([d.type for d in deltas])
         if len(distinct_delta_types) > 1:
             raise ValueError(
                 f"Deltas to merge must all share the same delta type "
-                f"(found {len(distinct_delta_types)} delta types).")
+                f"(found {len(distinct_delta_types)} delta types)."
+            )
         merged_manifest = Manifest.merge_manifests(
             manifests,
             manifest_author,
@@ -106,9 +111,7 @@ class Delta(dict):
         return val
 
     @manifest.setter
-    def manifest(
-            self,
-            manifest: Optional[Manifest]) -> None:
+    def manifest(self, manifest: Optional[Manifest]) -> None:
         self["manifest"] = manifest
 
     @property
@@ -119,9 +122,7 @@ class Delta(dict):
         return val
 
     @meta.setter
-    def meta(
-            self,
-            meta: Optional[ManifestMeta]) -> None:
+    def meta(self, meta: Optional[ManifestMeta]) -> None:
         self["meta"] = meta
 
     @property
@@ -129,9 +130,7 @@ class Delta(dict):
         return self.get("properties")
 
     @properties.setter
-    def properties(
-            self,
-            properties: Optional[Dict[str, str]]) -> None:
+    def properties(self, properties: Optional[Dict[str, str]]) -> None:
         self["properties"] = properties
 
     @property
@@ -140,9 +139,7 @@ class Delta(dict):
         return None if delta_type is None else DeltaType(delta_type)
 
     @type.setter
-    def type(
-            self,
-            delta_type: Optional[DeltaType]) -> None:
+    def type(self, delta_type: Optional[DeltaType]) -> None:
         self["type"] = delta_type
 
     @property
@@ -153,9 +150,7 @@ class Delta(dict):
         return val
 
     @locator.setter
-    def locator(
-            self,
-            delta_locator: Optional[DeltaLocator]) -> None:
+    def locator(self, delta_locator: Optional[DeltaLocator]) -> None:
         self["deltaLocator"] = delta_locator
 
     @property
@@ -163,9 +158,7 @@ class Delta(dict):
         return self.get("previousStreamPosition")
 
     @previous_stream_position.setter
-    def previous_stream_position(
-            self,
-            previous_stream_position: Optional[int]) -> None:
+    def previous_stream_position(self, previous_stream_position: Optional[int]) -> None:
         self["previousStreamPosition"] = previous_stream_position
 
     @property
@@ -262,8 +255,9 @@ class Delta(dict):
 
 class DeltaLocator(Locator, dict):
     @staticmethod
-    def of(partition_locator: Optional[PartitionLocator],
-           stream_position: Optional[int]) -> DeltaLocator:
+    def of(
+        partition_locator: Optional[PartitionLocator], stream_position: Optional[int]
+    ) -> DeltaLocator:
         """
         Creates a partition delta locator. Stream Position, if provided, should
         be greater than that of any prior delta in the partition.
@@ -274,14 +268,16 @@ class DeltaLocator(Locator, dict):
         return delta_locator
 
     @staticmethod
-    def at(namespace: Optional[str],
-           table_name: Optional[str],
-           table_version: Optional[str],
-           stream_id: Optional[str],
-           storage_type: Optional[str],
-           partition_values: Optional[List[Any]],
-           partition_id: Optional[str],
-           stream_position: Optional[int]) -> DeltaLocator:
+    def at(
+        namespace: Optional[str],
+        table_name: Optional[str],
+        table_version: Optional[str],
+        stream_id: Optional[str],
+        storage_type: Optional[str],
+        partition_values: Optional[List[Any]],
+        partition_id: Optional[str],
+        stream_position: Optional[int],
+    ) -> DeltaLocator:
         partition_locator = PartitionLocator.at(
             namespace,
             table_name,
@@ -304,9 +300,7 @@ class DeltaLocator(Locator, dict):
         return val
 
     @partition_locator.setter
-    def partition_locator(
-            self,
-            partition_locator: Optional[PartitionLocator]) -> None:
+    def partition_locator(self, partition_locator: Optional[PartitionLocator]) -> None:
         self["partitionLocator"] = partition_locator
 
     @property

deltacat/storage/model/list_result.py

@@ -1,18 +1,20 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-import ray
 from typing import Callable, Generic, List, Optional, TypeVar
 
-T = TypeVar('T')
+import ray
+
+T = TypeVar("T")
 
 
 class ListResult(dict, Generic[T]):
     @staticmethod
-    def of(items: Optional[List[T]],
-           pagination_key: Optional[str],
-           next_page_provider: Optional[Callable[..., ListResult[T]]]) \
-            -> ListResult:
+    def of(
+        items: Optional[List[T]],
+        pagination_key: Optional[str],
+        next_page_provider: Optional[Callable[..., ListResult[T]]],
+    ) -> ListResult:
         list_result = ListResult()
         list_result["items"] = items
         list_result["paginationKey"] = pagination_key
@@ -35,8 +37,10 @@ class ListResult(dict, Generic[T]):
         if pagination_key:
             next_page_provider = self.next_page_provider
             if next_page_provider is None:
-                raise ValueError(f"Pagination key ('{pagination_key}') "
-                                 f"specified without a next page provider!")
+                raise ValueError(
+                    f"Pagination key ('{pagination_key}') "
+                    f"specified without a next page provider!"
+                )
             next_list_result = next_page_provider(pagination_key)
             if next_list_result.next_page_provider is None:
                 next_list_result["nextPageProvider"] = next_page_provider

deltacat/storage/model/namespace.py

@@ -8,8 +8,9 @@ from deltacat.storage.model.locator import Locator
 
 class Namespace(dict):
     @staticmethod
-    def of(locator: Optional[NamespaceLocator],
-           permissions: Optional[Dict[str, Any]]) -> Namespace:
+    def of(
+        locator: Optional[NamespaceLocator], permissions: Optional[Dict[str, Any]]
+    ) -> Namespace:
         namespace = Namespace()
         namespace.locator = locator
         namespace.permissions = permissions
@@ -23,9 +24,7 @@ class Namespace(dict):
         return val
 
     @locator.setter
-    def locator(
-            self,
-            namespace_locator: Optional[NamespaceLocator]) -> None:
+    def locator(self, namespace_locator: Optional[NamespaceLocator]) -> None:
         self["namespaceLocator"] = namespace_locator
 
     @property

deltacat/storage/model/partition.py

@@ -1,29 +1,31 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+from typing import Any, Dict, List, Optional, Union
+
 import pyarrow as pa
 
+from deltacat.storage.model.locator import Locator
 from deltacat.storage.model.namespace import NamespaceLocator
 from deltacat.storage.model.stream import StreamLocator
 from deltacat.storage.model.table import TableLocator
 from deltacat.storage.model.table_version import TableVersionLocator
 from deltacat.storage.model.types import CommitState
-from deltacat.storage.model.locator import Locator
 from deltacat.types.media import ContentType
 
-from typing import Any, Dict, List, Optional, Union
-
 
 class Partition(dict):
     @staticmethod
-    def of(locator: Optional[PartitionLocator],
-           schema: Optional[Union[pa.Schema, str, bytes]],
-           content_types: Optional[List[ContentType]],
-           state: Optional[CommitState] = None,
-           previous_stream_position: Optional[int] = None,
-           previous_partition_id: Optional[str] = None,
-           stream_position: Optional[int] = None,
-           next_partition_id: Optional[str] = None) -> Partition:
+    def of(
+        locator: Optional[PartitionLocator],
+        schema: Optional[Union[pa.Schema, str, bytes]],
+        content_types: Optional[List[ContentType]],
+        state: Optional[CommitState] = None,
+        previous_stream_position: Optional[int] = None,
+        previous_partition_id: Optional[str] = None,
+        stream_position: Optional[int] = None,
+        next_partition_id: Optional[str] = None,
+    ) -> Partition:
         partition = Partition()
         partition.locator = locator
         partition.schema = schema
@@ -43,9 +45,7 @@ class Partition(dict):
         return val
 
     @locator.setter
-    def locator(
-            self,
-            partition_locator: Optional[PartitionLocator]) -> None:
+    def locator(self, partition_locator: Optional[PartitionLocator]) -> None:
         self["partitionLocator"] = partition_locator
 
     @property
@@ -59,13 +59,16 @@ class Partition(dict):
     @property
     def content_types(self) -> Optional[List[ContentType]]:
         content_types = self.get("contentTypes")
-        return None if content_types is None else \
-            [None if _ is None else ContentType(_) for _ in content_types]
+        return (
+            None
+            if content_types is None
+            else [None if _ is None else ContentType(_) for _ in content_types]
+        )
 
     @content_types.setter
     def content_types(
-            self,
-            supported_content_types: Optional[List[ContentType]]) -> None:
+        self, supported_content_types: Optional[List[ContentType]]
+    ) -> None:
         self["contentTypes"] = supported_content_types
 
     @property
@@ -82,9 +85,7 @@ class Partition(dict):
         return self.get("previousStreamPosition")
 
     @previous_stream_position.setter
-    def previous_stream_position(
-            self,
-            previous_stream_position: Optional[int]) -> None:
+    def previous_stream_position(self, previous_stream_position: Optional[int]) -> None:
         self["previousStreamPosition"] = previous_stream_position
 
     @property
@@ -92,9 +93,7 @@ class Partition(dict):
         return self.get("previousPartitionId")
 
     @previous_partition_id.setter
-    def previous_partition_id(
-            self,
-            previous_partition_id: Optional[str]) -> None:
+    def previous_partition_id(self, previous_partition_id: Optional[str]) -> None:
         self["previousPartitionId"] = previous_partition_id
 
     @property
@@ -191,15 +190,18 @@ class Partition(dict):
 
     def is_supported_content_type(self, content_type: ContentType) -> bool:
         supported_content_types = self.content_types
-        return (not supported_content_types) or \
-               (content_type in supported_content_types)
+        return (not supported_content_types) or (
+            content_type in supported_content_types
+        )
 
 
 class PartitionLocator(Locator, dict):
     @staticmethod
-    def of(stream_locator: Optional[StreamLocator],
-           partition_values: Optional[List[Any]],
-           partition_id: Optional[str]) -> PartitionLocator:
+    def of(
+        stream_locator: Optional[StreamLocator],
+        partition_values: Optional[List[Any]],
+        partition_id: Optional[str],
+    ) -> PartitionLocator:
         """
         Creates a stream partition locator. Partition ID is
         case-sensitive.
@@ -217,13 +219,15 @@ class PartitionLocator(Locator, dict):
         return partition_locator
 
     @staticmethod
-    def at(namespace: Optional[str],
-           table_name: Optional[str],
-           table_version: Optional[str],
-           stream_id: Optional[str],
-           storage_type: Optional[str],
-           partition_values: Optional[List[Any]],
-           partition_id: Optional[str]) -> PartitionLocator:
+    def at(
+        namespace: Optional[str],
+        table_name: Optional[str],
+        table_version: Optional[str],
+        stream_id: Optional[str],
+        storage_type: Optional[str],
+        partition_values: Optional[List[Any]],
+        partition_id: Optional[str],
+    ) -> PartitionLocator:
         stream_locator = StreamLocator.at(
             namespace,
             table_name,
@@ -245,9 +249,7 @@ class PartitionLocator(Locator, dict):
         return val
 
     @stream_locator.setter
-    def stream_locator(
-            self,
-            stream_locator: Optional[StreamLocator]) -> None:
+    def stream_locator(self, stream_locator: Optional[StreamLocator]) -> None:
         self["streamLocator"] = stream_locator
 
     @property
@@ -263,9 +265,7 @@ class PartitionLocator(Locator, dict):
         return self.get("partitionId")
 
     @partition_id.setter
-    def partition_id(
-            self,
-            partition_id: Optional[str]) -> None:
+    def partition_id(self, partition_id: Optional[str]) -> None:
         self["partitionId"] = partition_id
 
     @property

deltacat/storage/model/stream.py

@@ -1,21 +1,23 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+from typing import Any, Dict, List, Optional
+
+from deltacat.storage.model.locator import Locator
 from deltacat.storage.model.namespace import NamespaceLocator
 from deltacat.storage.model.table import TableLocator
 from deltacat.storage.model.table_version import TableVersionLocator
 from deltacat.storage.model.types import CommitState
-from deltacat.storage.model.locator import Locator
-
-from typing import Any, Dict, List, Optional
 
 
 class Stream(dict):
     @staticmethod
-    def of(locator: Optional[StreamLocator],
-           partition_keys: Optional[List[Dict[str, Any]]],
-           state: Optional[CommitState] = None,
-           previous_stream_digest: Optional[bytes] = None) -> Stream:
+    def of(
+        locator: Optional[StreamLocator],
+        partition_keys: Optional[List[Dict[str, Any]]],
+        state: Optional[CommitState] = None,
+        previous_stream_digest: Optional[bytes] = None,
+    ) -> Stream:
         stream = Stream()
         stream.locator = locator
         stream.partition_keys = partition_keys
@@ -31,9 +33,7 @@ class Stream(dict):
         return val
 
     @locator.setter
-    def locator(
-            self,
-            stream_locator: Optional[StreamLocator]) -> None:
+    def locator(self, stream_locator: Optional[StreamLocator]) -> None:
         self["streamLocator"] = stream_locator
 
     @property
@@ -41,9 +41,7 @@ class Stream(dict):
         return self.get("partitionKeys")
 
     @partition_keys.setter
-    def partition_keys(
-            self,
-            partition_keys: Optional[List[Dict[str, Any]]]) -> None:
+    def partition_keys(self, partition_keys: Optional[List[Dict[str, Any]]]) -> None:
         self["partitionKeys"] = partition_keys
 
     @property
@@ -51,9 +49,7 @@ class Stream(dict):
         return self.get("previousStreamDigest")
 
     @previous_stream_digest.setter
-    def previous_stream_digest(
-            self,
-            previous_stream_digest: Optional[str]) -> None:
+    def previous_stream_digest(self, previous_stream_digest: Optional[str]) -> None:
         self["previousStreamDigest"] = previous_stream_digest
 
     @property
@@ -114,9 +110,7 @@ class Stream(dict):
             return stream_locator.table_version
         return None
 
-    def validate_partition_values(
-            self,
-            partition_values: Optional[List[Any]]):
+    def validate_partition_values(self, partition_values: Optional[List[Any]]):
         # TODO (pdames): ensure value data types match key data types
         partition_keys = self.partition_keys
         num_keys = len(partition_keys) if partition_keys else 0
@@ -124,14 +118,17 @@ class Stream(dict):
         if num_values != num_keys:
             raise ValueError(
                 f"Found {num_values} partition values but "
-                f"{num_keys} partition keys: {self}")
+                f"{num_keys} partition keys: {self}"
+            )
 
 
 class StreamLocator(Locator, dict):
     @staticmethod
-    def of(table_version_locator: Optional[TableVersionLocator],
-           stream_id: Optional[str],
-           storage_type: Optional[str]) -> StreamLocator:
+    def of(
+        table_version_locator: Optional[TableVersionLocator],
+        stream_id: Optional[str],
+        storage_type: Optional[str],
+    ) -> StreamLocator:
         """
         Creates a table version Stream Locator. All input parameters are
         case-sensitive.
@@ -143,11 +140,13 @@ class StreamLocator(Locator, dict):
         return stream_locator
 
     @staticmethod
-    def at(namespace: Optional[str],
-           table_name: Optional[str],
-           table_version: Optional[str],
-           stream_id: Optional[str],
-           storage_type: Optional[str]) -> StreamLocator:
+    def at(
+        namespace: Optional[str],
+        table_name: Optional[str],
+        table_version: Optional[str],
+        stream_id: Optional[str],
+        storage_type: Optional[str],
+    ) -> StreamLocator:
         table_version_locator = TableVersionLocator.at(
             namespace,
             table_name,
@@ -168,8 +167,8 @@ class StreamLocator(Locator, dict):
 
     @table_version_locator.setter
     def table_version_locator(
-            self,
-            table_version_locator: Optional[TableVersionLocator]) -> None:
+        self, table_version_locator: Optional[TableVersionLocator]
+    ) -> None:
         self["tableVersionLocator"] = table_version_locator
 
     @property

deltacat/storage/model/table.py

@@ -1,18 +1,20 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from deltacat.storage.model.namespace import NamespaceLocator
-from deltacat.storage.model.locator import Locator
-
 from typing import Any, Dict, Optional
 
+from deltacat.storage.model.locator import Locator
+from deltacat.storage.model.namespace import NamespaceLocator
+
 
 class Table(dict):
     @staticmethod
-    def of(locator: Optional[TableLocator],
-           permissions: Optional[Dict[str, Any]] = None,
-           description: Optional[str] = None,
-           properties: Optional[Dict[str, str]] = None) -> Table:
+    def of(
+        locator: Optional[TableLocator],
+        permissions: Optional[Dict[str, Any]] = None,
+        description: Optional[str] = None,
+        properties: Optional[Dict[str, str]] = None,
+    ) -> Table:
         table = Table()
         table.locator = locator
         table.permissions = permissions
@@ -79,16 +81,16 @@ class Table(dict):
 
 class TableLocator(Locator, dict):
     @staticmethod
-    def of(namespace_locator: Optional[NamespaceLocator],
-           table_name: Optional[str]) -> TableLocator:
+    def of(
+        namespace_locator: Optional[NamespaceLocator], table_name: Optional[str]
+    ) -> TableLocator:
         table_locator = TableLocator()
         table_locator.namespace_locator = namespace_locator
         table_locator.table_name = table_name
         return table_locator
 
     @staticmethod
-    def at(namespace: Optional[str],
-           table_name: Optional[str]) -> TableLocator:
+    def at(namespace: Optional[str], table_name: Optional[str]) -> TableLocator:
         namespace_locator = NamespaceLocator.of(namespace)
         return TableLocator.of(namespace_locator, table_name)
 
@@ -100,9 +102,7 @@ class TableLocator(Locator, dict):
         return val
 
     @namespace_locator.setter
-    def namespace_locator(
-            self,
-            namespace_locator: Optional[NamespaceLocator]) -> None:
+    def namespace_locator(self, namespace_locator: Optional[NamespaceLocator]) -> None:
         self["namespaceLocator"] = namespace_locator
 
     @property