deltacat 1.1.36__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/storage/model/delta.py

@@ -1,31 +1,50 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+import posixpath
 from typing import Any, Dict, List, Optional
 
-from deltacat.aws.redshift import Manifest, ManifestAuthor, ManifestMeta
-from deltacat.storage.model.delete_parameters import DeleteParameters
-from deltacat.storage.model.locator import Locator
+import pyarrow
+
+from deltacat.storage.model.metafile import Metafile, MetafileRevisionInfo
+from deltacat.constants import TXN_DIR_NAME
+from deltacat.storage.model.manifest import (
+    Manifest,
+    ManifestMeta,
+    ManifestAuthor,
+)
+from deltacat.storage.model.locator import (
+    Locator,
+    LocatorName,
+)
 from deltacat.storage.model.namespace import NamespaceLocator
-from deltacat.storage.model.partition import PartitionLocator
+from deltacat.storage.model.partition import (
+    PartitionLocator,
+    PartitionValues,
+)
 from deltacat.storage.model.stream import StreamLocator
-from deltacat.storage.model.table import TableLocator
+from deltacat.storage.model.table import (
+    TableLocator,
+    Table,
+)
 from deltacat.storage.model.table_version import TableVersionLocator
-from deltacat.storage.model.types import DeltaType
-from deltacat.storage.model.partition_spec import DeltaPartitionSpec, PartitionValues
+from deltacat.storage.model.types import (
+    DeltaType,
+    StreamFormat,
+)
 
+DeltaProperties = Dict[str, Any]
 
-class Delta(dict):
+
+class Delta(Metafile):
     @staticmethod
     def of(
         locator: Optional[DeltaLocator],
         delta_type: Optional[DeltaType],
         meta: Optional[ManifestMeta],
-        properties: Optional[Dict[str, str]],
+        properties: Optional[DeltaProperties],
         manifest: Optional[Manifest],
         previous_stream_position: Optional[int] = None,
-        delete_parameters: Optional[DeleteParameters] = None,
-        partition_spec: Optional[DeltaPartitionSpec] = None,
     ) -> Delta:
         """
         Creates a Delta metadata model with the given Delta Locator, Delta Type,
@@ -39,8 +58,6 @@ class Delta(dict):
         delta.properties = properties
         delta.manifest = manifest
         delta.previous_stream_position = previous_stream_position
-        delta.delete_parameters = delete_parameters
-        delta.partition_spec = partition_spec
         return delta
 
     @staticmethod
@@ -48,7 +65,7 @@ class Delta(dict):
         deltas: List[Delta],
         manifest_author: Optional[ManifestAuthor] = None,
         stream_position: Optional[int] = None,
-        properties: Optional[Dict[str, str]] = None,
+        properties: Optional[DeltaProperties] = None,
     ) -> Delta:
         """
         Merges the input list of deltas into a single delta. All input deltas to
@@ -93,25 +110,10 @@ class Delta(dict):
                 f"Deltas to merge must all share the same delta type "
                 f"(found {len(distinct_delta_types)} delta types)."
             )
-        distinct_partition_spec = set([d.partition_spec for d in deltas])
-        if len(distinct_partition_spec) > 1:
-            raise ValueError(
-                f"Deltas to merge must all share the same partition spec "
-                f"(found {len(distinct_partition_spec)} partition specs)."
-            )
         merged_manifest = Manifest.merge_manifests(
             manifests,
             manifest_author,
         )
-        distinct_delta_type = list(distinct_delta_types)[0]
-        merged_delete_parameters = None
-        if distinct_delta_type is DeltaType.DELETE:
-            delete_parameters: List[DeleteParameters] = [
-                d.delete_parameters for d in deltas if d.delete_parameters
-            ]
-            merged_delete_parameters: Optional[
-                DeleteParameters
-            ] = DeleteParameters.merge_delete_parameters(delete_parameters)
         partition_locator = deltas[0].partition_locator
         prev_positions = [d.previous_stream_position for d in deltas]
         prev_position = None if None in prev_positions else max(prev_positions)
@@ -122,7 +124,6 @@ class Delta(dict):
             properties,
             merged_manifest,
             prev_position,
-            merged_delete_parameters,
         )
 
     @property
@@ -148,11 +149,11 @@ class Delta(dict):
         self["meta"] = meta
 
     @property
-    def properties(self) -> Optional[Dict[str, str]]:
+    def properties(self) -> Optional[DeltaProperties]:
         return self.get("properties")
 
     @properties.setter
-    def properties(self, properties: Optional[Dict[str, str]]) -> None:
+    def properties(self, properties: Optional[DeltaProperties]) -> None:
         self["properties"] = properties
 
     @property
@@ -222,7 +223,7 @@ class Delta(dict):
     def storage_type(self) -> Optional[str]:
         delta_locator = self.locator
         if delta_locator:
-            return delta_locator.storage_type
+            return delta_locator.stream_format
         return None
 
     @property
@@ -253,6 +254,13 @@ class Delta(dict):
             return delta_locator.stream_id
         return None
 
+    @property
+    def stream_format(self) -> Optional[str]:
+        delta_locator = self.locator
+        if delta_locator:
+            return delta_locator.stream_format
+        return None
+
     @property
     def partition_id(self) -> Optional[str]:
         delta_locator = self.locator
@@ -274,27 +282,64 @@ class Delta(dict):
             return delta_locator.stream_position
         return None
 
-    @property
-    def delete_parameters(self) -> Optional[DeleteParameters]:
-        delete_parameters = self.get("delete_parameters")
-        return (
-            None if delete_parameters is None else DeleteParameters(delete_parameters)
-        )
+    def to_serializable(self) -> Delta:
+        serializable = self
+        if serializable.table_locator:
+            serializable: Delta = Delta.update_for(self)
+            # remove the mutable table locator
+            serializable.table_version_locator.table_locator = TableLocator.at(
+                namespace=self.id,
+                table_name=self.id,
+            )
+        return serializable
+
+    def from_serializable(
+        self,
+        path: str,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    ) -> Delta:
+        # TODO(pdames): Lazily restore table locator on 1st property get.
+        #  Cache Metafile ID <-> Table/Namespace-Name map at Catalog Init, then
+        #  swap only Metafile IDs with Names here.
+        if self.table_locator and self.table_locator.table_name == self.id:
+            parent_rev_dir_path = Metafile._parent_metafile_rev_dir_path(
+                base_metafile_path=path,
+                parent_number=4,
+            )
+            txn_log_dir = posixpath.join(
+                posixpath.dirname(
+                    posixpath.dirname(
+                        posixpath.dirname(parent_rev_dir_path),
+                    )
+                ),
+                TXN_DIR_NAME,
+            )
+            table = Table.read(
+                MetafileRevisionInfo.latest_revision(
+                    revision_dir_path=parent_rev_dir_path,
+                    filesystem=filesystem,
+                    success_txn_log_dir=txn_log_dir,
+                ).path,
+                filesystem,
+            )
+            self.table_version_locator.table_locator = table.locator
+        return self
 
-    @delete_parameters.setter
-    def delete_parameters(self, delete_parameters: Optional[DeleteParameters]) -> None:
-        self["delete_parameters"] = delete_parameters
+
+class DeltaLocatorName(LocatorName):
+    def __init__(self, locator: DeltaLocator):
+        self.locator = locator
 
     @property
-    def partition_spec(self) -> Optional[DeltaPartitionSpec]:
-        val: Dict[str, Any] = self.get("partitionSpec")
-        if val is not None and not isinstance(val, DeltaPartitionSpec):
-            self.partition_spec = val = DeltaPartitionSpec(val)
-        return val
+    def immutable_id(self) -> Optional[str]:
+        return str(self.locator.stream_position)
 
-    @partition_spec.setter
-    def partition_spec(self, value: Optional[DeltaPartitionSpec]) -> None:
-        self["partitionSpec"] = value
+    @immutable_id.setter
+    def immutable_id(self, immutable_id: Optional[str]):
+        self.locator.stream_position = int(immutable_id)
+
+    def parts(self) -> List[str]:
+        return [str(self.locator.stream_position)]
 
 
 class DeltaLocator(Locator, dict):
@@ -318,25 +363,37 @@ class DeltaLocator(Locator, dict):
         table_name: Optional[str],
         table_version: Optional[str],
         stream_id: Optional[str],
-        storage_type: Optional[str],
+        stream_format: Optional[StreamFormat],
         partition_values: Optional[PartitionValues],
         partition_id: Optional[str],
         stream_position: Optional[int],
     ) -> DeltaLocator:
-        partition_locator = PartitionLocator.at(
-            namespace,
-            table_name,
-            table_version,
-            stream_id,
-            storage_type,
-            partition_values,
-            partition_id,
+        partition_locator = (
+            PartitionLocator.at(
+                namespace,
+                table_name,
+                table_version,
+                stream_id,
+                stream_format,
+                partition_values,
+                partition_id,
+            )
+            if partition_values and partition_id
+            else None
         )
         return DeltaLocator.of(
             partition_locator,
             stream_position,
         )
 
+    @property
+    def name(self):
+        return DeltaLocatorName(self)
+
+    @property
+    def parent(self) -> Optional[PartitionLocator]:
+        return self.partition_locator
+
     @property
     def partition_locator(self) -> Optional[PartitionLocator]:
         val: Dict[str, Any] = self.get("partitionLocator")
@@ -406,10 +463,10 @@ class DeltaLocator(Locator, dict):
         return None
 
     @property
-    def storage_type(self) -> Optional[str]:
+    def stream_format(self) -> Optional[str]:
         partition_locator = self.partition_locator
         if partition_locator:
-            return partition_locator.storage_type
+            return partition_locator.stream_format
         return None
 
     @property
@@ -432,13 +489,3 @@ class DeltaLocator(Locator, dict):
         if partition_locator:
             return partition_locator.table_version
         return None
-
-    def canonical_string(self) -> str:
-        """
-        Returns a unique string for the given locator that can be used
-        for equality checks (i.e. two locators are equal if they have
-        the same canonical string).
-        """
-        pl_hexdigest = self.partition_locator.hexdigest()
-        stream_position = self.stream_position
-        return f"{pl_hexdigest}|{stream_position}"

deltacat/storage/model/interop.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+from typing import Generic, Optional, TypeVar
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+
+class ModelMapper(ABC, Generic[T, U]):
+    @staticmethod
+    @abstractmethod
+    def map(obj: Optional[T], *args, **kwargs) -> Optional[U]:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def unmap(obj: Optional[U], **kwargs) -> Optional[T]:
+        pass
+
+
+class OneWayModelMapper(ABC, Generic[T, U]):
+    @staticmethod
+    @abstractmethod
+    def map(obj: Optional[T], **kwargs) -> Optional[U]:
+        pass

deltacat/storage/model/list_result.py

@@ -21,6 +21,14 @@ class ListResult(dict, Generic[T]):
         list_result["nextPageProvider"] = next_page_provider
         return list_result
 
+    @staticmethod
+    def empty() -> ListResult:
+        list_result = ListResult()
+        list_result["items"] = []
+        list_result["paginationKey"] = None
+        list_result["nextPageProvider"] = None
+        return list_result
+
     def read_page(self) -> Optional[List[T]]:
         return self.get("items")
 

deltacat/storage/model/locator.py

@@ -1,14 +1,104 @@
+# Allow classes to use self-referencing Type hints in Python 3.7.
+from __future__ import annotations
+
+from typing import Optional, List
+
 from deltacat.utils.common import sha1_digest, sha1_hexdigest
 
+DEFAULT_NAME_SEPARATOR = "|"
+DEFAULT_PATH_SEPARATOR = "/"
+
+
+class LocatorName:
+    """
+    Assigns a name to a catalog object. All sibling catalog objects must be
+    assigned unique names (e.g., all namespaces in a catalog must be assigned
+    unique locator names, all tables under a namespace must be assigned unique
+    locator names, etc.). Names may be mutable (e.g., namespace and table names)
+    or immutable (e.g., partition/stream IDs and delta stream positions). Names
+    may be single or multi-part.
+    """
+
+    @property
+    def immutable_id(self) -> Optional[str]:
+        """
+        If this locator name is immutable (i.e., if the object it refers to
+        can't be renamed) then returns an immutable ID suitable for use in
+        URLS or filesystem paths. Returns None if this locator name is mutable
+        (i.e., if the object it refers to can be renamed).
+        """
+        raise NotImplementedError()
+
+    @immutable_id.setter
+    def immutable_id(self, immutable_id: Optional[str]) -> None:
+        """
+        If this locator name is immutable (i.e., if the object it refers to
+        can't be renamed), then sets an immutable ID for this
+        locator name suitable for use in URLS or filesystem paths. Note that
+        the ID is only considered immutable in durable catalog storage, and
+        remains mutable in transient memory (i.e., this setter remains
+        functional regardless of whether an ID is already assigned, but each
+        update causes it to refer to a new, distinct object in durable storage).
+        """
+        raise NotImplementedError()
+
+    def parts(self) -> List[str]:
+        """
+        Returns the ordered parts of this locator's name.
+        """
+        raise NotImplementedError()
+
+    def join(self, separator: str = DEFAULT_NAME_SEPARATOR) -> str:
+        """
+        Returns this locator name as a string by joining its parts with the
+        given separator.
+        """
+        return separator.join(self.parts())
+
+    def exists(self) -> bool:
+        """
+        Returns True if this locator name is defined, False otherwise.
+        """
+        return self.immutable_id or all(self.parts())
+
 
 class Locator:
-    def canonical_string(self) -> str:
+    """
+    Creates a globally unique reference to any named catalog object. Locators
+    are composed of the name of the referenced catalog object and its parent
+    Locator (if any). Every Locator has a canonical string representation that
+    can be used for global equality checks. Cryptographic digests of this
+    canonical string can be used for uniform random hash distribution and
+    path-based references to the underlying catalog object in filesystems or
+    URLs.
+    """
+
+    @property
+    def name(self) -> LocatorName:
+        """
+        Returns the name of this locator.
+        """
+        raise NotImplementedError()
+
+    @property
+    def parent(self) -> Optional[Locator]:
+        """
+        Returns the parent of this locator, if any.
+        """
+        raise NotImplementedError()
+
+    def canonical_string(self, separator: str = DEFAULT_NAME_SEPARATOR) -> str:
         """
         Returns a unique string for the given locator that can be used
         for equality checks (i.e. two locators are equal if they have
         the same canonical string).
         """
-        raise NotImplementedError()
+        parts = []
+        parent_hexdigest = self.parent.hexdigest() if self.parent else None
+        if parent_hexdigest:
+            parts.append(parent_hexdigest)
+        parts.extend(self.name.parts())
+        return separator.join([str(part) for part in parts])
 
     def digest(self) -> bytes:
         """
@@ -26,7 +116,7 @@ class Locator:
         """
         return sha1_hexdigest(self.canonical_string().encode("utf-8"))
 
-    def path(self, root: str, separator: str = "/") -> str:
+    def path(self, root: str, separator: str = DEFAULT_PATH_SEPARATOR) -> str:
         """
         Returns a path for the locator of the form: "{root}/{hexdigest}", where
         the default path separator of "/" may optionally be overridden with