deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/storage/model/delta.py

@@ -1,36 +1,55 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+import json
+import posixpath
 from typing import Any, Dict, List, Optional
 
-from deltacat.aws.redshift import Manifest, ManifestAuthor, ManifestMeta
-from deltacat.storage.model.delete_parameters import DeleteParameters
-from deltacat.storage.model.locator import Locator
+import pyarrow
+
+from deltacat.storage.model.metafile import Metafile, MetafileRevisionInfo
+from deltacat.constants import TXN_DIR_NAME
+from deltacat.storage.model.manifest import (
+    Manifest,
+    ManifestMeta,
+    ManifestAuthor,
+)
+from deltacat.storage.model.locator import (
+    Locator,
+    LocatorName,
+)
 from deltacat.storage.model.namespace import NamespaceLocator
-from deltacat.storage.model.partition import PartitionLocator
+from deltacat.storage.model.partition import (
+    PartitionLocator,
+    PartitionValues,
+)
 from deltacat.storage.model.stream import StreamLocator
-from deltacat.storage.model.table import TableLocator
+from deltacat.storage.model.table import (
+    TableLocator,
+    Table,
+)
 from deltacat.storage.model.table_version import TableVersionLocator
-from deltacat.storage.model.types import DeltaType
-from deltacat.storage.model.partition_spec import DeltaPartitionSpec, PartitionValues
+from deltacat.storage.model.types import (
+    DeltaType,
+    StreamFormat,
+)
 
+DeltaProperties = Dict[str, Any]
 
-class Delta(dict):
+
+class Delta(Metafile):
     @staticmethod
     def of(
         locator: Optional[DeltaLocator],
         delta_type: Optional[DeltaType],
         meta: Optional[ManifestMeta],
-        properties: Optional[Dict[str, str]],
+        properties: Optional[DeltaProperties],
         manifest: Optional[Manifest],
         previous_stream_position: Optional[int] = None,
-        delete_parameters: Optional[DeleteParameters] = None,
-        partition_spec: Optional[DeltaPartitionSpec] = None,
     ) -> Delta:
         """
         Creates a Delta metadata model with the given Delta Locator, Delta Type,
-        manifest metadata, properties, manifest, and previous delta stream
-        position.
+        manifest metadata, properties, manifest, and previous delta stream position.
         """
         delta = Delta()
         delta.locator = locator
@@ -39,8 +58,6 @@ class Delta(dict):
         delta.properties = properties
         delta.manifest = manifest
         delta.previous_stream_position = previous_stream_position
-        delta.delete_parameters = delete_parameters
-        delta.partition_spec = partition_spec
         return delta
 
     @staticmethod
@@ -48,7 +65,7 @@ class Delta(dict):
         deltas: List[Delta],
         manifest_author: Optional[ManifestAuthor] = None,
         stream_position: Optional[int] = None,
-        properties: Optional[Dict[str, str]] = None,
+        properties: Optional[DeltaProperties] = None,
     ) -> Delta:
         """
         Merges the input list of deltas into a single delta. All input deltas to
@@ -93,25 +110,10 @@ class Delta(dict):
                 f"Deltas to merge must all share the same delta type "
                 f"(found {len(distinct_delta_types)} delta types)."
             )
-        distinct_partition_spec = set([d.partition_spec for d in deltas])
-        if len(distinct_partition_spec) > 1:
-            raise ValueError(
-                f"Deltas to merge must all share the same partition spec "
-                f"(found {len(distinct_partition_spec)} partition specs)."
-            )
         merged_manifest = Manifest.merge_manifests(
             manifests,
             manifest_author,
         )
-        distinct_delta_type = list(distinct_delta_types)[0]
-        merged_delete_parameters = None
-        if distinct_delta_type is DeltaType.DELETE:
-            delete_parameters: List[DeleteParameters] = [
-                d.delete_parameters for d in deltas if d.delete_parameters
-            ]
-            merged_delete_parameters: Optional[
-                DeleteParameters
-            ] = DeleteParameters.merge_delete_parameters(delete_parameters)
         partition_locator = deltas[0].partition_locator
         prev_positions = [d.previous_stream_position for d in deltas]
         prev_position = None if None in prev_positions else max(prev_positions)
@@ -122,7 +124,6 @@ class Delta(dict):
             properties,
             merged_manifest,
             prev_position,
-            merged_delete_parameters,
         )
 
     @property
@@ -148,11 +149,11 @@ class Delta(dict):
         self["meta"] = meta
 
     @property
-    def properties(self) -> Optional[Dict[str, str]]:
+    def properties(self) -> Optional[DeltaProperties]:
         return self.get("properties")
 
     @properties.setter
-    def properties(self, properties: Optional[Dict[str, str]]) -> None:
+    def properties(self, properties: Optional[DeltaProperties]) -> None:
         self["properties"] = properties
 
     @property
@@ -222,7 +223,7 @@ class Delta(dict):
     def storage_type(self) -> Optional[str]:
         delta_locator = self.locator
         if delta_locator:
-            return delta_locator.storage_type
+            return delta_locator.stream_format
         return None
 
     @property
@@ -253,6 +254,13 @@ class Delta(dict):
             return delta_locator.stream_id
         return None
 
+    @property
+    def stream_format(self) -> Optional[str]:
+        delta_locator = self.locator
+        if delta_locator:
+            return delta_locator.stream_format
+        return None
+
     @property
     def partition_id(self) -> Optional[str]:
         delta_locator = self.locator
@@ -267,6 +275,13 @@ class Delta(dict):
             return delta_locator.partition_values
         return None
 
+    @property
+    def partition_values_json(self) -> Optional[str]:
+        partition_values = (
+            self.partition_values if self.partition_values is not None else None
+        )
+        return json.dumps(partition_values)
+
     @property
     def stream_position(self) -> Optional[int]:
         delta_locator = self.locator
@@ -274,27 +289,71 @@ class Delta(dict):
             return delta_locator.stream_position
         return None
 
-    @property
-    def delete_parameters(self) -> Optional[DeleteParameters]:
-        delete_parameters = self.get("delete_parameters")
+    def url(self, catalog_name: Optional[str] = None) -> str:
         return (
-            None if delete_parameters is None else DeleteParameters(delete_parameters)
+            f"dc://{catalog_name}/{self.namespace}/{self.table_name}/{self.table_version}/{self.stream_format}/{self.partition_values_json}/{self.stream_position}/"
+            if catalog_name
+            else f"table://{self.namespace}/{self.table_name}/{self.table_version}/{self.stream_format}/{self.partition_values_json}/{self.stream_position}/"
         )
 
-    @delete_parameters.setter
-    def delete_parameters(self, delete_parameters: Optional[DeleteParameters]) -> None:
-        self["delete_parameters"] = delete_parameters
+    def to_serializable(self) -> Delta:
+        serializable = self
+        if serializable.table_locator:
+            serializable: Delta = Delta.update_for(self)
+            # remove the mutable table locator
+            serializable.table_version_locator.table_locator = TableLocator.at(
+                namespace=self.id,
+                table_name=self.id,
+            )
+        return serializable
+
+    def from_serializable(
+        self,
+        path: str,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    ) -> Delta:
+        # TODO(pdames): Lazily restore table locator on 1st property get.
+        #  Cache Metafile ID <-> Table/Namespace-Name map at Catalog Init, then
+        #  swap only Metafile IDs with Names here.
+        if self.table_locator and self.table_locator.table_name == self.id:
+            parent_rev_dir_path = Metafile._parent_metafile_rev_dir_path(
+                base_metafile_path=path,
+                parent_number=4,
+            )
+            txn_log_dir = posixpath.join(
+                posixpath.dirname(
+                    posixpath.dirname(
+                        posixpath.dirname(parent_rev_dir_path),
+                    )
+                ),
+                TXN_DIR_NAME,
+            )
+            table = Table.read(
+                MetafileRevisionInfo.latest_revision(
+                    revision_dir_path=parent_rev_dir_path,
+                    filesystem=filesystem,
+                    success_txn_log_dir=txn_log_dir,
+                ).path,
+                filesystem,
+            )
+            self.table_version_locator.table_locator = table.locator
+        return self
+
+
+class DeltaLocatorName(LocatorName):
+    def __init__(self, locator: DeltaLocator):
+        self.locator = locator
 
     @property
-    def partition_spec(self) -> Optional[DeltaPartitionSpec]:
-        val: Dict[str, Any] = self.get("partitionSpec")
-        if val is not None and not isinstance(val, DeltaPartitionSpec):
-            self.partition_spec = val = DeltaPartitionSpec(val)
-        return val
+    def immutable_id(self) -> Optional[str]:
+        return str(self.locator.stream_position)
+
+    @immutable_id.setter
+    def immutable_id(self, immutable_id: Optional[str]):
+        self.locator.stream_position = int(immutable_id)
 
-    @partition_spec.setter
-    def partition_spec(self, value: Optional[DeltaPartitionSpec]) -> None:
-        self["partitionSpec"] = value
+    def parts(self) -> List[str]:
+        return [str(self.locator.stream_position)]
 
 
 class DeltaLocator(Locator, dict):
@@ -318,25 +377,47 @@ class DeltaLocator(Locator, dict):
         table_name: Optional[str],
         table_version: Optional[str],
         stream_id: Optional[str],
-        storage_type: Optional[str],
+        stream_format: Optional[StreamFormat],
         partition_values: Optional[PartitionValues],
         partition_id: Optional[str],
         stream_position: Optional[int],
     ) -> DeltaLocator:
-        partition_locator = PartitionLocator.at(
-            namespace,
-            table_name,
-            table_version,
-            stream_id,
-            storage_type,
-            partition_values,
-            partition_id,
+        partition_locator = (
+            PartitionLocator.at(
+                namespace,
+                table_name,
+                table_version,
+                stream_id,
+                stream_format,
+                partition_values,
+                partition_id,
+            )
+            if any(
+                [
+                    partition_id,
+                    partition_values,
+                    stream_id,
+                    stream_format,
+                    table_name,
+                    table_version,
+                    namespace,
+                ]
+            )
+            else None
         )
         return DeltaLocator.of(
             partition_locator,
             stream_position,
         )
 
+    @property
+    def name(self):
+        return DeltaLocatorName(self)
+
+    @property
+    def parent(self) -> Optional[PartitionLocator]:
+        return self.partition_locator
+
     @property
     def partition_locator(self) -> Optional[PartitionLocator]:
         val: Dict[str, Any] = self.get("partitionLocator")
@@ -406,10 +487,10 @@ class DeltaLocator(Locator, dict):
         return None
 
     @property
-    def storage_type(self) -> Optional[str]:
+    def stream_format(self) -> Optional[str]:
         partition_locator = self.partition_locator
         if partition_locator:
-            return partition_locator.storage_type
+            return partition_locator.stream_format
         return None
 
     @property
@@ -432,13 +513,3 @@ class DeltaLocator(Locator, dict):
         if partition_locator:
             return partition_locator.table_version
         return None
-
-    def canonical_string(self) -> str:
-        """
-        Returns a unique string for the given locator that can be used
-        for equality checks (i.e. two locators are equal if they have
-        the same canonical string).
-        """
-        pl_hexdigest = self.partition_locator.hexdigest()
-        stream_position = self.stream_position
-        return f"{pl_hexdigest}|{stream_position}"

deltacat/storage/model/expression/__init__.py

@@ -0,0 +1,47 @@
+from deltacat.storage.model.expression.expression import (
+    Expression,
+    UnaryExpression,
+    BinaryExpression,
+    BooleanExpression,
+    Reference,
+    Literal,
+    Equal,
+    NotEqual,
+    GreaterThan,
+    LessThan,
+    GreaterThanEqual,
+    LessThanEqual,
+    And,
+    Or,
+    Not,
+    In,
+    Between,
+    Like,
+    IsNull,
+)
+
+from deltacat.storage.model.expression.visitor import ExpressionVisitor, DisplayVisitor
+
+__all__ = [
+    "Expression",
+    "UnaryExpression",
+    "BinaryExpression",
+    "BooleanExpression",
+    "Reference",
+    "Literal",
+    "Equal",
+    "NotEqual",
+    "GreaterThan",
+    "LessThan",
+    "GreaterThanEqual",
+    "LessThanEqual",
+    "And",
+    "Or",
+    "Not",
+    "In",
+    "Between",
+    "Like",
+    "IsNull",
+    "ExpressionVisitor",
+    "DisplayVisitor",
+]