deltacat 1.1.36__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

deltacat/storage/model/namespace.py

@@ -1,19 +1,23 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, List
 
-from deltacat.storage.model.locator import Locator
+from deltacat.storage.model.metafile import Metafile
+from deltacat.storage.model.locator import Locator, LocatorName
 
+NamespaceProperties = dict[str, Any]
 
-class Namespace(dict):
+
+class Namespace(Metafile):
     @staticmethod
     def of(
-        locator: Optional[NamespaceLocator], permissions: Optional[Dict[str, Any]]
+        locator: Optional[NamespaceLocator],
+        properties: Optional[NamespaceProperties] = None,
     ) -> Namespace:
         namespace = Namespace()
         namespace.locator = locator
-        namespace.permissions = permissions
+        namespace.properties = properties
         return namespace
 
     @property
@@ -35,12 +39,24 @@ class Namespace(dict):
         return None
 
     @property
-    def permissions(self) -> Optional[Dict[str, Any]]:
-        return self.get("permissions")
+    def properties(self) -> Optional[NamespaceProperties]:
+        return self.get("properties")
+
+    @properties.setter
+    def properties(self, properties: Optional[NamespaceProperties]) -> None:
+        self["properties"] = properties
+
+
+class NamespaceLocatorName(LocatorName):
+    def __init__(self, locator: NamespaceLocator):
+        self.locator = locator
+
+    @property
+    def immutable_id(self) -> Optional[str]:
+        return None
 
-    @permissions.setter
-    def permissions(self, permissions: Optional[Dict[str, Any]]) -> None:
-        self["permissions"] = permissions
+    def parts(self) -> List[str]:
+        return [self.locator.namespace]
 
 
 class NamespaceLocator(Locator, dict):
@@ -50,6 +66,14 @@ class NamespaceLocator(Locator, dict):
         namespace_locator.namespace = namespace
         return namespace_locator
 
+    @property
+    def name(self) -> NamespaceLocatorName:
+        return NamespaceLocatorName(self)
+
+    @property
+    def parent(self) -> Optional[Locator]:
+        return None
+
     @property
     def namespace(self) -> Optional[str]:
         return self.get("namespace")
@@ -57,11 +81,3 @@ class NamespaceLocator(Locator, dict):
     @namespace.setter
     def namespace(self, namespace: Optional[str]) -> None:
         self["namespace"] = namespace
-
-    def canonical_string(self) -> str:
-        """
-        Returns a unique string for the given locator that can be used
-        for equality checks (i.e. two locators are equal if they have
-        the same canonical string).
-        """
-        return self.namespace

deltacat/storage/model/partition.py

@@ -1,29 +1,58 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
-from typing import Any, Dict, List, Optional, Union
 
+import base64
+import posixpath
+
+import pyarrow
 import pyarrow as pa
-from deltacat.storage.model.partition_spec import PartitionValues
-from deltacat.storage.model.locator import Locator
+
+from typing import Any, Dict, List, Optional
+
+from deltacat.storage.model.metafile import Metafile, MetafileRevisionInfo
+from deltacat.constants import METAFILE_FORMAT, METAFILE_FORMAT_JSON, TXN_DIR_NAME
+from deltacat.storage.model.schema import (
+    FieldLocator,
+    Schema,
+)
+from deltacat.storage.model.locator import (
+    Locator,
+    LocatorName,
+)
 from deltacat.storage.model.namespace import NamespaceLocator
 from deltacat.storage.model.stream import StreamLocator
-from deltacat.storage.model.table import TableLocator
+from deltacat.storage.model.table import (
+    TableLocator,
+    Table,
+)
 from deltacat.storage.model.table_version import TableVersionLocator
-from deltacat.storage.model.types import CommitState
+from deltacat.storage.model.transform import Transform
+from deltacat.storage.model.types import (
+    CommitState,
+    StreamFormat,
+)
 from deltacat.types.media import ContentType
 
 
-class Partition(dict):
+"""
+An ordered list of partition values. Partition values are typically derived
+by applying one or more transforms to a table's fields.
+"""
+PartitionValues = List[Any]
+UNPARTITIONED_SCHEME_ID = "deadbeef-7277-49a4-a195-fdc8ed235d42"
+
+
+class Partition(Metafile):
     @staticmethod
     def of(
         locator: Optional[PartitionLocator],
-        schema: Optional[Union[pa.Schema, str, bytes]],
+        schema: Optional[Schema],
         content_types: Optional[List[ContentType]],
         state: Optional[CommitState] = None,
         previous_stream_position: Optional[int] = None,
         previous_partition_id: Optional[str] = None,
         stream_position: Optional[int] = None,
-        next_partition_id: Optional[str] = None,
+        partition_scheme_id: Optional[str] = None,
     ) -> Partition:
         partition = Partition()
         partition.locator = locator
@@ -33,7 +62,9 @@ class Partition(dict):
         partition.previous_stream_position = previous_stream_position
         partition.previous_partition_id = previous_partition_id
         partition.stream_position = stream_position
-        partition.next_partition_id = next_partition_id
+        partition.partition_scheme_id = (
+            partition_scheme_id if locator.partition_values else UNPARTITIONED_SCHEME_ID
+        )
         return partition
 
     @property
@@ -48,11 +79,18 @@ class Partition(dict):
         self["partitionLocator"] = partition_locator
 
     @property
-    def schema(self) -> Optional[Union[pa.Schema, str, bytes]]:
-        return self.get("schema")
+    def locator_alias(self) -> Optional[PartitionLocatorAlias]:
+        return PartitionLocatorAlias.of(self)
+
+    @property
+    def schema(self) -> Optional[Schema]:
+        val: Dict[str, Any] = self.get("schema")
+        if val is not None and not isinstance(val, Schema):
+            self.schema = val = Schema(val)
+        return val
 
     @schema.setter
-    def schema(self, schema: Optional[Union[pa.Schema, str, bytes]]) -> None:
+    def schema(self, schema: Optional[Schema]) -> None:
         self["schema"] = schema
 
     @property
@@ -104,12 +142,12 @@ class Partition(dict):
         self["streamPosition"] = stream_position
 
     @property
-    def next_partition_id(self) -> Optional[str]:
-        return self.get("nextPartitionId")
+    def partition_scheme_id(self) -> Optional[str]:
+        return self.get("partitionSchemeId")
 
-    @next_partition_id.setter
-    def next_partition_id(self, next_partition_id: Optional[str]):
-        self["nextPartitionId"] = next_partition_id
+    @partition_scheme_id.setter
+    def partition_scheme_id(self, partition_scheme_id: Optional[str]) -> None:
+        self["partitionSchemeId"] = partition_scheme_id
 
     @property
     def partition_id(self) -> Optional[str]:
@@ -125,6 +163,13 @@ class Partition(dict):
             return partition_locator.stream_id
         return None
 
+    @property
+    def stream_format(self) -> Optional[str]:
+        partition_locator = self.locator
+        if partition_locator:
+            return partition_locator.stream_format
+        return None
+
     @property
     def partition_values(self) -> Optional[PartitionValues]:
         partition_locator = self.locator
@@ -163,7 +208,7 @@ class Partition(dict):
     def storage_type(self) -> Optional[str]:
         partition_locator = self.locator
         if partition_locator:
-            return partition_locator.storage_type
+            return partition_locator.stream_format
         return None
 
     @property
@@ -193,6 +238,84 @@ class Partition(dict):
             content_type in supported_content_types
         )
 
+    def to_serializable(self) -> Partition:
+        serializable: Partition = Partition.update_for(self)
+        if serializable.schema:
+            schema_bytes = serializable.schema.serialize().to_pybytes()
+            serializable.schema = (
+                base64.b64encode(schema_bytes).decode("utf-8")
+                if METAFILE_FORMAT == METAFILE_FORMAT_JSON
+                else schema_bytes
+            )
+
+        if serializable.table_locator:
+            # replace the mutable table locator
+            serializable.table_version_locator.table_locator = TableLocator.at(
+                namespace=self.id,
+                table_name=self.id,
+            )
+        return serializable
+
+    def from_serializable(
+        self,
+        path: str,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    ) -> Partition:
+        if self.get("schema"):
+            schema_data = self["schema"]
+            schema_bytes = (
+                base64.b64decode(schema_data)
+                if METAFILE_FORMAT == METAFILE_FORMAT_JSON
+                else schema_data
+            )
+            self["schema"] = Schema.deserialize(pa.py_buffer(schema_bytes))
+        else:
+            self["schema"] = None
+
+        # restore the table locator from its mapped immutable metafile ID
+        if self.table_locator and self.table_locator.table_name == self.id:
+            parent_rev_dir_path = Metafile._parent_metafile_rev_dir_path(
+                base_metafile_path=path,
+                parent_number=3,
+            )
+            txn_log_dir = posixpath.join(
+                posixpath.dirname(
+                    posixpath.dirname(
+                        posixpath.dirname(parent_rev_dir_path),
+                    )
+                ),
+                TXN_DIR_NAME,
+            )
+            table = Table.read(
+                MetafileRevisionInfo.latest_revision(
+                    revision_dir_path=parent_rev_dir_path,
+                    filesystem=filesystem,
+                    success_txn_log_dir=txn_log_dir,
+                ).path,
+                filesystem,
+            )
+            self.table_version_locator.table_locator = table.locator
+        return self
+
+
+class PartitionLocatorName(LocatorName):
+    def __init__(self, locator: PartitionLocator):
+        self.locator = locator
+
+    @property
+    def immutable_id(self) -> Optional[str]:
+        return self.locator.partition_id
+
+    @immutable_id.setter
+    def immutable_id(self, immutable_id: Optional[str]):
+        self.locator.partition_id = immutable_id
+
+    def parts(self) -> List[str]:
+        return [
+            str(self.locator.partition_values),
+            self.locator.partition_id,
+        ]
+
 
 class PartitionLocator(Locator, dict):
     @staticmethod
@@ -223,16 +346,20 @@ class PartitionLocator(Locator, dict):
         table_name: Optional[str],
         table_version: Optional[str],
         stream_id: Optional[str],
-        storage_type: Optional[str],
+        stream_format: Optional[StreamFormat],
         partition_values: Optional[PartitionValues],
         partition_id: Optional[str],
     ) -> PartitionLocator:
-        stream_locator = StreamLocator.at(
-            namespace,
-            table_name,
-            table_version,
-            stream_id,
-            storage_type,
+        stream_locator = (
+            StreamLocator.at(
+                namespace,
+                table_name,
+                table_version,
+                stream_id,
+                stream_format,
+            )
+            if stream_id and stream_format
+            else None
         )
         return PartitionLocator.of(
             stream_locator,
@@ -240,6 +367,14 @@ class PartitionLocator(Locator, dict):
             partition_id,
         )
 
+    @property
+    def name(self) -> PartitionLocatorName:
+        return PartitionLocatorName(self)
+
+    @property
+    def parent(self) -> Optional[StreamLocator]:
+        return self.stream_locator
+
     @property
     def stream_locator(self) -> Optional[StreamLocator]:
         val: Dict[str, Any] = self.get("streamLocator")
@@ -296,10 +431,10 @@ class PartitionLocator(Locator, dict):
         return None
 
     @property
-    def storage_type(self) -> Optional[str]:
+    def stream_format(self) -> Optional[str]:
         stream_locator = self.stream_locator
         if stream_locator:
-            return stream_locator.storage_type
+            return stream_locator.format
         return None
 
     @property
@@ -323,13 +458,203 @@ class PartitionLocator(Locator, dict):
             return stream_locator.table_version
         return None
 
-    def canonical_string(self) -> str:
-        """
-        Returns a unique string for the given locator that can be used
-        for equality checks (i.e. two locators are equal if they have
-        the same canonical string).
-        """
-        sl_hexdigest = self.stream_locator.hexdigest()
-        partition_vals = str(self.partition_values)
-        partition_id = self.partition_id
-        return f"{sl_hexdigest}|{partition_vals}|{partition_id}"
+
+class PartitionKey(dict):
+    @staticmethod
+    def of(
+        key: List[FieldLocator],
+        name: Optional[str] = None,
+        field_id: Optional[int] = None,
+        transform: Optional[Transform] = None,
+        native_object: Optional[Any] = None,
+    ) -> PartitionKey:
+        return PartitionKey(
+            {
+                "key": key,
+                "name": name,
+                "fieldId": field_id,
+                "transform": transform,
+                "nativeObject": native_object,
+            }
+        )
+
+    def equivalent_to(
+        self,
+        other: PartitionKey,
+        check_identifiers: False,
+    ):
+        if other is None:
+            return False
+        if not isinstance(other, dict):
+            return False
+        if not isinstance(other, PartitionKey):
+            other = PartitionKey(other)
+        return (
+            self.key == other.key
+            and self.transform == other.transform
+            and not check_identifiers
+            or (self.name == other.name and self.id == other.id)
+        )
+
+    @property
+    def key(self) -> List[FieldLocator]:
+        return self.get("key")
+
+    @property
+    def name(self) -> Optional[str]:
+        return self.get("name")
+
+    @property
+    def id(self) -> Optional[int]:
+        return self.get("fieldId")
+
+    @property
+    def transform(self) -> Optional[Transform]:
+        val: Dict[str, Any] = self.get("transform")
+        if val is not None and not isinstance(val, Transform):
+            self["transform"] = val = Transform(val)
+        return val
+
+    @property
+    def native_object(self) -> Optional[Any]:
+        return self.get("nativeObject")
+
+
+class PartitionKeyList(List[PartitionKey]):
+    @staticmethod
+    def of(items: List[PartitionKey]) -> PartitionKeyList:
+        typed_items = PartitionKeyList()
+        for item in items:
+            if item is not None and not isinstance(item, PartitionKey):
+                item = PartitionKey(item)
+            typed_items.append(item)
+        return typed_items
+
+    def __getitem__(self, item):
+        val = super().__getitem__(item)
+        if val is not None and not isinstance(val, PartitionKey):
+            self[item] = val = PartitionKey(val)
+        return val
+
+
+class PartitionScheme(dict):
+    @staticmethod
+    def of(
+        keys: Optional[PartitionKeyList],
+        name: Optional[str] = None,
+        scheme_id: Optional[str] = None,
+        native_object: Optional[Any] = None,
+    ) -> PartitionScheme:
+        return PartitionScheme(
+            {
+                "keys": keys,
+                "name": name,
+                "id": scheme_id,
+                "nativeObject": native_object,
+            }
+        )
+
+    def equivalent_to(
+        self,
+        other: PartitionScheme,
+        check_identifiers: bool = False,
+    ) -> bool:
+        if other is None:
+            return False
+        if not isinstance(other, dict):
+            return False
+        if not isinstance(other, PartitionScheme):
+            other = PartitionScheme(other)
+        for i in range(len(self.keys)):
+            if not self.keys[i].equivalent_to(other.keys[i], check_identifiers):
+                return False
+        return not check_identifiers or (
+            self.name == other.name and self.id == other.id
+        )
+
+    @property
+    def keys(self) -> Optional[PartitionKeyList]:
+        val: List[PartitionKey] = self.get("keys")
+        if val is not None and not isinstance(val, PartitionKeyList):
+            self["keys"] = val = PartitionKeyList.of(val)
+        return val
+
+    @property
+    def name(self) -> Optional[str]:
+        return self.get("name")
+
+    @property
+    def id(self) -> Optional[str]:
+        return self.get("id")
+
+    @property
+    def native_object(self) -> Optional[Any]:
+        return self.get("nativeObject")
+
+
+class PartitionSchemeList(List[PartitionScheme]):
+    @staticmethod
+    def of(items: List[PartitionScheme]) -> PartitionSchemeList:
+        typed_items = PartitionSchemeList()
+        for item in items:
+            if item is not None and not isinstance(item, PartitionScheme):
+                item = PartitionScheme(item)
+            typed_items.append(item)
+        return typed_items
+
+    def __getitem__(self, item):
+        val = super().__getitem__(item)
+        if val is not None and not isinstance(val, PartitionScheme):
+            self[item] = val = PartitionScheme(val)
+        return val
+
+
+class PartitionLocatorAliasName(LocatorName):
+    def __init__(self, locator: PartitionLocatorAlias):
+        self.locator = locator
+
+    @property
+    def immutable_id(self) -> Optional[str]:
+        return None
+
+    def parts(self) -> List[str]:
+        return [
+            str(self.locator.partition_values),
+            self.locator.partition_scheme_id,
+        ]
+
+
+class PartitionLocatorAlias(Locator, dict):
+    @staticmethod
+    def of(parent_partition: Partition):
+        return (
+            PartitionLocatorAlias(
+                {
+                    "partition_values": parent_partition.partition_values,
+                    "partition_scheme_id": parent_partition.partition_scheme_id,
+                    "parent": (
+                        parent_partition.locator.parent
+                        if parent_partition.locator
+                        else None
+                    ),
+                }
+            )
+            if parent_partition.state == CommitState.COMMITTED
+            else None  # only committed partitions can be resolved by alias
+        )
+
+    @property
+    def partition_values(self) -> Optional[PartitionValues]:
+        return self.get("partition_values")
+
+    @property
+    def partition_scheme_id(self) -> Optional[str]:
+        return self.get("partition_scheme_id")
+
+    @property
+    def name(self) -> PartitionLocatorAliasName:
+        return PartitionLocatorAliasName(self)
+
+    @property
+    def parent(self) -> Optional[Locator]:
+        return self.get("parent")

deltacat/storage/model/scan/push_down.py

@@ -0,0 +1,19 @@
+class RowFilter:
+    ...
+
+
+class ColumnFilter:
+    ...
+
+
+class PartitionFilter:
+    ...
+
+
+class Pushdown:
+    """Represents pushdown predicates to be applied for DeltaCAT Tables"""
+
+    row_filter: RowFilter
+    column_filter: ColumnFilter
+    partition_filter: PartitionFilter
+    limit: int

deltacat/storage/model/scan/scan_plan.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+from deltacat.storage.model.scan.scan_task import ScanTask
+
+
+@dataclass
+class ScanPlan:
+    """Represents collection of ScanTasks to be passed to compute engine for query planning"""
+
+    scan_tasks: list[ScanTask]

deltacat/storage/model/scan/scan_task.py

@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class DataFile:
+    """Represents a data file, e.g. a S3 object, or a local file."""
+
+    file_path: str
+
+
+class ScanTask(ABC):
+    """Base class representing a unit of data to be read by a compute worker"""
+
+    @abstractmethod
+    def data_files(self) -> list[DataFile]:
+        pass
+
+
+@dataclass
+class FileScanTask(ScanTask):
+    """A unit of data in the form of data files"""
+
+    data_file_list: list[DataFile]
+
+    def data_files(self) -> list[DataFile]:
+        return self.data_file_list
+
+
+class ShardedScanTask(ScanTask):
+    """A unit of data in the form of shards (e.g. shard 1-10 each represents 1/10 of all data in a Table)"""
+
+    def data_files(self) -> list[DataFile]:
+        raise NotImplementedError("data_files is not implemented for ShardedScanTask")