deltacat 1.1.36__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

deltacat/{aws/redshift → storage}/model/manifest.py

@@ -1,20 +1,42 @@
-# Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-import itertools
 import logging
-from typing import Any, Dict, List, Optional
-from uuid import uuid4
+import itertools
+
 from enum import Enum
+from typing import Optional, List, Dict, Any
+from uuid import uuid4
 
 from deltacat import logs
 
+from deltacat.storage.model.schema import FieldLocator
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 class EntryType(str, Enum):
     """
-    Enum representing all possible content categories of an manifest entry file
+    Enum representing all possible content categories of a manifest entry file.
+
+    DATA: The entry contains fully qualified records compliant with the parent
+    table's schema to insert and/or update. Data files for upsert Deltas use
+    this entry's parameters to find matching fields to update. If no entry
+    parameters are specified, then the parent table's primary keys are used.
+    Only records from entries in Deltas with lower stream positions than this
+    entry will be targeted for update.
+
+    POSITIONAL_DELETE: The entry contains pointers to records in other entries
+    to delete. Deleted records will be filtered from query results at runtime.
+
+    EQUALITY_DELETE: The entry contains a subset of field values from the
+    table records to find and delete. The full record of any matching data
+    entries in Deltas with a lower stream position than this entry's Delta
+    will be deleted. The fields used for record discovery are controlled by
+    this entry's parameters. If no entry parameters are specified, then the
+    fields used for record discovery are linked to the parent table's merge
+    keys. The entry may contain additional fields not used for delete record
+    discovery which will be ignored. Deleted records will be filtered from
+    query results at runtime.
     """
 
     DATA = "data"
@@ -30,46 +52,40 @@ class EntryType(str, Enum):
         return [c.value for c in EntryType]
 
 
-class EntryFileParams(dict):
+class EntryParams(dict):
     """
-    Represents parameters relevant to the underlying contents of manifest entry. Contains all parameters required to support DELETEs
-    equality_column_names: List of column names that would be used to determine row equality for equality deletes.  Relevant only to equality deletes
-    position: Ordinal position of a deleted row in the target data file identified by uri, starting at 0. Relevant only to positional deletes
+    Parameters that control manifest entry interpretation.
+
+    For EQUALITY_DELETE manifest entry types, parameters include equality
+    field identifiers.
     """
 
     @staticmethod
     def of(
-        equality_column_names: Optional[List[str]] = None,
-        position: Optional[int] = None,
-    ) -> EntryFileParams:
-        entry_file_params = EntryFileParams()
-        if equality_column_names is not None:
-            entry_file_params["equality_column_names"] = equality_column_names
-        if position is not None:
-            entry_file_params["position"] = position
-        return entry_file_params
+        equality_field_locators: Optional[List[FieldLocator]] = None,
+    ) -> EntryParams:
+        params = EntryParams()
+        if equality_field_locators is not None:
+            params["equality_field_locators"] = equality_field_locators
+        return params
 
     @property
-    def equality_column_names(self) -> Optional[List[str]]:
-        return self.get("equality_column_names")
-
-    @property
-    def url(self) -> Optional[str]:
-        return self.get("url")
-
-    @property
-    def position(self) -> Optional[int]:
-        return self.get("position")
+    def equality_field_locators(self) -> Optional[List[FieldLocator]]:
+        return self.get("equality_field_locators")
 
 
 class Manifest(dict):
+    """
+    A DeltaCAT manifest contains metadata common to multiple manifest formats
+    like Amazon Redshift and Apache Iceberg to simplify dataset import/export.
+    """
+
     @staticmethod
     def _build_manifest(
         meta: Optional[ManifestMeta],
         entries: Optional[ManifestEntryList],
         author: Optional[ManifestAuthor] = None,
         uuid: str = None,
-        entry_type: Optional[EntryType] = None,
     ) -> Manifest:
         if not uuid:
             uuid = str(uuid4())
@@ -81,8 +97,6 @@ class Manifest(dict):
             manifest["entries"] = entries
         if author is not None:
             manifest["author"] = author
-        if entry_type is not None:
-            manifest["entry_type"] = entry_type.value
         return manifest
 
     @staticmethod
@@ -91,6 +105,7 @@ class Manifest(dict):
         author: Optional[ManifestAuthor] = None,
         uuid: str = None,
         entry_type: Optional[EntryType] = None,
+        entry_params: Optional[EntryParams] = None,
     ) -> Manifest:
         if not uuid:
             uuid = str(uuid4())
@@ -99,11 +114,13 @@ class Manifest(dict):
         total_source_content_length = 0
         content_type = None
         content_encoding = None
-        partition_values_set = set()
-        partition_values = None
+        credentials = None
+        content_type_params = None
         if entries:
             content_type = entries[0].meta.content_type
             content_encoding = entries[0].meta.content_encoding
+            credentials = entries[0].meta.credentials
+            content_type_params = entries[0].meta.content_type_parameters
             for entry in entries:
                 meta = entry.meta
                 if meta.content_type != content_type:
@@ -118,7 +135,7 @@ class Manifest(dict):
                         f"'{entry_content_type}'"
                     )
                     raise ValueError(msg)
-                entry_content_encoding = meta["content_encoding"]
+                entry_content_encoding = meta.get("content_encoding", None)
                 if entry_content_encoding != content_encoding:
                     msg = (
                         f"Expected all manifest entries to have content "
@@ -126,25 +143,53 @@ class Manifest(dict):
                         f"'{entry_content_encoding}'"
                     )
                     raise ValueError(msg)
+                actual_entry_type = meta.entry_type
+                if entry_type and (actual_entry_type != entry_type):
+                    msg = (
+                        f"Expected all manifest entries to have type "
+                        f"'{entry_type}' but found '{actual_entry_type}'"
+                    )
+                    raise ValueError(msg)
+                actual_entry_params = meta.entry_params
+                if entry_params and (actual_entry_params != entry_params):
+                    msg = (
+                        f"Expected all manifest entries to have params "
+                        f"'{entry_params}' but found '{actual_entry_params}'"
+                    )
+                    raise ValueError(msg)
+                actual_credentials = meta.credentials
+                if credentials and (actual_credentials != credentials):
+                    msg = (
+                        f"Expected all manifest entries to have credentials "
+                        f"'{credentials}' but found '{actual_credentials}'"
+                    )
+                    raise ValueError(msg)
+                actual_content_type_params = meta.content_type_parameters
+                if content_type_params and (
+                    actual_content_type_params != content_type_params
+                ):
+                    msg = (
+                        f"Expected all manifest entries to have content type params "
+                        f"'{content_type_params}' but found '{actual_content_type_params}'"
+                    )
+                    raise ValueError(msg)
+
                 total_record_count += meta.record_count or 0
                 total_content_length += meta.content_length or 0
                 total_source_content_length += meta.source_content_length or 0
-                if len(partition_values_set) <= 1:
-                    partition_values_set.add(entry.meta.partition_values)
-
-        if len(partition_values_set) == 1:
-            partition_values = partition_values_set.pop()
 
         meta = ManifestMeta.of(
-            total_record_count,
-            total_content_length,
-            content_type,
-            content_encoding,
-            total_source_content_length,
+            record_count=total_record_count,
+            content_length=total_content_length,
+            content_type=content_type,
+            content_encoding=content_encoding,
+            source_content_length=total_source_content_length,
+            credentials=credentials,
+            content_type_parameters=content_type_params,
             entry_type=entry_type,
-            partition_values=partition_values,
+            entry_params=entry_params,
         )
-        manifest = Manifest._build_manifest(meta, entries, author, uuid, entry_type)
+        manifest = Manifest._build_manifest(meta, entries, author, uuid)
         return manifest
 
     @staticmethod
@@ -194,7 +239,7 @@ class ManifestMeta(dict):
         credentials: Optional[Dict[str, str]] = None,
         content_type_parameters: Optional[List[Dict[str, str]]] = None,
         entry_type: Optional[EntryType] = None,
-        partition_values: Optional[List[str]] = None,
+        entry_params: Optional[EntryParams] = None,
     ) -> ManifestMeta:
         manifest_meta = ManifestMeta()
         if record_count is not None:
@@ -212,9 +257,11 @@ class ManifestMeta(dict):
         if credentials is not None:
             manifest_meta["credentials"] = credentials
         if entry_type is not None:
-            manifest_meta["entry_type"] = entry_type.value
-        if partition_values is not None:
-            manifest_meta["partition_values"] = partition_values
+            manifest_meta["entry_type"] = (
+                entry_type.value if isinstance(entry_type, EntryType) else entry_type
+            )
+        if entry_params is not None:
+            manifest_meta["entry_params"] = entry_params
         return manifest_meta
 
     @property
@@ -257,27 +304,11 @@ class ManifestMeta(dict):
         return val
 
     @property
-    def partition_values(self) -> Optional[List[str]]:
-        return self.get("partition_values")
-
-
-class ManifestAuthor(dict):
-    @staticmethod
-    def of(name: Optional[str], version: Optional[str]) -> ManifestAuthor:
-        manifest_author = ManifestAuthor()
-        if name is not None:
-            manifest_author["name"] = name
-        if version is not None:
-            manifest_author["version"] = version
-        return manifest_author
-
-    @property
-    def name(self) -> Optional[str]:
-        return self.get("name")
-
-    @property
-    def version(self) -> Optional[str]:
-        return self.get("version")
+    def entry_params(self) -> Optional[EntryParams]:
+        val: Dict[str, Any] = self.get("entry_params")
+        if val is not None and not isinstance(val, EntryParams):
+            self["entry_params"] = val = EntryParams(val)
+        return val
 
 
 class ManifestEntry(dict):
@@ -288,8 +319,6 @@ class ManifestEntry(dict):
         mandatory: bool = True,
         uri: Optional[str] = None,
         uuid: Optional[str] = None,
-        entry_type: Optional[EntryType] = None,
-        entry_file_params: Optional[EntryFileParams] = None,
     ) -> ManifestEntry:
         manifest_entry = ManifestEntry()
         if not (uri or url):
@@ -306,16 +335,6 @@ class ManifestEntry(dict):
             manifest_entry["mandatory"] = mandatory
         if uuid is not None:
             manifest_entry["id"] = uuid
-        if entry_type is not None:
-            manifest_entry["entry_type"] = entry_type.value
-        if entry_file_params is not None:
-            if entry_file_params.get("url") != manifest_entry.get("url"):
-                msg = (
-                    f"Expected manifest entry url: {manifest_entry.url}"
-                    f" and entry_file_params: '{entry_file_params.url}' to match"
-                )
-                raise ValueError(msg)
-            manifest_entry["entry_file_params"] = entry_file_params
         return manifest_entry
 
     @staticmethod
@@ -330,11 +349,11 @@ class ManifestEntry(dict):
         s3_obj = s3_utils.get_object_at_url(url, **s3_client_kwargs)
         logger.debug(f"Building manifest entry from {url}: {s3_obj}")
         manifest_entry_meta = ManifestMeta.of(
-            record_count,
-            s3_obj["ContentLength"],
-            s3_obj["ContentType"],
-            s3_obj["ContentEncoding"],
-            source_content_length,
+            record_count=record_count,
+            content_length=s3_obj["ContentLength"],
+            content_type=s3_obj["ContentType"],
+            content_encoding=s3_obj["ContentEncoding"],
+            source_content_length=source_content_length,
         )
         manifest_entry = ManifestEntry.of(url, manifest_entry_meta)
         return manifest_entry
@@ -362,19 +381,24 @@ class ManifestEntry(dict):
     def id(self) -> Optional[str]:
         return self.get("id")
 
+
+class ManifestAuthor(dict):
+    @staticmethod
+    def of(name: Optional[str], version: Optional[str]) -> ManifestAuthor:
+        manifest_author = ManifestAuthor()
+        if name is not None:
+            manifest_author["name"] = name
+        if version is not None:
+            manifest_author["version"] = version
+        return manifest_author
+
     @property
-    def entry_type(self) -> Optional[EntryType]:
-        val = self.get("entry_type")
-        if val is not None:
-            return EntryType(self["entry_type"])
-        return val
+    def name(self) -> Optional[str]:
+        return self.get("name")
 
     @property
-    def entry_file_params(self) -> Optional[EntryFileParams]:
-        val: Dict[str, Any] = self.get("entry_file_params")
-        if val is not None and not isinstance(val, EntryFileParams):
-            self["entry_file_params"] = val = EntryFileParams(val)
-        return val
+    def version(self) -> Optional[str]:
+        return self.get("version")
 
 
 class ManifestEntryList(List[ManifestEntry]):