deltacat 2.0.0b11__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/storage/model/delta.py

@@ -1,6 +1,7 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+import json
 import posixpath
 from typing import Any, Dict, List, Optional
 
@@ -48,8 +49,7 @@ class Delta(Metafile):
     ) -> Delta:
         """
         Creates a Delta metadata model with the given Delta Locator, Delta Type,
-        manifest metadata, properties, manifest, and previous delta stream
-        position.
+        manifest metadata, properties, manifest, and previous delta stream position.
         """
         delta = Delta()
         delta.locator = locator
@@ -275,6 +275,13 @@ class Delta(Metafile):
             return delta_locator.partition_values
         return None
 
+    @property
+    def partition_values_json(self) -> Optional[str]:
+        partition_values = (
+            self.partition_values if self.partition_values is not None else None
+        )
+        return json.dumps(partition_values)
+
     @property
     def stream_position(self) -> Optional[int]:
         delta_locator = self.locator
@@ -282,6 +289,13 @@ class Delta(Metafile):
             return delta_locator.stream_position
         return None
 
+    def url(self, catalog_name: Optional[str] = None) -> str:
+        return (
+            f"dc://{catalog_name}/{self.namespace}/{self.table_name}/{self.table_version}/{self.stream_format}/{self.partition_values_json}/{self.stream_position}/"
+            if catalog_name
+            else f"table://{self.namespace}/{self.table_name}/{self.table_version}/{self.stream_format}/{self.partition_values_json}/{self.stream_position}/"
+        )
+
     def to_serializable(self) -> Delta:
         serializable = self
         if serializable.table_locator:
@@ -378,7 +392,17 @@ class DeltaLocator(Locator, dict):
                 partition_values,
                 partition_id,
             )
-            if partition_values and partition_id
+            if any(
+                [
+                    partition_id,
+                    partition_values,
+                    stream_id,
+                    stream_format,
+                    table_name,
+                    table_version,
+                    namespace,
+                ]
+            )
             else None
         )
         return DeltaLocator.of(

deltacat/storage/model/locator.py

@@ -90,29 +90,23 @@ class Locator:
     def canonical_string(self, separator: str = DEFAULT_NAME_SEPARATOR) -> str:
         """
         Returns a unique string for the given locator that can be used
-        for equality checks (i.e. two locators are equal if they have
-        the same canonical string).
+        for equality checks between objects with the same parent.
         """
-        parts = []
-        parent_hexdigest = self.parent.hexdigest() if self.parent else None
-        if parent_hexdigest:
-            parts.append(parent_hexdigest)
-        parts.extend(self.name.parts())
-        return separator.join([str(part) for part in parts])
+        return separator.join([str(part) for part in self.name.parts()])
 
     def digest(self) -> bytes:
         """
         Return a digest of the given locator that can be used for
-        equality checks (i.e. two locators are equal if they have the
-        same digest) and uniform random hash distribution.
+        equality checks between objects with the same parent and uniform
+        random hash distribution.
         """
         return sha1_digest(self.canonical_string().encode("utf-8"))
 
     def hexdigest(self) -> str:
         """
         Returns a hexdigest of the given locator suitable
-        for use in equality (i.e. two locators are equal if they have the same
-        hexdigest) and inclusion in URLs.
+        equality checks between objects with the same parent and
+        inclusion in URLs.
         """
         return sha1_hexdigest(self.canonical_string().encode("utf-8"))
 

deltacat/storage/model/manifest.py

@@ -4,14 +4,26 @@ import logging
 import itertools
 
 from enum import Enum
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Dict, Any, TYPE_CHECKING
 from uuid import uuid4
 
+if TYPE_CHECKING:
+    from deltacat.storage.model.schema import FieldLocator
+
 from deltacat import logs
 
-from deltacat.storage.model.schema import FieldLocator
+from deltacat.types.media import (
+    ContentType,
+    ContentEncoding,
+    EXT_TO_CONTENT_TYPE,
+    EXT_TO_CONTENT_ENCODING,
+)
 
 import json
+import pyarrow as pa
+import posixpath
+
+from deltacat.utils.filesystem import get_file_info
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -64,7 +76,7 @@ class EntryParams(dict):
 
     @staticmethod
     def of(
-        equality_field_locators: Optional[List[FieldLocator]] = None,
+        equality_field_locators: Optional[List["FieldLocator"]] = None,
     ) -> EntryParams:
         params = EntryParams()
         if equality_field_locators is not None:
@@ -72,7 +84,7 @@ class EntryParams(dict):
         return params
 
     @property
-    def equality_field_locators(self) -> Optional[List[FieldLocator]]:
+    def equality_field_locators(self) -> Optional[List["FieldLocator"]]:
         return self.get("equality_field_locators")
 
 
@@ -118,11 +130,35 @@ class Manifest(dict):
         content_encoding = None
         credentials = None
         content_type_params = None
+        schema_id = None
+        sort_scheme_id = None
         if entries:
             content_type = entries[0].meta.content_type
             content_encoding = entries[0].meta.content_encoding
             credentials = entries[0].meta.credentials
             content_type_params = entries[0].meta.content_type_parameters
+
+            # Keep the latest schema ID
+            # Schema IDs are >= 0, and schema evolution always increments the last schema ID
+            entry_schema_ids = [
+                entry.meta.schema_id if entry.meta.schema_id is not None else -1
+                for entry in entries
+            ]
+            max_schema_id = max(entry_schema_ids) if entry_schema_ids else -1
+            schema_id = max_schema_id if max_schema_id >= 0 else None
+
+            # Handle sort_scheme_id: set to None if entries have multiple different sort_scheme_ids
+            entry_sort_scheme_ids = set(
+                entry.meta.sort_scheme_id
+                for entry in entries
+                if entry.meta.sort_scheme_id is not None
+            )
+            sort_scheme_id = (
+                list(entry_sort_scheme_ids)[0]
+                if len(entry_sort_scheme_ids) == 1
+                else None
+            )
+
             for entry in entries:
                 meta = entry.meta
                 if meta.content_type != content_type:
@@ -130,7 +166,7 @@ class Manifest(dict):
                 if meta.content_encoding != content_encoding:
                     content_encoding = None
                 entry_content_type = meta.content_type
-                if entry_content_type != content_type:
+                if content_type and entry_content_type != content_type:
                     msg = (
                         f"Expected all manifest entries to have content "
                         f"type '{content_type}' but found "
@@ -138,7 +174,7 @@ class Manifest(dict):
                     )
                     raise ValueError(msg)
                 entry_content_encoding = meta.get("content_encoding", None)
-                if entry_content_encoding != content_encoding:
+                if content_encoding and entry_content_encoding != content_encoding:
                     msg = (
                         f"Expected all manifest entries to have content "
                         f"encoding '{content_encoding}' but found "
@@ -190,6 +226,8 @@ class Manifest(dict):
             content_type_parameters=content_type_params,
             entry_type=entry_type,
             entry_params=entry_params,
+            schema_id=schema_id,
+            sort_scheme_id=sort_scheme_id,
         )
         manifest = Manifest._build_manifest(meta, entries, author, uuid)
         return manifest
@@ -256,6 +294,8 @@ class ManifestMeta(dict):
         content_type_parameters: Optional[List[Dict[str, str]]] = None,
         entry_type: Optional[EntryType] = None,
         entry_params: Optional[EntryParams] = None,
+        schema_id: Optional[int] = None,
+        sort_scheme_id: Optional[str] = None,
     ) -> ManifestMeta:
         manifest_meta = ManifestMeta()
         if record_count is not None:
@@ -278,6 +318,10 @@ class ManifestMeta(dict):
             )
         if entry_params is not None:
             manifest_meta["entry_params"] = entry_params
+        if schema_id is not None:
+            manifest_meta["schema_id"] = schema_id
+        if sort_scheme_id is not None:
+            manifest_meta["sort_scheme_id"] = sort_scheme_id
         return manifest_meta
 
     @staticmethod
@@ -295,6 +339,8 @@ class ManifestMeta(dict):
             content_type_parameters=obj.get("content_type_parameters"),
             entry_type=obj.get("entry_type"),
             entry_params=obj.get("entry_params"),
+            schema_id=obj.get("schema_id"),
+            sort_scheme_id=obj.get("sort_scheme_id"),
         )
 
     @property
@@ -343,6 +389,14 @@ class ManifestMeta(dict):
             self["entry_params"] = val = EntryParams(val)
         return val
 
+    @property
+    def schema_id(self) -> Optional[int]:
+        return self.get("schema_id")
+
+    @property
+    def sort_scheme_id(self) -> Optional[str]:
+        return self.get("sort_scheme_id")
+
 
 class ManifestEntry(dict):
     @staticmethod
@@ -375,6 +429,10 @@ class ManifestEntry(dict):
         url: str,
         record_count: int,
         source_content_length: Optional[int] = None,
+        credentials: Optional[Dict[str, str]] = None,
+        content_type_parameters: Optional[List[Dict[str, str]]] = None,
+        entry_type: Optional[EntryType] = None,
+        entry_params: Optional[EntryParams] = None,
         **s3_client_kwargs,
     ) -> ManifestEntry:
         from deltacat.aws import s3u as s3_utils
@@ -387,6 +445,10 @@ class ManifestEntry(dict):
             content_type=s3_obj["ContentType"],
             content_encoding=s3_obj["ContentEncoding"],
             source_content_length=source_content_length,
+            credentials=credentials,
+            content_type_parameters=content_type_parameters,
+            entry_type=entry_type,
+            entry_params=entry_params,
         )
         manifest_entry = ManifestEntry.of(url, manifest_entry_meta)
         return manifest_entry
@@ -401,6 +463,116 @@ class ManifestEntry(dict):
             uuid=obj.get("id"),
         )
 
+    @staticmethod
+    def from_path(
+        path: str,
+        filesystem: pa.fs.FileSystem,
+        record_count: int,
+        source_content_length: Optional[int] = None,
+        content_type: Optional[str] = None,
+        content_encoding: Optional[str] = None,
+        credentials: Optional[Dict[str, str]] = None,
+        content_type_parameters: Optional[List[Dict[str, str]]] = None,
+        entry_type: Optional[EntryType] = None,
+        entry_params: Optional[EntryParams] = None,
+        schema_id: Optional[int] = None,
+        sort_scheme_id: Optional[str] = None,
+    ) -> ManifestEntry:
+        """
+        Creates a manifest entry from a path using a pyarrow filesystem.
+
+        Args:
+            path: Path to the file
+            filesystem: PyArrow filesystem to use for accessing the file
+            record_count: Number of records in the file
+            source_content_length: Optional original content length in-memory
+                before writing to disk.
+            content_type: Optional content type override. If not provided, will
+                be derived from file extension.
+            content_encoding: Optional content encoding override. If not
+                provided, will be derived from file extension.
+            credentials: Optional credentials required to read this manifest entry.
+            content_type_parameters: Optional content type parameters.
+            entry_type: Optional entry type of this manifest entry. Defaults to DATA.
+            entry_params: Optional entry type parameters.
+            schema_id: Schema ID used to write this manifest entry.
+            sort_scheme_id: Sort scheme ID used to write this manifest entry.
+
+        Returns:
+            A ManifestEntry instance
+        """
+        file_info = get_file_info(path, filesystem)
+        if file_info.type != pa.fs.FileType.File:
+            raise FileNotFoundError(f"Path does not point to a file: {path}")
+
+        # Extract extensions from right to left
+        # First split will get potential encoding extension
+        base_path, ext1 = posixpath.splitext(path)
+
+        # Initialize with defaults for no extensions
+        derived_content_type = ContentType.BINARY
+        derived_content_encoding = ContentEncoding.IDENTITY
+
+        # Only proceed with extension checks if we found at least one extension
+        if ext1:
+            # Check if the first extension is a known encoding
+            derived_content_encoding = EXT_TO_CONTENT_ENCODING.get(
+                ext1,
+                ContentEncoding.IDENTITY,
+            )
+
+            # Get second extension only if first was an encoding
+            if derived_content_encoding != ContentEncoding.IDENTITY:
+                # Second split will get potential content type extension
+                _, ext2 = posixpath.splitext(base_path)
+                if ext2:
+                    derived_content_type = EXT_TO_CONTENT_TYPE.get(
+                        ext2,
+                        ContentType.BINARY,
+                    )
+            else:
+                # First extension wasn't an encoding, check if it's a
+                # content type
+                derived_content_type = EXT_TO_CONTENT_TYPE.get(
+                    ext1,
+                    ContentType.BINARY,
+                )
+
+        if (
+            derived_content_type == ContentType.BINARY
+            and derived_content_encoding != ContentEncoding.IDENTITY
+        ):
+            logger.debug(
+                f"Found encoding {derived_content_encoding.value} but no "
+                f"content type for {path}, assuming binary"
+            )
+
+        # Use provided values if available, otherwise use derived values
+        final_content_type = (
+            content_type if content_type is not None else derived_content_type.value
+        )
+        final_content_encoding = (
+            content_encoding
+            if content_encoding is not None
+            else derived_content_encoding.value
+        )
+
+        manifest_entry_meta = ManifestMeta.of(
+            record_count=record_count,
+            content_length=file_info.size,
+            content_type=final_content_type,
+            content_encoding=final_content_encoding,
+            source_content_length=source_content_length,
+            credentials=credentials,
+            content_type_parameters=content_type_parameters,
+            entry_type=entry_type,
+            entry_params=entry_params,
+            schema_id=schema_id,
+            sort_scheme_id=sort_scheme_id,
+        )
+        manifest_entry = ManifestEntry.of(path, manifest_entry_meta)
+        return manifest_entry
+
     @property
     def uri(self) -> Optional[str]:
         return self.get("uri")
@@ -465,3 +637,7 @@ class ManifestEntryList(List[ManifestEntry]):
         if val is not None and not isinstance(val, ManifestEntry):
             self[item] = val = ManifestEntry(val)
         return val
+
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self[i]  # This triggers __getitem__ conversion