deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/storage/model/expression/visitor.py

@@ -0,0 +1,248 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Generic, TypeVar, Callable, Optional
+from functools import singledispatchmethod
+import re
+
+from deltacat.storage.model.expression import (
+    Expression,
+    Reference,
+    Literal,
+    BinaryExpression,
+    UnaryExpression,
+    In,
+    Between,
+    Like,
+)
+
+
+C = TypeVar("C")  # Context type
+R = TypeVar("R")  # Return type
+
+
+class ExpressionVisitor(ABC, Generic[C, R]):
+    """
+    Visitor pattern for deltacat expressions.
+
+    This base class provides two ways to implement visitors:
+    1. Using a procedure dictionary (_PROCEDURES) - for simple, declarative visitors
+    2. Using specialized visit_xyz methods with snake_case naming - for more control
+
+    Subclasses need only implement visit_reference and visit_literal, plus either:
+    - Define _PROCEDURES dictionary with functions for handling different expression types
+    - Implement specific visit_xyz methods (using snake_case) for individual expressions
+    """
+
+    # Default procedure dictionary for subclasses to override
+    _PROCEDURES: Dict[str, Callable] = {}
+
+    def __init__(self):
+        """Initialize visitor and validate required methods."""
+        # Pre-check for required methods
+        if not hasattr(self, "visit_reference") or not callable(
+            getattr(self, "visit_reference")
+        ):
+            raise NotImplementedError("Subclasses must implement visit_reference")
+        if not hasattr(self, "visit_literal") or not callable(
+            getattr(self, "visit_literal")
+        ):
+            raise NotImplementedError("Subclasses must implement visit_literal")
+        self._setup_default_procedure_handlers()
+
+    def _to_snake_case(self, name: str) -> str:
+        """Convert PascalCase or camelCase to snake_case."""
+        pattern = re.compile(r"(?<!^)(?=[A-Z])")
+        return pattern.sub("_", name).lower()
+
+    def _setup_default_procedure_handlers(self):
+        """Set up default procedure application methods if not overridden."""
+        if not hasattr(self, "_apply_binary") or not callable(
+            getattr(self, "_apply_binary")
+        ):
+            self._apply_binary = lambda proc, left, right: proc(left, right)
+        if not hasattr(self, "_apply_unary") or not callable(
+            getattr(self, "_apply_unary")
+        ):
+            self._apply_unary = lambda proc, operand: proc(operand)
+        if not hasattr(self, "_apply_in") or not callable(getattr(self, "_apply_in")):
+            self._apply_in = lambda proc, value, values: proc(value, values)
+        if not hasattr(self, "_apply_between") or not callable(
+            getattr(self, "_apply_between")
+        ):
+            self._apply_between = lambda proc, value, lower, upper: proc(
+                value, lower, upper
+            )
+        if not hasattr(self, "_apply_like") or not callable(
+            getattr(self, "_apply_like")
+        ):
+            self._apply_like = lambda proc, value, pattern: proc(value, pattern)
+
+    @singledispatchmethod
+    def visit(self, expr: Expression, context: Optional[C] = None) -> R:
+        """
+        Generic visit method that dispatches to specific methods based on expression type.
+
+        Args:
+            expr: The expression to visit
+            context: Optional context to pass through the visitor
+
+        Returns:
+            Result of visiting the expression
+        """
+        expr_type = type(expr).__name__
+        raise NotImplementedError(f"No visit method for type {expr_type}")
+
+    @visit.register
+    def _visit_reference(self, expr: Reference, context: Optional[C] = None) -> R:
+        """Visit a Reference expression."""
+        return self.visit_reference(expr, context)
+
+    @visit.register
+    def _visit_literal(self, expr: Literal, context: Optional[C] = None) -> R:
+        """Visit a Literal expression."""
+        return self.visit_literal(expr, context)
+
+    @visit.register
+    def _visit_binary(self, expr: BinaryExpression, context: Optional[C] = None) -> R:
+        """Visit a binary expression using method specialization or procedures."""
+        expr_type = type(expr).__name__
+
+        left_result = self.visit(expr.left, context)
+        right_result = self.visit(expr.right, context)
+
+        method_name = f"visit_{self._to_snake_case(expr_type)}"
+        if hasattr(self, method_name):
+            method = getattr(self, method_name)
+            return method(expr, context)
+
+        if expr_type in self._PROCEDURES:
+            return self._apply_binary(
+                self._PROCEDURES[expr_type], left_result, right_result
+            )
+
+        try:
+            return self.visit_binary_expression(
+                expr, left_result, right_result, context
+            )
+        except NotImplementedError:
+            raise NotImplementedError(f"No handler for {expr_type}")
+
+    @visit.register
+    def _visit_unary(self, expr: UnaryExpression, context: Optional[C] = None) -> R:
+        """Visit a unary expression using method specialization or procedures."""
+        expr_type = type(expr).__name__
+
+        operand_result = self.visit(expr.operand, context)
+
+        method_name = f"visit_{self._to_snake_case(expr_type)}"
+        if hasattr(self, method_name):
+            method = getattr(self, method_name)
+            return method(expr, context)
+
+        if expr_type in self._PROCEDURES:
+            return self._apply_unary(self._PROCEDURES[expr_type], operand_result)
+
+        try:
+            return self.visit_unary_expression(expr, operand_result, context)
+        except NotImplementedError:
+            raise NotImplementedError(f"No handler for {expr_type}")
+
+    @visit.register
+    def _visit_in(self, expr: In, context: Optional[C] = None) -> R:
+        """Visit an In expression."""
+        if hasattr(self, "visit_in"):
+            return self.visit_in(expr, context)
+
+        if "In" in self._PROCEDURES:
+            value_result = self.visit(expr.value, context)
+            values_results = [self.visit(v, context) for v in expr.values]
+            return self._apply_in(self._PROCEDURES["In"], value_result, values_results)
+
+        raise NotImplementedError("No handler for In expression")
+
+    @visit.register
+    def _visit_between(self, expr: Between, context: Optional[C] = None) -> R:
+        """Visit a Between expression."""
+        if hasattr(self, "visit_between"):
+            return self.visit_between(expr, context)
+
+        if "Between" in self._PROCEDURES:
+            value_result = self.visit(expr.value, context)
+            lower_result = self.visit(expr.lower, context)
+            upper_result = self.visit(expr.upper, context)
+            return self._apply_between(
+                self._PROCEDURES["Between"], value_result, lower_result, upper_result
+            )
+
+        raise NotImplementedError("No handler for Between expression")
+
+    @visit.register
+    def _visit_like(self, expr: Like, context: Optional[C] = None) -> R:
+        """Visit a Like expression."""
+        if hasattr(self, "visit_like"):
+            return self.visit_like(expr, context)
+
+        if "Like" in self._PROCEDURES:
+            value_result = self.visit(expr.value, context)
+            pattern_result = self.visit(expr.pattern, context)
+            return self._apply_like(
+                self._PROCEDURES["Like"], value_result, pattern_result
+            )
+
+        raise NotImplementedError("No handler for Like expression")
+
+    @abstractmethod
+    def visit_reference(self, expr: Reference, context: Optional[C] = None) -> R:
+        """Visit a Reference expression."""
+        pass
+
+    @abstractmethod
+    def visit_literal(self, expr: Literal, context: Optional[C] = None) -> R:
+        """Visit a Literal expression."""
+        pass
+
+    def visit_binary_expression(
+        self, expr: BinaryExpression, left: R, right: R, context: Optional[C] = None
+    ) -> R:
+        """Default fallback handler for binary expressions."""
+        raise NotImplementedError(f"No handler for {type(expr).__name__}")
+
+    def visit_unary_expression(
+        self, expr: UnaryExpression, operand: R, context: Optional[C] = None
+    ) -> R:
+        """Default fallback handler for unary expressions."""
+        raise NotImplementedError(f"No handler for {type(expr).__name__}")
+
+
+class DisplayVisitor(ExpressionVisitor[Expression, str]):
+    """
+    Visitor implementation that formats expressions in standard infix notation.
+    For example: "a = b AND c > d" instead of "(AND (= a b) (> c d))".
+    """
+
+    # Map all expression types to their string formatting procedures with infix notation
+    _PROCEDURES = {
+        # Binary operations with infix notation
+        "Equal": lambda left, right: f"{left} = {right}",
+        "NotEqual": lambda left, right: f"{left} <> {right}",
+        "GreaterThan": lambda left, right: f"{left} > {right}",
+        "LessThan": lambda left, right: f"{left} < {right}",
+        "GreaterThanEqual": lambda left, right: f"{left} >= {right}",
+        "LessThanEqual": lambda left, right: f"{left} <= {right}",
+        "And": lambda left, right: f"({left} AND {right})",
+        "Or": lambda left, right: f"({left} OR {right})",
+        # Unary operations
+        "Not": lambda operand: f"NOT ({operand})",
+        "IsNull": lambda operand: f"({operand}) IS NULL",
+        # Special operations
+        "In": lambda value, values: f"{value} IN ({', '.join(values)})",
+        "Between": lambda value, lower, upper: f"{value} BETWEEN {lower} AND {upper}",
+        "Like": lambda value, pattern: f"{value} LIKE {pattern}",
+    }
+
+    def visit_reference(self, expr: Reference, context=None) -> str:
+        """Format a field reference."""
+        return expr.field
+
+    def visit_literal(self, expr: Literal, context=None) -> str:
+        """Format a literal value using its PyArrow representation."""
+        return str(expr.value)

deltacat/storage/model/locator.py

@@ -90,29 +90,23 @@ class Locator:
     def canonical_string(self, separator: str = DEFAULT_NAME_SEPARATOR) -> str:
         """
         Returns a unique string for the given locator that can be used
-        for equality checks (i.e. two locators are equal if they have
-        the same canonical string).
+        for equality checks between objects with the same parent.
         """
-        parts = []
-        parent_hexdigest = self.parent.hexdigest() if self.parent else None
-        if parent_hexdigest:
-            parts.append(parent_hexdigest)
-        parts.extend(self.name.parts())
-        return separator.join([str(part) for part in parts])
+        return separator.join([str(part) for part in self.name.parts()])
 
     def digest(self) -> bytes:
         """
         Return a digest of the given locator that can be used for
-        equality checks (i.e. two locators are equal if they have the
-        same digest) and uniform random hash distribution.
+        equality checks between objects with the same parent and uniform
+        random hash distribution.
         """
         return sha1_digest(self.canonical_string().encode("utf-8"))
 
     def hexdigest(self) -> str:
         """
         Returns a hexdigest of the given locator suitable
-        for use in equality (i.e. two locators are equal if they have the same
-        hexdigest) and inclusion in URLs.
+        equality checks between objects with the same parent and
+        inclusion in URLs.
         """
         return sha1_hexdigest(self.canonical_string().encode("utf-8"))
 

deltacat/storage/model/manifest.py

@@ -4,12 +4,26 @@ import logging
 import itertools
 
 from enum import Enum
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Dict, Any, TYPE_CHECKING
 from uuid import uuid4
 
+if TYPE_CHECKING:
+    from deltacat.storage.model.schema import FieldLocator
+
 from deltacat import logs
 
-from deltacat.storage.model.schema import FieldLocator
+from deltacat.types.media import (
+    ContentType,
+    ContentEncoding,
+    EXT_TO_CONTENT_TYPE,
+    EXT_TO_CONTENT_ENCODING,
+)
+
+import json
+import pyarrow as pa
+import posixpath
+
+from deltacat.utils.filesystem import get_file_info
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -62,7 +76,7 @@ class EntryParams(dict):
 
     @staticmethod
     def of(
-        equality_field_locators: Optional[List[FieldLocator]] = None,
+        equality_field_locators: Optional[List["FieldLocator"]] = None,
     ) -> EntryParams:
         params = EntryParams()
         if equality_field_locators is not None:
@@ -70,7 +84,7 @@ class EntryParams(dict):
         return params
 
     @property
-    def equality_field_locators(self) -> Optional[List[FieldLocator]]:
+    def equality_field_locators(self) -> Optional[List["FieldLocator"]]:
         return self.get("equality_field_locators")
 
 
@@ -116,11 +130,35 @@ class Manifest(dict):
         content_encoding = None
         credentials = None
         content_type_params = None
+        schema_id = None
+        sort_scheme_id = None
         if entries:
             content_type = entries[0].meta.content_type
             content_encoding = entries[0].meta.content_encoding
             credentials = entries[0].meta.credentials
             content_type_params = entries[0].meta.content_type_parameters
+
+            # Keep the latest schema ID
+            # Schema IDs are >= 0, and schema evolution always increments the last schema ID
+            entry_schema_ids = [
+                entry.meta.schema_id if entry.meta.schema_id is not None else -1
+                for entry in entries
+            ]
+            max_schema_id = max(entry_schema_ids) if entry_schema_ids else -1
+            schema_id = max_schema_id if max_schema_id >= 0 else None
+
+            # Handle sort_scheme_id: set to None if entries have multiple different sort_scheme_ids
+            entry_sort_scheme_ids = set(
+                entry.meta.sort_scheme_id
+                for entry in entries
+                if entry.meta.sort_scheme_id is not None
+            )
+            sort_scheme_id = (
+                list(entry_sort_scheme_ids)[0]
+                if len(entry_sort_scheme_ids) == 1
+                else None
+            )
+
             for entry in entries:
                 meta = entry.meta
                 if meta.content_type != content_type:
@@ -128,7 +166,7 @@ class Manifest(dict):
                 if meta.content_encoding != content_encoding:
                     content_encoding = None
                 entry_content_type = meta.content_type
-                if entry_content_type != content_type:
+                if content_type and entry_content_type != content_type:
                     msg = (
                         f"Expected all manifest entries to have content "
                         f"type '{content_type}' but found "
@@ -136,7 +174,7 @@ class Manifest(dict):
                     )
                     raise ValueError(msg)
                 entry_content_encoding = meta.get("content_encoding", None)
-                if entry_content_encoding != content_encoding:
+                if content_encoding and entry_content_encoding != content_encoding:
                     msg = (
                         f"Expected all manifest entries to have content "
                         f"encoding '{content_encoding}' but found "
@@ -188,10 +226,26 @@ class Manifest(dict):
             content_type_parameters=content_type_params,
             entry_type=entry_type,
             entry_params=entry_params,
+            schema_id=schema_id,
+            sort_scheme_id=sort_scheme_id,
         )
         manifest = Manifest._build_manifest(meta, entries, author, uuid)
         return manifest
 
+    @staticmethod
+    def from_json(json_string: str) -> Manifest:
+        parsed_dict = json.loads(json_string)
+        return Manifest.of(
+            entries=ManifestEntryList.of(
+                [
+                    ManifestEntry.from_dict(entry)
+                    for entry in parsed_dict.get("entries", [])
+                ]
+            ),
+            author=ManifestAuthor.from_dict(parsed_dict.get("author")),
+            uuid=parsed_dict.get("id"),
+        )
+
     @staticmethod
     def merge_manifests(
         manifests: List[Manifest], author: Optional[ManifestAuthor] = None
@@ -240,6 +294,8 @@ class ManifestMeta(dict):
         content_type_parameters: Optional[List[Dict[str, str]]] = None,
         entry_type: Optional[EntryType] = None,
         entry_params: Optional[EntryParams] = None,
+        schema_id: Optional[int] = None,
+        sort_scheme_id: Optional[str] = None,
     ) -> ManifestMeta:
         manifest_meta = ManifestMeta()
         if record_count is not None:
@@ -262,8 +318,31 @@ class ManifestMeta(dict):
             )
         if entry_params is not None:
             manifest_meta["entry_params"] = entry_params
+        if schema_id is not None:
+            manifest_meta["schema_id"] = schema_id
+        if sort_scheme_id is not None:
+            manifest_meta["sort_scheme_id"] = sort_scheme_id
         return manifest_meta
 
+    @staticmethod
+    def from_dict(obj: dict) -> Optional[ManifestMeta]:
+        if obj is None:
+            return None
+
+        return ManifestMeta.of(
+            record_count=obj.get("record_count"),
+            content_length=obj.get("content_length"),
+            content_type=obj.get("content_type"),
+            content_encoding=obj.get("content_encoding"),
+            source_content_length=obj.get("source_content_length"),
+            credentials=obj.get("credentials"),
+            content_type_parameters=obj.get("content_type_parameters"),
+            entry_type=obj.get("entry_type"),
+            entry_params=obj.get("entry_params"),
+            schema_id=obj.get("schema_id"),
+            sort_scheme_id=obj.get("sort_scheme_id"),
+        )
+
     @property
     def record_count(self) -> Optional[int]:
         return self.get("record_count")
@@ -310,6 +389,14 @@ class ManifestMeta(dict):
             self["entry_params"] = val = EntryParams(val)
         return val
 
+    @property
+    def schema_id(self) -> Optional[int]:
+        return self.get("schema_id")
+
+    @property
+    def sort_scheme_id(self) -> Optional[str]:
+        return self.get("sort_scheme_id")
+
 
 class ManifestEntry(dict):
     @staticmethod
@@ -342,6 +429,10 @@ class ManifestEntry(dict):
         url: str,
         record_count: int,
         source_content_length: Optional[int] = None,
+        credentials: Optional[Dict[str, str]] = None,
+        content_type_parameters: Optional[List[Dict[str, str]]] = None,
+        entry_type: Optional[EntryType] = None,
+        entry_params: Optional[EntryParams] = None,
         **s3_client_kwargs,
     ) -> ManifestEntry:
         from deltacat.aws import s3u as s3_utils
@@ -354,10 +445,134 @@ class ManifestEntry(dict):
             content_type=s3_obj["ContentType"],
             content_encoding=s3_obj["ContentEncoding"],
             source_content_length=source_content_length,
+            credentials=credentials,
+            content_type_parameters=content_type_parameters,
+            entry_type=entry_type,
+            entry_params=entry_params,
         )
         manifest_entry = ManifestEntry.of(url, manifest_entry_meta)
         return manifest_entry
 
+    @staticmethod
+    def from_dict(obj: dict) -> ManifestEntry:
+        return ManifestEntry.of(
+            url=obj.get("url"),
+            uri=obj.get("uri"),
+            meta=ManifestMeta.from_dict(obj.get("meta")),
+            mandatory=obj.get("mandatory", True),
+            uuid=obj.get("id"),
+        )
+
+    @staticmethod
+    def from_path(
+        path: str,
+        filesystem: pa.fs.FileSystem,
+        record_count: int,
+        source_content_length: Optional[int] = None,
+        content_type: Optional[str] = None,
+        content_encoding: Optional[str] = None,
+        credentials: Optional[Dict[str, str]] = None,
+        content_type_parameters: Optional[List[Dict[str, str]]] = None,
+        entry_type: Optional[EntryType] = None,
+        entry_params: Optional[EntryParams] = None,
+        schema_id: Optional[int] = None,
+        sort_scheme_id: Optional[str] = None,
+    ) -> ManifestEntry:
+        """
+        Creates a manifest entry from a path using a pyarrow filesystem.
+
+        Args:
+            path: Path to the file
+            filesystem: PyArrow filesystem to use for accessing the file
+            record_count: Number of records in the file
+            source_content_length: Optional original content length in-memory
+                before writing to disk.
+            content_type: Optional content type override. If not provided, will
+                be derived from file extension.
+            content_encoding: Optional content encoding override. If not
+                provided, will be derived from file extension.
+            credentials: Optional credentials required to read this manifest entry.
+            content_type_parameters: Optional content type parameters.
+            entry_type: Optional entry type of this manifest entry. Defaults to DATA.
+            entry_params: Optional entry type parameters.
+            schema_id: Schema ID used to write this manifest entry.
+            sort_scheme_id: Sort scheme ID used to write this manifest entry.
+
+        Returns:
+            A ManifestEntry instance
+        """
+        file_info = get_file_info(path, filesystem)
+        if file_info.type != pa.fs.FileType.File:
+            raise FileNotFoundError(f"Path does not point to a file: {path}")
+
+        # Extract extensions from right to left
+        # First split will get potential encoding extension
+        base_path, ext1 = posixpath.splitext(path)
+
+        # Initialize with defaults for no extensions
+        derived_content_type = ContentType.BINARY
+        derived_content_encoding = ContentEncoding.IDENTITY
+
+        # Only proceed with extension checks if we found at least one extension
+        if ext1:
+            # Check if the first extension is a known encoding
+            derived_content_encoding = EXT_TO_CONTENT_ENCODING.get(
+                ext1,
+                ContentEncoding.IDENTITY,
+            )
+
+            # Get second extension only if first was an encoding
+            if derived_content_encoding != ContentEncoding.IDENTITY:
+                # Second split will get potential content type extension
+                _, ext2 = posixpath.splitext(base_path)
+                if ext2:
+                    derived_content_type = EXT_TO_CONTENT_TYPE.get(
+                        ext2,
+                        ContentType.BINARY,
+                    )
+            else:
+                # First extension wasn't an encoding, check if it's a
+                # content type
+                derived_content_type = EXT_TO_CONTENT_TYPE.get(
+                    ext1,
+                    ContentType.BINARY,
+                )
+
+        if (
+            derived_content_type == ContentType.BINARY
+            and derived_content_encoding != ContentEncoding.IDENTITY
+        ):
+            logger.debug(
+                f"Found encoding {derived_content_encoding.value} but no "
+                f"content type for {path}, assuming binary"
+            )
+
+        # Use provided values if available, otherwise use derived values
+        final_content_type = (
+            content_type if content_type is not None else derived_content_type.value
+        )
+        final_content_encoding = (
+            content_encoding
+            if content_encoding is not None
+            else derived_content_encoding.value
+        )
+
+        manifest_entry_meta = ManifestMeta.of(
+            record_count=record_count,
+            content_length=file_info.size,
+            content_type=final_content_type,
+            content_encoding=final_content_encoding,
+            source_content_length=source_content_length,
+            credentials=credentials,
+            content_type_parameters=content_type_parameters,
+            entry_type=entry_type,
+            entry_params=entry_params,
+            schema_id=schema_id,
+            sort_scheme_id=sort_scheme_id,
+        )
+        manifest_entry = ManifestEntry.of(path, manifest_entry_meta)
+        return manifest_entry
+
     @property
     def uri(self) -> Optional[str]:
         return self.get("uri")
@@ -392,6 +607,12 @@ class ManifestAuthor(dict):
             manifest_author["version"] = version
         return manifest_author
 
+    @staticmethod
+    def from_dict(obj: dict) -> Optional[ManifestAuthor]:
+        if obj is None:
+            return None
+        return ManifestAuthor.of(obj.get("name"), obj.get("version"))
+
     @property
     def name(self) -> Optional[str]:
         return self.get("name")
@@ -416,3 +637,7 @@ class ManifestEntryList(List[ManifestEntry]):
         if val is not None and not isinstance(val, ManifestEntry):
             self[item] = val = ManifestEntry(val)
         return val
+
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self[i]  # This triggers __getitem__ conversion