deltacat 2.0.0b7__py3-none-any.whl → 2.0.0b10__py3-none-any.whl

deltacat/storage/model/expression/visitor.py

@@ -0,0 +1,248 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Generic, TypeVar, Callable, Optional
+from functools import singledispatchmethod
+import re
+
+from deltacat.storage.model.expression import (
+    Expression,
+    Reference,
+    Literal,
+    BinaryExpression,
+    UnaryExpression,
+    In,
+    Between,
+    Like,
+)
+
+
+C = TypeVar("C")  # Context type
+R = TypeVar("R")  # Return type
+
+
+class ExpressionVisitor(ABC, Generic[C, R]):
+    """
+    Visitor pattern for deltacat expressions.
+
+    This base class provides two ways to implement visitors:
+    1. Using a procedure dictionary (_PROCEDURES) - for simple, declarative visitors
+    2. Using specialized visit_xyz methods with snake_case naming - for more control
+
+    Subclasses need only implement visit_reference and visit_literal, plus either:
+    - Define _PROCEDURES dictionary with functions for handling different expression types
+    - Implement specific visit_xyz methods (using snake_case) for individual expressions
+    """
+
+    # Default procedure dictionary for subclasses to override
+    _PROCEDURES: Dict[str, Callable] = {}
+
+    def __init__(self):
+        """Initialize visitor and validate required methods."""
+        # Pre-check for required methods
+        if not hasattr(self, "visit_reference") or not callable(
+            getattr(self, "visit_reference")
+        ):
+            raise NotImplementedError("Subclasses must implement visit_reference")
+        if not hasattr(self, "visit_literal") or not callable(
+            getattr(self, "visit_literal")
+        ):
+            raise NotImplementedError("Subclasses must implement visit_literal")
+        self._setup_default_procedure_handlers()
+
+    def _to_snake_case(self, name: str) -> str:
+        """Convert PascalCase or camelCase to snake_case."""
+        pattern = re.compile(r"(?<!^)(?=[A-Z])")
+        return pattern.sub("_", name).lower()
+
+    def _setup_default_procedure_handlers(self):
+        """Set up default procedure application methods if not overridden."""
+        if not hasattr(self, "_apply_binary") or not callable(
+            getattr(self, "_apply_binary")
+        ):
+            self._apply_binary = lambda proc, left, right: proc(left, right)
+        if not hasattr(self, "_apply_unary") or not callable(
+            getattr(self, "_apply_unary")
+        ):
+            self._apply_unary = lambda proc, operand: proc(operand)
+        if not hasattr(self, "_apply_in") or not callable(getattr(self, "_apply_in")):
+            self._apply_in = lambda proc, value, values: proc(value, values)
+        if not hasattr(self, "_apply_between") or not callable(
+            getattr(self, "_apply_between")
+        ):
+            self._apply_between = lambda proc, value, lower, upper: proc(
+                value, lower, upper
+            )
+        if not hasattr(self, "_apply_like") or not callable(
+            getattr(self, "_apply_like")
+        ):
+            self._apply_like = lambda proc, value, pattern: proc(value, pattern)
+
+    @singledispatchmethod
+    def visit(self, expr: Expression, context: Optional[C] = None) -> R:
+        """
+        Generic visit method that dispatches to specific methods based on expression type.
+
+        Args:
+            expr: The expression to visit
+            context: Optional context to pass through the visitor
+
+        Returns:
+            Result of visiting the expression
+        """
+        expr_type = type(expr).__name__
+        raise NotImplementedError(f"No visit method for type {expr_type}")
+
+    @visit.register
+    def _visit_reference(self, expr: Reference, context: Optional[C] = None) -> R:
+        """Visit a Reference expression."""
+        return self.visit_reference(expr, context)
+
+    @visit.register
+    def _visit_literal(self, expr: Literal, context: Optional[C] = None) -> R:
+        """Visit a Literal expression."""
+        return self.visit_literal(expr, context)
+
+    @visit.register
+    def _visit_binary(self, expr: BinaryExpression, context: Optional[C] = None) -> R:
+        """Visit a binary expression using method specialization or procedures."""
+        expr_type = type(expr).__name__
+
+        left_result = self.visit(expr.left, context)
+        right_result = self.visit(expr.right, context)
+
+        method_name = f"visit_{self._to_snake_case(expr_type)}"
+        if hasattr(self, method_name):
+            method = getattr(self, method_name)
+            return method(expr, context)
+
+        if expr_type in self._PROCEDURES:
+            return self._apply_binary(
+                self._PROCEDURES[expr_type], left_result, right_result
+            )
+
+        try:
+            return self.visit_binary_expression(
+                expr, left_result, right_result, context
+            )
+        except NotImplementedError:
+            raise NotImplementedError(f"No handler for {expr_type}")
+
+    @visit.register
+    def _visit_unary(self, expr: UnaryExpression, context: Optional[C] = None) -> R:
+        """Visit a unary expression using method specialization or procedures."""
+        expr_type = type(expr).__name__
+
+        operand_result = self.visit(expr.operand, context)
+
+        method_name = f"visit_{self._to_snake_case(expr_type)}"
+        if hasattr(self, method_name):
+            method = getattr(self, method_name)
+            return method(expr, context)
+
+        if expr_type in self._PROCEDURES:
+            return self._apply_unary(self._PROCEDURES[expr_type], operand_result)
+
+        try:
+            return self.visit_unary_expression(expr, operand_result, context)
+        except NotImplementedError:
+            raise NotImplementedError(f"No handler for {expr_type}")
+
+    @visit.register
+    def _visit_in(self, expr: In, context: Optional[C] = None) -> R:
+        """Visit an In expression."""
+        if hasattr(self, "visit_in"):
+            return self.visit_in(expr, context)
+
+        if "In" in self._PROCEDURES:
+            value_result = self.visit(expr.value, context)
+            values_results = [self.visit(v, context) for v in expr.values]
+            return self._apply_in(self._PROCEDURES["In"], value_result, values_results)
+
+        raise NotImplementedError("No handler for In expression")
+
+    @visit.register
+    def _visit_between(self, expr: Between, context: Optional[C] = None) -> R:
+        """Visit a Between expression."""
+        if hasattr(self, "visit_between"):
+            return self.visit_between(expr, context)
+
+        if "Between" in self._PROCEDURES:
+            value_result = self.visit(expr.value, context)
+            lower_result = self.visit(expr.lower, context)
+            upper_result = self.visit(expr.upper, context)
+            return self._apply_between(
+                self._PROCEDURES["Between"], value_result, lower_result, upper_result
+            )
+
+        raise NotImplementedError("No handler for Between expression")
+
+    @visit.register
+    def _visit_like(self, expr: Like, context: Optional[C] = None) -> R:
+        """Visit a Like expression."""
+        if hasattr(self, "visit_like"):
+            return self.visit_like(expr, context)
+
+        if "Like" in self._PROCEDURES:
+            value_result = self.visit(expr.value, context)
+            pattern_result = self.visit(expr.pattern, context)
+            return self._apply_like(
+                self._PROCEDURES["Like"], value_result, pattern_result
+            )
+
+        raise NotImplementedError("No handler for Like expression")
+
+    @abstractmethod
+    def visit_reference(self, expr: Reference, context: Optional[C] = None) -> R:
+        """Visit a Reference expression."""
+        pass
+
+    @abstractmethod
+    def visit_literal(self, expr: Literal, context: Optional[C] = None) -> R:
+        """Visit a Literal expression."""
+        pass
+
+    def visit_binary_expression(
+        self, expr: BinaryExpression, left: R, right: R, context: Optional[C] = None
+    ) -> R:
+        """Default fallback handler for binary expressions."""
+        raise NotImplementedError(f"No handler for {type(expr).__name__}")
+
+    def visit_unary_expression(
+        self, expr: UnaryExpression, operand: R, context: Optional[C] = None
+    ) -> R:
+        """Default fallback handler for unary expressions."""
+        raise NotImplementedError(f"No handler for {type(expr).__name__}")
+
+
+class DisplayVisitor(ExpressionVisitor[Expression, str]):
+    """
+    Visitor implementation that formats expressions in standard infix notation.
+    For example: "a = b AND c > d" instead of "(AND (= a b) (> c d))".
+    """
+
+    # Map all expression types to their string formatting procedures with infix notation
+    _PROCEDURES = {
+        # Binary operations with infix notation
+        "Equal": lambda left, right: f"{left} = {right}",
+        "NotEqual": lambda left, right: f"{left} <> {right}",
+        "GreaterThan": lambda left, right: f"{left} > {right}",
+        "LessThan": lambda left, right: f"{left} < {right}",
+        "GreaterThanEqual": lambda left, right: f"{left} >= {right}",
+        "LessThanEqual": lambda left, right: f"{left} <= {right}",
+        "And": lambda left, right: f"({left} AND {right})",
+        "Or": lambda left, right: f"({left} OR {right})",
+        # Unary operations
+        "Not": lambda operand: f"NOT ({operand})",
+        "IsNull": lambda operand: f"({operand}) IS NULL",
+        # Special operations
+        "In": lambda value, values: f"{value} IN ({', '.join(values)})",
+        "Between": lambda value, lower, upper: f"{value} BETWEEN {lower} AND {upper}",
+        "Like": lambda value, pattern: f"{value} LIKE {pattern}",
+    }
+
+    def visit_reference(self, expr: Reference, context=None) -> str:
+        """Format a field reference."""
+        return expr.field
+
+    def visit_literal(self, expr: Literal, context=None) -> str:
+        """Format a literal value using its PyArrow representation."""
+        return str(expr.value)

deltacat/storage/model/metafile.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 import copy
 
-from typing import Optional, Tuple, List
+from typing import Optional, Tuple, List, Union
 
 import base64
 import json
@@ -412,7 +412,7 @@ class Metafile(dict):
     @staticmethod
     def based_on(
         other: Optional[Metafile],
-        new_id: Optional[Locator] = None,
+        new_id: Optional[str] = None,
     ) -> Optional[Metafile]:
         """
         Returns a new metafile equivalent to the input metafile, but with a new
@@ -539,29 +539,31 @@ class Metafile(dict):
                 f"${serialized_dict}"
             )
 
+    @staticmethod
+    def get_type_name(serialized_dict: dict):
+        """
+        Given a serialized dictionary of Metafile data, gets the type name of
+        the metafile class.
+        """
+        return Metafile.get_class(serialized_dict).__name__
+
     @classmethod
-    def read(
+    def deserialize(
         cls,
-        path: str,
-        filesystem: Optional[pyarrow.fs.FileSystem] = None,
-        format: Optional[str] = METAFILE_FORMAT,
+        serialized: Union[bytes, str],
+        meta_format: Optional[str] = METAFILE_FORMAT,
     ) -> Metafile:
         """
-        Read a metadata file and return the deserialized object.
-        :param path: Metadata file path to read.
-        :param filesystem: File system to use for reading the metadata file.
-        :param format: Format to use for deserializing the metadata file.
-        :return: Deserialized object from the metadata file.
+        Deserialize a metadata file from the given bytes or string.
+        :param serialized: Serialized metadata file data.
+        :param meta_format: Format to use for deserializing the metadata file.
+        :return: Deserialized metadata file.
         """
-        if format not in SUPPORTED_METAFILE_FORMATS:
+        if meta_format not in SUPPORTED_METAFILE_FORMATS:
             raise ValueError(
-                f"Unsupported format '{format}'. Supported formats include: {SUPPORTED_METAFILE_FORMATS}."
+                f"Unsupported format '{meta_format}'. "
+                f"Supported formats include: {SUPPORTED_METAFILE_FORMATS}."
             )
-
-        if not filesystem:
-            path, filesystem = resolve_path_and_filesystem(path, filesystem)
-        with filesystem.open_input_stream(path) as file:
-            binary = file.readall()
         reader = {
             "json": lambda b: json.loads(
                 b.decode("utf-8"),
@@ -573,12 +575,32 @@ class Metafile(dict):
                 },
             ),
             "msgpack": msgpack.loads,
-        }[format]
-        data = reader(binary)
+        }[meta_format]
+        data = reader(serialized)
         # cast this Metafile into the appropriate child class type
         clazz = Metafile.get_class(data)
-        obj = clazz(**data).from_serializable(path, filesystem)
-        return obj
+        return clazz(**data)
+
+    @classmethod
+    def read(
+        cls,
+        path: str,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+        meta_format: Optional[str] = METAFILE_FORMAT,
+    ) -> Metafile:
+        """
+        Read a metadata file and return the deserialized object.
+        :param path: Metadata file path to read.
+        :param filesystem: File system to use for reading the metadata file.
+        :param meta_format: Format to use for deserializing the metadata file.
+        :return: Deserialized object from the metadata file.
+        """
+        if not filesystem:
+            path, filesystem = resolve_path_and_filesystem(path, filesystem)
+        with filesystem.open_input_stream(path) as file:
+            serialized = file.readall()
+        metafile = Metafile.deserialize(serialized, meta_format)
+        return metafile.from_serializable(path, filesystem)
 
     def write_txn(
         self,
@@ -616,11 +638,37 @@ class Metafile(dict):
             filesystem=filesystem,
         )
 
+    def serialize(
+        self,
+        meta_format: Optional[str] = METAFILE_FORMAT,
+    ) -> Union[bytes, str]:
+        """
+        Serialize this object to the given metafile format.
+        :param meta_format: Format to use for serializing the metadata file.
+        :return: Serialized metadata file bytes or string (format dependent).
+        """
+        if meta_format not in SUPPORTED_METAFILE_FORMATS:
+            raise ValueError(
+                f"Unsupported format '{meta_format}'. "
+                f"Supported formats include: {SUPPORTED_METAFILE_FORMATS}."
+            )
+        serializer = {
+            "json": lambda data: json.dumps(
+                data,
+                indent=4,
+                default=lambda b: base64.b64encode(b).decode("utf-8")
+                if isinstance(b, bytes)
+                else b,
+            ).encode("utf-8"),
+            "msgpack": msgpack.dumps,
+        }[meta_format]
+        return serializer(self.to_serializable())
+
     def write(
         self,
         path: str,
         filesystem: Optional[pyarrow.fs.FileSystem] = None,
-        format: Optional[str] = METAFILE_FORMAT,
+        meta_format: Optional[str] = METAFILE_FORMAT,
     ) -> None:
         """
         Serialize and write this object to a metadata file.
@@ -628,31 +676,15 @@ class Metafile(dict):
         :param filesystem: File system to use for writing the metadata file. If
         not given, a default filesystem will be automatically selected based on
         the catalog root path.
-        param: format: Format to use for serializing the metadata file.
+        :param meta_format: Format to use for serializing the metadata file.
         """
-        if format not in SUPPORTED_METAFILE_FORMATS:
-            raise ValueError(
-                f"Unsupported format '{format}'. Supported formats include: {SUPPORTED_METAFILE_FORMATS}."
-            )
-
+        serialized = self.serialize(meta_format)
         if not filesystem:
             path, filesystem = resolve_path_and_filesystem(path, filesystem)
         revision_dir_path = posixpath.dirname(path)
         filesystem.create_dir(revision_dir_path, recursive=True)
-
-        writer = {
-            "json": lambda data: json.dumps(
-                data,
-                indent=4,
-                default=lambda b: base64.b64encode(b).decode("utf-8")
-                if isinstance(b, bytes)
-                else b,
-            ).encode("utf-8"),
-            "msgpack": msgpack.dumps,
-        }[format]
-
         with filesystem.open_output_stream(path) as file:
-            file.write(writer(self.to_serializable()))
+            file.write(serialized)
 
     def equivalent_to(self, other: Metafile) -> bool:
         """

deltacat/storage/model/scan/push_down.py

@@ -1,3 +1,10 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Optional
+
+from deltacat.storage.model.expression import Expression
+
+
 class RowFilter:
     ...
 
@@ -6,14 +13,34 @@ class ColumnFilter:
     ...
 
 
+@dataclass
 class PartitionFilter:
-    ...
+    expr: Expression
+
+    @staticmethod
+    def of(expr: Expression) -> PartitionFilter:
+        return PartitionFilter(expr)
 
 
+@dataclass
 class Pushdown:
     """Represents pushdown predicates to be applied for DeltaCAT Tables"""
 
-    row_filter: RowFilter
-    column_filter: ColumnFilter
-    partition_filter: PartitionFilter
-    limit: int
+    row_filter: Optional[RowFilter]
+    column_filter: Optional[ColumnFilter]
+    partition_filter: Optional[PartitionFilter]
+    limit: Optional[int]
+
+    @staticmethod
+    def of(
+        row_filter: Optional[RowFilter],
+        column_filter: Optional[ColumnFilter],
+        partition_filter: Optional[PartitionFilter],
+        limit: Optional[int],
+    ) -> Pushdown:
+        return Pushdown(
+            row_filter=row_filter,
+            column_filter=column_filter,
+            partition_filter=partition_filter,
+            limit=limit,
+        )

deltacat/storage/model/types.py

@@ -6,7 +6,7 @@ from typing import List, Union
 import numpy as np
 import pandas as pd
 import pyarrow as pa
-from ray.data.dataset import Dataset
+from ray.data.dataset import Dataset as RayDataset
 from daft import DataFrame as DaftDataFrame
 
 
@@ -16,13 +16,15 @@ LocalTable = Union[
     np.ndarray,
     pa.parquet.ParquetFile,
 ]
-LocalDataset = List[LocalTable]
-DistributedDataset = Union[Dataset, DaftDataFrame]
+LocalDataset = Union[LocalTable, List[LocalTable]]
+DistributedDataset = Union[RayDataset, DaftDataFrame]
+Dataset = Union[LocalDataset, DistributedDataset]
 
 
 class StreamFormat(str, Enum):
     DELTACAT = "deltacat"
     ICEBERG = "iceberg"
+    HIVE = "hive"
     HUDI = "hudi"
     DELTA_LAKE = "delta_lake"
     SQLITE3 = "SQLITE3"  # used by tests

deltacat/storage/rivulet/__init__.py

@@ -1,7 +1,7 @@
-from .schema.schema import Schema
-from .schema.schema import Field
-from .dataset import Dataset
-from .schema.schema import Datatype
+from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.storage.rivulet.schema.schema import Field
+from deltacat.storage.rivulet.dataset import Dataset
+from deltacat.storage.rivulet.schema.schema import Datatype
 
 __all__ = [
     "Schema",