deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/experimental/storage/iceberg/visitor.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Any
+
+import pyarrow
+from deltacat.storage.model.scan.push_down import PartitionFilter
+
+import deltacat.logs as logs
+from deltacat.storage.model.expression import Reference, Literal
+from deltacat.storage.model.expression.visitor import ExpressionVisitor
+from pyiceberg.expressions import (
+    And,
+    Or,
+    Not,
+    EqualTo,
+    NotEqualTo,
+    GreaterThan,
+    GreaterThanOrEqual,
+    LessThan,
+    LessThanOrEqual,
+    IsNull,
+    In,
+)
+
+# Initialize DeltaCAT logger
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+class IcebergExpressionVisitor(ExpressionVisitor[None, Any]):
+    """
+    Visitor that translates DeltaCAT expressions to PyIceberg expressions.
+    """
+
+    def visit(self, expr, context=None):
+        # Handle PartitionFilter by extracting and visiting the inner expression
+        if isinstance(expr, PartitionFilter):
+            return self.visit(expr.expr, context)
+        # Handle all other expressions using the parent's visit method
+        return super().visit(expr, context)
+
+    def visit_reference(self, expr: Reference, context=None) -> str:
+        return expr.field
+
+    def visit_literal(self, expr: Literal, context=None) -> Any:
+        # Convert PyArrow scalar to Python native type
+        return (
+            expr.value.as_py() if isinstance(expr.value, pyarrow.Scalar) else expr.value
+        )
+
+    def visit_and(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return And(left, right)
+
+    def visit_or(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return Or(left, right)
+
+    def visit_not(self, expr, context=None):
+        operand = self.visit(expr.operand, context)
+        return Not(operand)
+
+    def visit_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return EqualTo(left, right)
+
+    def visit_not_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return NotEqualTo(left, right)
+
+    def visit_greater_than(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return GreaterThan(left, right)
+
+    def visit_greater_than_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return GreaterThanOrEqual(left, right)
+
+    def visit_less_than(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return LessThan(left, right)
+
+    def visit_less_than_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return LessThanOrEqual(left, right)
+
+    def visit_is_null(self, expr, context=None):
+        operand = self.visit(expr.operand, context)
+        return IsNull(operand)
+
+    def visit_in(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        values = [self.visit(v, context) for v in expr.values]
+        return In(value, values)
+
+    def visit_between(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        lower = self.visit(expr.lower, context)
+        upper = self.visit(expr.upper, context)
+        return And(GreaterThanOrEqual(value, lower), LessThanOrEqual(value, upper))
+
+    # PyIceberg does not have a direct equivalent of LIKE
+    def visit_like(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        pattern = self.visit(expr.pattern, context)
+        logger.warning(
+            f"LIKE operation is not supported in PyIceberg. Ignoring LIKE filter: {value} LIKE '{pattern}'. "
+            "This may result in more data being returned than expected."
+        )
+        # Return None or a default expression that won't filter anything
+        return (
+            None  # or return NotEqualTo(value, None)  # matches everything except NULL
+        )

deltacat/experimental/storage/rivulet/__init__.py

@@ -0,0 +1,11 @@
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.schema.schema import Field
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.schema.schema import Datatype
+
+__all__ = [
+    "Schema",
+    "Field",
+    "Dataset",
+    "Datatype",
+]

deltacat/experimental/storage/rivulet/arrow/serializer.py

@@ -0,0 +1,78 @@
+from abc import ABC, abstractmethod
+from typing import Iterator, List, Any
+import pyarrow as pa
+
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.serializer import (
+    DataSerializer,
+    MEMTABLE_DATA,
+)
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+
+
+class ArrowSerializer(DataSerializer, ABC):
+    """
+    Utility class which can serialize data by first converting to arrow as intermediate format
+    and then using the provided serialization function
+    """
+
+    def __init__(self, file_provider: FileProvider, schema: Schema):
+        self.schema = schema
+        self.file_provider = file_provider
+        self.arrow_schema = self.schema.to_pyarrow()
+
+    @abstractmethod
+    def serialize(self, table: pa.Table) -> List[SSTableRow]:
+        """
+        Write an Arrow table to the specified output file
+
+        :param table: PyArrow table to write
+        :return: Number of row groups in the written file
+        """
+        pass
+
+    def _to_arrow_table(self, sorted_records: MEMTABLE_DATA) -> pa.Table:
+        """
+        Convert input records to an Arrow table
+        """
+        if isinstance(sorted_records, pa.Table):
+            return sorted_records
+        elif isinstance(sorted_records, (Iterator, List)):
+            return pa.Table.from_pylist(sorted_records, schema=self.arrow_schema)
+        else:
+            raise ValueError(f"Unsupported record type: {type(sorted_records)}")
+
+    def _get_min_max_key(self, table: pa.Table) -> (Any, Any):
+        """
+        Get min and max values for the merge key from the table
+        """
+        key_col = table[self.schema.get_merge_key()]
+        return key_col[0].as_py(), key_col[len(key_col) - 1].as_py()
+
+    def flush_batch(self, sorted_records: MEMTABLE_DATA) -> List[SSTableRow]:
+        """
+        Write records to new parquet file as row group
+        For now, we will depend on pyarrow to write to parquet
+
+        :param sorted_records: record batch in SORTED ORDER
+        :return: metadata for constructing SSTable
+        """
+        if not sorted_records:
+            return []
+
+        table = self._to_arrow_table(sorted_records)
+        return self.serialize(table)
+
+    def write(self, sorted_records: MEMTABLE_DATA) -> List[SSTableRow]:
+        """
+        Write records using the provided serialization function
+
+        :param sorted_records: record batch in SORTED ORDER
+        :return: metadata for constructing SSTable
+        """
+        if not sorted_records:
+            return []
+
+        table = self._to_arrow_table(sorted_records)
+        return self.serialize(table)