deltacat 2.0.0b11__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/experimental/storage/iceberg/iceberg_scan_planner.py

@@ -1,16 +1,50 @@
-from typing import Optional
+import logging
+from typing import Optional, Any, Set
 
 from pyiceberg.catalog import Catalog
-from deltacat.storage.model.scan.push_down import Pushdown
+from pyiceberg.table import Table
+import deltacat.logs as logs
+
+from deltacat.storage.model.scan.push_down import Pushdown, PartitionFilter
 from deltacat.storage.model.scan.scan_plan import ScanPlan
 from deltacat.storage.model.scan.scan_task import FileScanTask, DataFile
 from deltacat.storage.util.scan_planner import ScanPlanner
 from deltacat.experimental.storage.iceberg.impl import _try_load_iceberg_table
+from deltacat.experimental.storage.iceberg.visitor import IcebergExpressionVisitor
+
+# Initialize DeltaCAT logger
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 class IcebergScanPlanner(ScanPlanner):
     def __init__(self, catalog: Catalog):
         self.catalog = catalog
+        self.expression_visitor = IcebergExpressionVisitor()
+
+    @classmethod
+    def _collect_filter_fields(cls, expr: Any) -> Set[str]:
+        """
+        Collects all field names referenced in the filter expression.
+
+        Args:
+            expr: The expression to analyze
+
+        Returns:
+            Set of field names referenced in the expression
+        """
+        fields = set()
+        if hasattr(expr, "field"):
+            fields.add(expr.field)
+        if hasattr(expr, "left"):
+            fields.update(cls._collect_filter_fields(expr.left))
+        if hasattr(expr, "right"):
+            fields.update(cls._collect_filter_fields(expr.right))
+        if hasattr(expr, "expr"):
+            fields.update(cls._collect_filter_fields(expr.expr))
+        if hasattr(expr, "values"):
+            for value in expr.values:
+                fields.update(cls._collect_filter_fields(value))
+        return fields
 
     def create_scan_plan(
         self,
@@ -21,8 +55,75 @@ class IcebergScanPlanner(ScanPlanner):
         iceberg_table = _try_load_iceberg_table(
             self.catalog, namespace=namespace, table_name=table_name
         )
+
+        # TODO: implement row, column predicate pushdown to Iceberg
+
+        # Get the partition spec
+        partition_spec = iceberg_table.spec()
+
+        # Check if the table is partitioned
+        is_partitioned = len(partition_spec.fields) > 0
+
+        scan = iceberg_table.scan()
+        if is_partitioned:
+            if pushdown and pushdown.partition_filter:
+                filter_fields = self._collect_filter_fields(pushdown.partition_filter)
+                logger.info(
+                    f"Pushdown partition filter is enabled, converting to Iceberg. Fields discovered in filter: {', '.join(sorted(filter_fields))}"
+                )
+                # Handle partition filter if present, DeltaCAT only supports partition-level filters right now
+                iceberg_expression = self._convert_partition_filter(
+                    iceberg_table, pushdown.partition_filter
+                )
+                scan = scan.filter(iceberg_expression)
+
         file_scan_tasks = []
-        # TODO: implement predicate pushdown to Iceberg
-        for scan_task in iceberg_table.scan().plan_files():
+        for scan_task in scan.plan_files():
             file_scan_tasks.append(FileScanTask([DataFile(scan_task.file.file_path)]))
         return ScanPlan(file_scan_tasks)
+
+    @classmethod
+    def _validate_partition_references(
+        cls, expr: Any, partition_cols: Set[str]
+    ) -> None:
+        """
+        Validates that the expression only references partition columns.
+
+        Args:
+            expr: The expression to validate
+            partition_cols: Set of valid partition column names
+
+        Raises:
+            ValueError: If the expression references a non-partition column
+        """
+        if hasattr(expr, "field"):  # Reference type expression
+            if expr.field not in partition_cols:
+                raise ValueError(
+                    f"Filter references non-partition column: {expr.field}. "
+                    f"Partition columns are: {partition_cols}"
+                )
+        # Recursively validate nested expressions
+        if hasattr(expr, "left"):
+            cls._validate_partition_references(expr.left, partition_cols)
+        if hasattr(expr, "right"):
+            cls._validate_partition_references(expr.right, partition_cols)
+        if hasattr(expr, "expr"):
+            cls._validate_partition_references(expr.expr, partition_cols)
+        if hasattr(expr, "values"):
+            for value in expr.values:
+                cls._validate_partition_references(value, partition_cols)
+
+    def _convert_partition_filter(
+        self, table: Table, partition_filter: PartitionFilter
+    ):
+        """
+        Convert DeltaCAT partition filter to PyIceberg expression,
+        validating that only partition columns are referenced.
+        """
+        partition_cols = set(field.name for field in table.spec().fields)
+
+        # Validate before converting
+        self._validate_partition_references(partition_filter, partition_cols)
+
+        # Convert to PyIceberg expression
+        return self.expression_visitor.visit(partition_filter)

deltacat/experimental/storage/iceberg/impl.py

@@ -41,7 +41,7 @@ from deltacat.experimental.storage.iceberg.model import (
     NamespaceMapper,
     TableMapper,
 )
-from deltacat.types.media import ContentType, StorageType, TableType
+from deltacat.types.media import ContentType, StorageType, DatasetType
 from deltacat.utils.common import ReadKwargsProvider
 
 from pyiceberg.catalog import Catalog
@@ -281,7 +281,7 @@ def get_latest_delta(
 
 def download_delta(
     delta_like: Union[Delta, DeltaLocator],
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     storage_type: StorageType = StorageType.DISTRIBUTED,
     max_parallelism: Optional[int] = None,
     columns: Optional[List[str]] = None,
@@ -303,7 +303,7 @@ def download_delta(
 def download_delta_manifest_entry(
     delta_like: Union[Delta, DeltaLocator],
     entry_index: int,
-    table_type: TableType = TableType.PYARROW,
+    table_type: DatasetType = DatasetType.PYARROW,
     columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     *args,
@@ -603,6 +603,8 @@ def stage_delta(
     properties: Optional[DeltaProperties] = None,
     s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
+    schema: Optional[Schema] = None,
+    sort_scheme_id: Optional[str] = None,
     *args,
     **kwargs,
 ) -> Delta:

deltacat/experimental/storage/iceberg/model.py

@@ -66,6 +66,7 @@ from deltacat.storage import (
     TableVersionLocator,
     Transform,
     TransformName,
+    TruncateStrategy,
     TruncateTransform,
     TruncateTransformParameters,
     UnknownTransform,
@@ -227,7 +228,10 @@ class TransformMapper(ModelMapper[IcebergTransform, Transform]):
             )
         if isinstance(obj, IcebergTruncateTransform):
             return TruncateTransform.of(
-                TruncateTransformParameters.of(width=obj.width),
+                TruncateTransformParameters.of(
+                    width=obj.width,
+                    truncate_strategy=TruncateStrategy.ICEBERG,
+                ),
             )
         return UnknownTransform.of()
 
@@ -323,7 +327,7 @@ class PartitionSchemeMapper(ModelMapper[PartitionSpec, PartitionScheme]):
         elif not schema:
             err_msg = "Schema is required for Partition Spec conversion."
             raise ValueError(err_msg)
-        keys = [PartitionKeyMapper.map(field, schema) for field in obj.fields]
+        keys = [PartitionKeyMapper.map(field, schema) for field in obj.fields] or None
         return PartitionScheme.of(
             keys=keys,
             name=name,
@@ -425,7 +429,7 @@ class SortSchemeMapper(ModelMapper[IcebergSortOrder, SortScheme]):
         elif not schema:
             err_msg = "Schema is required for Sort Order conversion."
             raise ValueError(err_msg)
-        keys = [SortKeyMapper.map(field, schema) for field in obj.fields]
+        keys = [SortKeyMapper.map(field, schema) for field in obj.fields] or None
         return SortScheme.of(
             keys=keys,
             name=name,

deltacat/experimental/storage/iceberg/visitor.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Any
+
+import pyarrow
+from deltacat.storage.model.scan.push_down import PartitionFilter
+
+import deltacat.logs as logs
+from deltacat.storage.model.expression import Reference, Literal
+from deltacat.storage.model.expression.visitor import ExpressionVisitor
+from pyiceberg.expressions import (
+    And,
+    Or,
+    Not,
+    EqualTo,
+    NotEqualTo,
+    GreaterThan,
+    GreaterThanOrEqual,
+    LessThan,
+    LessThanOrEqual,
+    IsNull,
+    In,
+)
+
+# Initialize DeltaCAT logger
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+class IcebergExpressionVisitor(ExpressionVisitor[None, Any]):
+    """
+    Visitor that translates DeltaCAT expressions to PyIceberg expressions.
+    """
+
+    def visit(self, expr, context=None):
+        # Handle PartitionFilter by extracting and visiting the inner expression
+        if isinstance(expr, PartitionFilter):
+            return self.visit(expr.expr, context)
+        # Handle all other expressions using the parent's visit method
+        return super().visit(expr, context)
+
+    def visit_reference(self, expr: Reference, context=None) -> str:
+        return expr.field
+
+    def visit_literal(self, expr: Literal, context=None) -> Any:
+        # Convert PyArrow scalar to Python native type
+        return (
+            expr.value.as_py() if isinstance(expr.value, pyarrow.Scalar) else expr.value
+        )
+
+    def visit_and(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return And(left, right)
+
+    def visit_or(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return Or(left, right)
+
+    def visit_not(self, expr, context=None):
+        operand = self.visit(expr.operand, context)
+        return Not(operand)
+
+    def visit_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return EqualTo(left, right)
+
+    def visit_not_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return NotEqualTo(left, right)
+
+    def visit_greater_than(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return GreaterThan(left, right)
+
+    def visit_greater_than_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return GreaterThanOrEqual(left, right)
+
+    def visit_less_than(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return LessThan(left, right)
+
+    def visit_less_than_equal(self, expr, context=None):
+        left = self.visit(expr.left, context)
+        right = self.visit(expr.right, context)
+        return LessThanOrEqual(left, right)
+
+    def visit_is_null(self, expr, context=None):
+        operand = self.visit(expr.operand, context)
+        return IsNull(operand)
+
+    def visit_in(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        values = [self.visit(v, context) for v in expr.values]
+        return In(value, values)
+
+    def visit_between(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        lower = self.visit(expr.lower, context)
+        upper = self.visit(expr.upper, context)
+        return And(GreaterThanOrEqual(value, lower), LessThanOrEqual(value, upper))
+
+    # PyIceberg does not have a direct equivalent of LIKE
+    def visit_like(self, expr, context=None):
+        value = self.visit(expr.value, context)
+        pattern = self.visit(expr.pattern, context)
+        logger.warning(
+            f"LIKE operation is not supported in PyIceberg. Ignoring LIKE filter: {value} LIKE '{pattern}'. "
+            "This may result in more data being returned than expected."
+        )
+        # Return None or a default expression that won't filter anything
+        return (
+            None  # or return NotEqualTo(value, None)  # matches everything except NULL
+        )

deltacat/experimental/storage/rivulet/dataset.py

@@ -52,7 +52,6 @@ from deltacat.storage import (
     TableVersion,
     TableVersionLocator,
     Transaction,
-    TransactionType,
     TransactionOperation,
     TransactionOperationType,
 )
@@ -299,7 +298,6 @@ class Dataset:
                     partition_values=DEFAULT_PARTITION_VALUES,
                     partition_id=self._partition_id,
                 ),
-                schema=None,
                 content_types=None,
             ),
         ]
@@ -312,7 +310,6 @@ class Dataset:
         ]
 
         transaction = Transaction.of(
-            txn_type=TransactionType.APPEND,
             txn_operations=TransactionOperationList.of(txn_operations),
         )
 

deltacat/experimental/storage/rivulet/metastore/delta.py

@@ -10,7 +10,6 @@ from deltacat.storage import (
     Delta,
     DeltaType,
     Transaction,
-    TransactionType,
     TransactionOperation,
     TransactionOperationType,
 )
@@ -169,7 +168,6 @@ class DeltacatManifestIO(ManifestIO):
         delta["level"] = level
 
         tx_results = Transaction.of(
-            txn_type=TransactionType.APPEND,
             txn_operations=TransactionOperationList.of(
                 [
                     TransactionOperation.of(

deltacat/experimental/storage/rivulet/reader/dataset_metastore.py

@@ -5,6 +5,7 @@ from typing import Generator, Optional
 import pyarrow
 import pyarrow.fs
 
+from deltacat.constants import REV_DIR_NAME
 from deltacat.storage import Delta
 from deltacat.storage.model.partition import PartitionLocator
 from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
@@ -83,7 +84,7 @@ class DatasetMetastore:
         param: filesystem: The filesystem to search for the revisions.
         returns: The latest revision as a RivuletDelta.
         """
-        rev_directory = posixpath.join(delta_dir, "rev")
+        rev_directory = posixpath.join(delta_dir, REV_DIR_NAME)
         revisions = filesystem.get_file_info(
             pyarrow.fs.FileSelector(rev_directory, allow_not_found=True)
         )
@@ -128,7 +129,7 @@ class DatasetMetastore:
             return
 
         # Locate "rev" directory inside the partition
-        rev_directory = posixpath.join(partition_path, "rev")
+        rev_directory = posixpath.join(partition_path, REV_DIR_NAME)
         rev_info = filesystem.get_file_info(rev_directory)
 
         if rev_info.type != pyarrow.fs.FileType.Directory:

deltacat/io/datasource/deltacat_datasource.py

@@ -350,7 +350,6 @@ class DeltaCatDatasource(Datasource):
             ]
         elif self._deltacat_read_type == DeltacatReadType.METADATA_LIST:
             # do a shallow read of the top-level DeltaCAT metadata
-            print(f"listers: {self._reader.listers}")
             listers = copy.deepcopy(self._reader.listers)
             listers = [listers[0]]
             read_tasks = self._list_all_metafiles_read_tasks(

deltacat/storage/__init__.py

@@ -20,6 +20,9 @@ from deltacat.storage.model.metafile import (
 from deltacat.storage.model.transaction import (
     TransactionOperation,
     Transaction,
+    read_transaction,
+    transactions,
+    transaction,
 )
 from deltacat.storage.model.namespace import (
     Namespace,
@@ -31,6 +34,7 @@ from deltacat.storage.model.partition import (
     PartitionLocator,
     PartitionLocatorAlias,
     PartitionKey,
+    PartitionKeyList,
     PartitionScheme,
     PartitionSchemeList,
     PartitionValues,
@@ -43,6 +47,9 @@ from deltacat.storage.model.schema import (
     NestedFieldName,
     Schema,
     SchemaList,
+    SchemaUpdate,
+    SchemaUpdateOperation,
+    SchemaUpdateOperations,
 )
 from deltacat.storage.model.stream import (
     Stream,
@@ -75,6 +82,7 @@ from deltacat.storage.model.transform import (
     MonthTransform,
     YearTransform,
     TruncateTransform,
+    TruncateStrategy,
 )
 from deltacat.storage.model.types import (
     CommitState,
@@ -88,11 +96,12 @@ from deltacat.storage.model.types import (
     SchemaConsistencyType,
     StreamFormat,
     SortOrder,
-    TransactionType,
     TransactionOperationType,
+    TransactionStatus,
 )
 from deltacat.storage.model.sort_key import (
     SortKey,
+    SortKeyList,
     SortScheme,
     SortSchemeList,
 )
@@ -138,6 +147,7 @@ __all__ = [
     "NullOrder",
     "Partition",
     "PartitionKey",
+    "PartitionKeyList",
     "PartitionLocator",
     "PartitionLocatorAlias",
     "PartitionScheme",
@@ -145,8 +155,12 @@ __all__ = [
     "PartitionValues",
     "Schema",
     "SchemaList",
+    "SchemaUpdate",
+    "SchemaUpdateOperation",
+    "SchemaUpdateOperations",
     "SchemaConsistencyType",
     "SortKey",
+    "SortKeyList",
     "SortOrder",
     "SortScheme",
     "SortSchemeList",
@@ -163,13 +177,17 @@ __all__ = [
     "Transaction",
     "TransactionOperation",
     "TransactionOperationType",
-    "TransactionType",
+    "TransactionStatus",
     "Transform",
     "TransformName",
     "TransformParameters",
     "TruncateTransform",
     "TruncateTransformParameters",
+    "TruncateStrategy",
     "UnknownTransform",
     "VoidTransform",
     "YearTransform",
+    "read_transaction",
+    "transactions",
+    "transaction",
 ]