deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

deltacat/utils/daft.py

@@ -1,13 +1,42 @@
 import logging
-from typing import Optional, List, Any, Dict, Callable
+from typing import Optional, List, Any, Dict, Callable, Iterator
+
+from daft.daft import (
+    StorageConfig,
+    PartitionField,
+    Pushdowns as DaftRustPushdowns,
+    ScanTask,
+    FileFormatConfig,
+    ParquetSourceConfig,
+    PartitionTransform as DaftTransform,
+    PartitionField as DaftPartitionField,
+)
+from daft.expressions import Expression as DaftExpression
+from daft.expressions.visitor import PredicateVisitor
+from pyarrow import Field as PaField
+
 import daft
 import ray
-from daft import TimeUnit, DataFrame
+from daft import (
+    TimeUnit,
+    DataFrame,
+    Schema as DaftSchema,
+    DataType,
+)
+from daft.logical.schema import Field as DaftField
 from daft.recordbatch import read_parquet_into_pyarrow
-from daft.io import IOConfig, S3Config
+from daft.io import (
+    IOConfig,
+    S3Config,
+)
+from daft.io.scan import (
+    ScanOperator,
+    make_partition_field,
+)
 import pyarrow as pa
 
 from deltacat import logs
+from deltacat.catalog.model.table_definition import TableDefinition
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.schema import coerce_pyarrow_table_to_schema
 from deltacat.types.media import ContentType, ContentEncoding
@@ -22,11 +51,180 @@ from deltacat.utils.performance import timed_invocation
 from deltacat.types.partial_download import (
     PartialFileDownloadParams,
 )
-
+from deltacat.storage import (
+    Transform,
+    IdentityTransform,
+    HourTransform,
+    DayTransform,
+    MonthTransform,
+    YearTransform,
+    BucketTransform,
+    BucketingStrategy,
+    TruncateTransform,
+    PartitionKey,
+    Schema,
+)
+from deltacat.storage.model.interop import ModelMapper
+from deltacat.storage.model.expression import (
+    Expression,
+    Reference,
+    Literal,
+    Equal,
+    NotEqual,
+    GreaterThan,
+    LessThan,
+    GreaterThanEqual,
+    LessThanEqual,
+    And,
+    Or,
+    Not,
+    IsNull,
+)
+from deltacat.storage.model.scan.push_down import (
+    PartitionFilter,
+    Pushdown as DeltaCatPushdown,
+)
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
+def translate_pushdown(pushdown: DaftRustPushdowns) -> DeltaCatPushdown:
+    """
+    Helper method to translate a Daft Pushdowns object into a Deltacat Pushdown.
+    Args:
+        pushdown: Daft Daft Pushdowns object
+    Returns:
+        Pushdown: Deltacat Pushdown object with translated filters
+    """
+    translator = DaftToDeltacatVisitor()
+
+    partition_filters = None
+    if pushdown.partition_filters is not None:
+        daft_expr = DaftExpression._from_pyexpr(pushdown.partition_filters)
+        partition_filters = PartitionFilter.of(translator.visit(daft_expr))
+
+    filters = None
+    if pushdown.filters is not None:
+        daft_expr = DaftExpression._from_pyexpr(pushdown.filters)
+        # TODO: support deltacat row filters
+        # filters = RowFilter.of(translator.visit(daft_expr))
+
+    columns = None
+    limit = None
+
+    return DeltaCatPushdown.of(
+        partition_filter=partition_filters,
+        column_filter=columns,
+        row_filter=filters,
+        limit=limit,
+    )
+
+
+class DaftToDeltacatVisitor(PredicateVisitor[Expression]):
+    """PredicateVisitor implementation to translate Daft Expressions into Deltacat Expressions"""
+
+    def visit_col(self, name: str) -> Expression:
+        return Reference.of(name)
+
+    def visit_lit(self, value: Any) -> Expression:
+        return Literal.of(value)
+
+    def visit_cast(self, expr: DaftExpression, dtype: DataType) -> Expression:
+        # deltacat expressions do not support explicit casting
+        # pyarrow should handle any type casting
+        return self.visit(expr)
+
+    def visit_alias(self, expr: DaftExpression, alias: str) -> Expression:
+        return self.visit(expr)
+
+    def visit_function(self, name: str, args: List[DaftExpression]) -> Expression:
+        # TODO: Add Deltacat expression function support
+        raise ValueError("Function not supported")
+
+    def visit_and(self, left: DaftExpression, right: DaftExpression) -> Expression:
+        """Visit an 'and' expression."""
+        return And.of(self.visit(left), self.visit(right))
+
+    def visit_or(self, left: DaftExpression, right: DaftExpression) -> Expression:
+        """Visit an 'or' expression."""
+        return Or.of(self.visit(left), self.visit(right))
+
+    def visit_not(self, expr: DaftExpression) -> Expression:
+        """Visit a 'not' expression."""
+        return Not.of(self.visit(expr))
+
+    def visit_equal(self, left: DaftExpression, right: DaftExpression) -> Expression:
+        """Visit an 'equals' comparison predicate."""
+        return Equal.of(self.visit(left), self.visit(right))
+
+    def visit_not_equal(
+        self, left: DaftExpression, right: DaftExpression
+    ) -> Expression:
+        """Visit a 'not equals' comparison predicate."""
+        return NotEqual.of(self.visit(left), self.visit(right))
+
+    def visit_less_than(
+        self, left: DaftExpression, right: DaftExpression
+    ) -> Expression:
+        """Visit a 'less than' comparison predicate."""
+        return LessThan.of(self.visit(left), self.visit(right))
+
+    def visit_less_than_or_equal(
+        self, left: DaftExpression, right: DaftExpression
+    ) -> Expression:
+        """Visit a 'less than or equal' comparison predicate."""
+        return LessThanEqual.of(self.visit(left), self.visit(right))
+
+    def visit_greater_than(
+        self, left: DaftExpression, right: DaftExpression
+    ) -> Expression:
+        """Visit a 'greater than' comparison predicate."""
+        return GreaterThan.of(self.visit(left), self.visit(right))
+
+    def visit_greater_than_or_equal(
+        self, left: DaftExpression, right: DaftExpression
+    ) -> Expression:
+        """Visit a 'greater than or equal' comparison predicate."""
+        return GreaterThanEqual.of(self.visit(left), self.visit(right))
+
+    def visit_between(
+        self, expr: DaftExpression, lower: DaftExpression, upper: DaftExpression
+    ) -> Expression:
+        """Visit a 'between' predicate."""
+        # Implement BETWEEN as lower <= expr <= upper
+        lower_bound = LessThanEqual.of(self.visit(lower), self.visit(expr))
+        upper_bound = LessThanEqual.of(self.visit(expr), self.visit(upper))
+        return And.of(lower_bound, upper_bound)
+
+    def visit_is_in(
+        self, expr: DaftExpression, items: list[DaftExpression]
+    ) -> Expression:
+        """Visit an 'is_in' predicate."""
+        # For empty list, return false literal
+        if not items:
+            return Literal(pa.scalar(False))
+
+        # Implement IN as a series of equality checks combined with OR
+        visited_expr = self.visit(expr)
+        equals_exprs = [Equal.of(visited_expr, self.visit(item)) for item in items]
+
+        # Combine with OR
+        result = equals_exprs[0]
+        for eq_expr in equals_exprs[1:]:
+            result = Or.of(result, eq_expr)
+
+        return result
+
+    def visit_is_null(self, expr: DaftExpression) -> Expression:
+        """Visit an 'is_null' predicate."""
+        return IsNull.of(self.visit(expr))
+
+    def visit_not_null(self, expr: DaftExpression) -> Expression:
+        """Visit an 'not_null' predicate."""
+        # NOT NULL is implemented as NOT(IS NULL)
+        return Not.of(IsNull.of(self.visit(expr)))
+
+
 def s3_files_to_dataframe(
     uris: List[str],
     content_type: str,
@@ -167,3 +365,331 @@ def _get_s3_io_config(s3_client_kwargs) -> IOConfig:
             read_timeout_ms=10_000,  # Timeout for first byte from server
         )
     )
+
+
+class DeltaCatScanOperator(ScanOperator):
+    def __init__(self, table: TableDefinition, storage_config: StorageConfig) -> None:
+        super().__init__()
+        self.table = table
+        self._schema = self._infer_schema()
+        self.partition_keys = self._infer_partition_keys()
+        self.storage_config = storage_config
+
+    def schema(self) -> DaftSchema:
+        return self._schema
+
+    def name(self) -> str:
+        return "DeltaCatScanOperator"
+
+    def display_name(self) -> str:
+        return f"DeltaCATScanOperator({self.table.table.namespace}.{self.table.table.table_name})"
+
+    def partitioning_keys(self) -> list[PartitionField]:
+        return self.partition_keys
+
+    def multiline_display(self) -> list[str]:
+        return [
+            self.display_name(),
+            f"Schema = {self._schema}",
+            f"Partitioning keys = {self.partitioning_keys}",
+            f"Storage config = {self.storage_config}",
+        ]
+
+    def to_scan_tasks(self, pushdowns: DaftRustPushdowns) -> Iterator[ScanTask]:
+        dc_pushdown = translate_pushdown(pushdowns)
+        dc_scan_plan = self.table.create_scan_plan(pushdown=dc_pushdown)
+        scan_tasks = []
+        file_format_config = FileFormatConfig.from_parquet_config(
+            # maybe this: ParquetSourceConfig(field_id_mapping=self._field_id_mapping)
+            ParquetSourceConfig()
+        )
+        for dc_scan_task in dc_scan_plan.scan_tasks:
+            for data_file in dc_scan_task.data_files():
+                st = ScanTask.catalog_scan_task(
+                    file=data_file.file_path,
+                    file_format=file_format_config,
+                    schema=self._schema._schema,
+                    storage_config=self.storage_config,
+                    pushdowns=pushdowns,
+                )
+                scan_tasks.append(st)
+        return iter(scan_tasks)
+
+    def can_absorb_filter(self) -> bool:
+        return False
+
+    def can_absorb_limit(self) -> bool:
+        return False
+
+    def can_absorb_select(self) -> bool:
+        return True
+
+    def _infer_schema(self) -> DaftSchema:
+
+        if not (
+            self.table and self.table.table_version and self.table.table_version.schema
+        ):
+            raise RuntimeError(
+                f"Failed to infer schema for DeltaCAT Table "
+                f"{self.table.table.namespace}.{self.table.table.table_name}"
+            )
+
+        return DaftSchema.from_pyarrow_schema(self.table.table_version.schema.arrow)
+
+    def _infer_partition_keys(self) -> list[PartitionField]:
+        if not (
+            self.table
+            and self.table.table_version
+            and self.table.table_version.partition_scheme
+            and self.table.table_version.schema
+        ):
+            raise RuntimeError(
+                f"Failed to infer partition keys for DeltaCAT Table "
+                f"{self.table.table.namespace}.{self.table.table.table_name}"
+            )
+
+        schema = self.table.table_version.schema
+        partition_keys = self.table.table_version.partition_scheme.keys
+        if not partition_keys:
+            return []
+
+        partition_fields = []
+        for key in partition_keys:
+            field = DaftPartitionKeyMapper.unmap(key, schema)
+            # Assert that the returned value is not None.
+            assert field is not None, f"Unmapping failed for key {key}"
+            partition_fields.append(field)
+
+        return partition_fields
+
+
+class DaftFieldMapper(ModelMapper[DaftField, PaField]):
+    @staticmethod
+    def map(
+        obj: Optional[DaftField],
+        **kwargs,
+    ) -> Optional[PaField]:
+        """Convert Daft Field to PyArrow Field.
+
+        Args:
+            obj: The Daft Field to convert
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted PyArrow Field object
+        """
+        if obj is None:
+            return None
+
+        return pa.field(
+            name=obj.name,
+            type=obj.dtype.to_arrow_dtype(),
+        )
+
+    @staticmethod
+    def unmap(
+        obj: Optional[PaField],
+        **kwargs,
+    ) -> Optional[DaftField]:
+        """Convert PyArrow Field to Daft Field.
+
+        Args:
+            obj: The PyArrow Field to convert
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted Daft Field object
+        """
+        if obj is None:
+            return None
+
+        return DaftField.create(
+            name=obj.name,
+            dtype=DataType.from_arrow_type(obj.type),  # type: ignore
+        )
+
+
+class DaftTransformMapper(ModelMapper[DaftTransform, Transform]):
+    @staticmethod
+    def map(
+        obj: Optional[DaftTransform],
+        **kwargs,
+    ) -> Optional[Transform]:
+        """Convert DaftTransform to DeltaCAT Transform.
+
+        Args:
+            obj: The DaftTransform to convert
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted Transform object
+        """
+
+        # daft.PartitionTransform doesn't have a Python interface for accessing its attributes,
+        # thus conversion is not possible.
+        # TODO: request Daft to expose Python friendly interface for daft.PartitionTransform
+        raise NotImplementedError(
+            "Converting transform from Daft to DeltaCAT is not supported"
+        )
+
+    @staticmethod
+    def unmap(
+        obj: Optional[Transform],
+        **kwargs,
+    ) -> Optional[DaftTransform]:
+        """Convert DeltaCAT Transform to DaftTransform.
+
+        Args:
+            obj: The Transform to convert
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted DaftTransform object
+        """
+        if obj is None:
+            return None
+
+        # Map DeltaCAT transforms to Daft transforms using isinstance
+
+        if isinstance(obj, IdentityTransform):
+            return DaftTransform.identity()
+        elif isinstance(obj, HourTransform):
+            return DaftTransform.hour()
+        elif isinstance(obj, DayTransform):
+            return DaftTransform.day()
+        elif isinstance(obj, MonthTransform):
+            return DaftTransform.month()
+        elif isinstance(obj, YearTransform):
+            return DaftTransform.year()
+        elif isinstance(obj, BucketTransform):
+            if obj.parameters.bucketing_strategy == BucketingStrategy.ICEBERG:
+                return DaftTransform.iceberg_bucket(obj.parameters.num_buckets)
+            else:
+                raise ValueError(
+                    f"Unsupported Bucketing Strategy: {obj.parameters.bucketing_strategy}"
+                )
+        elif isinstance(obj, TruncateTransform):
+            return DaftTransform.iceberg_truncate(obj.parameters.width)
+
+        raise ValueError(f"Unsupported Transform: {obj}")
+
+
+class DaftPartitionKeyMapper(ModelMapper[DaftPartitionField, PartitionKey]):
+    @staticmethod
+    def map(
+        obj: Optional[DaftPartitionField],
+        schema: Optional[DaftSchema] = None,
+        **kwargs,
+    ) -> Optional[PartitionKey]:
+        """Convert DaftPartitionField to PartitionKey.
+
+        Args:
+            obj: The DaftPartitionField to convert
+            schema: The Daft schema containing field information
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted PartitionKey object
+        """
+        # Daft PartitionField only exposes 1 attribute `field` which is not enough
+        # to convert to DeltaCAT PartitionKey
+        # TODO: request Daft to expose more Python friendly interface for PartitionField
+        raise NotImplementedError(
+            f"Converting Daft PartitionField to DeltaCAT PartitionKey is not supported"
+        )
+
+    @staticmethod
+    def unmap(
+        obj: Optional[PartitionKey],
+        schema: Optional[Schema] = None,
+        **kwargs,
+    ) -> Optional[DaftPartitionField]:
+        """Convert PartitionKey to DaftPartitionField.
+
+        Args:
+            obj: The DeltaCAT PartitionKey to convert
+            schema: The Schema containing field information
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted DaftPartitionField object
+        """
+        if obj is None:
+            return None
+        if obj.name is None:
+            raise ValueError("Name is required for PartitionKey conversion")
+        if not schema:
+            raise ValueError("Schema is required for PartitionKey conversion")
+        if len(obj.key) < 1:
+            raise ValueError(
+                f"At least 1 PartitionKey FieldLocator is expected, instead got {len(obj.key)}. FieldLocators: {obj.key}."
+            )
+
+        # Get the source field from schema - FieldLocator in PartitionKey.key points to the source field of partition field
+        dc_source_field = schema.field(obj.key[0]).arrow
+        daft_source_field = DaftFieldMapper.unmap(obj=dc_source_field)
+        # Convert transform if present
+        daft_transform = DaftTransformMapper.unmap(obj.transform)
+        daft_partition_field = DaftPartitionKeyMapper.get_daft_partition_field(
+            partition_field_name=obj.name,
+            daft_source_field=daft_source_field,
+            dc_transform=obj.transform,
+        )
+
+        # Create DaftPartitionField
+        return make_partition_field(
+            field=daft_partition_field,
+            source_field=daft_source_field,
+            transform=daft_transform,
+        )
+
+    @staticmethod
+    def get_daft_partition_field(
+        partition_field_name: str,
+        daft_source_field: Optional[DaftField],
+        # TODO: replace DeltaCAT transform with Daft Transform for uniformality
+        # We cannot use Daft Transform here because Daft Transform doesn't have a Python interface for us to
+        # access its attributes.
+        # TODO: request Daft to provide a more python friendly interface for Daft Tranform
+        dc_transform: Optional[Transform],
+    ) -> DaftField:
+        """Generate Daft Partition Field given partition field name, source field and transform.
+        Partition field type is inferred using source field type and transform.
+
+        Args:
+            partition_field_name (str): the specified result field name
+            daft_source_field (DaftField): the source field of the partition field
+            daft_transform (DaftTransform): transform applied on the source field to create partition field
+
+        Returns:
+            DaftField: Daft Field representing the partition field
+        """
+        if daft_source_field is None:
+            raise ValueError("Source field is required for PartitionField conversion")
+        if dc_transform is None:
+            raise ValueError("Transform is required for PartitionField conversion")
+
+        result_type = None
+        # Below type conversion logic references Daft - Iceberg conversion logic:
+        # https://github.com/Eventual-Inc/Daft/blob/7f2e9b5fb50fdfe858be17572f132b37dd6e5ab2/daft/iceberg/iceberg_scan.py#L61-L85
+        if isinstance(dc_transform, IdentityTransform):
+            result_type = daft_source_field.dtype
+        elif isinstance(dc_transform, YearTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, MonthTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, DayTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, HourTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, BucketTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, TruncateTransform):
+            result_type = daft_source_field.dtype
+        else:
+            raise ValueError(f"Unsupported transform: {dc_transform}")
+
+        return DaftField.create(
+            name=partition_field_name,
+            dtype=result_type,
+        )

deltacat/utils/export.py

@@ -5,7 +5,9 @@ import pyarrow.parquet
 import pyarrow.feather
 from typing import Callable, Dict
 
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
 from deltacat import logs
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))

deltacat/utils/url.py

@@ -632,7 +632,7 @@ class DeltaCatUrl:
     Note that, for reads, each of the above URLs typically resolves directly
     to the equivalent :class:`deltacat.types.media.DatasetType` reader. For
     example, if Ray Data is the dataset type then the equivalent
-    ray.data.read_{} API is used. For example, a read referencing a URL of the
+    ray.data.read_{} API is used. In this case, a read referencing a URL of the
     form "audio+file:///my/audio.mp4" would resolve to a call to
     ray.data.read_audio("/my/audio.mp4").
     """

{deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 2.0.0b10
+Version: 2.0.0b11
 Summary: A portable, scalable, fast, and Pythonic Data Lakehouse for AI.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -19,6 +19,7 @@ Requires-Dist: aws-embedded-metrics==3.2.0
 Requires-Dist: boto3~=1.34
 Requires-Dist: google-cloud-storage
 Requires-Dist: gcsfs==2025.3.2
+Requires-Dist: daft==0.4.15
 Requires-Dist: intervaltree==3.1.0
 Requires-Dist: numpy==1.22.4
 Requires-Dist: pandas==2.2.3
@@ -26,7 +27,7 @@ Requires-Dist: polars==1.28.1
 Requires-Dist: pyarrow==16.0.0
 Requires-Dist: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3
 Requires-Dist: pymemcache==4.0.0
-Requires-Dist: ray[default]==2.43.0
+Requires-Dist: ray[default]==2.46.0
 Requires-Dist: tenacity==8.2.3
 Requires-Dist: typing-extensions==4.6.1
 Requires-Dist: redis==4.6.0
@@ -61,8 +62,6 @@ DeltaCAT provides four high-level components:
 
 DeltaCAT is rapidly evolving. Usage instructions will be posted here soon!
 
-For now, feel free to peruse some of our examples:
-* https://github.com/ray-project/deltacat/tree/2.0/deltacat/examples/rivulet
-* https://github.com/ray-project/deltacat/tree/2.0/deltacat/examples/iceberg
+For now, feel free to peruse some of our [examples](https://github.com/ray-project/deltacat/tree/2.0/deltacat/examples/).