deltacat 2.0.0b3__tar.gz → 2.0.0b7__tar.gz

{deltacat-2.0.0b3/deltacat.egg-info → deltacat-2.0.0b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 2.0.0b3
+Version: 2.0.0b7
 Summary: A portable, scalable, fast, and Pythonic Data Lakehouse for AI.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-2.0.0b3 → deltacat-2.0.0b7}/deltacat/__init__.py

@@ -67,7 +67,7 @@ if importlib.util.find_spec("pyiceberg") is not None:
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "2.0.0b3"
+__version__ = "2.0.0b7"
 
 
 __all__ = [

{deltacat-2.0.0b3 → deltacat-2.0.0b7}/deltacat/catalog/iceberg/impl.py

@@ -2,10 +2,13 @@ import logging
 
 from typing import Any, Dict, List, Optional, Union
 
-from daft import DataFrame
+from daft import DataFrame, context
+from daft.daft import ScanOperatorHandle, StorageConfig
+from daft.logical.builder import LogicalPlanBuilder
 
 from deltacat import logs
 from deltacat.catalog.model.table_definition import TableDefinition
+from deltacat.daft.daft_scan import DeltaCatScanOperator
 from deltacat.exceptions import TableAlreadyExistsError
 from deltacat.storage.iceberg.iceberg_scan_planner import IcebergScanPlanner
 from deltacat.storage.iceberg.model import PartitionSchemeMapper, SchemaMapper
@@ -144,7 +147,17 @@ def read_table(
     table: str, *args, namespace: Optional[str] = None, **kwargs
 ) -> DistributedDataset:
     """Read a table into a distributed dataset."""
-    raise NotImplementedError("read_table not implemented")
+    # TODO: more proper IO configuration
+    io_config = context.get_context().daft_planning_config.default_io_config
+    multithreaded_io = context.get_context().get_or_create_runner().name != "ray"
+
+    storage_config = StorageConfig(multithreaded_io, io_config)
+
+    dc_table = get_table(name=table, namespace=namespace, **kwargs)
+    dc_scan_operator = DeltaCatScanOperator(dc_table, storage_config)
+    handle = ScanOperatorHandle.from_python_scan_operator(dc_scan_operator)
+    builder = LogicalPlanBuilder.from_tabular_scan(scan_operator=handle)
+    return DataFrame(builder)
 
 
 def alter_table(

{deltacat-2.0.0b3 → deltacat-2.0.0b7}/deltacat/catalog/iceberg/overrides.py

@@ -5,12 +5,11 @@ from typing import Iterator, List
 from pyarrow.fs import FileSystem
 
 from pyiceberg.io.pyarrow import (
-    fill_parquet_file_metadata,
+    data_file_statistics_from_parquet_metadata,
     compute_statistics_plan,
     parquet_path_to_id_mapping,
 )
-from pyiceberg.table import Table, _MergingSnapshotProducer
-from pyiceberg.table.snapshots import Operation
+from pyiceberg.table import Table
 from pyiceberg.manifest import DataFile, DataFileContent, FileFormat
 from pyiceberg.types import StructType, NestedField, IntegerType
 from pyiceberg.typedef import Record
@@ -24,11 +23,10 @@ def append(table: Table, paths: List[str]) -> None:
     #    raise ValueError("Cannot write to tables with a sort-order")
 
     data_files = write_file(table, paths)
-    merge = _MergingSnapshotProducer(operation=Operation.APPEND, table=table)
-    for data_file in data_files:
-        merge.append_data_file(data_file)
-
-    merge.commit()
+    with table.transaction() as txn:
+        with txn.update_snapshot().fast_append() as snapshot_update:
+            for data_file in data_files:
+                snapshot_update.append_data_file(data_file)
 
 
 def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
@@ -41,6 +39,11 @@ def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
         fs_path = fs_tuple[1]
         with fs.open_input_file(fs_path) as native_file:
             parquet_metadata = pq.read_metadata(native_file)
+            statistics = data_file_statistics_from_parquet_metadata(
+                parquet_metadata=parquet_metadata,
+                stats_columns=compute_statistics_plan(table.schema(), table.properties),
+                parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
+            )
             data_file = DataFile(
                 content=DataFileContent.DATA,
                 file_path=file_path,
@@ -63,12 +66,7 @@ def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
                 spec_id=table.spec().spec_id,
                 equality_ids=None,
                 key_metadata=None,
-            )
-            fill_parquet_file_metadata(
-                data_file=data_file,
-                parquet_metadata=parquet_metadata,
-                stats_columns=compute_statistics_plan(table.schema(), table.properties),
-                parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
+                **statistics.to_serialized_dict(),
             )
             data_files.append(data_file)
     return data_files

{deltacat-2.0.0b3 → deltacat-2.0.0b7}/deltacat/catalog/main/impl.py

@@ -709,7 +709,7 @@ def _get_deltas_from_partition_filter(
 
 def _get_storage(**kwargs):
     """
-    Returns the implementation of `deltacat.storage.interface` to use with this catalog.
+    Returns the implementation of `deltacat.storage.interface` to use with this catalog
 
     This is configured in the `CatalogProperties` stored during initialization and passed through `delegate.py`
     """

deltacat-2.0.0b7/deltacat/daft/daft_scan.py

@@ -0,0 +1,111 @@
+from typing import Iterator
+
+from daft import Schema
+from daft.daft import (
+    StorageConfig,
+    PartitionField,
+    Pushdowns,
+    ScanTask,
+    FileFormatConfig,
+    ParquetSourceConfig,
+)
+from daft.io.scan import ScanOperator
+
+from deltacat.catalog.model.table_definition import TableDefinition
+from deltacat.daft.model import DaftPartitionKeyMapper
+
+
+class DeltaCatScanOperator(ScanOperator):
+    def __init__(self, table: TableDefinition, storage_config: StorageConfig) -> None:
+        super().__init__()
+        self.table = table
+        self._schema = self._infer_schema()
+        self.partition_keys = self._infer_partition_keys()
+        self.storage_config = storage_config
+
+    def schema(self) -> Schema:
+        return self._schema
+
+    def name(self) -> str:
+        return "DeltaCatScanOperator"
+
+    def display_name(self) -> str:
+        return f"DeltaCATScanOperator({self.table.table.namespace}.{self.table.table.table_name})"
+
+    def partitioning_keys(self) -> list[PartitionField]:
+        return self.partition_keys
+
+    def multiline_display(self) -> list[str]:
+        return [
+            self.display_name(),
+            f"Schema = {self._schema}",
+            f"Partitioning keys = {self.partitioning_keys}",
+            f"Storage config = {self.storage_config}",
+        ]
+
+    def to_scan_tasks(self, pushdowns: Pushdowns) -> Iterator[ScanTask]:
+        # TODO: implement pushdown predicate on DeltaCAT
+        dc_scan_plan = self.table.create_scan_plan()
+        scan_tasks = []
+        file_format_config = FileFormatConfig.from_parquet_config(
+            # maybe this: ParquetSourceConfig(field_id_mapping=self._field_id_mapping)
+            ParquetSourceConfig()
+        )
+        for dc_scan_task in dc_scan_plan.scan_tasks:
+            for data_file in dc_scan_task.data_files():
+                st = ScanTask.catalog_scan_task(
+                    file=data_file.file_path,
+                    file_format=file_format_config,
+                    schema=self._schema._schema,
+                    storage_config=self.storage_config,
+                    pushdowns=pushdowns,
+                )
+                scan_tasks.append(st)
+        return iter(scan_tasks)
+
+    def can_absorb_filter(self) -> bool:
+        return False
+
+    def can_absorb_limit(self) -> bool:
+        return False
+
+    def can_absorb_select(self) -> bool:
+        return True
+
+    def _infer_schema(self) -> Schema:
+
+        if not (
+            self.table and self.table.table_version and self.table.table_version.schema
+        ):
+            raise RuntimeError(
+                f"Failed to infer schema for DeltaCAT Table "
+                f"{self.table.table.namespace}.{self.table.table.table_name}"
+            )
+
+        return Schema.from_pyarrow_schema(self.table.table_version.schema.arrow)
+
+    def _infer_partition_keys(self) -> list[PartitionField]:
+        if not (
+            self.table
+            and self.table.table_version
+            and self.table.table_version.partition_scheme
+            and self.table.table_version.schema
+        ):
+            raise RuntimeError(
+                f"Failed to infer partition keys for DeltaCAT Table "
+                f"{self.table.table.namespace}.{self.table.table.table_name}"
+            )
+
+        schema = self.table.table_version.schema
+        partition_keys = self.table.table_version.partition_scheme.keys
+        if not partition_keys:
+            return []
+
+        partition_fields = []
+        for key in partition_keys:
+            field = DaftPartitionKeyMapper.unmap(key, schema)
+            # Assert that the returned value is not None.
+            assert field is not None, f"Unmapping failed for key {key}"
+            partition_fields.append(field)
+
+        return partition_fields

deltacat-2.0.0b7/deltacat/daft/model.py

@@ -0,0 +1,258 @@
+from typing import Optional
+
+import pyarrow as pa
+from pyarrow import Field as PaField
+from daft import Schema as DaftSchema, DataType
+from daft.daft import (
+    PartitionField as DaftPartitionField,
+    PartitionTransform as DaftTransform,
+)
+from daft.logical.schema import Field as DaftField
+from daft.io.scan import make_partition_field
+
+from deltacat.storage.model.schema import Schema
+from deltacat.storage.model.interop import ModelMapper
+from deltacat.storage.model.partition import PartitionKey
+from deltacat.storage.model.transform import (
+    BucketingStrategy,
+    Transform,
+    BucketTransform,
+    HourTransform,
+    DayTransform,
+    MonthTransform,
+    YearTransform,
+    IdentityTransform,
+    TruncateTransform,
+)
+
+
+class DaftFieldMapper(ModelMapper[DaftField, PaField]):
+    @staticmethod
+    def map(
+        obj: Optional[DaftField],
+        **kwargs,
+    ) -> Optional[PaField]:
+        """Convert Daft Field to PyArrow Field.
+
+        Args:
+            obj: The Daft Field to convert
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted PyArrow Field object
+        """
+        if obj is None:
+            return None
+
+        return pa.field(
+            name=obj.name,
+            type=obj.dtype.to_arrow_dtype(),
+        )
+
+    @staticmethod
+    def unmap(
+        obj: Optional[PaField],
+        **kwargs,
+    ) -> Optional[DaftField]:
+        """Convert PyArrow Field to Daft Field.
+
+        Args:
+            obj: The PyArrow Field to convert
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted Daft Field object
+        """
+        if obj is None:
+            return None
+
+        return DaftField.create(
+            name=obj.name,
+            dtype=DataType.from_arrow_type(obj.type),  # type: ignore
+        )
+
+
+class DaftTransformMapper(ModelMapper[DaftTransform, Transform]):
+    @staticmethod
+    def map(
+        obj: Optional[DaftTransform],
+        **kwargs,
+    ) -> Optional[Transform]:
+        """Convert DaftTransform to DeltaCAT Transform.
+
+        Args:
+            obj: The DaftTransform to convert
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted Transform object
+        """
+
+        # daft.PartitionTransform doesn't have a Python interface for accessing its attributes,
+        # thus conversion is not possible.
+        # TODO: request Daft to expose Python friendly interface for daft.PartitionTransform
+        raise NotImplementedError(
+            "Converting transform from Daft to DeltaCAT is not supported"
+        )
+
+    @staticmethod
+    def unmap(
+        obj: Optional[Transform],
+        **kwargs,
+    ) -> Optional[DaftTransform]:
+        """Convert DeltaCAT Transform to DaftTransform.
+
+        Args:
+            obj: The Transform to convert
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted DaftTransform object
+        """
+        if obj is None:
+            return None
+
+        # Map DeltaCAT transforms to Daft transforms using isinstance
+
+        if isinstance(obj, IdentityTransform):
+            return DaftTransform.identity()
+        elif isinstance(obj, HourTransform):
+            return DaftTransform.hour()
+        elif isinstance(obj, DayTransform):
+            return DaftTransform.day()
+        elif isinstance(obj, MonthTransform):
+            return DaftTransform.month()
+        elif isinstance(obj, YearTransform):
+            return DaftTransform.year()
+        elif isinstance(obj, BucketTransform):
+            if obj.parameters.bucketing_strategy == BucketingStrategy.ICEBERG:
+                return DaftTransform.iceberg_bucket(obj.parameters.num_buckets)
+            else:
+                raise ValueError(
+                    f"Unsupported Bucketing Strategy: {obj.parameters.bucketing_strategy}"
+                )
+        elif isinstance(obj, TruncateTransform):
+            return DaftTransform.iceberg_truncate(obj.parameters.width)
+
+        raise ValueError(f"Unsupported Transform: {obj}")
+
+
+class DaftPartitionKeyMapper(ModelMapper[DaftPartitionField, PartitionKey]):
+    @staticmethod
+    def map(
+        obj: Optional[DaftPartitionField],
+        schema: Optional[DaftSchema] = None,
+        **kwargs,
+    ) -> Optional[PartitionKey]:
+        """Convert DaftPartitionField to PartitionKey.
+
+        Args:
+            obj: The DaftPartitionField to convert
+            schema: The Daft schema containing field information
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted PartitionKey object
+        """
+        # Daft PartitionField only exposes 1 attribute `field` which is not enough
+        # to convert to DeltaCAT PartitionKey
+        # TODO: request Daft to expose more Python friendly interface for PartitionField
+        raise NotImplementedError(
+            f"Converting Daft PartitionField to DeltaCAT PartitionKey is not supported"
+        )
+
+    @staticmethod
+    def unmap(
+        obj: Optional[PartitionKey],
+        schema: Optional[Schema] = None,
+        **kwargs,
+    ) -> Optional[DaftPartitionField]:
+        """Convert PartitionKey to DaftPartitionField.
+
+        Args:
+            obj: The DeltaCAT PartitionKey to convert
+            schema: The Schema containing field information
+            **kwargs: Additional arguments
+
+        Returns:
+            Converted DaftPartitionField object
+        """
+        if obj is None:
+            return None
+        if obj.name is None:
+            raise ValueError("Name is required for PartitionKey conversion")
+        if not schema:
+            raise ValueError("Schema is required for PartitionKey conversion")
+        if len(obj.key) < 1:
+            raise ValueError(
+                f"At least 1 PartitionKey FieldLocator is expected, instead got {len(obj.key)}. FieldLocators: {obj.key}."
+            )
+
+        # Get the source field from schema - FieldLocator in PartitionKey.key points to the source field of partition field
+        dc_source_field = schema.field(obj.key[0]).arrow
+        daft_source_field = DaftFieldMapper.unmap(obj=dc_source_field)
+        # Convert transform if present
+        daft_transform = DaftTransformMapper.unmap(obj.transform)
+        daft_partition_field = DaftPartitionKeyMapper.get_daft_partition_field(
+            partition_field_name=obj.name,
+            daft_source_field=daft_source_field,
+            dc_transform=obj.transform,
+        )
+
+        # Create DaftPartitionField
+        return make_partition_field(
+            field=daft_partition_field,
+            source_field=daft_source_field,
+            transform=daft_transform,
+        )
+
+    @staticmethod
+    def get_daft_partition_field(
+        partition_field_name: str,
+        daft_source_field: Optional[DaftField],
+        # TODO: replace DeltaCAT transform with Daft Transform for uniformality
+        # We cannot use Daft Transform here because Daft Transform doesn't have a Python interface for us to
+        # access its attributes.
+        # TODO: request Daft to provide a more python friendly interface for Daft Tranform
+        dc_transform: Optional[Transform],
+    ) -> DaftField:
+        """Generate Daft Partition Field given partition field name, source field and transform.
+        Partition field type is inferred using source field type and transform.
+
+        Args:
+            partition_field_name (str): the specified result field name
+            daft_source_field (DaftField): the source field of the partition field
+            daft_transform (DaftTransform): transform applied on the source field to create partition field
+
+        Returns:
+            DaftField: Daft Field representing the partition field
+        """
+        if daft_source_field is None:
+            raise ValueError("Source field is required for PartitionField conversion")
+        if dc_transform is None:
+            raise ValueError("Transform is required for PartitionField conversion")
+
+        result_type = None
+        # Below type conversion logic references Daft - Iceberg conversion logic:
+        # https://github.com/Eventual-Inc/Daft/blob/7f2e9b5fb50fdfe858be17572f132b37dd6e5ab2/daft/iceberg/iceberg_scan.py#L61-L85
+        if isinstance(dc_transform, IdentityTransform):
+            result_type = daft_source_field.dtype
+        elif isinstance(dc_transform, YearTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, MonthTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, DayTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, HourTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, BucketTransform):
+            result_type = DataType.int32()
+        elif isinstance(dc_transform, TruncateTransform):
+            result_type = daft_source_field.dtype
+        else:
+            raise ValueError(f"Unsupported transform: {dc_transform}")
+
+        return DaftField.create(
+            name=partition_field_name,
+            dtype=result_type,
+        )

{deltacat-2.0.0b3 → deltacat-2.0.0b7}/deltacat/examples/iceberg/iceberg_bucket_writer.py

@@ -1,11 +1,15 @@
 import os
 import logging
 
+import uuid
 import daft
+from pyiceberg.catalog import CatalogType
+
 import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
+from deltacat.catalog.iceberg import IcebergCatalogConfig
 from deltacat.examples.common.fixtures import (
     store_cli_args_in_os_environ,
 )
@@ -30,6 +34,24 @@ driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
 
 
 def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
+    """
+    This is an e2e example that
+        1. creates a DeltaCAT Table (backed by an Iceberg Table) in Glue
+        2. writes data into the DeltaCAT Table
+        3. reads data from the DeltaCAT Table using Daft
+
+    To run the script:
+        1. prepare an AWS Account
+            1. prepare a S3 location where the data will be written to, which will be used in Step 3.
+            2. prepare an IAM Role that has access to the S3 location and Glue
+        2. retrieve the IAM Role AWS Credential and cache locally in ~/.aws/credentials
+        3. run below command to execute the example
+        ```
+        make venv && source venv/bin/activate
+        python -m deltacat.examples.iceberg.iceberg_bucket_writer --warehouse=s3://<YOUR_S3_LOCATION>
+        ```
+
+    """
     # create any runtime environment required to run the example
     runtime_env = create_ray_runtime_environment()
 
@@ -38,6 +60,7 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
     # Only the `iceberg` data catalog is provided so it will become the default.
     # If initializing multiple catalogs, use the `default_catalog_name` param
     # to specify which catalog should be the default.
+
     dc.init(
         catalogs={
             # the name of the DeltaCAT catalog is "iceberg"
@@ -49,11 +72,13 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
                 name="example-iceberg-catalog",
                 # for additional properties see:
                 # https://py.iceberg.apache.org/configuration/
-                properties={
-                    "type": "glue",
-                    "region_name": "us-east-1",
-                    "warehouse": warehouse,
-                },
+                config=IcebergCatalogConfig(
+                    type=CatalogType.GLUE,
+                    properties={
+                        "warehouse": warehouse,
+                        "region_name": "us-east-1",
+                    },
+                ),
             )
         },
         # pass the runtime environment into ray.init()
@@ -89,10 +114,10 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
         }
     )
 
-    # write to a table named `test_namespace.test_table_bucketed`
+    # write to a table named `test_namespace.test_table_bucketed-<SUFFIX>`
     # we don't need to specify which catalog to create this table in since
     # only the "iceberg" catalog is available
-    table_name = "test_table_bucketed"
+    table_name = f"test_table_bucketed-{uuid.uuid4().hex[:8]}"
     namespace = "test_namespace"
     print(f"Creating Glue Table: {namespace}.{table_name}")
     dc.write_to_table(
@@ -106,9 +131,40 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
     )
 
     print(f"Getting Glue Table: {namespace}.{table_name}")
-    table_definition = dc.get_table(table_name, namespace)
+    table_definition = dc.get_table(name=table_name, namespace=namespace)
     print(f"Retrieved Glue Table: {table_definition}")
 
+    # Read Data from DeltaCAT Table (backed by Iceberg) using Daft
+    daft_dataframe = dc.read_table(table=table_name, namespace=namespace)
+
+    daft_dataframe.where(df["bid"] > 200.0).show()
+    # Expected result:
+    # ╭────────┬─────────┬─────────╮
+    # │ symbol ┆ bid     ┆ ask     │
+    # │ ---    ┆ ---     ┆ ---     │
+    # │ Utf8   ┆ Float64 ┆ Float64 │
+    # ╞════════╪═════════╪═════════╡
+    # │ meta   ┆ 392.03  ┆ 392.09  │
+    # ├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
+    # │ msft   ┆ 403.25  ┆ 403.27  │
+    # ╰────────┴─────────┴─────────╯
+
+    daft_dataframe.select("symbol").show()
+    # Expected result:
+    # ╭────────╮
+    # │ symbol │
+    # │ ---    │
+    # │ Utf8   │
+    # ╞════════╡
+    # │ meta   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ amzn   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ goog   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ msft   │
+    # ╰────────╯
+
 
 if __name__ == "__main__":
     example_script_args = [
@@ -121,15 +177,6 @@ if __name__ == "__main__":
                 "type": str,
             },
         ),
-        (
-            [
-                "--STAGE",
-            ],
-            {
-                "help": "Example runtime environment stage (e.g. dev, alpha, beta, prod).",
-                "type": str,
-            },
-        ),
     ]
 
     # store any CLI args in the runtime environment

deltacat-2.0.0b7/deltacat/experimental/daft/__init__.py

@@ -0,0 +1,4 @@
+"""Daft integration package for DeltaCAT.
+
+This package provides integration between DeltaCAT and Daft.
+"""