deltacat 2.0.0b2__tar.gz → 2.0.0b6__tar.gz

{deltacat-2.0.0b2/deltacat.egg-info → deltacat-2.0.0b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 2.0.0b2
+Version: 2.0.0b6
 Summary: A portable, scalable, fast, and Pythonic Data Lakehouse for AI.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-2.0.0b2 → deltacat-2.0.0b6}/deltacat/__init__.py

@@ -59,7 +59,7 @@ from deltacat.types.tables import TableWriteMode
 
 __iceberg__ = []
 if importlib.util.find_spec("pyiceberg") is not None:
-    from deltacat.catalog.iceberg import impl as IcebergCatalog
+    from deltacat.catalog.iceberg import impl as IcebergCatalog  # noqa: F401
 
     __iceberg__ = [
         "IcebergCatalog",
@@ -67,7 +67,7 @@ if importlib.util.find_spec("pyiceberg") is not None:
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "2.0.0b2"
+__version__ = "2.0.0b6"
 
 
 __all__ = [
@@ -104,7 +104,6 @@ __all__ = [
     "Dataset",
     "Datatype",
     "Field",
-    "IcebergCatalog",
     "LifecycleState",
     "ListResult",
     "LocalDataset",

{deltacat-2.0.0b2 → deltacat-2.0.0b6}/deltacat/catalog/iceberg/impl.py

@@ -2,10 +2,13 @@ import logging
 
 from typing import Any, Dict, List, Optional, Union
 
-from daft import DataFrame
+from daft import DataFrame, context
+from daft.daft import ScanOperatorHandle, StorageConfig
+from daft.logical.builder import LogicalPlanBuilder
 
 from deltacat import logs
 from deltacat.catalog.model.table_definition import TableDefinition
+from deltacat.daft.daft_scan import DeltaCatScanOperator
 from deltacat.exceptions import TableAlreadyExistsError
 from deltacat.storage.iceberg.iceberg_scan_planner import IcebergScanPlanner
 from deltacat.storage.iceberg.model import PartitionSchemeMapper, SchemaMapper
@@ -144,7 +147,17 @@ def read_table(
     table: str, *args, namespace: Optional[str] = None, **kwargs
 ) -> DistributedDataset:
     """Read a table into a distributed dataset."""
-    raise NotImplementedError("read_table not implemented")
+    # TODO: more proper IO configuration
+    io_config = context.get_context().daft_planning_config.default_io_config
+    multithreaded_io = context.get_context().get_or_create_runner().name != "ray"
+
+    storage_config = StorageConfig(multithreaded_io, io_config)
+
+    dc_table = get_table(name=table, namespace=namespace, **kwargs)
+    dc_scan_operator = DeltaCatScanOperator(dc_table, storage_config)
+    handle = ScanOperatorHandle.from_python_scan_operator(dc_scan_operator)
+    builder = LogicalPlanBuilder.from_tabular_scan(scan_operator=handle)
+    return DataFrame(builder)
 
 
 def alter_table(

{deltacat-2.0.0b2 → deltacat-2.0.0b6}/deltacat/catalog/iceberg/overrides.py

@@ -5,12 +5,11 @@ from typing import Iterator, List
 from pyarrow.fs import FileSystem
 
 from pyiceberg.io.pyarrow import (
-    fill_parquet_file_metadata,
+    data_file_statistics_from_parquet_metadata,
     compute_statistics_plan,
     parquet_path_to_id_mapping,
 )
-from pyiceberg.table import Table, _MergingSnapshotProducer
-from pyiceberg.table.snapshots import Operation
+from pyiceberg.table import Table
 from pyiceberg.manifest import DataFile, DataFileContent, FileFormat
 from pyiceberg.types import StructType, NestedField, IntegerType
 from pyiceberg.typedef import Record
@@ -24,11 +23,10 @@ def append(table: Table, paths: List[str]) -> None:
     #    raise ValueError("Cannot write to tables with a sort-order")
 
     data_files = write_file(table, paths)
-    merge = _MergingSnapshotProducer(operation=Operation.APPEND, table=table)
-    for data_file in data_files:
-        merge.append_data_file(data_file)
-
-    merge.commit()
+    with table.transaction() as txn:
+        with txn.update_snapshot().fast_append() as snapshot_update:
+            for data_file in data_files:
+                snapshot_update.append_data_file(data_file)
 
 
 def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
@@ -41,6 +39,11 @@ def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
         fs_path = fs_tuple[1]
         with fs.open_input_file(fs_path) as native_file:
             parquet_metadata = pq.read_metadata(native_file)
+            statistics = data_file_statistics_from_parquet_metadata(
+                parquet_metadata=parquet_metadata,
+                stats_columns=compute_statistics_plan(table.schema(), table.properties),
+                parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
+            )
             data_file = DataFile(
                 content=DataFileContent.DATA,
                 file_path=file_path,
@@ -63,12 +66,7 @@ def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
                 spec_id=table.spec().spec_id,
                 equality_ids=None,
                 key_metadata=None,
-            )
-            fill_parquet_file_metadata(
-                data_file=data_file,
-                parquet_metadata=parquet_metadata,
-                stats_columns=compute_statistics_plan(table.schema(), table.properties),
-                parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
+                **statistics.to_serialized_dict(),
             )
             data_files.append(data_file)
     return data_files

{deltacat-2.0.0b2 → deltacat-2.0.0b6}/deltacat/catalog/main/impl.py

@@ -709,7 +709,7 @@ def _get_deltas_from_partition_filter(
 
 def _get_storage(**kwargs):
     """
-    Returns the implementation of `deltacat.storage.interface` to use with this catalog.
+    Returns the implementation of `deltacat.storage.interface` to use with this catalog
 
     This is configured in the `CatalogProperties` stored during initialization and passed through `delegate.py`
     """

{deltacat-2.0.0b2 → deltacat-2.0.0b6}/deltacat/examples/iceberg/iceberg_bucket_writer.py

@@ -1,11 +1,15 @@
 import os
 import logging
 
+import uuid
 import daft
+from pyiceberg.catalog import CatalogType
+
 import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
+from deltacat.catalog.iceberg import IcebergCatalogConfig
 from deltacat.examples.common.fixtures import (
     store_cli_args_in_os_environ,
 )
@@ -30,6 +34,24 @@ driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
 
 
 def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
+    """
+    This is an e2e example that
+        1. creates a DeltaCAT Table (backed by an Iceberg Table) in Glue
+        2. writes data into the DeltaCAT Table
+        3. reads data from the DeltaCAT Table using Daft
+
+    To run the script:
+        1. prepare an AWS Account
+            1. prepare a S3 location where the data will be written to, which will be used in Step 3.
+            2. prepare an IAM Role that has access to the S3 location and Glue
+        2. retrieve the IAM Role AWS Credential and cache locally in ~/.aws/credentials
+        3. run below command to execute the example
+        ```
+        make venv && source venv/bin/activate
+        python -m deltacat.examples.iceberg.iceberg_bucket_writer --warehouse=s3://<YOUR_S3_LOCATION>
+        ```
+
+    """
     # create any runtime environment required to run the example
     runtime_env = create_ray_runtime_environment()
 
@@ -38,6 +60,7 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
     # Only the `iceberg` data catalog is provided so it will become the default.
     # If initializing multiple catalogs, use the `default_catalog_name` param
     # to specify which catalog should be the default.
+
     dc.init(
         catalogs={
             # the name of the DeltaCAT catalog is "iceberg"
@@ -49,11 +72,13 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
                 name="example-iceberg-catalog",
                 # for additional properties see:
                 # https://py.iceberg.apache.org/configuration/
-                properties={
-                    "type": "glue",
-                    "region_name": "us-east-1",
-                    "warehouse": warehouse,
-                },
+                config=IcebergCatalogConfig(
+                    type=CatalogType.GLUE,
+                    properties={
+                        "warehouse": warehouse,
+                        "region_name": "us-east-1",
+                    },
+                ),
             )
         },
         # pass the runtime environment into ray.init()
@@ -89,10 +114,10 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
         }
     )
 
-    # write to a table named `test_namespace.test_table_bucketed`
+    # write to a table named `test_namespace.test_table_bucketed-<SUFFIX>`
     # we don't need to specify which catalog to create this table in since
     # only the "iceberg" catalog is available
-    table_name = "test_table_bucketed"
+    table_name = f"test_table_bucketed-{uuid.uuid4().hex[:8]}"
     namespace = "test_namespace"
     print(f"Creating Glue Table: {namespace}.{table_name}")
     dc.write_to_table(
@@ -106,9 +131,40 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
     )
 
     print(f"Getting Glue Table: {namespace}.{table_name}")
-    table_definition = dc.get_table(table_name, namespace)
+    table_definition = dc.get_table(name=table_name, namespace=namespace)
     print(f"Retrieved Glue Table: {table_definition}")
 
+    # Read Data from DeltaCAT Table (backed by Iceberg) using Daft
+    daft_dataframe = dc.read_table(table=table_name, namespace=namespace)
+
+    daft_dataframe.where(df["bid"] > 200.0).show()
+    # Expected result:
+    # ╭────────┬─────────┬─────────╮
+    # │ symbol ┆ bid     ┆ ask     │
+    # │ ---    ┆ ---     ┆ ---     │
+    # │ Utf8   ┆ Float64 ┆ Float64 │
+    # ╞════════╪═════════╪═════════╡
+    # │ meta   ┆ 392.03  ┆ 392.09  │
+    # ├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
+    # │ msft   ┆ 403.25  ┆ 403.27  │
+    # ╰────────┴─────────┴─────────╯
+
+    daft_dataframe.select("symbol").show()
+    # Expected result:
+    # ╭────────╮
+    # │ symbol │
+    # │ ---    │
+    # │ Utf8   │
+    # ╞════════╡
+    # │ meta   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ amzn   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ goog   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ msft   │
+    # ╰────────╯
+
 
 if __name__ == "__main__":
     example_script_args = [
@@ -121,15 +177,6 @@ if __name__ == "__main__":
                 "type": str,
             },
         ),
-        (
-            [
-                "--STAGE",
-            ],
-            {
-                "help": "Example runtime environment stage (e.g. dev, alpha, beta, prod).",
-                "type": str,
-            },
-        ),
     ]
 
     # store any CLI args in the runtime environment

deltacat-2.0.0b6/deltacat/experimental/daft/__init__.py

@@ -0,0 +1,4 @@
+"""Daft integration package for DeltaCAT.
+
+This package provides integration between DeltaCAT and Daft.
+"""

deltacat-2.0.0b6/deltacat/experimental/daft/daft_catalog.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+from typing import Tuple, Optional
+
+from deltacat.catalog.model.catalog import Catalog as DCCatalog
+from deltacat.catalog.model.table_definition import TableDefinition
+
+from daft.catalog import Catalog, Identifier, Table
+from daft.dataframe import DataFrame
+from daft.logical.schema import Schema
+from deltacat.constants import DEFAULT_NAMESPACE
+
+
+class DaftCatalog(Catalog):
+    """
+    Wrapper class to create a Daft catalog from a DeltaCAT catalog.
+
+    The initialization of DeltaCAT and Daft catalogs is managed in `deltacat.catalog.catalog.py`. The user
+    is just expected to initialize catalogs through the DeltaCAT public interface (init / put_catalog).
+
+    TODO (mccember) in follow up PR we need to consider how to keep the DeltaCAT Catalogs class and Daft session in sync,
+      and the user-facing entrypoint to get a Daft catalog
+
+    This class itself expects a `Catalog` and will invoke the underlying implementation
+    similar to `deltacat.catalog.delegate.py`, like:
+      catalog.impl.create_namespace(namespace, inner=catalog.inner)
+
+    We cannot route calls through the higher level catalog registry / delegate since this wrapper class is at a lower
+     layer and does not manage registering catalogs.
+    """
+
+    def __init__(self, catalog: DCCatalog, name: str):
+        """
+        Initialize given DeltaCAT catalog. This catalog is also registered with DeltaCAT (via deltacat.put_catalog) given the provided Name
+
+        :param catalog: DeltaCAT Catalog object. If None, the catalog will be fetched from `deltacat.Catalogs`
+            given the catalog name.
+
+        :param name: Name of DeltaCAT catalog. If the name is not yet registered with `deltacat.Catalogs`,
+            it will be registered upon creation to ensure that the DeltaCAT and Daft catalogs keep in sync.
+
+        :param kwargs: Additional keyword arguments passed to deltacat.get_catalog or deltacat.put_catalog,
+                       such as 'namespace' for tests.
+        """
+        self.dc_catalog = catalog
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    ###
+    # create_*
+    ###
+    def create_namespace(self, identifier: Identifier | str):
+        """Create a new namespace in the catalog."""
+        if isinstance(identifier, Identifier):
+            identifier = str(identifier)
+        self.dc_catalog.impl.create_namespace(identifier, inner=self.dc_catalog.inner)
+
+    def create_table(
+        self, identifier: Identifier | str, source: Schema | DataFrame, **kwargs
+    ) -> Table:
+        """
+        Create a DeltaCAT table via Daft catalog API
+
+        End users calling create_table through the daft table API may provide kwargs which will be plumbed through
+        to deltacat create_table. For full list of keyword arguments accepted by create_table.
+
+        Note: as of 4/22, Daft create_table does not yet support kwargs. Tracked at: https://github.com/Eventual-Inc/Daft/issues/4195
+
+        :param identifier: Daft table identifier. Sequence of strings of the format (namespace) or (namespace, table)
+            or (namespace, table, table version). If this is a string, it is a dot delimited string of the same format.
+            Identifiers can be created either like Identifier("namespace", "table", "version") OR
+                Identifier.from_str("namespace.table.version")
+
+        :param source: a TableSource, either a Daft DataFrame, Daft Schema, or str (filesystem path)
+        """
+        if isinstance(source, DataFrame):
+            return self._create_table_from_df(identifier, source)
+        elif isinstance(source, Schema):
+            return self._create_table_from_schema(identifier, source)
+        else:
+            raise Exception(
+                f"Expected table source to be Schema or DataFrame. Found: {type(source)}"
+            )
+
+    def _create_table_from_df(
+        self, ident: Identifier | str, source: DataFrame, **kwargs
+    ) -> Table:
+        """
+        Create a table from a DataFrame.
+        """
+        t = self._create_table_from_schema(ident, source.schema(), **kwargs)
+        # TODO (mccember) append data upon creation
+        return t
+
+    def _create_table_from_schema(
+        self, ident: Identifier | str, source: Schema, **kwargs
+    ) -> Table:
+        """
+        Create a table from a schema.
+        """
+        namespace, name, version = self._extract_namespace_name_version(ident)
+
+        # Convert the Daft schema to a DeltaCAT schema
+        # This is a simplified version, would need to be enhanced for production
+        deltacat_schema = self._convert_schema_to_deltacat(source)
+
+        # Create the table in DeltaCAT
+        table_def = self.dc_catalog.impl.create_table(
+            name,
+            namespace=namespace,
+            version=version,
+            schema=deltacat_schema,
+            inner=self.dc_catalog.inner,
+            **kwargs,
+        )
+
+        return DaftTable._from_obj(table_def)
+
+    ###
+    # drop_*
+    ###
+
+    def drop_namespace(self, identifier: Identifier | str):
+        raise NotImplementedError()
+
+    def drop_table(self, identifier: Identifier | str):
+        raise NotImplementedError()
+
+    ###
+    # get_*
+    ###
+
+    def get_table(self, identifier: Identifier | str, **kwargs) -> Table:
+        namespace, table, version = self._extract_namespace_name_version(identifier)
+
+        table_def = self.dc_catalog.impl.get_table(
+            table,
+            namespace=namespace,
+            table_version=version,
+            inner=self.dc_catalog.inner,
+            **kwargs,
+        )
+
+        if not table_def:
+            raise ValueError(f"Table {identifier} not found")
+
+        return DaftTable._from_obj(table_def)
+
+    ###
+    # list_*
+    ###
+
+    def list_namespaces(self, pattern: str | None = None) -> list[Identifier]:
+        raise NotImplementedError("Not implemented")
+
+    def list_tables(self, pattern: str | None = None) -> list[str]:
+        raise NotImplementedError("Not implemented")
+
+    def _extract_namespace_name_version(
+        self, ident: Identifier | str
+    ) -> Tuple[str, str, Optional[str]]:
+        """
+        Extract namespace, name,version from identifier
+
+        Returns a 3-tuple. If no namespace is provided, uses DeltaCAT defualt namespace
+        """
+        default_namespace = DEFAULT_NAMESPACE
+
+        if isinstance(ident, str):
+            ident = Identifier.from_str(ident)
+
+        if isinstance(ident, Identifier):
+            if len(ident) == 1:
+                return (default_namespace, ident[0], None)
+            elif len(ident) == 2:
+                return (ident[0], ident[1], None)
+            elif len(ident) == 3:
+                return (ident[0], ident[1], ident[2])
+            else:
+                raise ValueError(
+                    f"Expected table identifier to be in format (table) or (namespace, table)"
+                    f"or (namespace, table, version). Found: {ident}"
+                )
+
+    def _convert_schema_to_deltacat(self, schema: Schema):
+        """Convert Daft schema to DeltaCAT schema.
+        For now, just use PyArrow schema as intermediary
+        TODO look into how enhancements on schema can be propagated between Daft<=>DeltaCAT
+        """
+        from deltacat.storage.model.schema import Schema as DeltaCATSchema
+
+        return DeltaCATSchema.of(schema=schema.to_pyarrow_schema())
+
+
+class DaftTable(Table):
+    """
+    Wrapper class to create a Daft table from a DeltaCAT table
+    """
+
+    _inner: TableDefinition
+
+    _read_options = set()
+    _write_options = set()
+
+    def __init__(self, inner: TableDefinition):
+        self._inner = inner
+
+    @property
+    def name(self) -> str:
+        """Return the table name."""
+        return self._inner.table_version.table_name
+
+    @staticmethod
+    def _from_obj(obj: object) -> DaftTable:
+        """Returns a DeltaCATTable if the given object can be adapted so."""
+        if isinstance(obj, TableDefinition):
+            t = DaftTable.__new__(DaftTable)
+            t._inner = obj
+            return t
+        raise ValueError(f"Unsupported DeltaCAT table type: {type(obj)}")
+
+    def read(self, **options) -> DataFrame:
+        raise NotImplementedError("Not implemented")
+
+    def write(self, df: DataFrame | object, mode: str = "append", **options):
+        raise NotImplementedError("Not implemented")

{deltacat-2.0.0b2 → deltacat-2.0.0b6}/deltacat/storage/model/manifest.py

@@ -11,6 +11,8 @@ from deltacat import logs
 
 from deltacat.storage.model.schema import FieldLocator
 
+import json
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
@@ -192,6 +194,20 @@ class Manifest(dict):
         manifest = Manifest._build_manifest(meta, entries, author, uuid)
         return manifest
 
+    @staticmethod
+    def from_json(json_string: str) -> Manifest:
+        parsed_dict = json.loads(json_string)
+        return Manifest.of(
+            entries=ManifestEntryList.of(
+                [
+                    ManifestEntry.from_dict(entry)
+                    for entry in parsed_dict.get("entries", [])
+                ]
+            ),
+            author=ManifestAuthor.from_dict(parsed_dict.get("author")),
+            uuid=parsed_dict.get("id"),
+        )
+
     @staticmethod
     def merge_manifests(
         manifests: List[Manifest], author: Optional[ManifestAuthor] = None
@@ -264,6 +280,23 @@ class ManifestMeta(dict):
             manifest_meta["entry_params"] = entry_params
         return manifest_meta
 
+    @staticmethod
+    def from_dict(obj: dict) -> Optional[ManifestMeta]:
+        if obj is None:
+            return None
+
+        return ManifestMeta.of(
+            record_count=obj.get("record_count"),
+            content_length=obj.get("content_length"),
+            content_type=obj.get("content_type"),
+            content_encoding=obj.get("content_encoding"),
+            source_content_length=obj.get("source_content_length"),
+            credentials=obj.get("credentials"),
+            content_type_parameters=obj.get("content_type_parameters"),
+            entry_type=obj.get("entry_type"),
+            entry_params=obj.get("entry_params"),
+        )
+
     @property
     def record_count(self) -> Optional[int]:
         return self.get("record_count")
@@ -358,6 +391,16 @@ class ManifestEntry(dict):
         manifest_entry = ManifestEntry.of(url, manifest_entry_meta)
         return manifest_entry
 
+    @staticmethod
+    def from_dict(obj: dict) -> ManifestEntry:
+        return ManifestEntry.of(
+            url=obj.get("url"),
+            uri=obj.get("uri"),
+            meta=ManifestMeta.from_dict(obj.get("meta")),
+            mandatory=obj.get("mandatory", True),
+            uuid=obj.get("id"),
+        )
+
     @property
     def uri(self) -> Optional[str]:
         return self.get("uri")
@@ -392,6 +435,12 @@ class ManifestAuthor(dict):
             manifest_author["version"] = version
         return manifest_author
 
+    @staticmethod
+    def from_dict(obj: dict) -> Optional[ManifestAuthor]:
+        if obj is None:
+            return None
+        return ManifestAuthor.of(obj.get("name"), obj.get("version"))
+
     @property
     def name(self) -> Optional[str]:
         return self.get("name")