deltacat 2.0.0b2__py3-none-any.whl → 2.0.0b6__py3-none-any.whl

deltacat/__init__.py

@@ -59,7 +59,7 @@ from deltacat.types.tables import TableWriteMode
 
 __iceberg__ = []
 if importlib.util.find_spec("pyiceberg") is not None:
-    from deltacat.catalog.iceberg import impl as IcebergCatalog
+    from deltacat.catalog.iceberg import impl as IcebergCatalog  # noqa: F401
 
     __iceberg__ = [
         "IcebergCatalog",
@@ -67,7 +67,7 @@ if importlib.util.find_spec("pyiceberg") is not None:
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "2.0.0b2"
+__version__ = "2.0.0b6"
 
 
 __all__ = [
@@ -104,7 +104,6 @@ __all__ = [
     "Dataset",
     "Datatype",
     "Field",
-    "IcebergCatalog",
     "LifecycleState",
     "ListResult",
     "LocalDataset",

deltacat/catalog/iceberg/impl.py

@@ -2,10 +2,13 @@ import logging
 
 from typing import Any, Dict, List, Optional, Union
 
-from daft import DataFrame
+from daft import DataFrame, context
+from daft.daft import ScanOperatorHandle, StorageConfig
+from daft.logical.builder import LogicalPlanBuilder
 
 from deltacat import logs
 from deltacat.catalog.model.table_definition import TableDefinition
+from deltacat.daft.daft_scan import DeltaCatScanOperator
 from deltacat.exceptions import TableAlreadyExistsError
 from deltacat.storage.iceberg.iceberg_scan_planner import IcebergScanPlanner
 from deltacat.storage.iceberg.model import PartitionSchemeMapper, SchemaMapper
@@ -144,7 +147,17 @@ def read_table(
     table: str, *args, namespace: Optional[str] = None, **kwargs
 ) -> DistributedDataset:
     """Read a table into a distributed dataset."""
-    raise NotImplementedError("read_table not implemented")
+    # TODO: more proper IO configuration
+    io_config = context.get_context().daft_planning_config.default_io_config
+    multithreaded_io = context.get_context().get_or_create_runner().name != "ray"
+
+    storage_config = StorageConfig(multithreaded_io, io_config)
+
+    dc_table = get_table(name=table, namespace=namespace, **kwargs)
+    dc_scan_operator = DeltaCatScanOperator(dc_table, storage_config)
+    handle = ScanOperatorHandle.from_python_scan_operator(dc_scan_operator)
+    builder = LogicalPlanBuilder.from_tabular_scan(scan_operator=handle)
+    return DataFrame(builder)
 
 
 def alter_table(

deltacat/catalog/iceberg/overrides.py

@@ -5,12 +5,11 @@ from typing import Iterator, List
 from pyarrow.fs import FileSystem
 
 from pyiceberg.io.pyarrow import (
-    fill_parquet_file_metadata,
+    data_file_statistics_from_parquet_metadata,
     compute_statistics_plan,
     parquet_path_to_id_mapping,
 )
-from pyiceberg.table import Table, _MergingSnapshotProducer
-from pyiceberg.table.snapshots import Operation
+from pyiceberg.table import Table
 from pyiceberg.manifest import DataFile, DataFileContent, FileFormat
 from pyiceberg.types import StructType, NestedField, IntegerType
 from pyiceberg.typedef import Record
@@ -24,11 +23,10 @@ def append(table: Table, paths: List[str]) -> None:
     #    raise ValueError("Cannot write to tables with a sort-order")
 
     data_files = write_file(table, paths)
-    merge = _MergingSnapshotProducer(operation=Operation.APPEND, table=table)
-    for data_file in data_files:
-        merge.append_data_file(data_file)
-
-    merge.commit()
+    with table.transaction() as txn:
+        with txn.update_snapshot().fast_append() as snapshot_update:
+            for data_file in data_files:
+                snapshot_update.append_data_file(data_file)
 
 
 def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
@@ -41,6 +39,11 @@ def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
         fs_path = fs_tuple[1]
         with fs.open_input_file(fs_path) as native_file:
             parquet_metadata = pq.read_metadata(native_file)
+            statistics = data_file_statistics_from_parquet_metadata(
+                parquet_metadata=parquet_metadata,
+                stats_columns=compute_statistics_plan(table.schema(), table.properties),
+                parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
+            )
             data_file = DataFile(
                 content=DataFileContent.DATA,
                 file_path=file_path,
@@ -63,12 +66,7 @@ def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
                 spec_id=table.spec().spec_id,
                 equality_ids=None,
                 key_metadata=None,
-            )
-            fill_parquet_file_metadata(
-                data_file=data_file,
-                parquet_metadata=parquet_metadata,
-                stats_columns=compute_statistics_plan(table.schema(), table.properties),
-                parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
+                **statistics.to_serialized_dict(),
             )
             data_files.append(data_file)
     return data_files

deltacat/catalog/main/impl.py

@@ -709,7 +709,7 @@ def _get_deltas_from_partition_filter(
 
 def _get_storage(**kwargs):
     """
-    Returns the implementation of `deltacat.storage.interface` to use with this catalog.
+    Returns the implementation of `deltacat.storage.interface` to use with this catalog
 
     This is configured in the `CatalogProperties` stored during initialization and passed through `delegate.py`
     """

deltacat/examples/iceberg/iceberg_bucket_writer.py

@@ -1,11 +1,15 @@
 import os
 import logging
 
+import uuid
 import daft
+from pyiceberg.catalog import CatalogType
+
 import deltacat as dc
 
 from deltacat import logs
 from deltacat import IcebergCatalog
+from deltacat.catalog.iceberg import IcebergCatalogConfig
 from deltacat.examples.common.fixtures import (
     store_cli_args_in_os_environ,
 )
@@ -30,6 +34,24 @@ driver_logger = logs.configure_application_logger(logging.getLogger(__name__))
 
 
 def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
+    """
+    This is an e2e example that
+        1. creates a DeltaCAT Table (backed by an Iceberg Table) in Glue
+        2. writes data into the DeltaCAT Table
+        3. reads data from the DeltaCAT Table using Daft
+
+    To run the script:
+        1. prepare an AWS Account
+            1. prepare a S3 location where the data will be written to, which will be used in Step 3.
+            2. prepare an IAM Role that has access to the S3 location and Glue
+        2. retrieve the IAM Role AWS Credential and cache locally in ~/.aws/credentials
+        3. run below command to execute the example
+        ```
+        make venv && source venv/bin/activate
+        python -m deltacat.examples.iceberg.iceberg_bucket_writer --warehouse=s3://<YOUR_S3_LOCATION>
+        ```
+
+    """
     # create any runtime environment required to run the example
     runtime_env = create_ray_runtime_environment()
 
@@ -38,6 +60,7 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
     # Only the `iceberg` data catalog is provided so it will become the default.
     # If initializing multiple catalogs, use the `default_catalog_name` param
     # to specify which catalog should be the default.
+
     dc.init(
         catalogs={
             # the name of the DeltaCAT catalog is "iceberg"
@@ -49,11 +72,13 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
                 name="example-iceberg-catalog",
                 # for additional properties see:
                 # https://py.iceberg.apache.org/configuration/
-                properties={
-                    "type": "glue",
-                    "region_name": "us-east-1",
-                    "warehouse": warehouse,
-                },
+                config=IcebergCatalogConfig(
+                    type=CatalogType.GLUE,
+                    properties={
+                        "warehouse": warehouse,
+                        "region_name": "us-east-1",
+                    },
+                ),
             )
         },
         # pass the runtime environment into ray.init()
@@ -89,10 +114,10 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
         }
     )
 
-    # write to a table named `test_namespace.test_table_bucketed`
+    # write to a table named `test_namespace.test_table_bucketed-<SUFFIX>`
     # we don't need to specify which catalog to create this table in since
     # only the "iceberg" catalog is available
-    table_name = "test_table_bucketed"
+    table_name = f"test_table_bucketed-{uuid.uuid4().hex[:8]}"
     namespace = "test_namespace"
     print(f"Creating Glue Table: {namespace}.{table_name}")
     dc.write_to_table(
@@ -106,9 +131,40 @@ def run(warehouse="s3://my-bucket/my/key/prefix", **kwargs):
     )
 
     print(f"Getting Glue Table: {namespace}.{table_name}")
-    table_definition = dc.get_table(table_name, namespace)
+    table_definition = dc.get_table(name=table_name, namespace=namespace)
     print(f"Retrieved Glue Table: {table_definition}")
 
+    # Read Data from DeltaCAT Table (backed by Iceberg) using Daft
+    daft_dataframe = dc.read_table(table=table_name, namespace=namespace)
+
+    daft_dataframe.where(df["bid"] > 200.0).show()
+    # Expected result:
+    # ╭────────┬─────────┬─────────╮
+    # │ symbol ┆ bid     ┆ ask     │
+    # │ ---    ┆ ---     ┆ ---     │
+    # │ Utf8   ┆ Float64 ┆ Float64 │
+    # ╞════════╪═════════╪═════════╡
+    # │ meta   ┆ 392.03  ┆ 392.09  │
+    # ├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
+    # │ msft   ┆ 403.25  ┆ 403.27  │
+    # ╰────────┴─────────┴─────────╯
+
+    daft_dataframe.select("symbol").show()
+    # Expected result:
+    # ╭────────╮
+    # │ symbol │
+    # │ ---    │
+    # │ Utf8   │
+    # ╞════════╡
+    # │ meta   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ amzn   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ goog   │
+    # ├╌╌╌╌╌╌╌╌┤
+    # │ msft   │
+    # ╰────────╯
+
 
 if __name__ == "__main__":
     example_script_args = [
@@ -121,15 +177,6 @@ if __name__ == "__main__":
                 "type": str,
             },
         ),
-        (
-            [
-                "--STAGE",
-            ],
-            {
-                "help": "Example runtime environment stage (e.g. dev, alpha, beta, prod).",
-                "type": str,
-            },
-        ),
     ]
 
     # store any CLI args in the runtime environment

deltacat/experimental/daft/__init__.py

@@ -0,0 +1,4 @@
+"""Daft integration package for DeltaCAT.
+
+This package provides integration between DeltaCAT and Daft.
+"""

deltacat/experimental/daft/daft_catalog.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+from typing import Tuple, Optional
+
+from deltacat.catalog.model.catalog import Catalog as DCCatalog
+from deltacat.catalog.model.table_definition import TableDefinition
+
+from daft.catalog import Catalog, Identifier, Table
+from daft.dataframe import DataFrame
+from daft.logical.schema import Schema
+from deltacat.constants import DEFAULT_NAMESPACE
+
+
+class DaftCatalog(Catalog):
+    """
+    Wrapper class to create a Daft catalog from a DeltaCAT catalog.
+
+    The initialization of DeltaCAT and Daft catalogs is managed in `deltacat.catalog.catalog.py`. The user
+    is just expected to initialize catalogs through the DeltaCAT public interface (init / put_catalog).
+
+    TODO (mccember) in follow up PR we need to consider how to keep the DeltaCAT Catalogs class and Daft session in sync,
+      and the user-facing entrypoint to get a Daft catalog
+
+    This class itself expects a `Catalog` and will invoke the underlying implementation
+    similar to `deltacat.catalog.delegate.py`, like:
+      catalog.impl.create_namespace(namespace, inner=catalog.inner)
+
+    We cannot route calls through the higher level catalog registry / delegate since this wrapper class is at a lower
+     layer and does not manage registering catalogs.
+    """
+
+    def __init__(self, catalog: DCCatalog, name: str):
+        """
+        Initialize given DeltaCAT catalog. This catalog is also registered with DeltaCAT (via deltacat.put_catalog) given the provided Name
+
+        :param catalog: DeltaCAT Catalog object. If None, the catalog will be fetched from `deltacat.Catalogs`
+            given the catalog name.
+
+        :param name: Name of DeltaCAT catalog. If the name is not yet registered with `deltacat.Catalogs`,
+            it will be registered upon creation to ensure that the DeltaCAT and Daft catalogs keep in sync.
+
+        :param kwargs: Additional keyword arguments passed to deltacat.get_catalog or deltacat.put_catalog,
+                       such as 'namespace' for tests.
+        """
+        self.dc_catalog = catalog
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    ###
+    # create_*
+    ###
+    def create_namespace(self, identifier: Identifier | str):
+        """Create a new namespace in the catalog."""
+        if isinstance(identifier, Identifier):
+            identifier = str(identifier)
+        self.dc_catalog.impl.create_namespace(identifier, inner=self.dc_catalog.inner)
+
+    def create_table(
+        self, identifier: Identifier | str, source: Schema | DataFrame, **kwargs
+    ) -> Table:
+        """
+        Create a DeltaCAT table via Daft catalog API
+
+        End users calling create_table through the daft table API may provide kwargs which will be plumbed through
+        to deltacat create_table. For full list of keyword arguments accepted by create_table.
+
+        Note: as of 4/22, Daft create_table does not yet support kwargs. Tracked at: https://github.com/Eventual-Inc/Daft/issues/4195
+
+        :param identifier: Daft table identifier. Sequence of strings of the format (namespace) or (namespace, table)
+            or (namespace, table, table version). If this is a string, it is a dot delimited string of the same format.
+            Identifiers can be created either like Identifier("namespace", "table", "version") OR
+                Identifier.from_str("namespace.table.version")
+
+        :param source: a TableSource, either a Daft DataFrame, Daft Schema, or str (filesystem path)
+        """
+        if isinstance(source, DataFrame):
+            return self._create_table_from_df(identifier, source)
+        elif isinstance(source, Schema):
+            return self._create_table_from_schema(identifier, source)
+        else:
+            raise Exception(
+                f"Expected table source to be Schema or DataFrame. Found: {type(source)}"
+            )
+
+    def _create_table_from_df(
+        self, ident: Identifier | str, source: DataFrame, **kwargs
+    ) -> Table:
+        """
+        Create a table from a DataFrame.
+        """
+        t = self._create_table_from_schema(ident, source.schema(), **kwargs)
+        # TODO (mccember) append data upon creation
+        return t
+
+    def _create_table_from_schema(
+        self, ident: Identifier | str, source: Schema, **kwargs
+    ) -> Table:
+        """
+        Create a table from a schema.
+        """
+        namespace, name, version = self._extract_namespace_name_version(ident)
+
+        # Convert the Daft schema to a DeltaCAT schema
+        # This is a simplified version, would need to be enhanced for production
+        deltacat_schema = self._convert_schema_to_deltacat(source)
+
+        # Create the table in DeltaCAT
+        table_def = self.dc_catalog.impl.create_table(
+            name,
+            namespace=namespace,
+            version=version,
+            schema=deltacat_schema,
+            inner=self.dc_catalog.inner,
+            **kwargs,
+        )
+
+        return DaftTable._from_obj(table_def)
+
+    ###
+    # drop_*
+    ###
+
+    def drop_namespace(self, identifier: Identifier | str):
+        raise NotImplementedError()
+
+    def drop_table(self, identifier: Identifier | str):
+        raise NotImplementedError()
+
+    ###
+    # get_*
+    ###
+
+    def get_table(self, identifier: Identifier | str, **kwargs) -> Table:
+        namespace, table, version = self._extract_namespace_name_version(identifier)
+
+        table_def = self.dc_catalog.impl.get_table(
+            table,
+            namespace=namespace,
+            table_version=version,
+            inner=self.dc_catalog.inner,
+            **kwargs,
+        )
+
+        if not table_def:
+            raise ValueError(f"Table {identifier} not found")
+
+        return DaftTable._from_obj(table_def)
+
+    ###
+    # list_*
+    ###
+
+    def list_namespaces(self, pattern: str | None = None) -> list[Identifier]:
+        raise NotImplementedError("Not implemented")
+
+    def list_tables(self, pattern: str | None = None) -> list[str]:
+        raise NotImplementedError("Not implemented")
+
+    def _extract_namespace_name_version(
+        self, ident: Identifier | str
+    ) -> Tuple[str, str, Optional[str]]:
+        """
+        Extract namespace, name,version from identifier
+
+        Returns a 3-tuple. If no namespace is provided, uses DeltaCAT defualt namespace
+        """
+        default_namespace = DEFAULT_NAMESPACE
+
+        if isinstance(ident, str):
+            ident = Identifier.from_str(ident)
+
+        if isinstance(ident, Identifier):
+            if len(ident) == 1:
+                return (default_namespace, ident[0], None)
+            elif len(ident) == 2:
+                return (ident[0], ident[1], None)
+            elif len(ident) == 3:
+                return (ident[0], ident[1], ident[2])
+            else:
+                raise ValueError(
+                    f"Expected table identifier to be in format (table) or (namespace, table)"
+                    f"or (namespace, table, version). Found: {ident}"
+                )
+
+    def _convert_schema_to_deltacat(self, schema: Schema):
+        """Convert Daft schema to DeltaCAT schema.
+        For now, just use PyArrow schema as intermediary
+        TODO look into how enhancements on schema can be propagated between Daft<=>DeltaCAT
+        """
+        from deltacat.storage.model.schema import Schema as DeltaCATSchema
+
+        return DeltaCATSchema.of(schema=schema.to_pyarrow_schema())
+
+
+class DaftTable(Table):
+    """
+    Wrapper class to create a Daft table from a DeltaCAT table
+    """
+
+    _inner: TableDefinition
+
+    _read_options = set()
+    _write_options = set()
+
+    def __init__(self, inner: TableDefinition):
+        self._inner = inner
+
+    @property
+    def name(self) -> str:
+        """Return the table name."""
+        return self._inner.table_version.table_name
+
+    @staticmethod
+    def _from_obj(obj: object) -> DaftTable:
+        """Returns a DeltaCATTable if the given object can be adapted so."""
+        if isinstance(obj, TableDefinition):
+            t = DaftTable.__new__(DaftTable)
+            t._inner = obj
+            return t
+        raise ValueError(f"Unsupported DeltaCAT table type: {type(obj)}")
+
+    def read(self, **options) -> DataFrame:
+        raise NotImplementedError("Not implemented")
+
+    def write(self, df: DataFrame | object, mode: str = "append", **options):
+        raise NotImplementedError("Not implemented")

deltacat/storage/model/manifest.py

@@ -11,6 +11,8 @@ from deltacat import logs
 
 from deltacat.storage.model.schema import FieldLocator
 
+import json
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
@@ -192,6 +194,20 @@ class Manifest(dict):
         manifest = Manifest._build_manifest(meta, entries, author, uuid)
         return manifest
 
+    @staticmethod
+    def from_json(json_string: str) -> Manifest:
+        parsed_dict = json.loads(json_string)
+        return Manifest.of(
+            entries=ManifestEntryList.of(
+                [
+                    ManifestEntry.from_dict(entry)
+                    for entry in parsed_dict.get("entries", [])
+                ]
+            ),
+            author=ManifestAuthor.from_dict(parsed_dict.get("author")),
+            uuid=parsed_dict.get("id"),
+        )
+
     @staticmethod
     def merge_manifests(
         manifests: List[Manifest], author: Optional[ManifestAuthor] = None
@@ -264,6 +280,23 @@ class ManifestMeta(dict):
             manifest_meta["entry_params"] = entry_params
         return manifest_meta
 
+    @staticmethod
+    def from_dict(obj: dict) -> Optional[ManifestMeta]:
+        if obj is None:
+            return None
+
+        return ManifestMeta.of(
+            record_count=obj.get("record_count"),
+            content_length=obj.get("content_length"),
+            content_type=obj.get("content_type"),
+            content_encoding=obj.get("content_encoding"),
+            source_content_length=obj.get("source_content_length"),
+            credentials=obj.get("credentials"),
+            content_type_parameters=obj.get("content_type_parameters"),
+            entry_type=obj.get("entry_type"),
+            entry_params=obj.get("entry_params"),
+        )
+
     @property
     def record_count(self) -> Optional[int]:
         return self.get("record_count")
@@ -358,6 +391,16 @@ class ManifestEntry(dict):
         manifest_entry = ManifestEntry.of(url, manifest_entry_meta)
         return manifest_entry
 
+    @staticmethod
+    def from_dict(obj: dict) -> ManifestEntry:
+        return ManifestEntry.of(
+            url=obj.get("url"),
+            uri=obj.get("uri"),
+            meta=ManifestMeta.from_dict(obj.get("meta")),
+            mandatory=obj.get("mandatory", True),
+            uuid=obj.get("id"),
+        )
+
     @property
     def uri(self) -> Optional[str]:
         return self.get("uri")
@@ -392,6 +435,12 @@ class ManifestAuthor(dict):
             manifest_author["version"] = version
         return manifest_author
 
+    @staticmethod
+    def from_dict(obj: dict) -> Optional[ManifestAuthor]:
+        if obj is None:
+            return None
+        return ManifestAuthor.of(obj.get("name"), obj.get("version"))
+
     @property
     def name(self) -> Optional[str]:
         return self.get("name")

deltacat/tests/storage/model/test_manifest.py

@@ -0,0 +1,129 @@
+import json
+
+import pytest
+
+from deltacat.storage.model.manifest import Manifest, ManifestEntry
+
+
+@pytest.fixture
+def manifest_a():
+    return """
+        {
+          "entries":[
+            {
+              "uri":"s3://test_bucket/file1.tsv.gz",
+              "mandatory":true,
+              "meta":{
+                "record_count":0,
+                "content_length":123,
+                "source_content_length":0,
+                "content_type":"application/x-amzn-unescaped-tsv",
+                "content_encoding":"gzip"
+              }
+            },
+            {
+              "uri":"s3://test_bucket/file2.tsv.gz",
+              "mandatory":true,
+              "meta":{
+                "record_count":0,
+                "content_length":456,
+                "source_content_length":0,
+                "content_type":"application/x-amzn-unescaped-tsv",
+                "content_encoding":"gzip"
+              }
+            }
+          ],
+          "meta":{
+            "record_count":0,
+            "content_length":579,
+            "source_content_length":0,
+            "content_type":"application/x-amzn-unescaped-tsv",
+            "content_encoding":"gzip"
+          },
+          "id":"052f62c0-5082-4935-9937-18a705156123",
+          "author":{
+            "name":"Dave",
+            "version":"1.0"
+          }
+        }
+           """
+
+
+@pytest.fixture
+def manifest_no_author():
+    return """
+        {
+          "entries":[
+            {
+              "uri":"s3://test_bucket/file1.tsv.gz",
+              "mandatory":true,
+              "meta":{
+                "record_count":0,
+                "content_length":123,
+                "source_content_length":0,
+                "content_type":"application/x-amzn-unescaped-tsv",
+                "content_encoding":"gzip"
+              }
+            },
+            {
+              "uri":"s3://test_bucket/file2.tsv.gz",
+              "mandatory":true,
+              "meta":{
+                "record_count":0,
+                "content_length":456,
+                "source_content_length":0,
+                "content_type":"application/x-amzn-unescaped-tsv",
+                "content_encoding":"gzip"
+              }
+            }
+          ],
+          "meta":{
+            "record_count":0,
+            "content_length":579,
+            "source_content_length":0,
+            "content_type":"application/x-amzn-unescaped-tsv",
+            "content_encoding":"gzip"
+          },
+          "id":"052f62c0-5082-4935-9937-18a705156123"
+        }
+           """
+
+
+@pytest.fixture()
+def manifest_entry_no_meta():
+    return """
+            {
+              "uri":"s3://test_bucket/file1.tsv.gz",
+              "mandatory":true
+            }
+           """
+
+
+def test_manifest_from_json(manifest_a):
+    manifest = Manifest.from_json(manifest_a)
+
+    assert manifest.entries is not None
+    assert len(manifest.entries) == 2
+    assert manifest.entries[0].uri == "s3://test_bucket/file1.tsv.gz"
+    assert manifest.entries[0].meta.record_count == 0
+    assert manifest.meta.content_length == 579
+    assert manifest.author.name == "Dave"
+
+
+def test_manifest_from_json_no_author(manifest_no_author):
+    manifest = Manifest.from_json(manifest_no_author)
+
+    assert manifest.entries is not None
+    assert len(manifest.entries) == 2
+    assert manifest.entries[0].uri == "s3://test_bucket/file1.tsv.gz"
+    assert manifest.entries[0].meta is not None
+    assert manifest.author is None
+
+
+def test_manifest_entry_from_dict_no_meta(manifest_entry_no_meta):
+    entry = ManifestEntry.from_dict(json.loads(manifest_entry_no_meta))
+
+    assert entry is not None
+    assert entry.meta is None
+    assert entry.uri == "s3://test_bucket/file1.tsv.gz"
+    assert entry.mandatory is True

deltacat/utils/daft.py

@@ -2,7 +2,7 @@ import logging
 from typing import Optional, List, Any, Dict, Callable
 import daft
 import ray
-from daft.table import read_parquet_into_pyarrow
+from daft.recordbatch import read_parquet_into_pyarrow
 from daft import TimeUnit, DataFrame
 from daft.io import IOConfig, S3Config
 import pyarrow as pa
@@ -10,7 +10,6 @@ import pyarrow as pa
 from deltacat import logs
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.schema import coerce_pyarrow_table_to_schema
-
 from deltacat.types.media import ContentType, ContentEncoding
 from deltacat.aws.constants import (
     BOTO_MAX_RETRIES,
@@ -72,9 +71,7 @@ def s3_files_to_dataframe(
         f"Preparing to read S3 object from {len(uris)} files into daft dataframe"
     )
 
-    df, latency = timed_invocation(
-        daft.read_parquet, path=uris, io_config=io_config, use_native_downloader=True
-    )
+    df, latency = timed_invocation(daft.read_parquet, path=uris, io_config=io_config)
 
     logger.debug(f"Time to create daft dataframe from {len(uris)} files is {latency}s")
 

{deltacat-2.0.0b2.dist-info → deltacat-2.0.0b6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 2.0.0b2
+Version: 2.0.0b6
 Summary: A portable, scalable, fast, and Pythonic Data Lakehouse for AI.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -17,11 +17,11 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aws-embedded-metrics==3.2.0
 Requires-Dist: boto3~=1.34
-Requires-Dist: getdaft==0.3.6
+Requires-Dist: getdaft>=0.4.11
 Requires-Dist: intervaltree==3.1.0
 Requires-Dist: numpy==1.21.5
 Requires-Dist: pandas==1.3.5
-Requires-Dist: pyarrow==17.0.0
+Requires-Dist: pyarrow==16.0.0
 Requires-Dist: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3
 Requires-Dist: pymemcache==4.0.0
 Requires-Dist: ray>=2.20.0

{deltacat-2.0.0b2.dist-info → deltacat-2.0.0b6.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=VCSaUwZDmC1xQmNqKeRrtKuo0lVKH-BqouPvk9WdIZI,2482
+deltacat/__init__.py,sha256=gMoUv3PoGXHlkitSzr_bWcMYUfc5o1nn8-LdHDzLStU,2474
 deltacat/annotations.py,sha256=9lBi34DpIV_RPjCCK2Aiz_6nMyd-e-_CfQ1XtdRQQlM,1196
 deltacat/api.py,sha256=fYKurVlM97VKb_fh7kJ1rDcl-VAAuSflxPeqrsUt1u8,5257
 deltacat/constants.py,sha256=_JfHTRktDTM70Nls-LMnSmLeCRG17UwQYCmY6gQSGBg,3482
@@ -24,10 +24,10 @@ deltacat/catalog/delegate.py,sha256=x3jj_T61gyExuAnbDqhU6smbaAbIN4UxrVMZuBEOg0A,
 deltacat/catalog/interface.py,sha256=YB-qNBFsWupqyWJuHr7eQ-_MshhZZ5HpLphoZ64yn2g,12244
 deltacat/catalog/iceberg/__init__.py,sha256=LOENcLTQQlu_694MvRhMd2TQDLzwfg2vz0D8DuVO3M8,190
 deltacat/catalog/iceberg/iceberg_catalog_config.py,sha256=LfHxv8pk-YmTRQy5LvKFzwSqZ8ek2Y6v0KY7xihhIN0,786
-deltacat/catalog/iceberg/impl.py,sha256=hFAX0QGfWq25t9miYHACye_t_3fxUAmQXpQ9kf3w_xQ,13591
-deltacat/catalog/iceberg/overrides.py,sha256=HGev1Us2zJpavAoClCCMHrf6sQ8fG0poSxyLEJOB-Ss,2668
+deltacat/catalog/iceberg/impl.py,sha256=c_ONnLLyh8Vyqo5PusQSHySQ92iM4Qgk-rucHMfdd7s,14288
+deltacat/catalog/iceberg/overrides.py,sha256=WmM2mxf7ihDl8anb5GzBxo5-sxBkot8ZSRTxDpaauRA,2687
 deltacat/catalog/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/catalog/main/impl.py,sha256=y7sya4BVfYMbp0-smgs_00cktw7QHkJxXTWADSr0W3s,23093
+deltacat/catalog/main/impl.py,sha256=E9gCPaARJAaiIS2HTdXXz0-GwTjOaWIBX2TK2MsL194,23092
 deltacat/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/catalog/model/catalog.py,sha256=zGuNl1Czqbk2QQx9qGpMWCTK9ay4b3tm3SJzNkLlw-I,10198
 deltacat/catalog/model/properties.py,sha256=wdXjd39-JEj-zZLL5pH6wyIXAdpph-CD7yEIF96Wn-A,4110
@@ -136,8 +136,11 @@ deltacat/examples/hello_world.py,sha256=hXpMUvJINB2qWTpV3QFPlRNu0uE31BvEs2sLyQ3C
 deltacat/examples/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/examples/common/fixtures.py,sha256=MS0Hz1c__f9Axm3JgTajfWuMVeDAQmFmZ7KB7vz_1q4,430
 deltacat/examples/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/examples/iceberg/iceberg_bucket_writer.py,sha256=9i78x8WBgp-vvMBsvbCWkcRo6oEZ8SDtGfjMlNXAO30,4521
+deltacat/examples/iceberg/iceberg_bucket_writer.py,sha256=PdJG3jXcgPVds4UanfyNWB1egv-Os7LnZCPhdgv9Yyk,6586
 deltacat/examples/iceberg/iceberg_reader.py,sha256=mlF-277vT04at-2jibAjgRJG6Y-zle_NNy1-pXwS2YQ,5023
+deltacat/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/experimental/daft/__init__.py,sha256=0d1SsgjbDher8TKgS0gSBBdy5TGi01fewiwpG0BMwck,108
+deltacat/experimental/daft/daft_catalog.py,sha256=112wDqqzdtxmtZVwiZW59MektbRsFMjSRgqYHrUOuok,8396
 deltacat/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/file_object_store.py,sha256=YoNL3Qla8uLOHaWnyBmIgotjSGAy3Td3Tumah0kk73Y,1868
 deltacat/io/memcached_object_store.py,sha256=C96t77-4BQe0XZ4vC76Ygi2o1POUoMN4t4BiyPmulz0,10997
@@ -158,7 +161,7 @@ deltacat/storage/model/delta.py,sha256=PhkjME0dItGgPd37SrQbI8VjQcIaYW2OfIq0KJKgD
 deltacat/storage/model/interop.py,sha256=CzXdu1NuJF5ER3IjQJztkNECD6MRDwbmMezlfN4SRH0,536
 deltacat/storage/model/list_result.py,sha256=5DpRAu-c0M48cHtKdTRPSgQiq2nCWfjAY8LOVqp5wxI,2703
 deltacat/storage/model/locator.py,sha256=Q16y-eGSQSZpDPKDYQhOjSA9c5ajwg1jLw_13MIB4SM,4707
-deltacat/storage/model/manifest.py,sha256=iV53LLQY83pDv9YwUqlyzjfLiqFHWuJf9J0dZdR7yO4,15153
+deltacat/storage/model/manifest.py,sha256=3I4Vohd-PnEQ5NdQu9yN3jvFchqnzb8hQ3bq6w_tO4E,16808
 deltacat/storage/model/metafile.py,sha256=UVWPvvYvA0tj_pM8ig7NKfVFrVWU4l3eDP7I2n9Upeg,53404
 deltacat/storage/model/namespace.py,sha256=gLli1V64O9RHIf-FesmqWA29Wi7P1kwt01uz5sDdJR0,2409
 deltacat/storage/model/partition.py,sha256=qNCvc74o_4pmFVL-FCyKCZMH4lHSjRO560sb3vaF_H0,20759
@@ -277,6 +280,7 @@ deltacat/tests/storage/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 deltacat/tests/storage/main/test_main_storage.py,sha256=9dtsAcp9GZ4XQ5-8XhKnAcFF7upowJpTIuqZUB2EYig,58124
 deltacat/tests/storage/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/storage/model/test_delete_parameters.py,sha256=RcNRMIed0zUzkX9tRXDoYPXHb7721OEt8viY9tpWXZM,822
+deltacat/tests/storage/model/test_manifest.py,sha256=udp9YUNvIBpnT-NutjMaF25abEQOXEcPkQm8Aay_UCs,3733
 deltacat/tests/storage/model/test_metafile_io.py,sha256=116U9aNJPzR0JS6iadJyyx0_4KyAi3D47WCNbndag6o,101639
 deltacat/tests/storage/model/test_schema.py,sha256=5m4BscbxbbOiry-lDI8j4vQcnvkG2Y-f0ZfshncPiSI,9599
 deltacat/tests/storage/model/test_shard.py,sha256=6QBr-ws3zQkJjjGyB7QEOhtNC5ql0cdjOPB2wxGNW3Q,755
@@ -323,7 +327,7 @@ deltacat/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/utils/arguments.py,sha256=5y1Xz4HSAD8M8Jt83i6gOEKoYjy_fMQe1V43IhIE4hY,1191
 deltacat/utils/cloudpickle.py,sha256=XE7YDmQe56ksfl3NdYZkzOAhbHSuhNcBZGOehQpgZr0,1187
 deltacat/utils/common.py,sha256=RG_-enXNpLKaYrqyx1ne2lL10lxN9vK7F631oJP6SE8,1375
-deltacat/utils/daft.py,sha256=nd4XBKcZTFYxf_VH9jm-wqqbrIujKAeisCt2vVbW2BA,5807
+deltacat/utils/daft.py,sha256=RsOGzxI6ltsRcH6SfbK6PDBEaKyLZaUisCBXBlUvjbI,5770
 deltacat/utils/export.py,sha256=As5aiwOw9vLxtfolPLU0yak6W2RVR0rkuaYQ5YCy49U,1952
 deltacat/utils/filesystem.py,sha256=DthBgrVGzIcsQcGnyD3QYEQIpkYFxB19XmpF9DfCaeo,11709
 deltacat/utils/metafile_locator.py,sha256=_3yEW9n49jiEBuXHZmUKsFdYx6RxWWuS-Mu2gs_a1bw,2933
@@ -342,8 +346,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-2.0.0b2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-2.0.0b2.dist-info/METADATA,sha256=Gpme8jgbL6tI7uJp5YF7mqWnb9EmWIxCjiBd3fvZe4Q,2808
-deltacat-2.0.0b2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-deltacat-2.0.0b2.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-2.0.0b2.dist-info/RECORD,,
+deltacat-2.0.0b6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-2.0.0b6.dist-info/METADATA,sha256=USf1wawH_OzPK19QEAqDWvrHDYhymxpP6I4RHD97YsQ,2809
+deltacat-2.0.0b6.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+deltacat-2.0.0b6.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-2.0.0b6.dist-info/RECORD,,