deltacat 2.0.0b9__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py

@@ -3,10 +3,10 @@ from typing import List
 import pyarrow as pa
 from pyarrow import feather
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 
 class FeatherDataSerializer(ArrowSerializer):

deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py

@@ -3,9 +3,9 @@ import time
 from typing import List, Generator
 
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.fs.file_store import FileStore
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
 from deltacat.utils.metafile_locator import _find_partition_path
 
 

deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py

@@ -4,8 +4,8 @@ from pyarrow.fs import FileSystem, FileType, FileSelector
 # TODO(deltacat): Rely on deltacat implementation to resolve path and filesystem.
 from ray.data.datasource.path_util import _resolve_paths_and_filesystem
 
-from deltacat.storage.rivulet.fs.input_file import FSInputFile
-from deltacat.storage.rivulet.fs.output_file import FSOutputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import FSOutputFile
 
 
 class FileStore:

deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py

@@ -5,7 +5,7 @@ from typing import Protocol
 
 from pyarrow.fs import FileSystem, FileType
 
-from deltacat.storage.rivulet.fs.input_file import FSInputFile, InputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile, InputFile
 
 
 class OutputStream(Protocol):  # pragma: no cover

deltacat/{storage → experimental/storage}/rivulet/logical_plan.py

@@ -1,9 +1,9 @@
 from dataclasses import dataclass
 from typing import List, Callable, Any, Protocol
 
-from deltacat.storage.rivulet.dataset_executor import DatasetExecutor
-from deltacat.storage.rivulet.mvp.Table import MvpTable
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.dataset_executor import DatasetExecutor
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
 
 
 class DatasetOperation(Protocol):
@@ -99,7 +99,7 @@ class LogicalPlan:
         self.operations.append(CollectOperation())
         return self
 
-    def execute(self, executor: DatasetExecutor) -> "MvpTable":
+    def execute(self, executor: DatasetExecutor) -> MvpTable:
         for operation in self.operations:
             operation.visit(executor)
         return executor.output

deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py

@@ -19,7 +19,7 @@ from deltacat.storage.model.partition import PartitionLocator
 from deltacat.storage.model.transaction import TransactionOperationList
 
 from deltacat.storage.model.types import StreamFormat
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet import Schema
 
 StreamPosition = int
 """The stream position for creating a consistent ordering of manifests."""

deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py

@@ -4,9 +4,9 @@ import json
 from itertools import zip_longest
 from typing import List
 
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
-from deltacat.storage.rivulet.metastore.sst import (
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.metastore.sst import (
     SSTWriter,
     SSTableRow,
     SSTReader,

deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from typing import Protocol, Any, List
 
-from deltacat.storage.rivulet.fs.input_file import InputFile
-from deltacat.storage.rivulet.fs.output_file import OutputFile
+from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
+from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
 
 
 @dataclass(frozen=True)

deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py

@@ -8,9 +8,9 @@ from typing import Any, Dict, Set, List, FrozenSet, Iterable, TypeVar, NamedTupl
 
 from intervaltree import Interval, IntervalTree
 
-from deltacat.storage.rivulet.metastore.delta import DeltaContext
-from deltacat.storage.rivulet.metastore.sst import SSTable, SSTableRow
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTable, SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
 
 T = TypeVar("T")
 

deltacat/experimental/storage/rivulet/parquet/__init__.py

@@ -0,0 +1,7 @@
+# TODO later on this will be moved to a dedicated package
+from deltacat.experimental.storage.rivulet.parquet.file_reader import ParquetFileReader
+from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
+    FileReaderRegistrar,
+)
+
+FileReaderRegistrar.register_reader("parquet", ParquetFileReader)

deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py

@@ -4,15 +4,17 @@ from typing import Optional
 
 from pyarrow import RecordBatch
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.reader.data_reader import (
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
     RowAndKey,
     FileReader,
     FILE_FORMAT,
 )
-from deltacat.storage.rivulet.reader.pyarrow_data_reader import RecordBatchRowIndex
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
+    RecordBatchRowIndex,
+)
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 import pyarrow.parquet as pq
 import pyarrow as pa
 

deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py

@@ -3,11 +3,11 @@ from typing import List, Any
 import pyarrow as pa
 from pyarrow.parquet import FileMetaData
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 
 class ParquetDataSerializer(ArrowSerializer):

deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py

@@ -15,19 +15,30 @@ from typing import (
     AbstractSet,
 )
 
-from deltacat.storage.rivulet.metastore.delta import DeltaContext
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.metastore.sst_interval_tree import (
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
     OrderedBlockGroups,
     BlockGroup,
     Block,
 )
-from deltacat.storage.rivulet.reader.data_reader import RowAndKey, FileReader
-from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
-from deltacat.storage.rivulet.reader.pyarrow_data_reader import ArrowDataReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet.reader.reader_type_registrar import FileReaderRegistrar
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
+    RowAndKey,
+    FileReader,
+)
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
+    ArrowDataReader,
+)
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
+    FileReaderRegistrar,
+)
+from deltacat.experimental.storage.rivulet import Schema
 from deltacat import logs
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))

deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py

@@ -13,9 +13,9 @@ from typing import (
     Optional,
 )
 
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 
 FILE_FORMAT = TypeVar("FILE_FORMAT")
 MEMORY_FORMAT = TypeVar("MEMORY_FORMAT")

deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py

@@ -3,9 +3,11 @@ from typing import Generator, Dict, Optional
 import pyarrow as pa
 
 from deltacat.storage.model.shard import Shard
-from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet import Schema
 
 
 class DataScan:

deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py

@@ -7,16 +7,16 @@ import pyarrow.fs
 
 from deltacat.storage import Delta
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 from deltacat.utils.filesystem import resolve_path_and_filesystem
-from deltacat.storage.rivulet.metastore.json_sst import JsonSstReader
-from deltacat.storage.rivulet.metastore.delta import (
+from deltacat.experimental.storage.rivulet.metastore.json_sst import JsonSstReader
+from deltacat.experimental.storage.rivulet.metastore.delta import (
     ManifestIO,
     DeltaContext,
     RivuletDelta,
     DeltacatManifestIO,
 )
-from deltacat.storage.rivulet.metastore.sst import SSTReader, SSTable
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTReader, SSTable
 from deltacat.utils.metafile_locator import _find_table_path
 from deltacat import logs
 

deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py

@@ -2,18 +2,20 @@ import logging
 from typing import Generator, Optional, Set, Type, TypeVar, Any
 
 from deltacat.storage.model.shard import Shard
-from deltacat.storage.rivulet.metastore.sst import SSTableRow, SSTable
-from deltacat.storage.rivulet.metastore.sst_interval_tree import (
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow, SSTable
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
     BlockIntervalTree,
     OrderedBlockGroups,
 )
-from deltacat.storage.rivulet.reader.block_scanner import BlockScanner
-from deltacat.storage.rivulet.reader.dataset_metastore import (
+from deltacat.experimental.storage.rivulet.reader.block_scanner import BlockScanner
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
     DatasetMetastore,
     ManifestAccessor,
 )
-from deltacat.storage.rivulet.reader.query_expression import QueryExpression
-from deltacat.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+from deltacat.experimental.storage.rivulet import Schema
 
 # The type of data returned to reader
 T = TypeVar("T")

deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py

@@ -4,7 +4,10 @@ from typing import Generator, Dict, Type, NamedTuple, List
 
 from pyarrow import RecordBatch
 
-from deltacat.storage.rivulet.reader.data_reader import DataReader, MEMORY_FORMAT
+from deltacat.experimental.storage.rivulet.reader.data_reader import (
+    DataReader,
+    MEMORY_FORMAT,
+)
 import pyarrow as pa
 
 

deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py

@@ -1,9 +1,9 @@
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
-from deltacat.storage.rivulet.reader.data_reader import FileReader
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.reader.data_reader import FileReader
 from typing import Type, Dict
 
-from deltacat.storage.rivulet.schema.schema import Schema
+from deltacat.experimental.storage.rivulet.schema.schema import Schema
 
 
 class FileReaderRegistrar:

deltacat/{storage → experimental/storage}/rivulet/schema/schema.py

@@ -5,7 +5,7 @@ from typing import MutableMapping, Dict, Iterable, Tuple, Optional
 
 import pyarrow as pa
 
-from deltacat.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
 
 
 @dataclass(frozen=True)

deltacat/{storage → experimental/storage}/rivulet/serializer.py

@@ -1,6 +1,6 @@
 from typing import Protocol, Iterable, List, Union, Any, Dict
 
-from deltacat.storage.rivulet.metastore.sst import SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
 import pyarrow as pa
 
 MEMTABLE_DATA = Union[Iterable[Dict[str, Any]], pa.Table]

deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
-from deltacat.storage.rivulet.parquet.serializer import ParquetDataSerializer
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.serializer import DataSerializer
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.parquet.serializer import (
+    ParquetDataSerializer,
+)
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.serializer import DataSerializer
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
-from deltacat.storage.rivulet.feather.serializer import FeatherDataSerializer
+from deltacat.experimental.storage.rivulet.feather.serializer import (
+    FeatherDataSerializer,
+)
 
 
 class DataSerializerFactory:

deltacat/experimental/storage/rivulet/shard/range_shard.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+from typing import Generic, List, Union, Iterable
+from deltacat.storage.model.shard import T, Shard, ShardingStrategy
+from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
+    DatasetMetastore,
+)
+
+
+class RangeShard(Shard, Generic[T]):
+    """
+    Represents a range-based shard with minimum and maximum keys.
+
+    param: min_key: The minimum key for the shard.
+    param: max_key: The maximum key for the shard.
+    """
+
+    def __init__(self, min_key: T, max_key: T):
+        self.min_key = min_key
+        self.max_key = max_key
+
+    def __repr__(self) -> str:
+        return f"Shard(type=range, min_key={self.min_key}, max_key={self.max_key})"
+
+    @staticmethod
+    def split(
+        global_min: Union[int, str], global_max: Union[int, str], num_shards: int
+    ) -> List[RangeShard]:
+        """
+        Splits a range into `num_shards` shards.
+        Currently supports splitting ranges of integers and strings.
+
+        Note: If global_min == global_max or num_shards <= 1, a single shard is returned,
+              num_shards is ignored.
+
+        :param global_min: The minimum key for the entire range (int or str).
+        :param global_max: The maximum key for the entire range (int or str).
+        :param num_shards: The number of shards to create.
+        :return: A list of RangeShard objects.
+        """
+        if global_min == global_max or num_shards <= 1:
+            return [RangeShard(global_min, global_max)]
+
+        # Determine which interpolation function to use based on the type of min/max
+        if isinstance(global_min, int) and isinstance(global_max, int):
+            interpolate = RangeShard._interpolate_numeric
+        elif isinstance(global_min, str) and isinstance(global_max, str):
+            interpolate = RangeShard._interpolate_str
+        else:
+            raise ValueError(
+                "Unsupported combination of types for global_min and global_max."
+            )
+
+        shards: List[RangeShard] = []
+        for i in range(num_shards):
+            start = interpolate(global_min, global_max, i, num_shards)
+            end = interpolate(global_min, global_max, i + 1, num_shards)
+
+            if i > 0:
+                if isinstance(start, int):
+                    start = shards[-1].max_key + 1
+                elif isinstance(start, int):
+                    char_list = list(start)
+                    char_list[-1] = chr(ord(char_list[-1]) + 1)
+                    start = "".join(char_list)
+
+            shards.append(RangeShard(start, end))
+
+        return shards
+
+    @staticmethod
+    def _interpolate_numeric(start: int, end: int, step: int, total_steps: int) -> int:
+        """
+        Integer interpolation using integer (floor) division.
+
+        param: start (int): The starting number.
+        param: end (int): The ending number.
+        param: step (int): The current step in the interpolation (0-based).
+        param: total_steps (int): The total number of interpolation steps.
+
+        returns: int: The interpolated integer.
+        """
+        return start + (end - start) * step // total_steps
+
+    @staticmethod
+    def _interpolate_str(start: str, end: str, step: int, total_steps: int) -> str:
+        """
+        Interpolates between two strings lexicographically.
+
+        param: start (str): The starting string.
+        param: end (str): The ending string.
+        param: step (int): The current step in the interpolation (0-based).
+        param: total_steps (int): The total number of interpolation steps.
+
+        returns: str: The interpolated string.
+        """
+        max_len = max(len(start), len(end))
+
+        # Pad strings to the same length with spaces (smallest lexicographical character).
+        start = start.ljust(max_len, " ")
+        end = end.ljust(max_len, " ")
+
+        # Interpolate character by character based on ordinal values.
+        interpolated_chars = [
+            chr(round(ord(s) + (ord(e) - ord(s)) * step / total_steps))
+            for s, e in zip(start, end)
+        ]
+
+        return "".join(interpolated_chars).rstrip()
+
+
+class RangeShardingStrategy(ShardingStrategy, Generic[T]):
+    """
+    Implements a sharding strategy to divide a range of keys into shards.
+
+    method: shards: Generates a list of RangeShard objects based on the global range.
+    """
+
+    def shards(
+        self, num_shards: int, metastore: DatasetMetastore
+    ) -> Iterable[RangeShard[T]]:
+        """
+        Divides the global range of keys into evenly sized shards.
+
+        param: num_shards: The number of shards to divide the range into.
+        param: metastore: The dataset metastore providing access to manifests.
+        returns: A list of RangeShard objects representing the divided range.
+        """
+        min, max = metastore.get_min_max_keys()
+        return RangeShard.split(min, max, num_shards)

deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py

@@ -6,15 +6,26 @@ from typing import Any, List, Set, Protocol, TypeVar, Dict, Iterable
 
 from pyarrow import RecordBatch, Table
 from deltacat.storage.model.partition import PartitionLocator
-from deltacat.storage.rivulet.metastore.delta import ManifestIO, DeltacatManifestIO
-
-from deltacat.storage.rivulet import Schema
-from deltacat.storage.rivulet.metastore.json_sst import JsonSstWriter
-from deltacat.storage.rivulet.serializer import MEMTABLE_DATA, DataSerializer
-from deltacat.storage.rivulet.serializer_factory import DataSerializerFactory
-from deltacat.storage.rivulet.writer.dataset_writer import DatasetWriter, DATA
-from deltacat.storage.rivulet.metastore.sst import SSTWriter
-from deltacat.storage.rivulet.fs.file_provider import FileProvider
+from deltacat.experimental.storage.rivulet.metastore.delta import (
+    ManifestIO,
+    DeltacatManifestIO,
+)
+
+from deltacat.experimental.storage.rivulet import Schema
+from deltacat.experimental.storage.rivulet.metastore.json_sst import JsonSstWriter
+from deltacat.experimental.storage.rivulet.serializer import (
+    MEMTABLE_DATA,
+    DataSerializer,
+)
+from deltacat.experimental.storage.rivulet.serializer_factory import (
+    DataSerializerFactory,
+)
+from deltacat.experimental.storage.rivulet.writer.dataset_writer import (
+    DatasetWriter,
+    DATA,
+)
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTWriter
+from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
 
 INPUT_ROW = TypeVar("INPUT_ROW")
 

deltacat/io/__init__.py

@@ -0,0 +1,13 @@
+from deltacat.io.reader.deltacat_read_api import read_deltacat
+from deltacat.io.datasource.deltacat_datasource import DeltacatReadType
+from deltacat.io.datasource.deltacat_datasource import (
+    METAFILE_DATA_COLUMN_NAME,
+    METAFILE_TYPE_COLUMN_NAME,
+)
+
+__all__ = [
+    "read_deltacat",
+    "DeltacatReadType",
+    "METAFILE_DATA_COLUMN_NAME",
+    "METAFILE_TYPE_COLUMN_NAME",
+]

deltacat/io/dataset/deltacat_dataset.py

@@ -0,0 +1,91 @@
+# Allow classes to use self-referencing Type hints in Python 3.7.
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Optional, cast
+
+import pyarrow as pa
+from ray.data import Dataset
+
+from deltacat.utils.url import DeltaCatUrl
+from deltacat.io.datasink.deltacat_datasink import DeltaCatDatasink
+
+
+class DeltaCatDataset(Dataset):
+    @staticmethod
+    def from_dataset(dataset: Dataset) -> DeltaCatDataset:
+        # cast to DeltacatDataset in-place since it only adds new methods
+        dataset.__class__ = DeltaCatDataset
+        return cast(DeltaCatDataset, dataset)
+
+    def write_deltacat(
+        self,
+        url: DeltaCatUrl,
+        *,
+        # if the source dataset only contains DeltaCAT metadata, then only copy the metadata to the destination... if it contains external source file paths, then register them in a new Delta.
+        metadata_only: bool = False,
+        # merge all deltas as part of the write operation
+        copy_on_write: Optional[bool] = False,
+        filesystem: Optional[pa.fs.S3FileSystem] = None,
+        try_create_dir: bool = True,
+        arrow_open_stream_args: Optional[Dict[str, Any]] = None,
+        arrow_parquet_args_fn: Callable[[], Dict[str, Any]] = lambda: {},
+        min_rows_per_file: Optional[int] = None,
+        ray_remote_args: Dict[str, Any] = None,
+        concurrency: Optional[int] = None,
+        **arrow_parquet_args,
+    ) -> None:
+        """Writes the dataset to files and commits DeltaCAT metadata indexing
+        the files written.
+
+        This is only supported for datasets convertible to Arrow records.
+        To control the number of files, use ``.repartition()``.
+
+        Unless a custom block path provider is given, the format of the output
+        files will be {uuid}_{block_idx}.{extension}, where ``uuid`` is a
+        unique id for the dataset.
+
+        The DeltaCAT manifest will be written to ``f"{path}/manifest``
+
+        Examples:
+            >>> ds.write_deltacat("s3://catalog/root/path")
+
+        Time complexity: O(dataset size / parallelism)
+
+        Args:
+            url: The path to the root directory where materialized files and
+                DeltaCAT manifest will be written.
+            filesystem: The filesystem implementation to write to. This should
+                be either a PyArrow S3FileSystem.
+            try_create_dir: Try to create all directories in destination path
+                if True. Does nothing if all directories already exist.
+            arrow_open_stream_args: kwargs passed to
+                pyarrow.fs.S3FileSystem.open_output_stream
+            filename_provider: FilenameProvider implementation
+                to write each dataset block to a custom output path.
+            arrow_parquet_args_fn: Callable that returns a dictionary of write
+                arguments to use when writing each block to a file. Overrides
+                any duplicate keys from arrow_parquet_args. This should be used
+                instead of arrow_parquet_args if any of your write arguments
+                cannot be pickled, or if you'd like to lazily resolve the write
+                arguments for each dataset block.
+            arrow_parquet_args: Options to pass to
+                pyarrow.parquet.write_table(), which is used to write out each
+                block to a file.
+        """
+        datasink = DeltaCatDatasink(
+            url,
+            metadata_only=metadata_only,
+            copy_on_write=copy_on_write,
+            arrow_parquet_args_fn=arrow_parquet_args_fn,
+            arrow_parquet_args=arrow_parquet_args,
+            min_rows_per_file=min_rows_per_file,
+            filesystem=filesystem,
+            try_create_dir=try_create_dir,
+            open_stream_args=arrow_open_stream_args,
+            dataset_uuid=self._uuid,
+        )
+        self.write_datasink(
+            datasink,
+            ray_remote_args=ray_remote_args,
+            concurrency=concurrency,
+        )