deltacat 1.1.35__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/storage/rivulet/writer/memtable_dataset_writer.py

@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+import threading
+from threading import Thread
+from typing import Any, List, Set, Protocol, TypeVar, Dict, Iterable
+
+from pyarrow import RecordBatch, Table
+from deltacat.storage.model.partition import PartitionLocator
+from deltacat.storage.rivulet.metastore.delta import ManifestIO, DeltacatManifestIO
+
+from deltacat.storage.rivulet import Schema
+from deltacat.storage.rivulet.metastore.json_sst import JsonSstWriter
+from deltacat.storage.rivulet.serializer import MEMTABLE_DATA, DataSerializer
+from deltacat.storage.rivulet.serializer_factory import DataSerializerFactory
+from deltacat.storage.rivulet.writer.dataset_writer import DatasetWriter, DATA
+from deltacat.storage.rivulet.metastore.sst import SSTWriter
+from deltacat.storage.rivulet.fs.file_provider import FileProvider
+
+INPUT_ROW = TypeVar("INPUT_ROW")
+
+
+class Memtable(Protocol[INPUT_ROW]):
+    """
+    Protocol defining the interface for a memtable that can store and sort records of type T.
+    """
+
+    def add_record(self, record: INPUT_ROW) -> bool:
+        """
+        Add a record to the memtable.
+
+        Args:
+            record: The record to add of type INPUT_ROW
+
+        Returns:
+            bool: True if the memtable is full after adding the record, False otherwise
+        """
+        ...
+
+    def get_sorted_records(self, schema: Schema) -> MEMTABLE_DATA:
+        """
+        Get all records in the memtable in sorted order.
+
+        Returns:
+            List[T]: A list of sorted records
+        """
+        ...
+
+
+class DictMemTable(Memtable[Dict[str, Any]]):
+    """
+    Unit of in memory buffering of sorted records before records are written to file
+
+    TODO future improvements:
+        1. build b+ tree of record indexes on insertion
+             OR If we end up using arrow as intermediate format, we can use
+          pyarrow compute sort
+        2. Probably we will re-write in rust
+    """
+
+    def __init__(self, merge_key: str):
+        self.row_size = 0
+        self.merge_key = merge_key
+
+        self._records: List[Dict[str, Any]] = []
+        self.lock = threading.Lock()
+
+    def add_record(self, record: Dict[str, Any]):
+        with self.lock:
+            self._records.append(record)
+            self.row_size += 1
+
+        if self.row_size >= MemtableDatasetWriter.MAX_ROW_SIZE:
+            return True
+        return False
+
+    def get_sorted_records(self, schema: Schema) -> List[Dict[str, Any]]:
+        """
+        Gets sorted records
+
+        :return: iterator over sorted record
+        """
+        with self.lock:
+            self._records.sort(key=lambda x: x.__getitem__(self.merge_key))
+            return self._records
+
+
+class RecordBatchMemTable(Memtable[RecordBatch]):
+    """
+    Note that this will not respect max row size.
+    """
+
+    def __init__(self, merge_key: str):
+        self.row_size = 0
+        self.merge_key = merge_key
+
+        # list of full record batches in memtable
+        self._records_batches: List[RecordBatch] = []
+        self.lock = threading.Lock()
+
+    def add_record(self, record: RecordBatch):
+        with self.lock:
+            self._records_batches.append(record)
+            self.row_size += record.num_rows
+
+        if self.row_size >= MemtableDatasetWriter.MAX_ROW_SIZE:
+            return True
+        return False
+
+    def get_sorted_records(self, schema: Schema) -> Table:
+        """
+        Gets sorted records
+
+        :return: iterator over sorted record
+        """
+        with self.lock:
+            # Note that we are providing schema so that pyarrow does not infer it
+            table = Table.from_batches(self._records_batches, schema.to_pyarrow())
+            return table.sort_by(self.merge_key)
+
+
+class MemtableDatasetWriter(DatasetWriter):
+    # Note that this max row size is not respected when PyArrow RecordBatches are used
+    # In that case, the entire record batch is written within one memtable even if the row count overflows
+    MAX_ROW_SIZE = 1000000
+    """
+    Buffers data into rotating memtables. When a memtable reaches a certain size, it is flushed to disk and a new memtable is allocated
+
+    Uses DataWriter which will be format specific for writing data
+    Uses MetadataWriter for writing metadata
+
+    TODO Future Improvements
+    1. Maybe we should re-write this class in Rust (pending testing)
+    """
+
+    def __init__(
+        self,
+        file_provider: FileProvider,
+        schema: Schema,
+        locator: PartitionLocator,
+        file_format: str | None = None,
+        sst_writer: SSTWriter = None,
+        manifest_io: ManifestIO = None,
+    ):
+
+        if not sst_writer:
+            sst_writer = JsonSstWriter()
+        if not manifest_io:
+            manifest_io = DeltacatManifestIO(file_provider.uri, locator)
+
+        self.schema = schema
+
+        self.file_provider = file_provider
+        self.data_serializer: DataSerializer = DataSerializerFactory.get_serializer(
+            self.schema, self.file_provider, file_format
+        )
+        self.sst_writer = sst_writer
+        self.manifest_io = manifest_io
+
+        self._sst_files: Set[str] = set()
+        self.__curr_memtable = None
+        self.__open_memtables = []
+        self.__rlock = threading.RLock()
+        self.__open_threads: List[Thread] = []
+        self._locator = locator
+
+    def write_dict(self, record: Dict[str, Any]) -> None:
+
+        # Construct memtable if doesn't exist. If previous memtable wrong type, rotate
+        memtable_ctor = lambda: DictMemTable(self.schema.get_merge_key())
+        if not self.__curr_memtable:
+            self.__curr_memtable = memtable_ctor()
+        try:
+            isinstance(self.__curr_memtable, DictMemTable)
+        except TypeError:
+            self.__rotate_memtable(memtable_ctor)
+
+        # Write record(s). If memtable is full, rotate
+        if self.__curr_memtable.add_record(record):
+            self.__rotate_memtable(memtable_ctor)
+
+    def write_record_batch(self, record: RecordBatch) -> None:
+        # Construct memtable if doesn't exist. If previous memtable wrong type, rotate
+        memtable_ctor = lambda: RecordBatchMemTable(self.schema.get_merge_key())
+        if not self.__curr_memtable:
+            self.__curr_memtable = memtable_ctor()
+
+        try:
+            isinstance(self.__curr_memtable, RecordBatchMemTable)
+        except TypeError:
+            self.__rotate_memtable(memtable_ctor)
+
+        # Write record(s). If memtable is full, rotate
+        if self.__curr_memtable.add_record(record):
+            self.__rotate_memtable(memtable_ctor)
+
+    def write(self, data: DATA) -> None:
+        if isinstance(data, RecordBatch):
+            self.write_record_batch(data)
+        elif isinstance(data, Iterable):
+            for x in data:
+                if isinstance(x, dict):
+                    self.write_dict(x)
+                elif isinstance(x, RecordBatch):
+                    self.write_record_batch(x)
+                else:
+                    raise ValueError(
+                        f"Iterable contained unsupported type {type(x).__name__}."
+                        f" Supported data types to write are: {DATA}"
+                    )
+        else:
+            raise ValueError(
+                f"Unsupported data type {type(data).__name__}. Supported data types to write are: {DATA}"
+            )
+
+    def flush(self) -> str:
+        """
+        Explicitly flush any data and metadata and commit to dataset
+        """
+        self.__flush_memtable(self.__curr_memtable)
+        for thread in [t for t in self.__open_threads if t.is_alive()]:
+            thread.join()
+
+        manifest_location = self.__write_manifest_file()
+        self._sst_files.clear()
+
+        return manifest_location
+
+    def __enter__(self) -> Any:
+        """
+        Enter and exit method allows python "with" statement
+        """
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """
+        Closes all open memtables and ensures all data is flushed.
+        """
+        self.flush()
+        # return False to propogate up error messages
+        return False
+
+    def __rotate_memtable(self, memtable_constructor_closure):
+        """
+        Replace the active memtable
+        :return:
+        """
+        with self.__rlock:
+            self.__flush_memtable(self.__curr_memtable)
+            self.__curr_memtable = memtable_constructor_closure()
+            self.__open_memtables.append(self.__curr_memtable)
+
+            # Reap dead threads
+            self.__open_threads = [t for t in self.__open_threads if t.is_alive()]
+
+    def __flush_memtable(self, memtable):
+        thread = threading.Thread(target=self.__flush_memtable_async, args=(memtable,))
+        thread.start()
+        with self.__rlock:
+            self.__open_threads.append(thread)
+
+    def __flush_memtable_async(self, memtable: Memtable):
+        """
+        Flushes data and metadata for a given memtable
+        Called asynchronously in background thread
+        """
+        if not memtable:
+            return
+
+        sst_metadata_list = self.data_serializer.flush_batch(
+            memtable.get_sorted_records(self.schema)
+        )
+
+        # short circuit if no data/metadata written
+        if not sst_metadata_list:
+            with self.__rlock:
+                self.__open_memtables.remove(memtable)
+            return
+
+        # Write SST. Each memtable is going to have a dedicated L0 SST file because that is the unit at which
+        # we have contiguously sorted data
+        sst_file = self.file_provider.provide_l0_sst_file()
+
+        with self.__rlock:
+            self.sst_writer.write(sst_file, sst_metadata_list)
+            self._sst_files.add(sst_file.location)
+
+            if memtable in self.__open_memtables:
+                self.__open_memtables.remove(memtable)
+
+    def __write_manifest_file(self) -> str:
+        """
+        Write the manifest file to the filesystem at the given URI.
+        """
+        return self.manifest_io.write(list(self._sst_files), self.schema, 0)

deltacat/tests/_io/__init__.py

@@ -0,0 +1 @@
+# NOTE - this module is renamed because it is shadowing the stdlib io module when running tests in Pycharm

deltacat/tests/catalog/test_catalogs.py

@@ -0,0 +1,324 @@
+import unittest
+import pytest
+import ray
+import tempfile
+import shutil
+import uuid
+from unittest import mock
+import os
+
+from deltacat.catalog import CatalogProperties
+from pyiceberg.catalog import Catalog as IcebergCatalog
+
+from deltacat.catalog.model.catalog import (
+    Catalog,
+    init,
+    get_catalog,
+    put_catalog,
+    is_initialized,
+)
+from deltacat.catalog.iceberg.iceberg_catalog_config import IcebergCatalogConfig
+
+from pyiceberg.catalog import CatalogType
+
+
+# Test module to mock a catalog implementation
+class MockCatalogImpl:
+    @staticmethod
+    def initialize(*args, **kwargs):
+        # Return some state that the catalog would normally maintain
+        return {"initialized": True, "args": args, "kwargs": kwargs}
+
+
+@pytest.fixture(scope="function")
+def reset_catalogs_ray_actor():
+    """
+    Setup and teardown for Ray environment for tests.
+
+    This will kill the actor all_catalogs, essentially wiping global state for catalogs
+
+    NOTE: tests using this fixture must be run serially. As of April 7 2025, the unit test suite had various
+      failures if run in parallel, in part because the state of all_catalogs in ray is shared across tests.
+
+    NOTE: when using this fixture, ensure you pass ray_init_args={"ignore_reinit_error": True} into all
+        functions which may re-initialize ray. This is because the production code checks the all_catalogs actor
+        in order to determine whether it needs to initialize Ray
+    """
+    # Reset the global catalog_actor state before each test
+    import deltacat.catalog.model.catalog as catalog_module
+
+    # Initialize Ray if not already initialized
+    if not ray.is_initialized():
+        ray.init(ignore_reinit_error=True)
+    yield
+
+    # Clean up the actor if it exists
+    if catalog_module.all_catalogs is not None:
+        try:
+            ray.kill(catalog_module.all_catalogs)
+        except Exception:
+            pass
+        finally:
+            catalog_module.all_catalogs = None
+
+
+class TestCatalog(unittest.TestCase):
+    """Tests for the Catalog class itself, without Ray initialization."""
+
+    def test_catalog_constructor(self):
+        """Test that the Catalog constructor correctly initializes with the given implementation."""
+        catalog = Catalog(impl=MockCatalogImpl)
+
+        self.assertEqual(catalog.impl, MockCatalogImpl)
+
+        # Check that inner state was correctly initialized
+        # This just asserts that kwargs were plumbed through from Catalog constructor
+        self.assertTrue(catalog.inner["initialized"])
+        self.assertEqual(catalog.inner["args"], ())
+        self.assertEqual(catalog.inner["kwargs"], {})
+
+    def test_iceberg_factory_method(self):
+        """Test the iceberg factory method correctly creates an Iceberg catalog."""
+        # Create a mock for the Iceberg catalog module
+        with mock.patch(
+            "deltacat.catalog.model.catalog.IcebergCatalog"
+        ) as mock_iceberg_catalog:
+            # Configure the mock to return a known value when initialize is called
+            mock_iceberg_catalog.initialize.return_value = {"iceberg": True}
+
+            # Create an Iceberg catalog config and invoke iceberg factory method
+            config = IcebergCatalogConfig(type=CatalogType.IN_MEMORY, properties={})
+            catalog = Catalog.iceberg(config)
+
+            # Check that the implementation is set to iceberg_catalog
+            self.assertEqual(catalog.impl, mock_iceberg_catalog)
+            # Check that the inner state is set to the output of initialize
+            self.assertEqual(catalog.inner, {"iceberg": True})
+
+
+class TestCatalogsIntegration:
+    """Integration tests for Default catalog functionality."""
+
+    temp_dir = None
+
+    @classmethod
+    def setup_class(cls):
+        cls.temp_dir = tempfile.mkdtemp()
+        # Other tests are going to have initialized ray catalog. Initialize here to ensure
+        # that when this test class is run individuall it mimicks running with other tests
+        catalog = Catalog(impl=MockCatalogImpl)
+        init(
+            catalog,
+            ray_init_args={"ignore_reinit_error": True},
+            **{"force_reinitialize": True},
+        )
+
+    @classmethod
+    def teardown_class(cls):
+        if cls.temp_dir and os.path.exists(cls.temp_dir):
+            shutil.rmtree(cls.temp_dir)
+
+    def test_init_single_catalog(self, reset_catalogs_ray_actor):
+        """Test initializing a single catalog."""
+
+        catalog = Catalog(impl=MockCatalogImpl)
+
+        # Initialize with a single catalog and Ray init args including the namespace
+        init(
+            catalog,
+            ray_init_args={"ignore_reinit_error": True},
+            **{"force_reinitialize": True},
+        )
+
+        assert is_initialized()
+
+        # Get the default catalog and check it's the same one we initialized with
+        retrieved_catalog = get_catalog()
+        assert retrieved_catalog.impl == MockCatalogImpl
+        assert retrieved_catalog.inner["initialized"]
+
+    def test_init_multiple_catalogs(self, reset_catalogs_ray_actor):
+        """Test initializing multiple catalogs."""
+        # Create catalogs
+        catalog1 = Catalog(impl=MockCatalogImpl, id=1)
+        catalog2 = Catalog(impl=MockCatalogImpl, id=2)
+
+        # Initialize with multiple catalogs and Ray init args including the namespace
+        catalogs_dict = {"catalog1": catalog1, "catalog2": catalog2}
+        init(
+            catalogs_dict,
+            ray_init_args={"ignore_reinit_error": True},
+            **{"force_reinitialize": True},
+        )
+
+        assert is_initialized()
+
+        # Get catalogs by name and check they're the same ones we initialized with
+        retrieved_catalog1 = get_catalog("catalog1")
+        assert retrieved_catalog1.impl == MockCatalogImpl
+        assert retrieved_catalog1.inner["kwargs"]["id"] == 1
+
+        retrieved_catalog2 = get_catalog("catalog2")
+        assert retrieved_catalog2.impl == MockCatalogImpl
+        assert retrieved_catalog2.inner["kwargs"]["id"] == 2
+
+    def test_init_with_default_catalog_name(self, reset_catalogs_ray_actor):
+        """Test initializing with a specified default catalog name."""
+        # Create catalogs
+        catalog1 = Catalog(impl=MockCatalogImpl, id=1)
+        catalog2 = Catalog(impl=MockCatalogImpl, id=2)
+
+        # Initialize with multiple catalogs and specify a default
+        catalogs_dict = {"catalog1": catalog1, "catalog2": catalog2}
+        init(
+            catalogs_dict,
+            default="catalog2",
+            ray_init_args={"ignore_reinit_error": True},
+            **{"force_reinitialize": True},
+        )
+
+        # Get the default catalog and check it's catalog2
+        default_catalog = get_catalog()
+        assert default_catalog.impl == MockCatalogImpl
+        assert default_catalog.inner["kwargs"]["id"] == 2
+
+    def test_put_catalog(self, reset_catalogs_ray_actor):
+        """Test adding a catalog after initialization."""
+        # Initialize with a single catalog
+        catalog1 = Catalog(impl=MockCatalogImpl, id=1)
+        catalog2 = Catalog(impl=MockCatalogImpl, id=2)
+        init(
+            {"catalog1": catalog1},
+            ray_init_args={"ignore_reinit_error": True},
+            **{"force_reinitialize": True},
+        )
+
+        # Add a second catalog
+        put_catalog("catalog2", catalog2)
+
+        # Check both catalogs are available
+        retrieved_catalog1 = get_catalog("catalog1")
+        assert retrieved_catalog1.inner["kwargs"]["id"] == 1
+
+        retrieved_catalog2 = get_catalog("catalog2")
+        assert retrieved_catalog2.inner["kwargs"]["id"] == 2
+
+    def test_put_catalog_that_already_exists(self, reset_catalogs_ray_actor):
+        catalog = Catalog(impl=MockCatalogImpl, id=1)
+        catalog2 = Catalog(impl=MockCatalogImpl, id=2)
+        put_catalog(
+            "test_catalog",
+            catalog,
+            id=1,
+            ray_init_args={"ignore_reinit_error": True},
+        )
+
+        # Try to add another catalog with the same name. Should not error
+        put_catalog(
+            "test_catalog",
+            catalog2,
+            ray_init_args={"ignore_reinit_error": True},
+        )
+
+        retrieved_catalog = get_catalog("test_catalog")
+        assert retrieved_catalog.inner["kwargs"]["id"] == 2
+
+        # If fail_if_exists, put call should fail
+        with pytest.raises(ValueError):
+            put_catalog(
+                "test_catalog",
+                catalog,
+                ray_init_args={"ignore_reinit_error": True},
+                fail_if_exists=True,
+            )
+
+    def test_get_catalog_nonexistent(self, reset_catalogs_ray_actor):
+        """Test that trying to get a nonexistent catalog raises an error."""
+        # Initialize with a catalog
+        catalog = Catalog(impl=MockCatalogImpl)
+        init(
+            {"test_catalog": catalog},
+            ray_init_args={"ignore_reinit_error": True},
+            **{"force_reinitialize": True},
+        )
+
+        # Try to get a nonexistent catalog
+        with pytest.raises(ValueError):
+            get_catalog("nonexistent")
+
+    def test_get_catalog_no_default(self, reset_catalogs_ray_actor):
+        """Test that trying to get the default catalog when none is set raises an error."""
+        # Initialize with multiple catalogs but no default
+        catalog1 = Catalog(impl=MockCatalogImpl, id=1)
+        catalog2 = Catalog(impl=MockCatalogImpl, id=2)
+        init(
+            {"catalog1": catalog1, "catalog2": catalog2},
+            ray_init_args={"ignore_reinit_error": True},
+            **{"force_reinitialize": True},
+        )
+
+        # Try to get the default catalog
+        with pytest.raises(ValueError):
+            get_catalog()
+
+    def test_default_catalog_initialization(self, reset_catalogs_ray_actor):
+        """Test that a Default catalog can be initialized and accessed using the factory method."""
+        from deltacat.catalog.model.properties import CatalogProperties
+
+        catalog_name = str(uuid.uuid4())
+
+        # Create the catalog properties
+        config = CatalogProperties(root=self.temp_dir)
+
+        # Create the catalog using the factory method
+        catalog = Catalog.default(config)
+
+        # Initialize DeltaCAT with this catalog
+        init(
+            {catalog_name: catalog},
+            ray_init_args={"ignore_reinit_error": True},
+            **{"force_reinitialize": True},
+        )
+
+        # Retrieve the catalog and verify it's the same one
+        retrieved_catalog = get_catalog(catalog_name)
+        assert retrieved_catalog.impl.__name__ == "deltacat.catalog.main.impl"
+        assert isinstance(retrieved_catalog.inner, CatalogProperties)
+        assert retrieved_catalog.inner.root == self.temp_dir
+
+    def test_default_catalog_initialization_from_kwargs(self, reset_catalogs_ray_actor):
+
+        catalog_name = str(uuid.uuid4())
+        # Initialize DeltaCAT with this catalog
+        from deltacat.catalog.main import impl as DeltacatCatalog
+
+        put_catalog(
+            catalog_name,
+            Catalog(DeltacatCatalog, **{"root": "test_root"}),
+            ray_init_args={"ignore_reinit_error": True},
+        )
+
+        # Retrieve the catalog and verify it's the same one
+        retrieved_catalog = get_catalog(catalog_name)
+        assert retrieved_catalog.impl.__name__ == "deltacat.catalog.main.impl"
+        assert isinstance(retrieved_catalog.inner, CatalogProperties)
+        assert retrieved_catalog.inner.root == "test_root"
+
+    def test_iceberg_catalog_initialization(self, reset_catalogs_ray_actor):
+        """Test that an Iceberg catalog can be initialized and accessed."""
+        catalog_name = str(uuid.uuid4())
+
+        # Create the Iceberg catalog config
+        config = IcebergCatalogConfig(
+            type=CatalogType.IN_MEMORY, properties={"warehouse": self.temp_dir}
+        )
+
+        # Create the catalog using the factory method
+        catalog = Catalog.iceberg(config)
+
+        put_catalog(catalog_name, catalog, ray_init_args={"ignore_reinit_error": True})
+
+        # Retrieve the catalog and verify it's the same one
+        retrieved_catalog = get_catalog(catalog_name)
+        assert retrieved_catalog.impl.__name__ == "deltacat.catalog.iceberg.impl"
+        assert isinstance(retrieved_catalog.inner, IcebergCatalog)

deltacat/tests/catalog/test_default_catalog_impl.py

@@ -1,20 +1,23 @@
 import unittest
 import sqlite3
+import uuid
+
 import ray
 import os
 import deltacat.tests.local_deltacat_storage as ds
+from deltacat import Catalog
+from deltacat.catalog import CatalogProperties
 from deltacat.utils.common import current_time_ms
 from deltacat.tests.test_utils.pyarrow import (
     create_delta_from_csv_file,
     commit_delta_to_partition,
 )
 from deltacat.types.media import DistributedDatasetType, ContentType
-from deltacat.catalog import default_catalog_impl as dc
+import deltacat as dc
 
 
 class TestReadTable(unittest.TestCase):
     READ_TABLE_NAMESPACE = "catalog_read_table_namespace"
-    LOCAL_CATALOG_NAME = "local_catalog"
     DB_FILE_PATH = f"{current_time_ms()}.db"
     SAMPLE_FILE_PATH = "deltacat/tests/catalog/data/sample_table.csv"
 
@@ -31,6 +34,13 @@ class TestReadTable(unittest.TestCase):
         }
         cls.deltacat_storage_kwargs = {ds.DB_FILE_PATH_ARG: cls.DB_FILE_PATH}
 
+        cls.catalog_name = str(uuid.uuid4())
+        catalog_config = CatalogProperties(storage=ds)
+        dc.put_catalog(
+            cls.catalog_name,
+            catalog=Catalog.default(config=catalog_config),
+            ray_init_args={"ignore_reinit_error": True},
+        )
         super().setUpClass()
 
     @classmethod
@@ -49,13 +59,12 @@ class TestReadTable(unittest.TestCase):
             **self.kwargs,
         )
 
-        dc.initialize(ds=ds)
         df = dc.read_table(
             table=READ_TABLE_TABLE_NAME,
             namespace=self.READ_TABLE_NAMESPACE,
-            catalog=self.LOCAL_CATALOG_NAME,
+            catalog=self.catalog_name,
             distributed_dataset_type=DistributedDatasetType.DAFT,
-            deltacat_storage_kwargs=self.kwargs,
+            **self.kwargs,
         )
 
         # verify
@@ -81,14 +90,13 @@ class TestReadTable(unittest.TestCase):
         )
 
         # action
-        dc.initialize(ds=ds)
         df = dc.read_table(
             table=READ_TABLE_TABLE_NAME,
             namespace=self.READ_TABLE_NAMESPACE,
-            catalog=self.LOCAL_CATALOG_NAME,
+            catalog=self.catalog_name,
             distributed_dataset_type=DistributedDatasetType.DAFT,
             merge_on_read=False,
-            deltacat_storage_kwargs=self.kwargs,
+            **self.kwargs,
         )
 
         # verify