deltacat 1.1.9__py3-none-any.whl → 1.1.11__py3-none-any.whl

deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -1,8 +1,12 @@
-import unittest
-import sqlite3
+from typing import Dict, Any
 import ray
 import os
-from unittest.mock import patch
+import pytest
+import boto3
+from deltacat.compute.compactor.model.compaction_session_audit_info import (
+    CompactionSessionAuditInfo,
+)
+from boto3.resources.base import ServiceResource
 import deltacat.tests.local_deltacat_storage as ds
 from deltacat.types.media import ContentType
 from deltacat.compute.compactor_v2.compaction_session import (
@@ -11,80 +15,241 @@ from deltacat.compute.compactor_v2.compaction_session import (
 from deltacat.compute.compactor.model.compact_partition_params import (
     CompactPartitionParams,
 )
-from deltacat.utils.common import current_time_ms
-from deltacat.tests.test_utils.pyarrow import stage_partition_from_file_paths
+from deltacat.tests.test_utils.utils import read_s3_contents
+from deltacat.tests.compute.test_util_constant import (
+    TEST_S3_RCF_BUCKET_NAME,
+)
+from deltacat.tests.compute.test_util_common import get_rcf
+from deltacat.tests.test_utils.pyarrow import (
+    stage_partition_from_file_paths,
+    commit_delta_to_staged_partition,
+    commit_delta_to_partition,
+)
+from moto import mock_s3
+
+DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
+    "db_file_path",
+    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
+)
+
+
+@pytest.fixture(autouse=True, scope="module")
+def setup_ray_cluster():
+    ray.init(local_mode=True, ignore_reinit_error=True)
+    yield
+    ray.shutdown()
+
+
+@pytest.fixture(autouse=True, scope="module")
+def mock_aws_credential():
+    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+    os.environ["AWS_SECRET_ACCESS_ID"] = "testing"
+    os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    os.environ["AWS_SESSION_TOKEN"] = "testing"
+    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
+    yield
+
+
+@pytest.fixture(scope="module")
+def s3_resource(mock_aws_credential):
+    with mock_s3():
+        yield boto3.resource("s3")
 
 
-class TestCompactionSession(unittest.TestCase):
+@pytest.fixture(autouse=True, scope="module")
+def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
+    s3_resource.create_bucket(
+        ACL="authenticated-read",
+        Bucket=TEST_S3_RCF_BUCKET_NAME,
+    )
+    yield
+
+
+@pytest.fixture(scope="function")
+def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
+    kwargs_for_local_deltacat_storage: Dict[str, Any] = {
+        DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
+    }
+    yield kwargs_for_local_deltacat_storage
+    if os.path.exists(DATABASE_FILE_PATH_VALUE):
+        os.remove(DATABASE_FILE_PATH_VALUE)
+
+
+class TestCompactionSession:
     """
     This class adds specific tests that aren't part of the parametrized test suite.
     """
 
-    DB_FILE_PATH = f"{current_time_ms()}.db"
     NAMESPACE = "compact_partition_v2_namespace"
+    BACKFILL_FILE_PATH = (
+        "deltacat/tests/compute/compactor_v2/data/backfill_source_date_pk.csv"
+    )
+    INCREMENTAL_FILE_PATH = (
+        "deltacat/tests/compute/compactor_v2/data/incremental_source_date_pk.csv"
+    )
 
-    @classmethod
-    def setUpClass(cls):
-        ray.init(local_mode=True, ignore_reinit_error=True)
+    def test_compact_partition_when_no_input_deltas_to_compact(
+        self, local_deltacat_storage_kwargs
+    ):
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["test"], **local_deltacat_storage_kwargs
+        )
+        source_partition = ds.commit_partition(
+            staged_source, **local_deltacat_storage_kwargs
+        )
 
-        con = sqlite3.connect(cls.DB_FILE_PATH)
-        cur = con.cursor()
-        cls.kwargs = {ds.SQLITE_CON_ARG: con, ds.SQLITE_CUR_ARG: cur}
-        cls.deltacat_storage_kwargs = {ds.DB_FILE_PATH_ARG: cls.DB_FILE_PATH}
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
 
-        super().setUpClass()
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 2,
+                    "last_stream_position_to_compact": source_partition.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": None,
+                    "rebase_source_partition_high_watermark": None,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_partition.locator,
+                }
+            )
+        )
 
-    @classmethod
-    def doClassCleanups(cls) -> None:
-        os.remove(cls.DB_FILE_PATH)
-        ray.shutdown()
-        super().tearDownClass()
+        # verify that no RCF is written
+        assert rcf_url is None
+
+    def test_compact_partition_when_rcf_was_written_by_past_commit(
+        self, s3_resource, local_deltacat_storage_kwargs
+    ):
+        """
+        Backward compatibility test for when a RCF was written by a previous commit.
+        """
 
-    @patch("deltacat.compute.compactor_v2.compaction_session.rcf")
-    @patch("deltacat.compute.compactor_v2.compaction_session.s3_utils")
-    def test_compact_partition_when_no_input_deltas_to_compact(self, s3_utils, rcf_url):
         # setup
-        rcf_url.read_round_completion_file.return_value = None
         staged_source = stage_partition_from_file_paths(
-            self.NAMESPACE, ["test"], **self.deltacat_storage_kwargs
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
         )
-        source_partition = ds.commit_partition(
-            staged_source, **self.deltacat_storage_kwargs
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
         )
 
         staged_dest = stage_partition_from_file_paths(
-            self.NAMESPACE, ["destination"], **self.deltacat_storage_kwargs
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
         )
         dest_partition = ds.commit_partition(
-            staged_dest, **self.deltacat_storage_kwargs
+            staged_dest, **local_deltacat_storage_kwargs
         )
 
         # action
         rcf_url = compact_partition(
             CompactPartitionParams.of(
                 {
-                    "compaction_artifact_s3_bucket": "test_bucket",
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
                     "compacted_file_content_type": ContentType.PARQUET,
                     "dd_max_parallelism_ratio": 1.0,
                     "deltacat_storage": ds,
-                    "deltacat_storage_kwargs": self.deltacat_storage_kwargs,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
                     "destination_partition_locator": dest_partition.locator,
                     "drop_duplicates": True,
                     "hash_bucket_count": 1,
-                    "last_stream_position_to_compact": source_partition.stream_position,
+                    "last_stream_position_to_compact": source_delta.stream_position,
                     "list_deltas_kwargs": {
-                        **self.deltacat_storage_kwargs,
+                        **local_deltacat_storage_kwargs,
                         **{"equivalent_table_types": []},
                     },
                     "primary_keys": [],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": None,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        bucket, backfill_key1, backfill_key2 = rcf_url.strip("s3://").split("/")
+        assert bucket == TEST_S3_RCF_BUCKET_NAME
+
+        # Now delete the RCF at new location and copy it to old location
+        # Copy the RCF from rcf_url to another location
+        s3_resource.Object(TEST_S3_RCF_BUCKET_NAME, f"{backfill_key1}.json").copy_from(
+            CopySource=f"{TEST_S3_RCF_BUCKET_NAME}/{backfill_key1}/{backfill_key2}"
+        )
+
+        s3_resource.Object(
+            TEST_S3_RCF_BUCKET_NAME, f"{backfill_key1}/{backfill_key2}"
+        ).delete()
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 1,
+                    "last_stream_position_to_compact": new_source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
                     "rebase_source_partition_locator": None,
                     "rebase_source_partition_high_watermark": None,
                     "records_per_compacted_file": 4000,
                     "s3_client_kwargs": {},
-                    "source_partition_locator": source_partition.locator,
+                    "source_partition_locator": new_source_delta.partition_locator,
                 }
             )
         )
 
-        # verify that no RCF is written
-        self.assertIsNone(rcf_url)
+        new_bucket, incremental_key1, incremental_key2 = new_rcf_url.strip(
+            "s3://"
+        ).split("/")
+
+        assert new_bucket == TEST_S3_RCF_BUCKET_NAME
+        assert backfill_key1 == incremental_key1
+        assert backfill_key2 != incremental_key2
+
+        rcf = get_rcf(s3_resource, new_rcf_url)
+
+        _, compaction_audit_key = rcf.compaction_audit_url.strip("s3://").split("/", 1)
+        compaction_audit = CompactionSessionAuditInfo(
+            **read_s3_contents(
+                s3_resource, TEST_S3_RCF_BUCKET_NAME, compaction_audit_key
+            )
+        )
+
+        # as it should be running incremental
+        assert compaction_audit.uniform_deltas_created == 1
+        assert compaction_audit.input_records == 6

deltacat/tests/compute/test_compact_partition_rebase.py

@@ -254,7 +254,7 @@ def test_compact_partition_rebase_same_source_and_destination(
         }
     )
 
-    from deltacat.compute.compactor_v2.model.compaction_session import (
+    from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
         ExecutionCompactionResult,
     )
 

deltacat/tests/compute/test_util_common.py

@@ -24,6 +24,12 @@ from deltacat.compute.compactor import (
     RoundCompletionInfo,
 )
 
+from deltacat.storage.model.partition import PartitionLocator
+from deltacat.storage.model.stream import StreamLocator
+from deltacat.storage.model.table_version import TableVersionLocator
+from deltacat.storage.model.table import TableLocator
+from deltacat.storage.model.namespace import NamespaceLocator
+
 
 class PartitionKeyType(str, Enum):
     INT = "int"
@@ -51,6 +57,18 @@ UTILS
 """
 
 
+def get_test_partition_locator(partition_id):
+    tv_locator = TableVersionLocator.of(
+        TableLocator.of(NamespaceLocator.of("default"), "test_table"), "1"
+    )
+    stream_locator = StreamLocator.of(tv_locator, "test_stream_id", "local")
+    partition_locator = PartitionLocator.of(
+        stream_locator, partition_id=partition_id, partition_values=[]
+    )
+
+    return partition_locator
+
+
 def _create_table(
     namespace: str,
     table_name: str,
@@ -140,7 +158,7 @@ def create_rebase_table(
 def get_rcf(s3_resource, rcf_file_s3_uri: str) -> RoundCompletionInfo:
     from deltacat.tests.test_utils.utils import read_s3_contents
 
-    _, rcf_object_key = rcf_file_s3_uri.rsplit("/", 1)
+    _, rcf_object_key = rcf_file_s3_uri.strip("s3://").split("/", 1)
     rcf_file_output: Dict[str, Any] = read_s3_contents(
         s3_resource, TEST_S3_RCF_BUCKET_NAME, rcf_object_key
     )
@@ -151,9 +169,6 @@ def get_compacted_delta_locator_from_rcf(
     s3_resource: ServiceResource, rcf_file_s3_uri: str
 ):
     from deltacat.storage import DeltaLocator
-    from deltacat.compute.compactor import (
-        RoundCompletionInfo,
-    )
 
     round_completion_info: RoundCompletionInfo = get_rcf(s3_resource, rcf_file_s3_uri)
 

deltacat/tests/local_deltacat_storage/__init__.py

@@ -41,6 +41,12 @@ from deltacat.storage import (
     ManifestEntry,
     ManifestEntryList,
     DeleteParameters,
+    PartitionFilter,
+    PartitionValues,
+    DeltaPartitionSpec,
+    StreamPartitionSpec,
+    TransformName,
+    IdentityTransformParameters,
 )
 from deltacat.types.media import (
     ContentType,
@@ -194,12 +200,13 @@ def list_stream_partitions(stream: Stream, *args, **kwargs) -> ListResult[Partit
 def list_deltas(
     namespace: str,
     table_name: str,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     table_version: Optional[str] = None,
     first_stream_position: Optional[int] = None,
     last_stream_position: Optional[int] = None,
     ascending_order: Optional[bool] = None,
     include_manifest: bool = False,
+    partition_filter: Optional[PartitionFilter] = None,
     *args,
     **kwargs,
 ) -> ListResult[Delta]:
@@ -207,6 +214,13 @@ def list_deltas(
     if stream is None:
         return ListResult.of([], None, None)
 
+    if partition_values is not None and partition_filter is not None:
+        raise ValueError(
+            "Only one of partition_values or partition_filter must be provided"
+        )
+    if partition_filter is not None:
+        partition_values = partition_filter.partition_values
+
     partition = get_partition(stream.locator, partition_values, *args, **kwargs)
 
     all_deltas = list_partition_deltas(
@@ -297,15 +311,25 @@ def get_delta(
     namespace: str,
     table_name: str,
     stream_position: int,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     table_version: Optional[str] = None,
     include_manifest: bool = False,
+    partition_filter: Optional[PartitionFilter] = None,
     *args,
     **kwargs,
 ) -> Optional[Delta]:
     cur, con = _get_sqlite3_cursor_con(kwargs)
 
     stream = get_stream(namespace, table_name, table_version, *args, **kwargs)
+
+    if partition_values is not None and partition_filter is not None:
+        raise ValueError(
+            "Only one of partition_values or partition_filter must be provided"
+        )
+
+    if partition_filter is not None:
+        partition_values = partition_filter.partition_values
+
     partition = get_partition(stream.locator, partition_values, *args, **kwargs)
     delta_locator = DeltaLocator.of(partition.locator, stream_position)
 
@@ -328,22 +352,24 @@ def get_delta(
 def get_latest_delta(
     namespace: str,
     table_name: str,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     table_version: Optional[str] = None,
     include_manifest: bool = False,
+    partition_filter: Optional[PartitionFilter] = None,
     *args,
     **kwargs,
 ) -> Optional[Delta]:
 
     deltas = list_deltas(
-        namespace,
-        table_name,
-        partition_values,
-        table_version,
-        None,
-        None,
-        False,
-        include_manifest,
+        namespace=namespace,
+        table_name=table_name,
+        partition_values=partition_values,
+        table_version=table_version,
+        first_stream_position=None,
+        last_stream_position=None,
+        ascending_order=False,
+        include_manifest=include_manifest,
+        partition_filter=partition_filter,
         *args,
         **kwargs,
     ).all_items()
@@ -363,13 +389,24 @@ def download_delta(
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
     ray_options_provider: Callable[[int, Any], Dict[str, Any]] = None,
     distributed_dataset_type: DistributedDatasetType = DistributedDatasetType.RAY_DATASET,
+    partition_filter: Optional[PartitionFilter] = None,
     *args,
     **kwargs,
 ) -> Union[LocalDataset, DistributedDataset]:  # type: ignore
     result = []
     manifest = get_delta_manifest(delta_like, *args, **kwargs)
 
+    partition_values: PartitionValues = None
+    if partition_filter is not None:
+        partition_values = partition_filter.partition_values
+
     for entry_index in range(len(manifest.entries)):
+        if (
+            partition_values is not None
+            and partition_values != manifest.entries[entry_index].meta.partition_values
+        ):
+            continue
+
         result.append(
             download_delta_manifest_entry(
                 delta_like=delta_like,
@@ -524,11 +561,29 @@ def create_table_version(
     table_description: Optional[str] = None,
     table_properties: Optional[Dict[str, str]] = None,
     supported_content_types: Optional[List[ContentType]] = None,
+    partition_spec: Optional[StreamPartitionSpec] = None,
     *args,
     **kwargs,
 ) -> Stream:
     cur, con = _get_sqlite3_cursor_con(kwargs)
 
+    if partition_keys is not None and partition_spec is not None:
+        raise ValueError(
+            "Only one of partition_keys or partition_spec must be provided"
+        )
+    if partition_spec is not None:
+        assert (
+            partition_spec.ordered_transforms is not None
+        ), "Ordered transforms must be specified when partition_spec is specified"
+        partition_keys = []
+        for transform in partition_spec.ordered_transforms:
+            assert transform.name == TransformName.IDENTITY, (
+                "Local DeltaCAT storage does not support creating table versions "
+                "with non identity transform partition spec"
+            )
+            transform_params: IdentityTransformParameters = transform.parameters
+            partition_keys.append(transform_params.column_name)
+
     latest_version = get_latest_table_version(namespace, table_name, *args, **kwargs)
     if (
         table_version is not None
@@ -776,7 +831,7 @@ def delete_stream(
 
 
 def stage_partition(
-    stream: Stream, partition_values: Optional[List[Any]] = None, *args, **kwargs
+    stream: Stream, partition_values: Optional[PartitionValues] = None, *args, **kwargs
 ) -> Partition:
     cur, con = _get_sqlite3_cursor_con(kwargs)
     partition_id = uuid.uuid4().__str__()
@@ -877,7 +932,7 @@ def delete_partition(
     namespace: str,
     table_name: str,
     table_version: Optional[str] = None,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     *args,
     **kwargs,
 ) -> None:
@@ -894,7 +949,7 @@ def delete_partition(
 
 def get_partition(
     stream_locator: StreamLocator,
-    partition_values: Optional[List[Any]] = None,
+    partition_values: Optional[PartitionValues] = None,
     *args,
     **kwargs,
 ) -> Optional[Partition]:
@@ -935,12 +990,14 @@ def stage_delta(
     s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
     delete_parameters: Optional[DeleteParameters] = None,
+    partition_spec: Optional[DeltaPartitionSpec] = None,
+    partition_values: Optional[PartitionValues] = None,
     *args,
     **kwargs,
 ) -> Delta:
     cur, con = _get_sqlite3_cursor_con(kwargs)
-    manifest_entry_id = uuid.uuid4().__str__()
-    uri = _get_manifest_entry_uri(manifest_entry_id)
+    manifest_id = uuid.uuid4().__str__()
+    uri = _get_manifest_entry_uri(manifest_id)
 
     if data is None:
         delta = create_empty_delta(
@@ -948,7 +1005,7 @@ def stage_delta(
             delta_type,
             author,
             properties=properties,
-            manifest_entry_id=manifest_entry_id,
+            manifest_entry_id=manifest_id,
         )
         cur.execute("INSERT OR IGNORE INTO data VALUES (?, ?)", (uri, None))
         params = (delta.locator.canonical_string(), "staged_delta", json.dumps(delta))
@@ -956,6 +1013,12 @@ def stage_delta(
         con.commit()
         return delta
 
+    if partition_spec:
+        assert partition_values is not None, (
+            "partition_values must be provided as local "
+            "storage does not support computing it from input data"
+        )
+
     serialized_data = None
     if content_type == ContentType.PARQUET:
         buffer = io.BytesIO()
@@ -980,18 +1043,19 @@ def stage_delta(
         content_type=content_type,
         content_encoding=ContentEncoding.IDENTITY,
         source_content_length=data.nbytes,
+        partition_values=partition_values,
     )
 
     manifest = Manifest.of(
         entries=ManifestEntryList.of(
             [
                 ManifestEntry.of(
-                    uri=uri, url=uri, meta=meta, mandatory=True, uuid=manifest_entry_id
+                    uri=uri, url=uri, meta=meta, mandatory=True, uuid=manifest_id
                 )
             ]
         ),
         author=author,
-        uuid=manifest_entry_id,
+        uuid=manifest_id,
     )
 
     delta = Delta.of(

deltacat/tests/test_utils/pyarrow.py

@@ -66,7 +66,10 @@ def download_delta(delta_like: Union[Delta, DeltaLocator], *args, **kwargs) -> D
 
 
 def commit_delta_to_partition(
-    partition: Partition, file_paths: List[str], *args, **kwargs
+    partition: Union[Partition, PartitionLocator],
+    file_paths: List[str],
+    *args,
+    **kwargs,
 ) -> Delta:
     tables = []
 

deltacat/tests/utils/ray_utils/test_dataset.py

@@ -0,0 +1,66 @@
+from ray.data import from_items
+from typing import Any
+import pytest
+import fsspec
+from fsspec import AbstractFileSystem
+from ray.data.datasource import FilenameProvider
+from deltacat.types.media import ContentType
+import ray
+
+
+class TestDatasetToFile:
+
+    BASE_PATH = "/tmp"
+    SUB_PATH = "abcd"
+
+    @pytest.fixture(autouse=True, scope="module")
+    def ensure_ray_down(self):
+        # ray.data fails when ray is instantiated in local mode
+        ray.shutdown()
+
+    @pytest.fixture(scope="module")
+    def mock_dataset(self):
+        return from_items([{"col1": i, "col2": i * 2} for i in range(1000)])
+
+    @pytest.fixture(scope="module")
+    def mock_filename_provider(self):
+        class MockFilenameProvider(FilenameProvider):
+            def get_filename_for_block(
+                self, block: Any, task_index: int, block_index: int
+            ) -> str:
+                return TestDatasetToFile.SUB_PATH
+
+        return MockFilenameProvider()
+
+    def test_parquet_sanity(self, mock_dataset, mock_filename_provider):
+        from deltacat.utils.ray_utils.dataset import dataset_to_file
+
+        fs: AbstractFileSystem = fsspec.filesystem("local")
+
+        dataset_to_file(
+            mock_dataset,
+            self.BASE_PATH,
+            file_system=fs,
+            block_path_provider=mock_filename_provider,
+        )
+
+        file_expected_at = f"{self.BASE_PATH}/{self.SUB_PATH}"
+        assert fs.exists(file_expected_at), "file was not written"
+        fs.delete(file_expected_at)
+
+    def test_csv_sanity(self, mock_dataset, mock_filename_provider):
+        from deltacat.utils.ray_utils.dataset import dataset_to_file
+
+        fs: AbstractFileSystem = fsspec.filesystem("local")
+
+        dataset_to_file(
+            mock_dataset,
+            self.BASE_PATH,
+            file_system=fs,
+            block_path_provider=mock_filename_provider,
+            content_type=ContentType.CSV.value,
+        )
+
+        file_expected_at = f"{self.BASE_PATH}/{self.SUB_PATH}"
+        assert fs.exists(file_expected_at), "file was not written"
+        fs.delete(file_expected_at)