deltacat 1.1.8__py3-none-any.whl → 1.1.9__py3-none-any.whl

deltacat/tests/compute/test_compact_partition_rebase.py

@@ -0,0 +1,289 @@
+import ray
+import os
+from moto import mock_s3
+import pytest
+import boto3
+from boto3.resources.base import ServiceResource
+import pyarrow as pa
+from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
+from pytest_benchmark.fixture import BenchmarkFixture
+
+from deltacat.tests.compute.test_util_constant import (
+    TEST_S3_RCF_BUCKET_NAME,
+    DEFAULT_NUM_WORKERS,
+    DEFAULT_WORKER_INSTANCE_CPUS,
+)
+from deltacat.compute.compactor.model.compactor_version import CompactorVersion
+from deltacat.tests.compute.test_util_common import (
+    get_compacted_delta_locator_from_rcf,
+)
+from deltacat.tests.compute.test_util_create_table_deltas_repo import (
+    create_src_w_deltas_destination_rebase_w_deltas_strategy,
+)
+from deltacat.tests.compute.compact_partition_rebase_test_cases import (
+    REBASE_TEST_CASES,
+)
+from typing import Any, Callable, Dict, List, Optional, Set
+from deltacat.types.media import StorageType
+from deltacat.storage import (
+    DeltaLocator,
+    Partition,
+)
+from deltacat.types.media import ContentType
+from deltacat.compute.compactor.model.compact_partition_params import (
+    CompactPartitionParams,
+)
+from deltacat.utils.placement import (
+    PlacementGroupManager,
+)
+
+DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
+    "db_file_path",
+    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
+)
+
+
+"""
+MODULE scoped fixtures
+"""
+
+
+@pytest.fixture(autouse=True, scope="module")
+def setup_ray_cluster():
+    ray.init(local_mode=True, ignore_reinit_error=True)
+    yield
+    ray.shutdown()
+
+
+@pytest.fixture(autouse=True, scope="module")
+def mock_aws_credential():
+    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+    os.environ["AWS_SECRET_ACCESS_ID"] = "testing"
+    os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    os.environ["AWS_SESSION_TOKEN"] = "testing"
+    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
+    yield
+
+
+@pytest.fixture(autouse=True, scope="module")
+def cleanup_the_database_file_after_all_compaction_session_package_tests_complete():
+    # make sure the database file is deleted after all the compactor package tests are completed
+    if os.path.exists(DATABASE_FILE_PATH_VALUE):
+        os.remove(DATABASE_FILE_PATH_VALUE)
+
+
+@pytest.fixture(scope="module")
+def s3_resource(mock_aws_credential):
+    with mock_s3():
+        yield boto3.resource("s3")
+
+
+@pytest.fixture(autouse=True, scope="module")
+def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
+    s3_resource.create_bucket(
+        ACL="authenticated-read",
+        Bucket=TEST_S3_RCF_BUCKET_NAME,
+    )
+    yield
+
+
+"""
+FUNCTION scoped fixtures
+"""
+
+
+@pytest.fixture(scope="function")
+def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
+    # see deltacat/tests/local_deltacat_storage/README.md for documentation
+    kwargs_for_local_deltacat_storage: Dict[str, Any] = {
+        DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
+    }
+    yield kwargs_for_local_deltacat_storage
+    if os.path.exists(DATABASE_FILE_PATH_VALUE):
+        os.remove(DATABASE_FILE_PATH_VALUE)
+
+
+@pytest.mark.parametrize(
+    [
+        "test_name",
+        "primary_keys",
+        "sort_keys",
+        "partition_keys_param",
+        "partition_values_param",
+        "input_deltas_param",
+        "input_deltas_delta_type",
+        "expected_terminal_compact_partition_result",
+        "expected_terminal_exception",
+        "expected_terminal_exception_message",
+        "create_placement_group_param",
+        "records_per_compacted_file_param",
+        "hash_bucket_count_param",
+        "read_kwargs_provider_param",
+        "drop_duplicates_param",
+        "skip_enabled_compact_partition_drivers",
+        "rebase_expected_compact_partition_result",
+        "compact_partition_func",
+    ],
+    [
+        (
+            test_name,
+            primary_keys,
+            sort_keys,
+            partition_keys_param,
+            partition_values_param,
+            input_deltas,
+            input_deltas_delta_type,
+            expected_terminal_compact_partition_result,
+            expected_terminal_exception,
+            expected_terminal_exception_message,
+            create_placement_group_param,
+            records_per_compacted_file_param,
+            hash_bucket_count_param,
+            drop_duplicates_param,
+            read_kwargs_provider,
+            skip_enabled_compact_partition_drivers,
+            rebase_expected_compact_partition_result,
+            compact_partition_func,
+        )
+        for test_name, (
+            primary_keys,
+            sort_keys,
+            partition_keys_param,
+            partition_values_param,
+            input_deltas,
+            input_deltas_delta_type,
+            expected_terminal_compact_partition_result,
+            expected_terminal_exception,
+            expected_terminal_exception_message,
+            create_placement_group_param,
+            records_per_compacted_file_param,
+            hash_bucket_count_param,
+            drop_duplicates_param,
+            read_kwargs_provider,
+            skip_enabled_compact_partition_drivers,
+            rebase_expected_compact_partition_result,
+            compact_partition_func,
+        ) in REBASE_TEST_CASES.items()
+    ],
+    ids=[test_name for test_name in REBASE_TEST_CASES],
+)
+def test_compact_partition_rebase_same_source_and_destination(
+    mocker,
+    s3_resource: ServiceResource,
+    local_deltacat_storage_kwargs: Dict[str, Any],
+    test_name: str,
+    primary_keys: Set[str],
+    sort_keys: List[Optional[Any]],
+    partition_keys_param: Optional[List[Any]],
+    partition_values_param: List[Optional[str]],
+    input_deltas_param: List[pa.Array],
+    input_deltas_delta_type: str,
+    expected_terminal_compact_partition_result: pa.Table,
+    expected_terminal_exception: BaseException,
+    expected_terminal_exception_message: Optional[str],
+    create_placement_group_param: bool,
+    records_per_compacted_file_param: int,
+    hash_bucket_count_param: int,
+    drop_duplicates_param: bool,
+    read_kwargs_provider_param: Any,
+    rebase_expected_compact_partition_result: pa.Table,
+    skip_enabled_compact_partition_drivers: List[CompactorVersion],
+    compact_partition_func: Callable,
+    benchmark: BenchmarkFixture,
+):
+    import deltacat.tests.local_deltacat_storage as ds
+
+    ds_mock_kwargs = local_deltacat_storage_kwargs
+    """
+    This test tests the scenario where source partition locator == destination partition locator,
+    but rebase source partition locator is different.
+    This scenario could occur when hash bucket count changes.
+    """
+    partition_keys = partition_keys_param
+    (
+        source_table_stream,
+        _,
+        rebased_table_stream,
+    ) = create_src_w_deltas_destination_rebase_w_deltas_strategy(
+        primary_keys,
+        sort_keys,
+        partition_keys,
+        input_deltas_param,
+        input_deltas_delta_type,
+        partition_values_param,
+        ds_mock_kwargs,
+    )
+    source_partition: Partition = ds.get_partition(
+        source_table_stream.locator,
+        partition_values_param,
+        **ds_mock_kwargs,
+    )
+    rebased_partition: Partition = ds.get_partition(
+        rebased_table_stream.locator,
+        partition_values_param,
+        **ds_mock_kwargs,
+    )
+    num_workers, worker_instance_cpu = DEFAULT_NUM_WORKERS, DEFAULT_WORKER_INSTANCE_CPUS
+    total_cpus = num_workers * worker_instance_cpu
+    pgm = None
+    create_placement_group_param = False
+    if create_placement_group_param:
+        pgm = PlacementGroupManager(
+            1, total_cpus, worker_instance_cpu, memory_per_bundle=4000000
+        ).pgs[0]
+    compact_partition_params = CompactPartitionParams.of(
+        {
+            "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+            "compacted_file_content_type": ContentType.PARQUET,
+            "dd_max_parallelism_ratio": 1.0,
+            "deltacat_storage": ds,
+            "deltacat_storage_kwargs": ds_mock_kwargs,
+            "destination_partition_locator": rebased_partition.locator,
+            "hash_bucket_count": hash_bucket_count_param,
+            "last_stream_position_to_compact": source_partition.stream_position,
+            "list_deltas_kwargs": {**ds_mock_kwargs, **{"equivalent_table_types": []}},
+            "object_store": RayPlasmaObjectStore(),
+            "pg_config": pgm,
+            "primary_keys": primary_keys,
+            "read_kwargs_provider": read_kwargs_provider_param,
+            "rebase_source_partition_locator": source_partition.locator,
+            "records_per_compacted_file": records_per_compacted_file_param,
+            "s3_client_kwargs": {},
+            "source_partition_locator": rebased_partition.locator,
+            "sort_keys": sort_keys if sort_keys else None,
+        }
+    )
+
+    from deltacat.compute.compactor_v2.model.compaction_session import (
+        ExecutionCompactionResult,
+    )
+
+    execute_compaction_result_spy = mocker.spy(ExecutionCompactionResult, "__init__")
+
+    # execute
+    rcf_file_s3_uri = compact_partition_func(compact_partition_params)
+
+    # Assert not in-place compacted
+    assert (
+        execute_compaction_result_spy.call_args.args[-1] is False
+    ), "Table version erroneously marked as in-place compacted!"
+    compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
+        s3_resource, rcf_file_s3_uri
+    )
+    tables = ds.download_delta(
+        compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
+    )
+    actual_rebase_compacted_table = pa.concat_tables(tables)
+    # if no primary key is specified then sort by sort_key for consistent assertion
+    sorting_cols: List[Any] = (
+        [(val, "ascending") for val in primary_keys] if primary_keys else sort_keys
+    )
+    rebase_expected_compact_partition_result = (
+        rebase_expected_compact_partition_result.combine_chunks().sort_by(sorting_cols)
+    )
+    actual_rebase_compacted_table = (
+        actual_rebase_compacted_table.combine_chunks().sort_by(sorting_cols)
+    )
+    assert actual_rebase_compacted_table.equals(
+        rebase_expected_compact_partition_result
+    ), f"{actual_rebase_compacted_table} does not match {rebase_expected_compact_partition_result}"

deltacat/tests/compute/test_util_create_table_deltas_repo.py

@@ -251,6 +251,7 @@ def create_src_w_deltas_destination_rebase_w_deltas_strategy(
     ds.commit_partition(staged_partition, **ds_mock_kwargs)
 
     # get streams
+    # TODO: Add deltas to destination stream
     destination_table_stream: Stream = ds.get_stream(
         namespace=destination_table_namespace,
         table_name=destination_table_name,

deltacat/tests/io/test_memcached_object_store.py

@@ -1,6 +1,9 @@
 import unittest
 import numpy as np
 from unittest import mock
+from deltacat.exceptions import (
+    PymemcachedPutObjectError,
+)
 
 
 class MockPyMemcacheClient:
@@ -86,7 +89,7 @@ class TestMemcachedObjectStore(unittest.TestCase):
         mock_retrying_client.return_value = mock_client.return_value
         mock_client.return_value.set_many.return_value = ["abcd"]
 
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(PymemcachedPutObjectError):
             self.object_store.put_many(["a", "b"])
 
         self.assertEqual(1, mock_client.return_value.set_many.call_count)
@@ -169,7 +172,7 @@ class TestMemcachedObjectStore(unittest.TestCase):
         mock_client.return_value.set.return_value = False
         mock_retrying_client.return_value = mock_client.return_value
 
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(PymemcachedPutObjectError):
             self.object_store.put("test_ip")
 
         self.assertEqual(1, mock_client.return_value.set.call_count)

deltacat/tests/local_deltacat_storage/__init__.py

@@ -12,6 +12,7 @@ import io
 from deltacat.tests.test_utils.storage import create_empty_delta
 from deltacat.utils.common import current_time_ms
 
+
 from deltacat.storage import (
     Delta,
     DeltaLocator,
@@ -49,6 +50,10 @@ from deltacat.types.media import (
     DistributedDatasetType,
 )
 from deltacat.utils.common import ReadKwargsProvider
+from deltacat.tests.local_deltacat_storage.exceptions import (
+    InvalidNamespaceError,
+    LocalStorageValidationError,
+)
 
 SQLITE_CUR_ARG = "sqlite3_cur"
 SQLITE_CON_ARG = "sqlite3_con"
@@ -97,6 +102,19 @@ def _get_manifest_entry_uri(manifest_entry_id: str) -> str:
     return f"cloudpickle://{manifest_entry_id}"
 
 
+def _merge_and_promote(
+    partition_deltas: List[Delta], previous_partition_deltas: List[Delta]
+):
+    previous_partition_deltas_spos_gt: List[Delta] = [
+        delta
+        for delta in previous_partition_deltas
+        if delta.stream_position > partition_deltas[0].stream_position
+    ]
+    # handle the case if the previous partition deltas have a greater stream position than the partition_delta
+    partition_deltas = previous_partition_deltas_spos_gt + partition_deltas
+    return partition_deltas
+
+
 def list_namespaces(*args, **kwargs) -> ListResult[Namespace]:
     cur, con = _get_sqlite3_cursor_con(kwargs)
     res = cur.execute("SELECT * FROM namespaces")
@@ -820,19 +838,19 @@ def commit_partition(
         ).all_items()
         or []
     )
+
     partition_deltas: Optional[List[Delta]] = (
         list_partition_deltas(
             partition, ascending_order=False, *args, **kwargs
         ).all_items()
         or []
     )
-    previous_partition_deltas_spos_gt: List[Delta] = [
-        delta
-        for delta in previous_partition_deltas
-        if delta.stream_position > partition_deltas[0].stream_position
-    ]
-    # handle the case if the previous partition deltas have a greater stream position than the partition_delta
-    partition_deltas = previous_partition_deltas_spos_gt + partition_deltas
+
+    # if previous_partition is passed in, table is in-place compacted and we need to run merge-and-promote
+    if previous_partition:
+        partition_deltas = _merge_and_promote(
+            partition_deltas, previous_partition_deltas
+        )
 
     stream_position = (
         partition_deltas[0].stream_position
@@ -840,13 +858,14 @@ def commit_partition(
         else partition.stream_position
     )
 
-    partition.state = CommitState.COMMITTED
     partition.stream_position = stream_position
+    if partition_deltas:
+        partition.locator = partition_deltas[0].partition_locator
+
+    partition.state = CommitState.COMMITTED
     partition.previous_stream_position = (
         pv_partition.stream_position if pv_partition else None
     )
-    if partition_deltas:
-        partition.locator = partition_deltas[0].partition_locator
     params = (json.dumps(partition), partition.locator.canonical_string())
     cur.execute("UPDATE partitions SET value = ? WHERE locator = ?", params)
     con.commit()
@@ -1162,3 +1181,15 @@ def get_table_version_column_names(
     **kwargs,
 ) -> Optional[List[str]]:
     raise NotImplementedError("Fetching column names is not supported")
+
+
+def can_categorize(e: BaseException, **kwargs) -> bool:
+    if isinstance(e, InvalidNamespaceError):
+        return True
+    else:
+        return False
+
+
+def raise_categorized_error(e: BaseException, **kwargs):
+    if isinstance(e, InvalidNamespaceError):
+        raise LocalStorageValidationError("Namespace provided is invalid!")

deltacat/tests/local_deltacat_storage/exceptions.py

@@ -0,0 +1,10 @@
+class InvalidNamespaceError(Exception):
+    error_name = "InvalidNamespaceError"
+
+
+class LocalStorageValidationError(Exception):
+    error_name = "LocalStorageValidationError"
+
+
+class LocalStorageError(Exception):
+    error_name = "LocalStorageError"

deltacat/tests/test_exceptions.py

@@ -0,0 +1,100 @@
+import unittest
+from deltacat.exceptions import categorize_errors
+import ray
+from deltacat.exceptions import (
+    DependencyPyarrowCapacityError,
+    NonRetryableDownloadTableError,
+    RetryableError,
+    NonRetryableError,
+    DeltaCatTransientError,
+    DependencyDaftTransientError,
+    UnclassifiedDeltaCatError,
+)
+from daft.exceptions import DaftTransientError
+from deltacat.tests.local_deltacat_storage.exceptions import (
+    InvalidNamespaceError,
+    LocalStorageValidationError,
+)
+from botocore.exceptions import NoCredentialsError
+from tenacity import retry, retry_if_exception_type, stop_after_attempt
+
+from pyarrow.lib import ArrowCapacityError
+import deltacat.tests.local_deltacat_storage as ds
+
+
+class MockUnknownException(Exception):
+    pass
+
+
+@categorize_errors
+def mock_raise_exception(exception_to_raise, deltacat_storage=ds):
+    raise exception_to_raise
+
+
+@retry(retry=retry_if_exception_type(NoCredentialsError), stop=stop_after_attempt(2))
+def mock_tenacity_wrapped_method(exception_to_raise):
+    mock_raise_exception(exception_to_raise)
+
+
+@ray.remote
+def mock_remote_task(exception_to_raise):
+    mock_raise_exception(exception_to_raise)
+
+
+class TestCategorizeErrors(unittest.TestCase):
+    def test_pyarrow_exception_categorizer(self):
+        self.assertRaises(
+            DependencyPyarrowCapacityError,
+            lambda: mock_raise_exception(ArrowCapacityError),
+        )
+
+    def test_storage_exception_categorizer(self):
+        self.assertRaises(
+            LocalStorageValidationError,
+            lambda: mock_raise_exception(InvalidNamespaceError, deltacat_storage=ds),
+        )
+
+    def test_non_retryable_error(self):
+        self.assertRaises(
+            NonRetryableError,
+            lambda: mock_raise_exception(NonRetryableDownloadTableError),
+        )
+
+    def test_retryable_error(self):
+        self.assertRaises(RetryableError, lambda: mock_raise_exception(ConnectionError))
+
+    def test_ray_task_returns_wrapped_exception(self):
+        self.assertRaises(
+            DeltaCatTransientError,
+            lambda: ray.get(mock_remote_task.remote(ConnectionError)),
+        )
+
+    def test_daft_transient_error(self):
+        self.assertRaises(
+            DependencyDaftTransientError,
+            lambda: ray.get(mock_remote_task.remote(DaftTransientError)),
+        )
+
+    def test_tenacity_underlying_error_returned(self):
+        self.assertRaises(
+            DeltaCatTransientError,
+            lambda: mock_tenacity_wrapped_method(NoCredentialsError),
+        )
+
+    def test_unclassified_error_when_error_cannot_be_categorized(self):
+        self.assertRaises(
+            UnclassifiedDeltaCatError,
+            lambda: ray.get(mock_remote_task.remote(MockUnknownException)),
+        )
+
+    def test_deltacat_exception_contains_attributes(self):
+
+        try:
+            mock_raise_exception(ConnectionError)
+        except DeltaCatTransientError as e:
+            self.assertTrue(hasattr(e, "is_retryable"))
+            self.assertTrue(hasattr(e, "error_name"))
+            assert e.error_name == "DeltaCatTransientError"
+            return
+
+        self.assertFalse(True)

deltacat/tests/test_logs.py

@@ -38,6 +38,7 @@ class TestJsonFormatter(unittest.TestCase):
         self.assertEqual({"message": "test_message"}, result)
 
     def test_format_sanity(self):
+        ray.shutdown()
         formatter = JsonFormatter({"message": "msg"})
 
         record = LogRecord(

deltacat/tests/utils/test_daft.py

@@ -1,7 +1,6 @@
 import unittest
 from deltacat.types.media import ContentEncoding, ContentType
 from deltacat.utils.daft import daft_s3_file_to_table, s3_files_to_dataframe
-
 from deltacat.utils.pyarrow import ReadKwargsProviderPyArrowSchemaOverride
 from deltacat.types.partial_download import PartialParquetParameters
 import pyarrow as pa

deltacat/tests/utils/test_resources.py

@@ -1,8 +1,6 @@
 import unittest
 from unittest import mock
 import time
-from multiprocessing import Pool
-import platform
 
 
 class TestGetCurrentClusterUtilization(unittest.TestCase):
@@ -72,29 +70,3 @@ class TestProcessUtilizationOverTimeRange(unittest.TestCase):
             nu.schedule_callback(test_callback, 1)
             time.sleep(3)
             self.assertTrue(nu.test_field_set)
-
-
-class TestTimeoutDecorator(unittest.TestCase):
-    from deltacat.utils.resources import timeout
-
-    @staticmethod
-    @timeout(2)
-    def something_that_runs_xs(x, *args, **kwargs):
-        time.sleep(x)
-
-    def test_timeout(self):
-        if platform.system() != "Windows":
-            self.assertRaises(
-                TimeoutError, lambda: self.something_that_runs_xs(3, test=10)
-            )
-
-    def test_sanity_in_multiprocess(self):
-        if platform.system() != "Windows":
-            # An alarm works per process
-            # https://pubs.opengroup.org/onlinepubs/9699919799/functions/alarm.html
-            with Pool(3) as p:
-                p.map(self.something_that_runs_xs, [1, 1.1, 1.2])
-
-    def test_sanity(self):
-        if platform.system() != "Windows":
-            self.something_that_runs_xs(1, test=10)

deltacat/utils/daft.py

@@ -16,6 +16,7 @@ from deltacat.aws.constants import (
     BOTO_MAX_RETRIES,
     DAFT_MAX_S3_CONNECTIONS_PER_FILE,
     AWS_REGION,
+    DEFAULT_FILE_READ_TIMEOUT_MS,
 )
 from deltacat.utils.performance import timed_invocation
 
@@ -112,6 +113,7 @@ def daft_s3_file_to_table(
     coerce_int96_timestamp_unit = TimeUnit.from_str(
         kwargs.get("coerce_int96_timestamp_unit", "ms")
     )
+    file_timeout_ms = kwargs.get("file_timeout_ms", DEFAULT_FILE_READ_TIMEOUT_MS)
 
     row_groups = None
     if (
@@ -132,6 +134,7 @@ def daft_s3_file_to_table(
         io_config=io_config,
         coerce_int96_timestamp_unit=coerce_int96_timestamp_unit,
         multithreaded_io=False,
+        file_timeout_ms=file_timeout_ms,
     )
 
     logger.debug(f"Time to read S3 object from {s3_url} into daft table: {latency}s")

deltacat/utils/pyarrow.py

@@ -8,7 +8,7 @@ import logging
 from functools import partial
 from typing import Any, Callable, Dict, Iterable, List, Optional
 from pyarrow.parquet import ParquetFile
-from deltacat.exceptions import ValidationError
+from deltacat.exceptions import ContentTypeValidationError
 
 import pyarrow as pa
 import numpy as np
@@ -245,6 +245,7 @@ class ReadKwargsProviderPyArrowSchemaOverride(ContentTypeKwargsProvider):
         schema: Optional[pa.Schema] = None,
         pq_coerce_int96_timestamp_unit: Optional[str] = None,
         parquet_reader_type: Optional[str] = None,
+        file_read_timeout_ms: Optional[int] = None,
     ):
         """
 
@@ -258,6 +259,7 @@ class ReadKwargsProviderPyArrowSchemaOverride(ContentTypeKwargsProvider):
         self.schema = schema
         self.pq_coerce_int96_timestamp_unit = pq_coerce_int96_timestamp_unit
         self.parquet_reader_type = parquet_reader_type
+        self.file_read_timeout_ms = file_read_timeout_ms
 
     def _get_kwargs(self, content_type: str, kwargs: Dict[str, Any]) -> Dict[str, Any]:
         if content_type in DELIMITED_TEXT_CONTENT_TYPES:
@@ -282,6 +284,8 @@ class ReadKwargsProviderPyArrowSchemaOverride(ContentTypeKwargsProvider):
             else:
                 kwargs["reader_type"] = "daft"
 
+            kwargs["file_timeout_ms"] = self.file_read_timeout_ms
+
         return kwargs
 
 
@@ -476,10 +480,9 @@ def s3_file_to_parquet(
         content_type != ContentType.PARQUET.value
         or content_encoding != ContentEncoding.IDENTITY
     ):
-        raise ValidationError(
-            f"S3 file with content type: {content_type} and "
-            f"content encoding: {content_encoding} cannot be read"
-            "into pyarrow.parquet.ParquetFile"
+        raise ContentTypeValidationError(
+            f"S3 file with content type: {content_type} and content encoding: {content_encoding} "
+            "cannot be read into pyarrow.parquet.ParquetFile"
         )
 
     if s3_client_kwargs is None:

deltacat/utils/ray_utils/runtime.py

@@ -1,6 +1,6 @@
 import logging
 import time
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 
 import ray
 
@@ -120,7 +120,7 @@ def log_cluster_resources() -> None:
     logger.info(f"Cluster Nodes: {ray.nodes()}")
 
 
-def get_current_ray_task_id() -> str:
+def get_current_ray_task_id() -> Optional[str]:
     return ray.get_runtime_context().get_task_id()