deltacat 1.1.30__tar.gz → 1.1.32__tar.gz

{deltacat-1.1.30/deltacat.egg-info → deltacat-1.1.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.30
+Version: 1.1.32
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.30 → deltacat-1.1.32}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.30"
+__version__ = "1.1.32"
 
 
 __all__ = [

{deltacat-1.1.30 → deltacat-1.1.32}/deltacat/compute/compactor_v2/utils/task_options.py

@@ -1,11 +1,16 @@
 import logging
 from typing import Dict, Optional, List, Tuple, Any
 from deltacat import logs
+from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
+from deltacat.compute.compactor_v2.constants import (
+    AVERAGE_RECORD_SIZE_BYTES as DEFAULT_AVERAGE_RECORD_SIZE_BYTES,
+)
 from deltacat.compute.compactor_v2.model.merge_file_group import (
     LocalMergeFileGroupsProvider,
 )
 from deltacat.storage import (
     Manifest,
+    ManifestEntry,
     interface as unimplemented_deltacat_storage,
 )
 from deltacat.compute.compactor.model.delta_annotated import DeltaAnnotated
@@ -81,16 +86,27 @@ def _get_merge_task_options(
         and compacted_delta_manifest
         and round_completion_info.hb_index_to_entry_range
     ):
-
-        previous_inflation = (
-            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-            / round_completion_info.compacted_pyarrow_write_result.file_bytes
+        logger.debug_conditional(
+            f"[Merge task {index}]: Using previous compaction rounds to calculate merge memory: {round_completion_info.compacted_pyarrow_write_result}",
+            memory_logs_enabled,
+        )
+        previous_inflation: float = (
+            (
+                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+                / round_completion_info.compacted_pyarrow_write_result.file_bytes
+            )
+            if round_completion_info.compacted_pyarrow_write_result.file_bytes
+            else PYARROW_INFLATION_MULTIPLIER
         )
         debug_memory_params["previous_inflation"] = previous_inflation
 
-        average_record_size = (
-            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-            / round_completion_info.compacted_pyarrow_write_result.records
+        average_record_size: float = (
+            (
+                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+                / round_completion_info.compacted_pyarrow_write_result.records
+            )
+            if round_completion_info.compacted_pyarrow_write_result.records
+            else DEFAULT_AVERAGE_RECORD_SIZE_BYTES
         )
         debug_memory_params["average_record_size"] = average_record_size
 
@@ -106,31 +122,36 @@ def _get_merge_task_options(
                 str(hb_idx)
             ]
             for entry_index in range(entry_start, entry_end):
-                entry = compacted_delta_manifest.entries[entry_index]
-
-                current_entry_size = estimate_manifest_entry_size_bytes(
-                    entry=entry,
-                    operation_type=OperationType.PYARROW_DOWNLOAD,
-                    estimate_resources_params=estimate_resources_params,
+                entry: ManifestEntry = compacted_delta_manifest.entries[entry_index]
+                current_entry_size: float = (
+                    estimate_manifest_entry_size_bytes(
+                        entry=entry,
+                        operation_type=OperationType.PYARROW_DOWNLOAD,
+                        estimate_resources_params=estimate_resources_params,
+                    )
+                    or 0.0
                 )
-                current_entry_rows = estimate_manifest_entry_num_rows(
-                    entry=entry,
-                    operation_type=OperationType.PYARROW_DOWNLOAD,
-                    estimate_resources_params=estimate_resources_params,
+                current_entry_rows: int = (
+                    estimate_manifest_entry_num_rows(
+                        entry=entry,
+                        operation_type=OperationType.PYARROW_DOWNLOAD,
+                        estimate_resources_params=estimate_resources_params,
+                    )
+                    or 0
                 )
-
+                # NOTE: We can treat the current_entry_size and current_entry_rows as 0 as a None estimated entry size implies a 0 value
                 data_size += current_entry_size
                 num_rows += current_entry_rows
-
                 if primary_keys:
-                    pk_size = estimate_manifest_entry_column_size_bytes(
+                    pk_size: Optional[
+                        float
+                    ] = estimate_manifest_entry_column_size_bytes(
                         entry=entry,
                         columns=primary_keys,
                         operation_type=OperationType.PYARROW_DOWNLOAD,
                         estimate_resources_params=estimate_resources_params,
                     )
-
-                    if pk_size is None:
+                    if not pk_size:
                         pk_size_bytes += current_entry_size
                     else:
                         pk_size_bytes += pk_size
@@ -159,7 +180,6 @@ def _get_merge_task_options(
         f"[Merge task {index}]: Params used for calculating merge memory: {debug_memory_params}",
         memory_logs_enabled,
     )
-
     return _get_task_options(0.01, total_memory, ray_custom_resources)
 
 

deltacat-1.1.32/deltacat/tests/compute/compactor_v2/utils/test_task_options.py

@@ -0,0 +1,305 @@
+import unittest
+import ray
+from deltacat.compute.compactor_v2.utils.task_options import (
+    _get_task_options,
+    _get_merge_task_options,
+    logger,
+)
+from deltacat.compute.resource_estimation.model import (
+    EstimateResourcesParams,
+    ResourceEstimationMethod,
+)
+from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
+from deltacat.compute.compactor import (
+    PyArrowWriteResult,
+    RoundCompletionInfo,
+)
+from deltacat.types.media import (
+    ContentType,
+    ContentEncoding,
+)
+from deltacat.storage import (
+    DeltaLocator,
+    Manifest,
+    ManifestMeta,
+    ManifestEntry,
+    ManifestEntryList,
+    PartitionValues,
+)
+from unittest.mock import MagicMock
+from typing import Optional
+
+from deltacat.compute.compactor_v2.constants import (
+    AVERAGE_RECORD_SIZE_BYTES as DEFAULT_AVERAGE_RECORD_SIZE_BYTES,
+)
+
+
+@ray.remote
+def valid_func():
+    return 2
+
+
+@ray.remote
+def throwing_func():
+    raise ConnectionAbortedError()
+
+
+class TestTaskOptions(unittest.TestCase):
+    TEST_INDEX = 0
+    TEST_HB_GROUP_IDX = 0
+    TEST_STREAM_POSITION = 1_000_000
+    TEST_NUM_HASH_GROUPS = 1
+
+    @classmethod
+    def setUpClass(cls):
+        ray.init(local_mode=True, ignore_reinit_error=True)
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        ray.shutdown()
+
+    def _make_estimate_resource_params(
+        cls,
+        resource_estimation_method: Optional[
+            ResourceEstimationMethod
+        ] = ResourceEstimationMethod.DEFAULT,
+        previous_inflation: Optional[int] = 7,
+        average_record_size_bytes: Optional[int] = 1000,
+    ):
+        return EstimateResourcesParams.of(
+            resource_estimation_method=resource_estimation_method,
+            previous_inflation=previous_inflation,
+            average_record_size_bytes=average_record_size_bytes,
+        )
+
+    def _make_manifest(
+        self,
+        source_content_length: Optional[int] = 1000,
+        content_type: Optional[ContentType] = ContentType.PARQUET,
+        content_encoding: Optional[ContentEncoding] = ContentEncoding.IDENTITY,
+        partition_values: Optional[PartitionValues] = None,
+        uri: Optional[str] = "test",
+        url: Optional[str] = "test",
+        author: Optional[str] = "foo",
+        entry_uuid: Optional[str] = "foo",
+        manifest_uuid: Optional[str] = "bar",
+    ) -> Manifest:
+        meta = ManifestMeta.of(
+            10,
+            10,
+            content_type=content_type,
+            content_encoding=content_encoding,
+            source_content_length=source_content_length,
+            partition_values=partition_values,
+        )
+
+        return Manifest.of(
+            entries=ManifestEntryList.of(
+                [
+                    ManifestEntry.of(
+                        uri=uri, url=url, meta=meta, mandatory=True, uuid=entry_uuid
+                    )
+                ]
+            ),
+            author=author,
+            uuid=manifest_uuid,
+        )
+
+    def make_round_completion_info(
+        self,
+        high_watermark: Optional[int] = 1_000_000,
+        compacted_delta_locator: Optional[DeltaLocator] = None,
+        records_written: Optional[int] = 10,
+        bytes_written: Optional[int] = 10,
+        files_written: Optional[int] = 10,
+        rows_dropped: Optional[int] = 10,
+        sort_keys_bit_width: Optional[int] = 0,
+        hash_bucket_count: Optional[int] = 1,
+        hb_index_to_entry_range: Optional[dict] = None,
+    ) -> RoundCompletionInfo:
+        if compacted_delta_locator is None:
+            compacted_delta_locator = MagicMock(spec=DeltaLocator)
+
+        hb_index_to_entry_range = hb_index_to_entry_range or {"0": (0, 1)}
+
+        return RoundCompletionInfo.of(
+            compacted_delta_locator=compacted_delta_locator,
+            high_watermark=high_watermark,
+            compacted_pyarrow_write_result=PyArrowWriteResult.of(
+                records_written, bytes_written, files_written, rows_dropped
+            ),
+            sort_keys_bit_width=sort_keys_bit_width,
+            hb_index_to_entry_range=hb_index_to_entry_range,
+            hash_bucket_count=hash_bucket_count,
+        )
+
+    def test_get_task_options_sanity(self):
+        opts = _get_task_options(0.01, 0.01)
+        result_ref = valid_func.options(**opts).remote()
+        result = ray.get(result_ref)
+
+        self.assertEqual(result, 2)
+
+    def test_get_task_options_when_exception_is_thrown(self):
+        opts = _get_task_options(0.01, 0.01)
+        result_ref = throwing_func.options(**opts).remote()
+
+        self.assertRaises(ConnectionAbortedError, lambda: ray.get(result_ref))
+
+    def test_get_merge_task_options_memory_logs_enabled_sanity(self):
+        test_index = 0
+        test_hb_group_idx = 0
+        test_debug_memory_params = {"merge_task_index": test_index}
+        test_estimate_memory_params = self._make_estimate_resource_params()
+        test_ray_custom_resources = {}
+        test_rcf = self.make_round_completion_info()
+        test_manifest = self._make_manifest()
+        expected_task_opts = {
+            "max_retries": 3,
+            "memory": 1680.64,
+            "num_cpus": 0.01,
+            "scheduling_strategy": "SPREAD",
+        }
+        expected_previous_inflation = 1.0
+        expected_average_record_size = 1.0
+        with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+            # At least one log of level DEBUG must be emitted
+            actual_merge_tasks_opts = _get_merge_task_options(
+                index=test_index,
+                hb_group_idx=test_hb_group_idx,
+                data_size=1,
+                pk_size_bytes=1,
+                num_rows=1,
+                num_hash_groups=1,
+                total_memory_buffer_percentage=1,
+                incremental_index_array_size=1,
+                debug_memory_params=test_debug_memory_params,
+                ray_custom_resources=test_ray_custom_resources,
+                estimate_resources_params=test_estimate_memory_params,
+                round_completion_info=test_rcf,
+                compacted_delta_manifest=test_manifest,
+                memory_logs_enabled=True,
+            )
+            assert {k: actual_merge_tasks_opts[k] for k in expected_task_opts}
+        log_message_round_completion_info = cm.records[0].getMessage()
+        log_message_debug_memory_params = cm.records[1].getMessage()
+        self.assertIn(
+            f"[Merge task {test_index}]: Using previous compaction rounds to calculate merge memory",
+            log_message_round_completion_info,
+        )
+        self.assertIn(
+            f"[Merge task {test_index}]: Params used for calculating merge memory",
+            log_message_debug_memory_params,
+        )
+        self.assertIn(
+            f"'previous_inflation': {expected_previous_inflation}",
+            log_message_debug_memory_params,
+        )
+        self.assertIn(
+            f"'average_record_size': {expected_average_record_size}",
+            log_message_debug_memory_params,
+        )
+
+    def test_get_merge_task_options_memory_logs_enabled_fallback_previous_inflation_fallback_average_record_size(
+        self,
+    ):
+        test_index = 0
+        test_hb_group_idx = 0
+        test_debug_memory_params = {"merge_task_index": test_index}
+        test_estimate_memory_params = self._make_estimate_resource_params()
+        test_ray_custom_resources = {}
+        test_rcf = self.make_round_completion_info(
+            bytes_written=0, records_written=0, files_written=0, rows_dropped=0
+        )
+        test_manifest = self._make_manifest()
+        expected_task_opts = {
+            "max_retries": 3,
+            "memory": 1680.64,
+            "num_cpus": 0.01,
+            "scheduling_strategy": "SPREAD",
+        }
+        expected_previous_inflation = PYARROW_INFLATION_MULTIPLIER
+        expected_average_record_size = DEFAULT_AVERAGE_RECORD_SIZE_BYTES
+        with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+            # At least one log of level DEBUG must be emitted
+            actual_merge_tasks_opts = _get_merge_task_options(
+                index=test_index,
+                hb_group_idx=test_hb_group_idx,
+                data_size=1,
+                pk_size_bytes=1,
+                num_rows=1,
+                num_hash_groups=1,
+                total_memory_buffer_percentage=1,
+                incremental_index_array_size=1,
+                debug_memory_params=test_debug_memory_params,
+                ray_custom_resources=test_ray_custom_resources,
+                estimate_resources_params=test_estimate_memory_params,
+                round_completion_info=test_rcf,
+                compacted_delta_manifest=test_manifest,
+                memory_logs_enabled=True,
+            )
+            assert {k: actual_merge_tasks_opts[k] for k in expected_task_opts}
+        log_message_round_completion_info = cm.records[0].getMessage()
+        log_message_debug_memory_params = cm.records[1].getMessage()
+        self.assertIn(
+            f"[Merge task {test_index}]: Using previous compaction rounds to calculate merge memory",
+            log_message_round_completion_info,
+        )
+        self.assertIn(
+            f"[Merge task {test_index}]: Params used for calculating merge memory",
+            log_message_debug_memory_params,
+        )
+        self.assertIn(
+            f"'previous_inflation': {expected_previous_inflation}",
+            log_message_debug_memory_params,
+        )
+        self.assertIn(
+            f"'average_record_size': {expected_average_record_size}",
+            log_message_debug_memory_params,
+        )
+
+    def test_get_merge_task_options_memory_logs_enabled_not_using_previous_round_completion_info(
+        self,
+    ):
+        test_index = 0
+        test_hb_group_idx = 0
+        test_debug_memory_params = {"merge_task_index": test_index}
+        test_estimate_memory_params = self._make_estimate_resource_params()
+        test_ray_custom_resources = {}
+        test_rcf = None
+        test_manifest = self._make_manifest()
+        expected_task_opts = {
+            "max_retries": 3,
+            "memory": 1680.64,
+            "num_cpus": 0.01,
+            "scheduling_strategy": "SPREAD",
+        }
+        with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+            # At least one log of level DEBUG must be emitted
+            actual_merge_tasks_opts = _get_merge_task_options(
+                index=test_index,
+                hb_group_idx=test_hb_group_idx,
+                data_size=1,
+                pk_size_bytes=1,
+                num_rows=1,
+                num_hash_groups=1,
+                total_memory_buffer_percentage=1,
+                incremental_index_array_size=1,
+                debug_memory_params=test_debug_memory_params,
+                ray_custom_resources=test_ray_custom_resources,
+                estimate_resources_params=test_estimate_memory_params,
+                round_completion_info=test_rcf,
+                compacted_delta_manifest=test_manifest,
+                memory_logs_enabled=True,
+            )
+            assert {k: actual_merge_tasks_opts[k] for k in expected_task_opts}
+        log_message_debug_memory_params = cm.records[0].getMessage()
+        self.assertIn(
+            f"[Merge task {test_index}]: Params used for calculating merge memory",
+            log_message_debug_memory_params,
+        )
+        self.assertNotIn(
+            "'average_record_size'",
+            log_message_debug_memory_params,
+        )

{deltacat-1.1.30 → deltacat-1.1.32}/deltacat/tests/utils/test_pyarrow.py

@@ -2,9 +2,12 @@ from unittest import TestCase
 from deltacat.utils.pyarrow import (
     s3_partial_parquet_file_to_table,
     pyarrow_read_csv,
+    ContentTypeValidationError,
     content_type_to_reader_kwargs,
     _add_column_kwargs,
+    logger,
     s3_file_to_table,
+    s3_file_to_parquet,
     ReadKwargsProviderPyArrowSchemaOverride,
     RAISE_ON_EMPTY_CSV_KWARG,
     RAISE_ON_DECIMAL_OVERFLOW,
@@ -435,7 +438,7 @@ class TestReadCSV(TestCase):
             pa.lib.ArrowInvalid,
             lambda: pyarrow_read_csv(
                 OVERFLOWING_DECIMAL_PRECISION_UTSV_PATH,
-                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True}
+                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True},
             ),
         )
 
@@ -479,7 +482,7 @@ class TestReadCSV(TestCase):
             pa.lib.ArrowInvalid,
             lambda: pyarrow_read_csv(
                 OVERFLOWING_DECIMAL_SCALE_UTSV_PATH,
-                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True}
+                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True},
             ),
         )
 
@@ -590,7 +593,7 @@ class TestReadCSV(TestCase):
             pa.lib.ArrowNotImplementedError,
             lambda: pyarrow_read_csv(
                 OVERFLOWING_DECIMAL_SCALE_UTSV_PATH,
-                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True}
+                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True},
             ),
         )
 
@@ -818,8 +821,11 @@ class TestS3FileToTable(TestCase):
         schema = pa.schema(
             [("is_active", pa.string()), ("ship_datetime_utc", pa.timestamp("us"))]
         )
-
         # OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG has no effect on uTSV files
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            **kwargs,
+        }
         pa_kwargs_provider = lambda content_type, kwargs: {
             "reader_type": "pyarrow",
             OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG: ContentEncoding.IDENTITY.value,
@@ -864,3 +870,99 @@ class TestS3FileToTable(TestCase):
         schema = result.schema
         schema_index = schema.get_field_index("n_legs")
         self.assertEqual(schema.field(schema_index).type, "int64")
+
+
+class TestS3FileToParquet(TestCase):
+    def test_s3_file_to_parquet_sanity(self):
+        test_s3_url = PARQUET_FILE_PATH
+        test_content_type = ContentType.PARQUET.value
+        test_content_encoding = ContentEncoding.IDENTITY.value
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            **kwargs,
+        }
+        with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+            result_parquet_file: ParquetFile = s3_file_to_parquet(
+                test_s3_url,
+                test_content_type,
+                test_content_encoding,
+                ["n_legs", "animal"],
+                ["n_legs"],
+                pa_read_func_kwargs_provider=pa_kwargs_provider,
+            )
+        log_message_log_args = cm.records[0].getMessage()
+        log_message_presanitize_kwargs = cm.records[1].getMessage()
+        self.assertIn(
+            f"Reading {test_s3_url} to PyArrow ParquetFile. Content type: {test_content_type}. Encoding: {test_content_encoding}",
+            log_message_log_args,
+        )
+        self.assertIn("{'reader_type': 'pyarrow'}", log_message_presanitize_kwargs)
+        for index, field in enumerate(result_parquet_file.schema_arrow):
+            self.assertEqual(
+                field.name, result_parquet_file.schema_arrow.field(index).name
+            )
+        self.assertEqual(result_parquet_file.schema_arrow.field(0).type, "int64")
+
+    def test_s3_file_to_parquet_when_parquet_gzip_encoding_and_overridden_returns_success(
+        self,
+    ):
+        test_s3_url = PARQUET_FILE_PATH
+        test_content_type = ContentType.PARQUET.value
+        test_content_encoding = ContentEncoding.GZIP.value
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG: ContentEncoding.IDENTITY.value,
+            **kwargs,
+        }
+        with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+            result_parquet_file: ParquetFile = s3_file_to_parquet(
+                test_s3_url,
+                test_content_type,
+                test_content_encoding,
+                ["n_legs", "animal"],
+                ["n_legs"],
+                pa_read_func_kwargs_provider=pa_kwargs_provider,
+            )
+        log_message_log_args = cm.records[0].getMessage()
+        log_message_log_new_content_encoding = cm.records[1].getMessage()
+        log_message_presanitize_kwargs = cm.records[2].getMessage()
+        self.assertIn(
+            f"Reading {test_s3_url} to PyArrow ParquetFile. Content type: {test_content_type}. Encoding: {test_content_encoding}",
+            log_message_log_args,
+        )
+        self.assertIn(
+            f"Overriding {test_s3_url} content encoding from {ContentEncoding.GZIP.value} to {ContentEncoding.IDENTITY.value}",
+            log_message_log_new_content_encoding,
+        )
+        self.assertIn("{'reader_type': 'pyarrow'}", log_message_presanitize_kwargs)
+        for index, field in enumerate(result_parquet_file.schema_arrow):
+            self.assertEqual(
+                field.name, result_parquet_file.schema_arrow.field(index).name
+            )
+        self.assertEqual(result_parquet_file.schema_arrow.field(0).type, "int64")
+
+    def test_s3_file_to_parquet_when_parquet_gzip_encoding_not_overridden_throws_error(
+        self,
+    ):
+        test_s3_url = PARQUET_FILE_PATH
+        test_content_type = ContentType.PARQUET.value
+        test_content_encoding = ContentEncoding.GZIP.value
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            **kwargs,
+        }
+        with self.assertRaises(ContentTypeValidationError):
+            with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+                s3_file_to_parquet(
+                    test_s3_url,
+                    test_content_type,
+                    test_content_encoding,
+                    ["n_legs", "animal"],
+                    ["n_legs"],
+                    pa_read_func_kwargs_provider=pa_kwargs_provider,
+                )
+        log_message_log_args = cm.records[0].getMessage()
+        self.assertIn(
+            f"Reading {test_s3_url} to PyArrow ParquetFile. Content type: {test_content_type}. Encoding: {test_content_encoding}",
+            log_message_log_args,
+        )

{deltacat-1.1.30 → deltacat-1.1.32}/deltacat/utils/pyarrow.py

@@ -617,7 +617,18 @@ def s3_file_to_parquet(
         f"Reading {s3_url} to PyArrow ParquetFile. "
         f"Content type: {content_type}. Encoding: {content_encoding}"
     )
+    kwargs = {}
+    if pa_read_func_kwargs_provider:
+        kwargs = pa_read_func_kwargs_provider(content_type, kwargs)
 
+    if OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG in kwargs:
+        new_content_encoding = kwargs.pop(OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG)
+        if content_type == ContentType.PARQUET.value:
+            logger.debug(
+                f"Overriding {s3_url} content encoding from {content_encoding} "
+                f"to {new_content_encoding}"
+            )
+            content_encoding = new_content_encoding
     if (
         content_type != ContentType.PARQUET.value
         or content_encoding != ContentEncoding.IDENTITY
@@ -630,15 +641,10 @@ def s3_file_to_parquet(
     if s3_client_kwargs is None:
         s3_client_kwargs = {}
 
-    kwargs = {}
-
     if s3_url.startswith("s3://"):
         s3_file_system = create_s3_file_system(s3_client_kwargs)
         kwargs["filesystem"] = s3_file_system
 
-    if pa_read_func_kwargs_provider:
-        kwargs = pa_read_func_kwargs_provider(content_type, kwargs)
-
     logger.debug(f"Pre-sanitize kwargs for {s3_url}: {kwargs}")
 
     kwargs = sanitize_kwargs_to_callable(ParquetFile.__init__, kwargs)

{deltacat-1.1.30 → deltacat-1.1.32/deltacat.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.30
+Version: 1.1.32
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

deltacat-1.1.30/deltacat/tests/compute/compactor_v2/utils/test_task_options.py

@@ -1,33 +0,0 @@
-import unittest
-import ray
-from deltacat.compute.compactor_v2.utils.task_options import _get_task_options
-
-
-@ray.remote
-def valid_func():
-    return 2
-
-
-@ray.remote
-def throwing_func():
-    raise ConnectionAbortedError()
-
-
-class TestTaskOptions(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        ray.init(local_mode=True, ignore_reinit_error=True)
-        super().setUpClass()
-
-    def test_get_task_options_sanity(self):
-        opts = _get_task_options(0.01, 0.01)
-        result_ref = valid_func.options(**opts).remote()
-        result = ray.get(result_ref)
-
-        self.assertEqual(result, 2)
-
-    def test_get_task_options_when_exception_is_thrown(self):
-        opts = _get_task_options(0.01, 0.01)
-        result_ref = throwing_func.options(**opts).remote()
-
-        self.assertRaises(ConnectionAbortedError, lambda: ray.get(result_ref))