deltacat 1.1.33__tar.gz → 1.1.34__tar.gz

{deltacat-1.1.33/deltacat.egg-info → deltacat-1.1.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.33
+Version: 1.1.34
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.33 → deltacat-1.1.34}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.33"
+__version__ = "1.1.34"
 
 
 __all__ = [

{deltacat-1.1.33 → deltacat-1.1.34}/deltacat/compute/compactor_v2/utils/content_type_params.py

@@ -5,6 +5,7 @@ from deltacat.compute.compactor_v2.constants import (
     TASK_MAX_PARALLELISM,
     MAX_PARQUET_METADATA_SIZE,
 )
+from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.ray_utils.concurrency import invoke_parallel
 from deltacat import logs
 from deltacat.storage import (
@@ -75,11 +76,21 @@ def _download_parquet_metadata_for_manifest_entry(
     entry_index: int,
     deltacat_storage: unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[Any, Any]] = {},
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> Dict[str, Any]:
+    logger.info(
+        f"Downloading the parquet metadata for Delta with locator {delta.locator} and entry_index: {entry_index}"
+    )
+    if "file_reader_kwargs_provider" in deltacat_storage_kwargs:
+        logger.info(
+            "'file_reader_kwargs_provider' is also present in deltacat_storage_kwargs. Removing to prevent multiple values for keyword argument"
+        )
+        deltacat_storage_kwargs.pop("file_reader_kwargs_provider")
     pq_file = deltacat_storage.download_delta_manifest_entry(
         delta,
         entry_index=entry_index,
         table_type=TableType.PYARROW_PARQUET,
+        file_reader_kwargs_provider=file_reader_kwargs_provider,
         **deltacat_storage_kwargs,
     )
 
@@ -97,11 +108,15 @@ def append_content_type_params(
     max_parquet_meta_size_bytes: Optional[int] = MAX_PARQUET_METADATA_SIZE,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> bool:
     """
     This operation appends content type params into the delta entry. Note
     that this operation can be time consuming, hence we cache it in a Ray actor.
     """
+    logger.info(
+        f"Appending the content type params for Delta with locator {delta.locator}..."
+    )
 
     if not delta.meta:
         logger.warning(f"Delta with locator {delta.locator} doesn't contain meta.")
@@ -159,6 +174,7 @@ def append_content_type_params(
 
     def input_provider(index, item) -> Dict:
         return {
+            "file_reader_kwargs_provider": file_reader_kwargs_provider,
             "deltacat_storage_kwargs": deltacat_storage_kwargs,
             "deltacat_storage": deltacat_storage,
             "delta": delta,
@@ -168,6 +184,7 @@ def append_content_type_params(
     logger.info(
         f"Downloading parquet meta for {len(entry_indices_to_download)} manifest entries..."
     )
+
     pq_files_promise = invoke_parallel(
         entry_indices_to_download,
         ray_task=_download_parquet_metadata_for_manifest_entry,

{deltacat-1.1.33 → deltacat-1.1.34}/deltacat/compute/compactor_v2/utils/io.py

@@ -101,7 +101,6 @@ def create_uniform_input_deltas(
     delta_manifest_entries_count = 0
     estimated_da_bytes = 0
     input_da_list = []
-
     for delta in input_deltas:
         if (
             compact_partition_params.enable_input_split
@@ -118,6 +117,7 @@ def create_uniform_input_deltas(
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
                 task_max_parallelism=compact_partition_params.task_max_parallelism,
                 max_parquet_meta_size_bytes=compact_partition_params.max_parquet_meta_size_bytes,
+                file_reader_kwargs_provider=compact_partition_params.read_kwargs_provider,
             )
 
         manifest_entries = delta.manifest.entries

{deltacat-1.1.33 → deltacat-1.1.34}/deltacat/compute/resource_estimation/delta.py

@@ -93,11 +93,29 @@ def _estimate_resources_required_to_process_delta_using_type_params(
                 on_disk_size_bytes=delta.meta.content_length,
             ),
         )
-
+    file_reader_kwargs_provider = kwargs.get(
+        "file_reader_kwargs_provider"
+    ) or deltacat_storage_kwargs.get("file_reader_kwargs_provider")
+
+    """
+    NOTE: The file_reader_kwargs_provider parameter can be passed in two ways:
+    1. Nested within deltacat_storage_kwargs during resource estimation
+    2. As a top-level attribute of CompactPartitionsParams during compaction
+
+    This creates an inconsistent parameter path between resource estimation and compaction flows.
+    As a long-term solution, this should be unified to use a single consistent path (either always
+    nested in deltacat_storage_kwargs or always as a top-level parameter).
+
+    For now, this implementation handles the resource estimation case by:
+    1. First checking for file_reader_kwargs_provider as a direct kwarg
+    2. Falling back to deltacat_storage_kwargs if not found
+    This approach maintains backward compatibility by not modifying the DELTA_RESOURCE_ESTIMATION_FUNCTIONS signatures.
+    """
     appended = append_content_type_params(
         delta=delta,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,
+        file_reader_kwargs_provider=file_reader_kwargs_provider,
     )
 
     if not appended:

deltacat-1.1.34/deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py

@@ -0,0 +1,253 @@
+import ray
+from typing import Dict, Any
+from deltacat.types.media import ContentType
+import pyarrow as pa
+
+import pytest
+import deltacat.tests.local_deltacat_storage as ds
+import os
+from deltacat.tests.test_utils.pyarrow import (
+    stage_partition_from_file_paths,
+    commit_delta_to_staged_partition,
+)
+from deltacat.utils.pyarrow import (
+    ReadKwargsProviderPyArrowCsvPureUtf8,
+    ReadKwargsProviderPyArrowSchemaOverride,
+)
+
+DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
+    "db_file_path",
+    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
+)
+
+
+class TestContentTypeParams:
+    TEST_NAMESPACE = "test_content_type_params"
+    TEST_ENTRY_INDEX = 0
+    DEDUPE_BASE_COMPACTED_TABLE_STRING_PK = "deltacat/tests/compute/compactor_v2/steps/data/dedupe_base_compacted_table_string_pk.csv"
+    DEDUPE_NO_DUPLICATION_STRING_PK = "deltacat/tests/compute/compactor_v2/steps/data/dedupe_table_no_duplication_string_pk.csv"
+
+    @pytest.fixture(scope="module", autouse=True)
+    def setup_ray_cluster(self):
+        ray.init(local_mode=True, ignore_reinit_error=True)
+        yield
+        ray.shutdown()
+
+    @pytest.fixture(scope="function")
+    def local_deltacat_storage_kwargs(self, request: pytest.FixtureRequest):
+        # see deltacat/tests/local_deltacat_storage/README.md for documentation
+        kwargs_for_local_deltacat_storage: Dict[str, Any] = {
+            DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
+        }
+        yield kwargs_for_local_deltacat_storage
+        if os.path.exists(DATABASE_FILE_PATH_VALUE):
+            os.remove(DATABASE_FILE_PATH_VALUE)
+
+    def test__download_parquet_metadata_for_manifest_entry_sanity(
+        self, local_deltacat_storage_kwargs
+    ):
+        from deltacat.compute.compactor_v2.utils.content_type_params import (
+            _download_parquet_metadata_for_manifest_entry,
+        )
+        from deltacat.types.partial_download import PartialParquetParameters
+
+        partition = stage_partition_from_file_paths(
+            self.TEST_NAMESPACE,
+            [self.DEDUPE_BASE_COMPACTED_TABLE_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_delta = commit_delta_to_staged_partition(
+            partition,
+            [self.DEDUPE_BASE_COMPACTED_TABLE_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_entry_index = 0
+        obj_ref = _download_parquet_metadata_for_manifest_entry.remote(
+            test_delta, test_entry_index, ds, local_deltacat_storage_kwargs
+        )
+        parquet_metadata = ray.get(obj_ref)
+        partial_parquet_params = parquet_metadata["partial_parquet_params"]
+
+        # validate
+        assert isinstance(parquet_metadata, dict)
+        assert "entry_index" in parquet_metadata
+        assert "partial_parquet_params" in parquet_metadata
+        assert parquet_metadata["entry_index"] == test_entry_index
+        assert isinstance(partial_parquet_params, PartialParquetParameters)
+
+        assert partial_parquet_params.row_groups_to_download == [0]
+        assert partial_parquet_params.num_row_groups == 1
+        assert partial_parquet_params.num_rows == 8
+        assert isinstance(partial_parquet_params.in_memory_size_bytes, float)
+        assert partial_parquet_params.in_memory_size_bytes > 0
+
+        pq_metadata = partial_parquet_params.pq_metadata
+        assert pq_metadata.num_columns == 2
+        assert pq_metadata.num_rows == 8
+        assert pq_metadata.num_row_groups == 1
+        assert pq_metadata.format_version == "2.6"
+
+        assert (
+            test_delta.manifest.entries[self.TEST_ENTRY_INDEX].meta.content_type
+            == ContentType.PARQUET.value
+        )
+
+    @pytest.mark.parametrize(
+        "read_kwargs_provider,expected_values",
+        [
+            (
+                ReadKwargsProviderPyArrowCsvPureUtf8(),
+                {
+                    "num_rows": 6,
+                    "num_columns": 2,
+                    "num_row_groups": 1,
+                    "format_version": "2.6",
+                    "column_types": [pa.string(), pa.string()],
+                },
+            ),
+            (
+                ReadKwargsProviderPyArrowSchemaOverride(
+                    schema=pa.schema(
+                        [
+                            ("id", pa.string()),
+                            ("value", pa.int64()),
+                        ]
+                    )
+                ),
+                {
+                    "num_rows": 6,
+                    "num_columns": 2,
+                    "num_row_groups": 1,
+                    "format_version": "2.6",
+                    "column_types": [pa.string(), pa.int64()],
+                },
+            ),
+            (
+                ReadKwargsProviderPyArrowSchemaOverride(
+                    schema=None,
+                    pq_coerce_int96_timestamp_unit="ms",
+                    parquet_reader_type="daft",
+                ),
+                {
+                    "num_rows": 6,
+                    "num_columns": 2,
+                    "num_row_groups": 1,
+                    "format_version": "2.6",
+                    "column_types": None,  # Will use default type inference
+                },
+            ),
+        ],
+    )
+    def test__download_parquet_metadata_for_manifest_entry_with_read_kwargs_provider(
+        self, read_kwargs_provider, expected_values, local_deltacat_storage_kwargs
+    ):
+        from deltacat.compute.compactor_v2.utils.content_type_params import (
+            _download_parquet_metadata_for_manifest_entry,
+        )
+
+        partition = stage_partition_from_file_paths(
+            self.TEST_NAMESPACE,
+            [self.DEDUPE_NO_DUPLICATION_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_delta = commit_delta_to_staged_partition(
+            partition,
+            [self.DEDUPE_NO_DUPLICATION_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_entry_index = 0
+        read_kwargs_provider = ReadKwargsProviderPyArrowCsvPureUtf8
+        obj_ref = _download_parquet_metadata_for_manifest_entry.remote(
+            test_delta,
+            test_entry_index,
+            ds,
+            local_deltacat_storage_kwargs,
+            read_kwargs_provider,
+        )
+        parquet_metadata = ray.get(obj_ref)
+        partial_parquet_params = parquet_metadata["partial_parquet_params"]
+
+        # validate
+        assert isinstance(parquet_metadata, dict)
+        assert "entry_index" in parquet_metadata
+        assert "partial_parquet_params" in parquet_metadata
+        assert parquet_metadata["entry_index"] == self.TEST_ENTRY_INDEX
+
+        assert partial_parquet_params.row_groups_to_download == [0]
+        assert (
+            partial_parquet_params.num_row_groups == expected_values["num_row_groups"]
+        )
+        assert partial_parquet_params.num_rows == expected_values["num_rows"]
+        assert isinstance(partial_parquet_params.in_memory_size_bytes, float)
+        assert partial_parquet_params.in_memory_size_bytes > 0
+
+        pq_metadata = partial_parquet_params.pq_metadata
+        assert pq_metadata.num_columns == expected_values["num_columns"]
+        assert pq_metadata.num_rows == expected_values["num_rows"]
+        assert pq_metadata.num_row_groups == expected_values["num_row_groups"]
+        assert pq_metadata.format_version == expected_values["format_version"]
+
+        assert (
+            test_delta.manifest.entries[self.TEST_ENTRY_INDEX].meta.content_type
+            == ContentType.PARQUET.value
+        )
+
+    def test_download_parquet_metadata_for_manifest_entry_file_reader_kwargs_present_top_level_and_deltacat_storage_kwarg(
+        self, local_deltacat_storage_kwargs, caplog
+    ):
+        from deltacat.compute.compactor_v2.utils.content_type_params import (
+            _download_parquet_metadata_for_manifest_entry,
+        )
+        from deltacat.types.partial_download import PartialParquetParameters
+
+        test_file_reader_kwargs_provider = ReadKwargsProviderPyArrowCsvPureUtf8()
+
+        local_deltacat_storage_kwargs[
+            "file_reader_kwargs_provider"
+        ] = ReadKwargsProviderPyArrowCsvPureUtf8()
+
+        partition = stage_partition_from_file_paths(
+            self.TEST_NAMESPACE,
+            [self.DEDUPE_BASE_COMPACTED_TABLE_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_delta = commit_delta_to_staged_partition(
+            partition,
+            [self.DEDUPE_BASE_COMPACTED_TABLE_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+
+        test_entry_index = 0
+        obj_ref = _download_parquet_metadata_for_manifest_entry.remote(
+            test_delta,
+            test_entry_index,
+            ds,
+            local_deltacat_storage_kwargs,
+            test_file_reader_kwargs_provider,
+        )
+        parquet_metadata = ray.get(obj_ref)
+        partial_parquet_params = parquet_metadata["partial_parquet_params"]
+
+        # validate
+        assert isinstance(parquet_metadata, dict)
+        assert "entry_index" in parquet_metadata
+        assert "partial_parquet_params" in parquet_metadata
+        assert parquet_metadata["entry_index"] == test_entry_index
+        assert isinstance(partial_parquet_params, PartialParquetParameters)
+
+        assert partial_parquet_params.row_groups_to_download == [0]
+        assert partial_parquet_params.num_row_groups == 1
+        assert partial_parquet_params.num_rows == 8
+        assert isinstance(partial_parquet_params.in_memory_size_bytes, float)
+        assert partial_parquet_params.in_memory_size_bytes > 0
+
+        pq_metadata = partial_parquet_params.pq_metadata
+        assert pq_metadata.num_columns == 2
+        assert pq_metadata.num_rows == 8
+        assert pq_metadata.num_row_groups == 1
+        assert pq_metadata.format_version == "2.6"
+
+        assert (
+            test_delta.manifest.entries[self.TEST_ENTRY_INDEX].meta.content_type
+            == ContentType.PARQUET.value
+        )

{deltacat-1.1.33 → deltacat-1.1.34/deltacat.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.33
+Version: 1.1.34
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.33 → deltacat-1.1.34}/deltacat.egg-info/SOURCES.txt

@@ -164,6 +164,7 @@ deltacat/tests/compute/compactor_v2/__init__.py
 deltacat/tests/compute/compactor_v2/test_compaction_session.py
 deltacat/tests/compute/compactor_v2/test_hashlib.py
 deltacat/tests/compute/compactor_v2/utils/__init__.py
+deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py
 deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py
 deltacat/tests/compute/compactor_v2/utils/test_task_options.py
 deltacat/tests/compute/resource_estimation/__init__.py