deltacat 1.1.33__py3-none-any.whl → 1.1.35__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.33"
+__version__ = "1.1.35"
 
 
 __all__ = [

deltacat/compute/compactor_v2/constants.py

@@ -1,4 +1,4 @@
-from deltacat.utils.common import env_bool, env_integer
+from deltacat.utils.common import env_bool, env_integer, env_string
 
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
@@ -92,3 +92,18 @@ DEFAULT_NUM_ROUNDS = 1
 SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED = env_bool(
     "SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED", False
 )
+
+# This env variable specifies whether to check bucketing spec
+# compliance of the existing compacted table.
+# PRINT_LOG: Enable logging if any partition is found
+# to be non-compliant with the bucketing spec.
+# ASSERT: Fail the job with ValidationError if the
+# current compacted partition is found to be non-compliant
+# with bucketing spec. Note, logging is implicitly enabled
+# in this case.
+BUCKETING_SPEC_COMPLIANCE_PROFILE = env_string(
+    "BUCKETING_SPEC_COMPLIANCE_PROFILE", None
+)
+
+BUCKETING_SPEC_COMPLIANCE_PRINT_LOG = "PRINT_LOG"
+BUCKETING_SPEC_COMPLIANCE_ASSERT = "ASSERT"

deltacat/compute/compactor_v2/steps/merge.py

@@ -32,6 +32,7 @@ from deltacat.utils.resources import (
 )
 from deltacat.compute.compactor_v2.utils.primary_key_index import (
     generate_pk_hash_column,
+    pk_digest_to_hash_bucket_index,
 )
 from deltacat.storage import (
     Delta,
@@ -47,6 +48,9 @@ from deltacat.compute.compactor_v2.constants import (
     MERGE_TIME_IN_SECONDS,
     MERGE_SUCCESS_COUNT,
     MERGE_FAILURE_COUNT,
+    BUCKETING_SPEC_COMPLIANCE_PROFILE,
+    BUCKETING_SPEC_COMPLIANCE_ASSERT,
+    BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
 )
 from deltacat.exceptions import (
     categorize_errors,
@@ -188,9 +192,34 @@ def _merge_tables(
     return final_table
 
 
+def _validate_bucketing_spec_compliance(
+    table: pa.Table, rcf: RoundCompletionInfo, hb_index: int, primary_keys: List[str]
+) -> None:
+    pki_table = generate_pk_hash_column(
+        [table], primary_keys=primary_keys, requires_hash=True
+    )[0]
+    for index, hash_value in enumerate(sc.pk_hash_string_column_np(pki_table)):
+        hash_bucket = pk_digest_to_hash_bucket_index(hash_value, rcf.hash_bucket_count)
+        if hash_bucket != hb_index:
+            logger.info(
+                f"{rcf.compacted_delta_locator.namespace}.{rcf.compacted_delta_locator.table_name}"
+                f".{rcf.compacted_delta_locator.table_version}.{rcf.compacted_delta_locator.partition_id}"
+                f".{rcf.compacted_delta_locator.partition_values} has non-compliant bucketing spec. "
+                f"Expected hash bucket is {hb_index} but found {hash_bucket}."
+            )
+            if BUCKETING_SPEC_COMPLIANCE_PROFILE == BUCKETING_SPEC_COMPLIANCE_ASSERT:
+                raise AssertionError(
+                    "Hash bucket drift detected. Expected hash bucket index"
+                    f" to be {hb_index} but found {hash_bucket}"
+                )
+            # No further checks necessary
+            break
+
+
 def _download_compacted_table(
     hb_index: int,
     rcf: RoundCompletionInfo,
+    primary_keys: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[dict] = None,
@@ -214,7 +243,23 @@ def _download_compacted_table(
 
         tables.append(table)
 
-    return pa.concat_tables(tables)
+    compacted_table = pa.concat_tables(tables)
+    check_bucketing_spec = BUCKETING_SPEC_COMPLIANCE_PROFILE in [
+        BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
+        BUCKETING_SPEC_COMPLIANCE_ASSERT,
+    ]
+
+    logger.debug(
+        f"Value of BUCKETING_SPEC_COMPLIANCE_PROFILE, check_bucketing_spec:"
+        f" {BUCKETING_SPEC_COMPLIANCE_PROFILE}, {check_bucketing_spec}"
+    )
+
+    # Bucketing spec compliance isn't required without primary keys
+    if primary_keys and check_bucketing_spec:
+        _validate_bucketing_spec_compliance(
+            compacted_table, rcf, hb_index, primary_keys
+        )
+    return compacted_table
 
 
 def _copy_all_manifest_files_from_old_hash_buckets(
@@ -543,6 +588,7 @@ def _timed_merge(input: MergeInput) -> MergeResult:
                 compacted_table = _download_compacted_table(
                     hb_index=merge_file_group.hb_index,
                     rcf=input.round_completion_info,
+                    primary_keys=input.primary_keys,
                     read_kwargs_provider=input.read_kwargs_provider,
                     deltacat_storage=input.deltacat_storage,
                     deltacat_storage_kwargs=input.deltacat_storage_kwargs,

deltacat/compute/compactor_v2/utils/content_type_params.py

@@ -5,6 +5,7 @@ from deltacat.compute.compactor_v2.constants import (
     TASK_MAX_PARALLELISM,
     MAX_PARQUET_METADATA_SIZE,
 )
+from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.ray_utils.concurrency import invoke_parallel
 from deltacat import logs
 from deltacat.storage import (
@@ -75,11 +76,21 @@ def _download_parquet_metadata_for_manifest_entry(
     entry_index: int,
     deltacat_storage: unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[Any, Any]] = {},
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> Dict[str, Any]:
+    logger.info(
+        f"Downloading the parquet metadata for Delta with locator {delta.locator} and entry_index: {entry_index}"
+    )
+    if "file_reader_kwargs_provider" in deltacat_storage_kwargs:
+        logger.info(
+            "'file_reader_kwargs_provider' is also present in deltacat_storage_kwargs. Removing to prevent multiple values for keyword argument"
+        )
+        deltacat_storage_kwargs.pop("file_reader_kwargs_provider")
     pq_file = deltacat_storage.download_delta_manifest_entry(
         delta,
         entry_index=entry_index,
         table_type=TableType.PYARROW_PARQUET,
+        file_reader_kwargs_provider=file_reader_kwargs_provider,
         **deltacat_storage_kwargs,
     )
 
@@ -97,11 +108,15 @@ def append_content_type_params(
     max_parquet_meta_size_bytes: Optional[int] = MAX_PARQUET_METADATA_SIZE,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
+    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> bool:
     """
     This operation appends content type params into the delta entry. Note
     that this operation can be time consuming, hence we cache it in a Ray actor.
     """
+    logger.info(
+        f"Appending the content type params for Delta with locator {delta.locator}..."
+    )
 
     if not delta.meta:
         logger.warning(f"Delta with locator {delta.locator} doesn't contain meta.")
@@ -159,6 +174,7 @@ def append_content_type_params(
 
     def input_provider(index, item) -> Dict:
         return {
+            "file_reader_kwargs_provider": file_reader_kwargs_provider,
             "deltacat_storage_kwargs": deltacat_storage_kwargs,
             "deltacat_storage": deltacat_storage,
             "delta": delta,
@@ -168,6 +184,7 @@ def append_content_type_params(
     logger.info(
         f"Downloading parquet meta for {len(entry_indices_to_download)} manifest entries..."
     )
+
     pq_files_promise = invoke_parallel(
         entry_indices_to_download,
         ray_task=_download_parquet_metadata_for_manifest_entry,

deltacat/compute/compactor_v2/utils/io.py

@@ -101,7 +101,6 @@ def create_uniform_input_deltas(
     delta_manifest_entries_count = 0
     estimated_da_bytes = 0
     input_da_list = []
-
     for delta in input_deltas:
         if (
             compact_partition_params.enable_input_split
@@ -118,6 +117,7 @@ def create_uniform_input_deltas(
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
                 task_max_parallelism=compact_partition_params.task_max_parallelism,
                 max_parquet_meta_size_bytes=compact_partition_params.max_parquet_meta_size_bytes,
+                file_reader_kwargs_provider=compact_partition_params.read_kwargs_provider,
             )
 
         manifest_entries = delta.manifest.entries

deltacat/compute/resource_estimation/delta.py

@@ -93,11 +93,29 @@ def _estimate_resources_required_to_process_delta_using_type_params(
                 on_disk_size_bytes=delta.meta.content_length,
             ),
         )
-
+    file_reader_kwargs_provider = kwargs.get(
+        "file_reader_kwargs_provider"
+    ) or deltacat_storage_kwargs.get("file_reader_kwargs_provider")
+
+    """
+    NOTE: The file_reader_kwargs_provider parameter can be passed in two ways:
+    1. Nested within deltacat_storage_kwargs during resource estimation
+    2. As a top-level attribute of CompactPartitionsParams during compaction
+
+    This creates an inconsistent parameter path between resource estimation and compaction flows.
+    As a long-term solution, this should be unified to use a single consistent path (either always
+    nested in deltacat_storage_kwargs or always as a top-level parameter).
+
+    For now, this implementation handles the resource estimation case by:
+    1. First checking for file_reader_kwargs_provider as a direct kwarg
+    2. Falling back to deltacat_storage_kwargs if not found
+    This approach maintains backward compatibility by not modifying the DELTA_RESOURCE_ESTIMATION_FUNCTIONS signatures.
+    """
     appended = append_content_type_params(
         delta=delta,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,
+        file_reader_kwargs_provider=file_reader_kwargs_provider,
     )
 
     if not appended:

deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -4,9 +4,11 @@ import os
 import pyarrow as pa
 import pytest
 import boto3
+import json
 from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
 )
+from deltacat.exceptions import ValidationError
 from boto3.resources.base import ServiceResource
 import deltacat.tests.local_deltacat_storage as ds
 from deltacat.types.media import ContentType
@@ -88,6 +90,17 @@ def disable_sha1(monkeypatch):
     )
 
 
+@pytest.fixture(scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 class TestCompactionSession:
     """
     This class adds specific tests that aren't part of the parametrized test suite.
@@ -689,3 +702,307 @@ class TestCompactionSession:
             incremental_rcf.compacted_pyarrow_write_result.pyarrow_bytes >= 2300000000
         )
         assert incremental_rcf.compacted_pyarrow_write_result.records == 4
+
+    def test_compact_partition_when_bucket_spec_validation_fails(
+        self,
+        s3_resource,
+        local_deltacat_storage_kwargs,
+        enable_bucketing_spec_validation,
+    ):
+        """
+        A test case which asserts the bucketing spec validation throws an assertion error
+        when the validation has failed.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 1,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        backfill_rcf = get_rcf(s3_resource, rcf_url)
+        bucket, backfill_key1, backfill_key2 = rcf_url.strip("s3://").split("/")
+        # Move the records to different hash buckets to simulate a validation failure.
+        backfill_rcf["hbIndexToEntryRange"] = {"1": [0, 3]}
+        s3_resource.Bucket(bucket).put_object(
+            Key=f"{backfill_key1}/{backfill_key2}", Body=json.dumps(backfill_rcf)
+        )
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_destination_partition = ds.get_partition(
+            dest_partition.stream_locator, [], **local_deltacat_storage_kwargs
+        )
+
+        with pytest.raises(ValidationError) as excinfo:
+            compact_partition(
+                CompactPartitionParams.of(
+                    {
+                        "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                        "compacted_file_content_type": ContentType.PARQUET,
+                        "dd_max_parallelism_ratio": 1.0,
+                        "deltacat_storage": ds,
+                        "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                        "destination_partition_locator": new_destination_partition.locator,
+                        "drop_duplicates": True,
+                        "hash_bucket_count": 4,
+                        "last_stream_position_to_compact": new_source_delta.stream_position,
+                        "list_deltas_kwargs": {
+                            **local_deltacat_storage_kwargs,
+                            **{"equivalent_table_types": []},
+                        },
+                        "primary_keys": ["pk"],
+                        "rebase_source_partition_locator": None,
+                        "rebase_source_partition_high_watermark": None,
+                        "records_per_compacted_file": 4000,
+                        "s3_client_kwargs": {},
+                        "source_partition_locator": new_source_delta.partition_locator,
+                    }
+                )
+            )
+
+        assert (
+            "Hash bucket drift detected. Expected hash bucket index to be 1 but found 0"
+            in str(excinfo.value)
+        )
+
+    def test_compact_partition_when_bucket_spec_validation_fails_but_env_variable_disabled(
+        self,
+        s3_resource,
+        local_deltacat_storage_kwargs,
+    ):
+        """
+        A test case which asserts even if bucketing spec validation fails, compaction doesn't
+        throw an error if the feature is not enabled.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 1,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        backfill_rcf = get_rcf(s3_resource, rcf_url)
+        bucket, backfill_key1, backfill_key2 = rcf_url.strip("s3://").split("/")
+        # Move the records to different hash buckets to simulate a validation failure.
+        backfill_rcf["hbIndexToEntryRange"] = {"1": [0, 3]}
+        s3_resource.Bucket(bucket).put_object(
+            Key=f"{backfill_key1}/{backfill_key2}", Body=json.dumps(backfill_rcf)
+        )
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_destination_partition = ds.get_partition(
+            dest_partition.stream_locator, [], **local_deltacat_storage_kwargs
+        )
+
+        new_rcf = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": new_destination_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": new_source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": None,
+                    "rebase_source_partition_high_watermark": None,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": new_source_delta.partition_locator,
+                }
+            )
+        )
+
+        incremental_rcf = get_rcf(s3_resource, new_rcf)
+        assert incremental_rcf.hash_bucket_count == 4
+        assert len(incremental_rcf.hb_index_to_entry_range) == 2
+
+    def test_compact_partition_when_bucket_spec_validation_succeeds(
+        self,
+        s3_resource,
+        local_deltacat_storage_kwargs,
+        enable_bucketing_spec_validation,
+    ):
+        """
+        A test case which asserts the bucketing spec validation does not throw
+        and error when the validation succeeds.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 1,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        rcf = get_rcf(s3_resource, rcf_url)
+        assert rcf.hash_bucket_count == 4
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_destination_partition = ds.get_partition(
+            dest_partition.stream_locator, [], **local_deltacat_storage_kwargs
+        )
+
+        new_uri = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": new_destination_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": new_source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": None,
+                    "rebase_source_partition_high_watermark": None,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": new_source_delta.partition_locator,
+                }
+            )
+        )
+
+        rcf = get_rcf(s3_resource, new_uri)
+        assert rcf.hash_bucket_count == 4

deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py

@@ -0,0 +1,253 @@
+import ray
+from typing import Dict, Any
+from deltacat.types.media import ContentType
+import pyarrow as pa
+
+import pytest
+import deltacat.tests.local_deltacat_storage as ds
+import os
+from deltacat.tests.test_utils.pyarrow import (
+    stage_partition_from_file_paths,
+    commit_delta_to_staged_partition,
+)
+from deltacat.utils.pyarrow import (
+    ReadKwargsProviderPyArrowCsvPureUtf8,
+    ReadKwargsProviderPyArrowSchemaOverride,
+)
+
+DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
+    "db_file_path",
+    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
+)
+
+
+class TestContentTypeParams:
+    TEST_NAMESPACE = "test_content_type_params"
+    TEST_ENTRY_INDEX = 0
+    DEDUPE_BASE_COMPACTED_TABLE_STRING_PK = "deltacat/tests/compute/compactor_v2/steps/data/dedupe_base_compacted_table_string_pk.csv"
+    DEDUPE_NO_DUPLICATION_STRING_PK = "deltacat/tests/compute/compactor_v2/steps/data/dedupe_table_no_duplication_string_pk.csv"
+
+    @pytest.fixture(scope="module", autouse=True)
+    def setup_ray_cluster(self):
+        ray.init(local_mode=True, ignore_reinit_error=True)
+        yield
+        ray.shutdown()
+
+    @pytest.fixture(scope="function")
+    def local_deltacat_storage_kwargs(self, request: pytest.FixtureRequest):
+        # see deltacat/tests/local_deltacat_storage/README.md for documentation
+        kwargs_for_local_deltacat_storage: Dict[str, Any] = {
+            DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
+        }
+        yield kwargs_for_local_deltacat_storage
+        if os.path.exists(DATABASE_FILE_PATH_VALUE):
+            os.remove(DATABASE_FILE_PATH_VALUE)
+
+    def test__download_parquet_metadata_for_manifest_entry_sanity(
+        self, local_deltacat_storage_kwargs
+    ):
+        from deltacat.compute.compactor_v2.utils.content_type_params import (
+            _download_parquet_metadata_for_manifest_entry,
+        )
+        from deltacat.types.partial_download import PartialParquetParameters
+
+        partition = stage_partition_from_file_paths(
+            self.TEST_NAMESPACE,
+            [self.DEDUPE_BASE_COMPACTED_TABLE_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_delta = commit_delta_to_staged_partition(
+            partition,
+            [self.DEDUPE_BASE_COMPACTED_TABLE_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_entry_index = 0
+        obj_ref = _download_parquet_metadata_for_manifest_entry.remote(
+            test_delta, test_entry_index, ds, local_deltacat_storage_kwargs
+        )
+        parquet_metadata = ray.get(obj_ref)
+        partial_parquet_params = parquet_metadata["partial_parquet_params"]
+
+        # validate
+        assert isinstance(parquet_metadata, dict)
+        assert "entry_index" in parquet_metadata
+        assert "partial_parquet_params" in parquet_metadata
+        assert parquet_metadata["entry_index"] == test_entry_index
+        assert isinstance(partial_parquet_params, PartialParquetParameters)
+
+        assert partial_parquet_params.row_groups_to_download == [0]
+        assert partial_parquet_params.num_row_groups == 1
+        assert partial_parquet_params.num_rows == 8
+        assert isinstance(partial_parquet_params.in_memory_size_bytes, float)
+        assert partial_parquet_params.in_memory_size_bytes > 0
+
+        pq_metadata = partial_parquet_params.pq_metadata
+        assert pq_metadata.num_columns == 2
+        assert pq_metadata.num_rows == 8
+        assert pq_metadata.num_row_groups == 1
+        assert pq_metadata.format_version == "2.6"
+
+        assert (
+            test_delta.manifest.entries[self.TEST_ENTRY_INDEX].meta.content_type
+            == ContentType.PARQUET.value
+        )
+
+    @pytest.mark.parametrize(
+        "read_kwargs_provider,expected_values",
+        [
+            (
+                ReadKwargsProviderPyArrowCsvPureUtf8(),
+                {
+                    "num_rows": 6,
+                    "num_columns": 2,
+                    "num_row_groups": 1,
+                    "format_version": "2.6",
+                    "column_types": [pa.string(), pa.string()],
+                },
+            ),
+            (
+                ReadKwargsProviderPyArrowSchemaOverride(
+                    schema=pa.schema(
+                        [
+                            ("id", pa.string()),
+                            ("value", pa.int64()),
+                        ]
+                    )
+                ),
+                {
+                    "num_rows": 6,
+                    "num_columns": 2,
+                    "num_row_groups": 1,
+                    "format_version": "2.6",
+                    "column_types": [pa.string(), pa.int64()],
+                },
+            ),
+            (
+                ReadKwargsProviderPyArrowSchemaOverride(
+                    schema=None,
+                    pq_coerce_int96_timestamp_unit="ms",
+                    parquet_reader_type="daft",
+                ),
+                {
+                    "num_rows": 6,
+                    "num_columns": 2,
+                    "num_row_groups": 1,
+                    "format_version": "2.6",
+                    "column_types": None,  # Will use default type inference
+                },
+            ),
+        ],
+    )
+    def test__download_parquet_metadata_for_manifest_entry_with_read_kwargs_provider(
+        self, read_kwargs_provider, expected_values, local_deltacat_storage_kwargs
+    ):
+        from deltacat.compute.compactor_v2.utils.content_type_params import (
+            _download_parquet_metadata_for_manifest_entry,
+        )
+
+        partition = stage_partition_from_file_paths(
+            self.TEST_NAMESPACE,
+            [self.DEDUPE_NO_DUPLICATION_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_delta = commit_delta_to_staged_partition(
+            partition,
+            [self.DEDUPE_NO_DUPLICATION_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_entry_index = 0
+        read_kwargs_provider = ReadKwargsProviderPyArrowCsvPureUtf8
+        obj_ref = _download_parquet_metadata_for_manifest_entry.remote(
+            test_delta,
+            test_entry_index,
+            ds,
+            local_deltacat_storage_kwargs,
+            read_kwargs_provider,
+        )
+        parquet_metadata = ray.get(obj_ref)
+        partial_parquet_params = parquet_metadata["partial_parquet_params"]
+
+        # validate
+        assert isinstance(parquet_metadata, dict)
+        assert "entry_index" in parquet_metadata
+        assert "partial_parquet_params" in parquet_metadata
+        assert parquet_metadata["entry_index"] == self.TEST_ENTRY_INDEX
+
+        assert partial_parquet_params.row_groups_to_download == [0]
+        assert (
+            partial_parquet_params.num_row_groups == expected_values["num_row_groups"]
+        )
+        assert partial_parquet_params.num_rows == expected_values["num_rows"]
+        assert isinstance(partial_parquet_params.in_memory_size_bytes, float)
+        assert partial_parquet_params.in_memory_size_bytes > 0
+
+        pq_metadata = partial_parquet_params.pq_metadata
+        assert pq_metadata.num_columns == expected_values["num_columns"]
+        assert pq_metadata.num_rows == expected_values["num_rows"]
+        assert pq_metadata.num_row_groups == expected_values["num_row_groups"]
+        assert pq_metadata.format_version == expected_values["format_version"]
+
+        assert (
+            test_delta.manifest.entries[self.TEST_ENTRY_INDEX].meta.content_type
+            == ContentType.PARQUET.value
+        )
+
+    def test_download_parquet_metadata_for_manifest_entry_file_reader_kwargs_present_top_level_and_deltacat_storage_kwarg(
+        self, local_deltacat_storage_kwargs, caplog
+    ):
+        from deltacat.compute.compactor_v2.utils.content_type_params import (
+            _download_parquet_metadata_for_manifest_entry,
+        )
+        from deltacat.types.partial_download import PartialParquetParameters
+
+        test_file_reader_kwargs_provider = ReadKwargsProviderPyArrowCsvPureUtf8()
+
+        local_deltacat_storage_kwargs[
+            "file_reader_kwargs_provider"
+        ] = ReadKwargsProviderPyArrowCsvPureUtf8()
+
+        partition = stage_partition_from_file_paths(
+            self.TEST_NAMESPACE,
+            [self.DEDUPE_BASE_COMPACTED_TABLE_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+        test_delta = commit_delta_to_staged_partition(
+            partition,
+            [self.DEDUPE_BASE_COMPACTED_TABLE_STRING_PK],
+            **local_deltacat_storage_kwargs,
+        )
+
+        test_entry_index = 0
+        obj_ref = _download_parquet_metadata_for_manifest_entry.remote(
+            test_delta,
+            test_entry_index,
+            ds,
+            local_deltacat_storage_kwargs,
+            test_file_reader_kwargs_provider,
+        )
+        parquet_metadata = ray.get(obj_ref)
+        partial_parquet_params = parquet_metadata["partial_parquet_params"]
+
+        # validate
+        assert isinstance(parquet_metadata, dict)
+        assert "entry_index" in parquet_metadata
+        assert "partial_parquet_params" in parquet_metadata
+        assert parquet_metadata["entry_index"] == test_entry_index
+        assert isinstance(partial_parquet_params, PartialParquetParameters)
+
+        assert partial_parquet_params.row_groups_to_download == [0]
+        assert partial_parquet_params.num_row_groups == 1
+        assert partial_parquet_params.num_rows == 8
+        assert isinstance(partial_parquet_params.in_memory_size_bytes, float)
+        assert partial_parquet_params.in_memory_size_bytes > 0
+
+        pq_metadata = partial_parquet_params.pq_metadata
+        assert pq_metadata.num_columns == 2
+        assert pq_metadata.num_rows == 8
+        assert pq_metadata.num_row_groups == 1
+        assert pq_metadata.format_version == "2.6"
+
+        assert (
+            test_delta.manifest.entries[self.TEST_ENTRY_INDEX].meta.content_type
+            == ContentType.PARQUET.value
+        )

deltacat/tests/compute/test_compact_partition_incremental.py

@@ -119,6 +119,21 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    """
+    Enable the bucketing spec validation for all tests.
+    This will help catch hash bucket drift in testing.
+    """
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 @pytest.mark.parametrize(
     [
         "test_name",

deltacat/tests/compute/test_compact_partition_multiple_rounds.py

@@ -114,6 +114,21 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    """
+    Enable the bucketing spec validation for all tests.
+    This will help catch hash bucket drift in testing.
+    """
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 @pytest.mark.parametrize(
     [
         "test_name",

deltacat/tests/compute/test_compact_partition_rebase.py

@@ -114,6 +114,21 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    """
+    Enable the bucketing spec validation for all tests.
+    This will help catch hash bucket drift in testing.
+    """
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 @pytest.mark.parametrize(
     [
         "test_name",

deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -118,6 +118,21 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    """
+    Enable the bucketing spec validation for all tests.
+    This will help catch hash bucket drift in testing.
+    """
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 @pytest.mark.parametrize(
     [
         "test_name",

{deltacat-1.1.33.dist-info → deltacat-1.1.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.33
+Version: 1.1.35
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.33.dist-info → deltacat-1.1.35.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=Zo8lJd_CSkfzbM25rTFXXFIIu7hsQfgEeRkHV7_vej0,1778
+deltacat/__init__.py,sha256=br2aQSDj5eFS_j0mwGUSEQF386HRAXjiYg421vB9pME,1778
 deltacat/constants.py,sha256=TUJLXUJ9xq1Ryil72yLkKR8EDH_Irp5wUg56QstbRNE,2181
 deltacat/exceptions.py,sha256=7sjk3BuMY5Oo-6OvAfHncZx_OcvtEL47BblWr2F7waE,12740
 deltacat/logs.py,sha256=EQSDin1deehzz5xlLV1_TrFJrO_IBZ9Ahp7MdL-4cK8,9363
@@ -51,7 +51,7 @@ deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZR
 deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
 deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/compaction_session.py,sha256=COtol2s63DRPbd-AN9KCiWr4exLX8x5Tvxea_7cOGEQ,8078
-deltacat/compute/compactor_v2/constants.py,sha256=wvd34d7RGdniGbbiJcMljxRrRas4_uy9F9UaqXfS_Ag,3034
+deltacat/compute/compactor_v2/constants.py,sha256=F5Phrh-2JgnWvtjHXacxOG5Z2ivKcHnboerI12rc1zk,3632
 deltacat/compute/compactor_v2/deletes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/deletes/delete_file_envelope.py,sha256=AeuH9JRMwp6mvQf6P2cqL92hUEtResQq6qUTS0kIKac,3111
 deltacat/compute/compactor_v2/deletes/delete_strategy.py,sha256=SMEJOxR-5r92kvKNqtu2w6HmwtmhljcZX1wcNEuS-4w,2833
@@ -69,12 +69,12 @@ deltacat/compute/compactor_v2/private/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
 deltacat/compute/compactor_v2/private/compaction_utils.py,sha256=fMWXg1SCIIgjk9p_OFYrcm760dOKNbFO1Lj3_JI3GCY,30929
 deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=1R5xLUkl7GqL1nY-apAgY1czKDEHjIVYSRi9qLOMass,6726
-deltacat/compute/compactor_v2/steps/merge.py,sha256=LliCkWxWZ5Yh7UxVGSDJ1aRViw5hUZhEzlWxoXftbxA,22909
+deltacat/compute/compactor_v2/steps/merge.py,sha256=T2G2AaVsezYzo6oJtpuXH-bYv8nt-yFHA5ZbDIGodQg,24971
 deltacat/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=1P9CDpuWErsFcTTlRCeuUQHDokVI92he_MsL82uRAdA,7424
+deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=t2j9H9IdFRH9EfpL-9g5XvZs9WK9HybqBGA7fDi82EM,8310
 deltacat/compute/compactor_v2/utils/dedupe.py,sha256=Jz1QbBOdZJwT8K1vD9q01eOn7hdLNZ_AF7bJ0wficr0,1949
 deltacat/compute/compactor_v2/utils/delta.py,sha256=I7Yvda8NVbpKXG3nM2Ku1utvR2r2OpHvUMqUL2ja3aw,3626
-deltacat/compute/compactor_v2/utils/io.py,sha256=3m4dorxj-WD6Yu9_3gRE6gz3C-eNJA7nn02sHKwo-J8,6018
+deltacat/compute/compactor_v2/utils/io.py,sha256=Xjs7_D-0xKSetvllIe4o96aM1elfdjt1Ii7YfsHPvZs,6108
 deltacat/compute/compactor_v2/utils/merge.py,sha256=EV_iKhNc3WflgfLW1Q46dXUvyClx8VebWHGtninEfsI,5311
 deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=SbQ97M1Cxld-zZik2QMSzlj20g6JlENaQx_0PhlCIP8,12034
 deltacat/compute/compactor_v2/utils/task_options.py,sha256=0GoB_DLkCN1q8CVKTlWlDYt55qnpTDIa9fPyXJwB-cU,13801
@@ -85,7 +85,7 @@ deltacat/compute/merge_on_read/model/merge_on_read_params.py,sha256=Q51znagh8PtL
 deltacat/compute/merge_on_read/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/merge_on_read/utils/delta.py,sha256=e4BtOHa5XPpUnR4r0HqBKjXckBsTI8qBwdUWwpJfkWQ,1367
 deltacat/compute/resource_estimation/__init__.py,sha256=4bfBXcq-VAt9JCmjvj3yAmn0lEHVGdGsUCCoMGxjEqA,799
-deltacat/compute/resource_estimation/delta.py,sha256=8oRy1rgGUimwMqPB5At81AS-AsjPHdcvLHzJ9TW8RpM,9522
+deltacat/compute/resource_estimation/delta.py,sha256=dN64jbUQ8OI1BTz4fYGbulJLWjKjdT-XvwDJNLM__Oo,10583
 deltacat/compute/resource_estimation/manifest.py,sha256=gSqOyIda-pYq3vRsKFq3IiZvwhV3mMqrWPtsmUH9dD8,13035
 deltacat/compute/resource_estimation/model.py,sha256=psyagFXdpLGt8DfDqy7c8DWiuXCacr0Swe5f0M7DdO4,5465
 deltacat/compute/resource_estimation/parquet.py,sha256=5_apma4EKbKcm-nfV73-qN2nfnCeyhFW23ZHX3jz0Kw,3158
@@ -137,11 +137,11 @@ deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py,sha256=kW
 deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=8HVr3EIFYFqNaJoqeCuj9xIBjM4Ch2bx-mJcO4BRrLo,16839
 deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py,sha256=l_6-pAKOsRY3NbtfHsYmEaJEkq6IJueYuLsjyJxNgz4,81564
 deltacat/tests/compute/compact_partition_test_cases.py,sha256=HJ15Xyawv8ImFju8wDwt22fH5okoPhyS-QAygkXDb7Q,27422
-deltacat/tests/compute/test_compact_partition_incremental.py,sha256=lkfAraOJmEmieesf7b1BqlfTS26YjYM5xXOXoTMrsos,14989
-deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=xXBA66TTfARR90m5KQs31nmiokuMy9iGQt7Z9evyG7M,12950
+deltacat/tests/compute/test_compact_partition_incremental.py,sha256=8hUqnzeGIhAENcBxLL0R_yfjAaNTmRds6OWxQOmVqD8,15416
+deltacat/tests/compute/test_compact_partition_multiple_rounds.py,sha256=6d3F9E_4eO2Okh97v8NWFbEptPkzKoO0Qq8O6yAXrIs,13377
 deltacat/tests/compute/test_compact_partition_params.py,sha256=Dm5eLyHo8oGMeO3XBbpj1rZqHtPZ1hAB7z2qvzc4Lxk,8497
-deltacat/tests/compute/test_compact_partition_rebase.py,sha256=DNcpmnBo5QoZ23BiIhJCC3zaDK0xClZLUb2-ZEEp5s4,13108
-deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=Rxen3QGIaxVPa8lcO7NDMRxQ0aBjrOKn46LK5ZsfQTo,15073
+deltacat/tests/compute/test_compact_partition_rebase.py,sha256=vOF8wgTpdaWJKo47mK9aii3NKtwVwWgujoQyS8C3YyA,13535
+deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py,sha256=BimvU-iHiF78WlE4xbfk4dzHm0a-frwpE7H7Kh4XkbE,15500
 deltacat/tests/compute/test_util_common.py,sha256=0mEHo38bgH64y0XZ_zgUL_aZgQMgJOSTlOYvIJxG_MM,11825
 deltacat/tests/compute/test_util_constant.py,sha256=4o-W3E7r7jhFl1A3OFLLrdKnwcF46zx4lEIDY8ONJ3c,929
 deltacat/tests/compute/test_util_create_table_deltas_repo.py,sha256=Q3HJj1fjoe2JwRUOW8KEjbTqPIIoP2o_T3ZGH6SJnCM,13244
@@ -152,9 +152,10 @@ deltacat/tests/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 deltacat/tests/compute/compactor/utils/test_io.py,sha256=st5mlU4cVU-eQl7B4mvPgNA3izuNwbVawYOp-NcoyrI,4326
 deltacat/tests/compute/compactor/utils/test_round_completion_file.py,sha256=LAQ4usiRF4oTx4cA85L0eOcBa_Z-febc-CuzUijSGrI,7439
 deltacat/tests/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compute/compactor_v2/test_compaction_session.py,sha256=y8nNHq9ADHENzUKMQYguB45zOD7F2lZgcBYYTvbTsdM,28957
+deltacat/tests/compute/compactor_v2/test_compaction_session.py,sha256=zEXOIilybDpKuQt1ZRxGg4x_kUacBOcHE8KWcOmL01s,42563
 deltacat/tests/compute/compactor_v2/test_hashlib.py,sha256=8csF2hFWtBvY2MbX3-6iphCsVXxRp0zP1NTnKhfdmkg,328
 deltacat/tests/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py,sha256=eoiDuBUhgCmc3DYKCXL1g4QWtmROhZ0RJCQgePMY9as,9959
 deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py,sha256=aFb9rzT_EK9k8qAMHPtpqd5btyEmll1So1loDmZkotQ,1769
 deltacat/tests/compute/compactor_v2/utils/test_task_options.py,sha256=YDQKUKv3Vv8S1fe0YQmjHTrwnWSliqKHIWGu0fEdKnI,11478
 deltacat/tests/compute/resource_estimation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -211,8 +212,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.33.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.33.dist-info/METADATA,sha256=VCMvgoSMp8vuySFpact8lJHN3JchevV3Vd8Qbrdswek,1733
-deltacat-1.1.33.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-deltacat-1.1.33.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.33.dist-info/RECORD,,
+deltacat-1.1.35.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.35.dist-info/METADATA,sha256=b8Z4aVdNYjBoy0_uh0m4yoU_8h2w8v7I2AZOwacv5Es,1733
+deltacat-1.1.35.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+deltacat-1.1.35.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.35.dist-info/RECORD,,