deltacat 1.1.34__tar.gz → 1.1.35__tar.gz

{deltacat-1.1.34/deltacat.egg-info → deltacat-1.1.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.34
+Version: 1.1.35
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.34 → deltacat-1.1.35}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.34"
+__version__ = "1.1.35"
 
 
 __all__ = [

{deltacat-1.1.34 → deltacat-1.1.35}/deltacat/compute/compactor_v2/constants.py

@@ -1,4 +1,4 @@
-from deltacat.utils.common import env_bool, env_integer
+from deltacat.utils.common import env_bool, env_integer, env_string
 
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
@@ -92,3 +92,18 @@ DEFAULT_NUM_ROUNDS = 1
 SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED = env_bool(
     "SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED", False
 )
+
+# This env variable specifies whether to check bucketing spec
+# compliance of the existing compacted table.
+# PRINT_LOG: Enable logging if any partition is found
+# to be non-compliant with the bucketing spec.
+# ASSERT: Fail the job with ValidationError if the
+# current compacted partition is found to be non-compliant
+# with bucketing spec. Note, logging is implicitly enabled
+# in this case.
+BUCKETING_SPEC_COMPLIANCE_PROFILE = env_string(
+    "BUCKETING_SPEC_COMPLIANCE_PROFILE", None
+)
+
+BUCKETING_SPEC_COMPLIANCE_PRINT_LOG = "PRINT_LOG"
+BUCKETING_SPEC_COMPLIANCE_ASSERT = "ASSERT"

{deltacat-1.1.34 → deltacat-1.1.35}/deltacat/compute/compactor_v2/steps/merge.py

@@ -32,6 +32,7 @@ from deltacat.utils.resources import (
 )
 from deltacat.compute.compactor_v2.utils.primary_key_index import (
     generate_pk_hash_column,
+    pk_digest_to_hash_bucket_index,
 )
 from deltacat.storage import (
     Delta,
@@ -47,6 +48,9 @@ from deltacat.compute.compactor_v2.constants import (
     MERGE_TIME_IN_SECONDS,
     MERGE_SUCCESS_COUNT,
     MERGE_FAILURE_COUNT,
+    BUCKETING_SPEC_COMPLIANCE_PROFILE,
+    BUCKETING_SPEC_COMPLIANCE_ASSERT,
+    BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
 )
 from deltacat.exceptions import (
     categorize_errors,
@@ -188,9 +192,34 @@ def _merge_tables(
     return final_table
 
 
+def _validate_bucketing_spec_compliance(
+    table: pa.Table, rcf: RoundCompletionInfo, hb_index: int, primary_keys: List[str]
+) -> None:
+    pki_table = generate_pk_hash_column(
+        [table], primary_keys=primary_keys, requires_hash=True
+    )[0]
+    for index, hash_value in enumerate(sc.pk_hash_string_column_np(pki_table)):
+        hash_bucket = pk_digest_to_hash_bucket_index(hash_value, rcf.hash_bucket_count)
+        if hash_bucket != hb_index:
+            logger.info(
+                f"{rcf.compacted_delta_locator.namespace}.{rcf.compacted_delta_locator.table_name}"
+                f".{rcf.compacted_delta_locator.table_version}.{rcf.compacted_delta_locator.partition_id}"
+                f".{rcf.compacted_delta_locator.partition_values} has non-compliant bucketing spec. "
+                f"Expected hash bucket is {hb_index} but found {hash_bucket}."
+            )
+            if BUCKETING_SPEC_COMPLIANCE_PROFILE == BUCKETING_SPEC_COMPLIANCE_ASSERT:
+                raise AssertionError(
+                    "Hash bucket drift detected. Expected hash bucket index"
+                    f" to be {hb_index} but found {hash_bucket}"
+                )
+            # No further checks necessary
+            break
+
+
 def _download_compacted_table(
     hb_index: int,
     rcf: RoundCompletionInfo,
+    primary_keys: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[dict] = None,
@@ -214,7 +243,23 @@ def _download_compacted_table(
 
         tables.append(table)
 
-    return pa.concat_tables(tables)
+    compacted_table = pa.concat_tables(tables)
+    check_bucketing_spec = BUCKETING_SPEC_COMPLIANCE_PROFILE in [
+        BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
+        BUCKETING_SPEC_COMPLIANCE_ASSERT,
+    ]
+
+    logger.debug(
+        f"Value of BUCKETING_SPEC_COMPLIANCE_PROFILE, check_bucketing_spec:"
+        f" {BUCKETING_SPEC_COMPLIANCE_PROFILE}, {check_bucketing_spec}"
+    )
+
+    # Bucketing spec compliance isn't required without primary keys
+    if primary_keys and check_bucketing_spec:
+        _validate_bucketing_spec_compliance(
+            compacted_table, rcf, hb_index, primary_keys
+        )
+    return compacted_table
 
 
 def _copy_all_manifest_files_from_old_hash_buckets(
@@ -543,6 +588,7 @@ def _timed_merge(input: MergeInput) -> MergeResult:
                 compacted_table = _download_compacted_table(
                     hb_index=merge_file_group.hb_index,
                     rcf=input.round_completion_info,
+                    primary_keys=input.primary_keys,
                     read_kwargs_provider=input.read_kwargs_provider,
                     deltacat_storage=input.deltacat_storage,
                     deltacat_storage_kwargs=input.deltacat_storage_kwargs,

{deltacat-1.1.34 → deltacat-1.1.35}/deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -4,9 +4,11 @@ import os
 import pyarrow as pa
 import pytest
 import boto3
+import json
 from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
 )
+from deltacat.exceptions import ValidationError
 from boto3.resources.base import ServiceResource
 import deltacat.tests.local_deltacat_storage as ds
 from deltacat.types.media import ContentType
@@ -88,6 +90,17 @@ def disable_sha1(monkeypatch):
     )
 
 
+@pytest.fixture(scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 class TestCompactionSession:
     """
     This class adds specific tests that aren't part of the parametrized test suite.
@@ -689,3 +702,307 @@ class TestCompactionSession:
             incremental_rcf.compacted_pyarrow_write_result.pyarrow_bytes >= 2300000000
         )
         assert incremental_rcf.compacted_pyarrow_write_result.records == 4
+
+    def test_compact_partition_when_bucket_spec_validation_fails(
+        self,
+        s3_resource,
+        local_deltacat_storage_kwargs,
+        enable_bucketing_spec_validation,
+    ):
+        """
+        A test case which asserts the bucketing spec validation throws an assertion error
+        when the validation has failed.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 1,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        backfill_rcf = get_rcf(s3_resource, rcf_url)
+        bucket, backfill_key1, backfill_key2 = rcf_url.strip("s3://").split("/")
+        # Move the records to different hash buckets to simulate a validation failure.
+        backfill_rcf["hbIndexToEntryRange"] = {"1": [0, 3]}
+        s3_resource.Bucket(bucket).put_object(
+            Key=f"{backfill_key1}/{backfill_key2}", Body=json.dumps(backfill_rcf)
+        )
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_destination_partition = ds.get_partition(
+            dest_partition.stream_locator, [], **local_deltacat_storage_kwargs
+        )
+
+        with pytest.raises(ValidationError) as excinfo:
+            compact_partition(
+                CompactPartitionParams.of(
+                    {
+                        "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                        "compacted_file_content_type": ContentType.PARQUET,
+                        "dd_max_parallelism_ratio": 1.0,
+                        "deltacat_storage": ds,
+                        "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                        "destination_partition_locator": new_destination_partition.locator,
+                        "drop_duplicates": True,
+                        "hash_bucket_count": 4,
+                        "last_stream_position_to_compact": new_source_delta.stream_position,
+                        "list_deltas_kwargs": {
+                            **local_deltacat_storage_kwargs,
+                            **{"equivalent_table_types": []},
+                        },
+                        "primary_keys": ["pk"],
+                        "rebase_source_partition_locator": None,
+                        "rebase_source_partition_high_watermark": None,
+                        "records_per_compacted_file": 4000,
+                        "s3_client_kwargs": {},
+                        "source_partition_locator": new_source_delta.partition_locator,
+                    }
+                )
+            )
+
+        assert (
+            "Hash bucket drift detected. Expected hash bucket index to be 1 but found 0"
+            in str(excinfo.value)
+        )
+
+    def test_compact_partition_when_bucket_spec_validation_fails_but_env_variable_disabled(
+        self,
+        s3_resource,
+        local_deltacat_storage_kwargs,
+    ):
+        """
+        A test case which asserts even if bucketing spec validation fails, compaction doesn't
+        throw an error if the feature is not enabled.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 1,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        backfill_rcf = get_rcf(s3_resource, rcf_url)
+        bucket, backfill_key1, backfill_key2 = rcf_url.strip("s3://").split("/")
+        # Move the records to different hash buckets to simulate a validation failure.
+        backfill_rcf["hbIndexToEntryRange"] = {"1": [0, 3]}
+        s3_resource.Bucket(bucket).put_object(
+            Key=f"{backfill_key1}/{backfill_key2}", Body=json.dumps(backfill_rcf)
+        )
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_destination_partition = ds.get_partition(
+            dest_partition.stream_locator, [], **local_deltacat_storage_kwargs
+        )
+
+        new_rcf = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": new_destination_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": new_source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": None,
+                    "rebase_source_partition_high_watermark": None,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": new_source_delta.partition_locator,
+                }
+            )
+        )
+
+        incremental_rcf = get_rcf(s3_resource, new_rcf)
+        assert incremental_rcf.hash_bucket_count == 4
+        assert len(incremental_rcf.hb_index_to_entry_range) == 2
+
+    def test_compact_partition_when_bucket_spec_validation_succeeds(
+        self,
+        s3_resource,
+        local_deltacat_storage_kwargs,
+        enable_bucketing_spec_validation,
+    ):
+        """
+        A test case which asserts the bucketing spec validation does not throw
+        and error when the validation succeeds.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 1,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        rcf = get_rcf(s3_resource, rcf_url)
+        assert rcf.hash_bucket_count == 4
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_destination_partition = ds.get_partition(
+            dest_partition.stream_locator, [], **local_deltacat_storage_kwargs
+        )
+
+        new_uri = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": new_destination_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 4,
+                    "last_stream_position_to_compact": new_source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": None,
+                    "rebase_source_partition_high_watermark": None,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": new_source_delta.partition_locator,
+                }
+            )
+        )
+
+        rcf = get_rcf(s3_resource, new_uri)
+        assert rcf.hash_bucket_count == 4

{deltacat-1.1.34 → deltacat-1.1.35}/deltacat/tests/compute/test_compact_partition_incremental.py

@@ -119,6 +119,21 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    """
+    Enable the bucketing spec validation for all tests.
+    This will help catch hash bucket drift in testing.
+    """
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 @pytest.mark.parametrize(
     [
         "test_name",

{deltacat-1.1.34 → deltacat-1.1.35}/deltacat/tests/compute/test_compact_partition_multiple_rounds.py

@@ -114,6 +114,21 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    """
+    Enable the bucketing spec validation for all tests.
+    This will help catch hash bucket drift in testing.
+    """
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 @pytest.mark.parametrize(
     [
         "test_name",

{deltacat-1.1.34 → deltacat-1.1.35}/deltacat/tests/compute/test_compact_partition_rebase.py

@@ -114,6 +114,21 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    """
+    Enable the bucketing spec validation for all tests.
+    This will help catch hash bucket drift in testing.
+    """
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 @pytest.mark.parametrize(
     [
         "test_name",

{deltacat-1.1.34 → deltacat-1.1.35}/deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -118,6 +118,21 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def enable_bucketing_spec_validation(monkeypatch):
+    """
+    Enable the bucketing spec validation for all tests.
+    This will help catch hash bucket drift in testing.
+    """
+    import deltacat.compute.compactor_v2.steps.merge
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.steps.merge,
+        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
+        "ASSERT",
+    )
+
+
 @pytest.mark.parametrize(
     [
         "test_name",

{deltacat-1.1.34 → deltacat-1.1.35/deltacat.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.34
+Version: 1.1.35
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team