deltacat 1.1.27__tar.gz → 1.1.29__tar.gz

{deltacat-1.1.27/deltacat.egg-info → deltacat-1.1.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.27
+Version: 1.1.29
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.27 → deltacat-1.1.29}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.27"
+__version__ = "1.1.29"
 
 
 __all__ = [

{deltacat-1.1.27 → deltacat-1.1.29}/deltacat/compute/compactor_v2/constants.py

@@ -1,3 +1,5 @@
+from deltacat.utils.common import env_bool, env_integer
+
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
 PK_DELIMITER = "L6kl7u5f"
@@ -31,7 +33,9 @@ TOTAL_MEMORY_BUFFER_PERCENTAGE = 30
 # The total size of records that will be hash bucketed at once
 # Since, sorting is nlogn, we ensure that is not performed
 # on a very large dataset for best performance.
-MAX_SIZE_OF_RECORD_BATCH_IN_GIB = 2 * 1024 * 1024 * 1024
+MAX_SIZE_OF_RECORD_BATCH_IN_GIB = env_integer(
+    "MAX_SIZE_OF_RECORD_BATCH_IN_GIB", 2 * 1024 * 1024 * 1024
+)
 
 # Whether to drop duplicates during merge.
 DROP_DUPLICATES = True
@@ -78,3 +82,13 @@ COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"
 # Number of rounds to run hash/merge for a single
 # partition. (For large table support)
 DEFAULT_NUM_ROUNDS = 1
+
+# Whether to perform sha1 hashing when required to
+# optimize memory. For example, hashing is always
+# required for bucketing where it's not mandatory
+# when dropping duplicates. Setting this to True
+# will disable sha1 hashing in cases where it isn't
+# mandatory. This flag is False by default.
+SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED = env_bool(
+    "SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED", False
+)

{deltacat-1.1.27 → deltacat-1.1.29}/deltacat/compute/compactor_v2/steps/merge.py

@@ -7,6 +7,7 @@ import ray
 import itertools
 import time
 import pyarrow.compute as pc
+from deltacat.utils.pyarrow import MAX_INT_BYTES
 import deltacat.compute.compactor_v2.utils.merge as merge_utils
 from uuid import uuid4
 from deltacat import logs
@@ -147,10 +148,32 @@ def _merge_tables(
     if compacted_table:
         compacted_table = all_tables[0]
 
+        compacted_pk_hash_str = compacted_table[sc._PK_HASH_STRING_COLUMN_NAME]
+        incremental_pk_hash_str = incremental_table[sc._PK_HASH_STRING_COLUMN_NAME]
+
+        logger.info(
+            f"Size of compacted pk hash={compacted_pk_hash_str.nbytes} "
+            f"and incremental pk hash={incremental_pk_hash_str.nbytes}."
+        )
+
+        if (
+            compacted_table[sc._PK_HASH_STRING_COLUMN_NAME].nbytes >= MAX_INT_BYTES
+            or incremental_table[sc._PK_HASH_STRING_COLUMN_NAME].nbytes >= MAX_INT_BYTES
+        ):
+            logger.info("Casting compacted and incremental pk hash to large_string...")
+            # is_in combines the chunks of the chunked array passed which can cause
+            # ArrowCapacityError if the total size of string array is over 2GB.
+            # Using a large_string would resolve this issue.
+            # The cast here should be zero-copy in most cases.
+            compacted_pk_hash_str = pc.cast(compacted_pk_hash_str, pa.large_string())
+            incremental_pk_hash_str = pc.cast(
+                incremental_pk_hash_str, pa.large_string()
+            )
+
         records_to_keep = pc.invert(
             pc.is_in(
-                compacted_table[sc._PK_HASH_STRING_COLUMN_NAME],
-                incremental_table[sc._PK_HASH_STRING_COLUMN_NAME],
+                compacted_pk_hash_str,
+                incremental_pk_hash_str,
             )
         )
 
@@ -492,9 +515,11 @@ def _copy_manifests_from_hash_bucketing(
 def _timed_merge(input: MergeInput) -> MergeResult:
     task_id = get_current_ray_task_id()
     worker_id = get_current_ray_worker_id()
-    with memray.Tracker(
-        f"merge_{worker_id}_{task_id}.bin"
-    ) if input.enable_profiler else nullcontext():
+    with (
+        memray.Tracker(f"merge_{worker_id}_{task_id}.bin")
+        if input.enable_profiler
+        else nullcontext()
+    ):
         total_input_records, total_deduped_records = 0, 0
         total_dropped_records = 0
         materialized_results: List[MaterializeResult] = []

{deltacat-1.1.27 → deltacat-1.1.29}/deltacat/compute/compactor_v2/utils/dedupe.py

@@ -25,7 +25,7 @@ def _create_chunked_index_array(array: pa.Array) -> pa.Array:
         result[index] = np.arange(cl, dtype="int32")
 
     chunk_lengths = ([0] + chunk_lengths)[:-1]
-    result = pa.chunked_array(result + np.cumsum(chunk_lengths))
+    result = pa.chunked_array(result + np.cumsum(chunk_lengths), type=pa.int32())
     return result
 
 

{deltacat-1.1.27 → deltacat-1.1.29}/deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -10,6 +10,7 @@ from deltacat.compute.compactor_v2.constants import (
     TOTAL_BYTES_IN_SHA1_HASH,
     PK_DELIMITER,
     MAX_SIZE_OF_RECORD_BATCH_IN_GIB,
+    SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED,
 )
 import time
 from deltacat.compute.compactor.model.delta_file_envelope import DeltaFileEnvelope
@@ -48,6 +49,13 @@ def _is_sha1_desired(hash_columns: List[pa.Array]) -> bool:
         f"Found total length of hash column={total_len} and total_size={total_size}"
     )
 
+    if SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED:
+        logger.info(
+            f"SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED is True. "
+            f"Returning False for is_sha1_desired"
+        )
+        return False
+
     return total_size > TOTAL_BYTES_IN_SHA1_HASH * total_len
 
 
@@ -108,9 +116,10 @@ def _optimized_group_record_batches_by_hash_bucket(
     record_batches = []
     result_len = 0
     for record_batch in table_batches:
-        current_bytes += record_batch.nbytes
-        record_batches.append(record_batch)
-        if current_bytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB:
+        if (
+            record_batches
+            and current_bytes + record_batch.nbytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB
+        ):
             logger.info(
                 f"Total number of record batches without exceeding {MAX_SIZE_OF_RECORD_BATCH_IN_GIB} "
                 f"is {len(record_batches)} and size {current_bytes}"
@@ -128,6 +137,9 @@ def _optimized_group_record_batches_by_hash_bucket(
             current_bytes = 0
             record_batches.clear()
 
+        current_bytes += record_batch.nbytes
+        record_batches.append(record_batch)
+
     if record_batches:
         appended_len, append_latency = timed_invocation(
             _append_table_by_hash_bucket,

{deltacat-1.1.27 → deltacat-1.1.29}/deltacat/tests/compute/compact_partition_test_cases.py

@@ -601,6 +601,38 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
         assert_compaction_audit=None,
     ),
+    "15-incremental-empty-input-with-single-hash-bucket": IncrementalCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[SortKey.of(key_name="sk_col_1")],
+        partition_keys=ZERO_VALUED_PARTITION_KEYS_PARAM,
+        partition_values=ZERO_VALUED_PARTITION_VALUES_PARAM,
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([]),
+                pa.array([]),
+            ],
+            names=["pk_col_1", "sk_col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([]),
+                pa.array([]),
+            ],
+            names=["pk_col_1", "sk_col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=1,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        is_inplace=False,
+        add_late_deltas=None,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
+    ),
 }
 
 INCREMENTAL_TEST_CASES = with_compactor_version_func_test_param(INCREMENTAL_TEST_CASES)

{deltacat-1.1.27 → deltacat-1.1.29}/deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -1,6 +1,7 @@
 from typing import Dict, Any
 import ray
 import os
+import pyarrow as pa
 import pytest
 import boto3
 from deltacat.compute.compactor.model.compaction_session_audit_info import (
@@ -76,6 +77,17 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
+@pytest.fixture(scope="function")
+def disable_sha1(monkeypatch):
+    import deltacat.compute.compactor_v2.utils.primary_key_index
+
+    monkeypatch.setattr(
+        deltacat.compute.compactor_v2.utils.primary_key_index,
+        "SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED",
+        True,
+    )
+
+
 class TestCompactionSession:
     """
     This class adds specific tests that aren't part of the parametrized test suite.
@@ -556,3 +568,124 @@ class TestCompactionSession:
                 }
             )
         )
+
+    def test_compact_partition_when_incremental_pk_hash_is_over_2gb(
+        self, s3_resource, local_deltacat_storage_kwargs, disable_sha1
+    ):
+        """
+        A test case which ensures the compaction succeeds even if the incremental
+        arrow table size is over 2GB. It is added to prevent ArrowCapacityError
+        when running is_in operation during merge.
+
+        Note that we set SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED to bypass sha1 hashing
+        which truncates the lengths of pk strings when deduping.
+        """
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+        # we create chunked array to avoid ArrowCapacityError
+        chunked_pk_array = pa.chunked_array([["13bytesstring"], ["12bytestring"]])
+        table = pa.table([chunked_pk_array], names=["pk"])
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, pa_table=table, **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # rebase first
+        rebase_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 1,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                    "resource_estimation_method": ResourceEstimationMethod.PREVIOUS_INFLATION,
+                }
+            )
+        )
+
+        rebased_rcf = get_rcf(s3_resource, rebase_url)
+
+        assert rebased_rcf.compacted_pyarrow_write_result.files == 1
+        assert rebased_rcf.compacted_pyarrow_write_result.records == 2
+
+        # Run incremental with a small delta on source
+        chunked_pk_array = pa.chunked_array(
+            [["13bytesstring" * 95_000_000], ["12bytestring" * 95_000_000]]
+        )  # 2.3GB
+        table = pa.table([chunked_pk_array], names=["pk"])
+
+        incremental_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            pa_table=table,
+            **local_deltacat_storage_kwargs,
+        )
+        assert (
+            incremental_source_delta.partition_locator == source_delta.partition_locator
+        ), "source partition locator should not change"
+        dest_partition = ds.get_partition(
+            dest_partition.stream_locator,
+            dest_partition.partition_values,
+            **local_deltacat_storage_kwargs,
+        )
+
+        assert (
+            dest_partition.locator
+            == rebased_rcf.compacted_delta_locator.partition_locator
+        ), "The new destination partition should be same as compacted partition"
+
+        # Run incremental
+        incremental_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 1,
+                    "last_stream_position_to_compact": incremental_source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": incremental_source_delta.partition_locator,
+                    "resource_estimation_method": ResourceEstimationMethod.PREVIOUS_INFLATION,
+                }
+            )
+        )
+
+        incremental_rcf = get_rcf(s3_resource, incremental_url)
+
+        assert incremental_rcf.compacted_pyarrow_write_result.files == 1
+        assert (
+            incremental_rcf.compacted_pyarrow_write_result.pyarrow_bytes >= 2300000000
+        )
+        assert incremental_rcf.compacted_pyarrow_write_result.records == 4

deltacat-1.1.29/deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py

@@ -0,0 +1,45 @@
+import pyarrow as pa
+from deltacat.compute.compactor_v2.utils.primary_key_index import (
+    group_by_pk_hash_bucket,
+)
+
+
+class TestGroupByPkHashBucket:
+    def test_sanity(self):
+        record = pa.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+        pk = pa.array(["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"])
+        record_batch = pa.RecordBatch.from_arrays([record, pk], names=["record", "pk"])
+        table = pa.Table.from_batches([record_batch])
+        grouped_array = group_by_pk_hash_bucket(table, 3, ["pk"])
+
+        assert len(grouped_array) == 3
+        total_records = 0
+        for arr in grouped_array:
+            if arr is not None:
+                total_records += len(arr[1])
+
+        assert total_records == len(table)
+
+    def test_when_record_batches_exceed_int_max_size(self):
+        record = pa.array(["12bytestring" * 90_000_000])
+        record_batch = pa.RecordBatch.from_arrays([record], names=["pk"])
+        table = pa.Table.from_batches([record_batch, record_batch])
+
+        grouped_array = group_by_pk_hash_bucket(table, 3, ["pk"])
+
+        assert len(grouped_array) == 3
+        # two record batches are preserved as combining them
+        # would exceed 2GB.
+        assert len(grouped_array[2].to_batches()) == 2
+
+    def test_when_record_batches_less_than_int_max_size(self):
+        record = pa.array(["12bytestring" * 90_000])
+        record_batch = pa.RecordBatch.from_arrays([record], names=["pk"])
+        table = pa.Table.from_batches([record_batch, record_batch])
+
+        grouped_array = group_by_pk_hash_bucket(table, 3, ["pk"])
+
+        assert len(grouped_array) == 3
+        # Combined the arrays into one record batch as the size
+        # would not exceed 2GB.
+        assert len(grouped_array[1].to_batches()) == 1

{deltacat-1.1.27 → deltacat-1.1.29}/deltacat/tests/test_utils/pyarrow.py

@@ -47,7 +47,8 @@ def stage_partition_from_file_paths(
 
 def commit_delta_to_staged_partition(
     staged_partition,
-    file_paths: List[str],
+    file_paths: List[str] = None,
+    pa_table: pa.Table = None,
     content_type: ContentType = ContentType.PARQUET,
     *args,
     **kwargs,
@@ -57,6 +58,7 @@ def commit_delta_to_staged_partition(
         *args,
         file_paths=file_paths,
         content_type=content_type,
+        pa_table=pa_table,
         **kwargs,
     )
     ds.commit_partition(staged_partition, **kwargs)
@@ -76,23 +78,28 @@ def download_delta(delta_like: Union[Delta, DeltaLocator], *args, **kwargs) -> D
 
 def commit_delta_to_partition(
     partition: Union[Partition, PartitionLocator],
-    file_paths: List[str],
+    file_paths: List[str] = None,
+    pa_table: pa.Table = None,
     content_type: ContentType = ContentType.PARQUET,
     *args,
     **kwargs,
 ) -> Delta:
-    tables = []
 
     if isinstance(partition, PartitionLocator):
         partition = ds.get_partition(
             partition.stream_locator, partition.partition_values, *args, **kwargs
         )
+    if pa_table is None:
+        assert file_paths is not None, "One of pa_table or file_paths must be passed."
+        tables = []
+        for file_path in file_paths:
+            table = pa.csv.read_csv(file_path)
+            tables.append(table)
 
-    for file_path in file_paths:
-        table = pa.csv.read_csv(file_path)
-        tables.append(table)
+        pa_table = pa.concat_tables(tables)
 
-    table = pa.concat_tables(tables)
-    staged_delta = ds.stage_delta(table, partition, content_type=content_type, **kwargs)
+    staged_delta = ds.stage_delta(
+        pa_table, partition, content_type=content_type, **kwargs
+    )
 
     return ds.commit_delta(staged_delta, **kwargs)