deltacat 1.1.26__tar.gz → 1.1.27__tar.gz

{deltacat-1.1.26/deltacat.egg-info → deltacat-1.1.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.26
+Version: 1.1.27
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.26 → deltacat-1.1.27}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.26"
+__version__ = "1.1.27"
 
 
 __all__ = [

{deltacat-1.1.26 → deltacat-1.1.27}/deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -584,8 +584,11 @@ def _process_merge_results(
             f"Duplicate record count ({duplicate_hash_bucket_mat_results}) is as large "
             f"as or greater than params.num_rounds, which is {params.num_rounds}"
         )
+        # ensure start index is the first file index if task index is same
         hb_id_to_entry_indices_range[str(mat_result.task_index)] = (
-            file_index,
+            hb_id_to_entry_indices_range.get(str(mat_result.task_index), [file_index])[
+                0
+            ],
             file_index + mat_result.pyarrow_write_result.files,
         )
 

{deltacat-1.1.26 → deltacat-1.1.27}/deltacat/compute/resource_estimation/delta.py

@@ -188,7 +188,7 @@ def _estimate_resources_required_to_process_delta_using_file_sampling(
         sampled_on_disk_size += delta.manifest.entries[entry_index].meta.content_length
         sampled_num_rows += len(tbl)
 
-    if not sampled_on_disk_size:
+    if not sampled_on_disk_size or not sampled_in_memory_size:
         return EstimatedResources.of(
             memory_bytes=0,
             statistics=Statistics.of(

{deltacat-1.1.26 → deltacat-1.1.27}/deltacat/tests/compute/resource_estimation/test_delta.py

@@ -437,6 +437,43 @@ class TestEstimateResourcesRequiredToProcessDelta:
             == parquet_delta_with_manifest.meta.content_length
         )
 
+    def test_parquet_delta_when_file_sampling_and_arrow_size_zero(
+        self,
+        local_deltacat_storage_kwargs,
+        parquet_delta_with_manifest: Delta,
+        monkeypatch,
+    ):
+        params = EstimateResourcesParams.of(
+            resource_estimation_method=ResourceEstimationMethod.FILE_SAMPLING,
+            max_files_to_sample=2,
+        )
+
+        def mock_func(*args, **kwargs):
+            class MockedValue:
+                nbytes = 0
+
+                def __len__(self):
+                    return 0
+
+            return MockedValue()
+
+        monkeypatch.setattr(ds, "download_delta_manifest_entry", mock_func)
+
+        result = estimate_resources_required_to_process_delta(
+            delta=parquet_delta_with_manifest,
+            operation_type=OperationType.PYARROW_DOWNLOAD,
+            deltacat_storage=ds,
+            deltacat_storage_kwargs=local_deltacat_storage_kwargs,
+            estimate_resources_params=params,
+        )
+
+        assert parquet_delta_with_manifest.manifest is not None
+        assert result.memory_bytes == 0
+        assert (
+            result.statistics.on_disk_size_bytes
+            == parquet_delta_with_manifest.meta.content_length
+        )
+
     def test_delta_manifest_utsv_when_file_sampling(
         self, local_deltacat_storage_kwargs, utsv_delta_with_manifest: Delta
     ):

{deltacat-1.1.26 → deltacat-1.1.27}/deltacat/tests/compute/test_compact_partition_incremental.py

@@ -328,6 +328,16 @@ def test_compact_partition_incremental(
         **compaction_audit_obj
     )
 
+    # assert if RCF covers all files
+    if compactor_version != CompactorVersion.V1.value:
+        previous_end = None
+        for start, end in round_completion_info.hb_index_to_entry_range.values():
+            assert (previous_end is None and start == 0) or start == previous_end
+            previous_end = end
+        assert (
+            previous_end == round_completion_info.compacted_pyarrow_write_result.files
+        )
+
     tables = ds.download_delta(
         compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
     )

{deltacat-1.1.26 → deltacat-1.1.27}/deltacat/tests/compute/test_compact_partition_multiple_rounds.py

@@ -309,6 +309,16 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
             **compaction_audit_obj
         )
 
+        # assert if RCF covers all files
+        # multiple rounds feature is only supported in V2 compactor
+        previous_end = None
+        for start, end in round_completion_info.hb_index_to_entry_range.values():
+            assert (previous_end is None and start == 0) or start == previous_end
+            previous_end = end
+        assert (
+            previous_end == round_completion_info.compacted_pyarrow_write_result.files
+        )
+
         # Assert not in-place compacted
         assert (
             execute_compaction_result_spy.call_args.args[-1] is False

{deltacat-1.1.26 → deltacat-1.1.27}/deltacat/tests/compute/test_compact_partition_rebase.py

@@ -299,6 +299,17 @@ def test_compact_partition_rebase_same_source_and_destination(
             round_completion_info.compaction_audit_url
         )
 
+        # assert if RCF covers all files
+        if compactor_version != CompactorVersion.V1.value:
+            previous_end = None
+            for start, end in round_completion_info.hb_index_to_entry_range.values():
+                assert (previous_end is None and start == 0) or start == previous_end
+                previous_end = end
+            assert (
+                previous_end
+                == round_completion_info.compacted_pyarrow_write_result.files
+            )
+
         compaction_audit_obj: Dict[str, Any] = read_s3_contents(
             s3_resource, audit_bucket, audit_key
         )

{deltacat-1.1.26 → deltacat-1.1.27}/deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -355,6 +355,16 @@ def test_compact_partition_rebase_then_incremental(
     compacted_delta_locator_incremental: DeltaLocator = (
         round_completion_info.compacted_delta_locator
     )
+    # assert if RCF covers all files
+    if compactor_version != CompactorVersion.V1.value:
+        previous_end = None
+        for start, end in round_completion_info.hb_index_to_entry_range.values():
+            assert (previous_end is None and start == 0) or start == previous_end
+            previous_end = end
+        assert (
+            previous_end == round_completion_info.compacted_pyarrow_write_result.files
+        )
+
     audit_bucket, audit_key = round_completion_info.compaction_audit_url.replace(
         "s3://", ""
     ).split("/", 1)

{deltacat-1.1.26 → deltacat-1.1.27/deltacat.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.26
+Version: 1.1.27
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team