deltacat 1.1.36__py3-none-any.whl → 1.1.37__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.36"
+__version__ = "1.1.37"
 
 
 __all__ = [

deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -78,13 +78,25 @@ def _append_table_by_hash_bucket(
         f"Grouping a pki table of length {len(pki_table)} took {groupby_latency}s"
     )
 
+    hb_pk_grouped_by = hb_pk_grouped_by.sort_by(sc._HASH_BUCKET_IDX_COLUMN_NAME)
     group_count_array = hb_pk_grouped_by[f"{sc._HASH_BUCKET_IDX_COLUMN_NAME}_count"]
     hb_group_array = hb_pk_grouped_by[sc._HASH_BUCKET_IDX_COLUMN_NAME]
 
     result_len = 0
     for i, group_count in enumerate(group_count_array):
         hb_idx = hb_group_array[i].as_py()
-        pyarrow_table = hb_pk_table.slice(offset=result_len, length=group_count.as_py())
+        group_count_py = group_count.as_py()
+        pyarrow_table = hb_pk_table.slice(offset=result_len, length=group_count_py)
+        assert group_count_py == len(
+            pyarrow_table
+        ), f"Group count {group_count_py} not equal to {len(pyarrow_table)}"
+        all_buckets = pc.unique(pyarrow_table[sc._HASH_BUCKET_IDX_COLUMN_NAME])
+        assert (
+            len(all_buckets) == 1
+        ), f"Only one hash bucket is allowed by found {len(all_buckets)}"
+        assert (
+            all_buckets[0].as_py() == hb_idx
+        ), f"Hash bucket not equal, {all_buckets[0]} and {hb_idx}"
         pyarrow_table = pyarrow_table.drop(
             [sc._HASH_BUCKET_IDX_COLUMN_NAME, sc._PK_HASH_STRING_COLUMN_NAME]
         )
@@ -141,6 +153,7 @@ def _optimized_group_record_batches_by_hash_bucket(
         record_batches.append(record_batch)
 
     if record_batches:
+        print(f"{len(record_batches)} -- END")
         appended_len, append_latency = timed_invocation(
             _append_table_by_hash_bucket,
             pa.Table.from_batches(record_batches),

deltacat/compute/resource_estimation/delta.py

@@ -170,6 +170,10 @@ def _estimate_resources_required_to_process_delta_using_file_sampling(
         operation_type == OperationType.PYARROW_DOWNLOAD
     ), "Number of rows can only be estimated for PYARROW_DOWNLOAD operation"
 
+    if not estimate_resources_params.max_files_to_sample:
+        # we cannot calculate if we cannot sample
+        return None
+
     if not delta.manifest:
         delta.manifest = deltacat_storage.get_delta_manifest(
             delta.locator,
@@ -186,10 +190,6 @@ def _estimate_resources_required_to_process_delta_using_file_sampling(
             ),
         )
 
-    if not estimate_resources_params.max_files_to_sample:
-        # we cannot calculate if we cannot sample
-        return None
-
     sampled_in_memory_size = 0.0
     sampled_on_disk_size = 0.0
     sampled_num_rows = 0
@@ -252,6 +252,10 @@ RESOURCE_ESTIMATION_METHOD_TO_DELTA_RESOURCE_ESTIMATION_FUNCTIONS = {
         _estimate_resources_required_to_process_delta_using_file_sampling,
         _estimate_resources_required_to_process_delta_using_previous_inflation,
     ],
+    ResourceEstimationMethod.FILE_SAMPLING_WITH_PREVIOUS_INFLATION: [
+        _estimate_resources_required_to_process_delta_using_file_sampling,
+        _estimate_resources_required_to_process_delta_using_previous_inflation,
+    ],
 }
 
 

deltacat/compute/resource_estimation/model.py

@@ -23,6 +23,14 @@ class ResourceEstimationMethod(str, Enum):
     """
     DEFAULT_V2 = "DEFAULT_V2"
 
+    """
+    This approach combines file sampling estimation and inflation based methods
+    and runs them in the order specified below:
+    1. FILE_SAMPLING
+    2. PREVIOUS_INFLATION
+    """
+    FILE_SAMPLING_WITH_PREVIOUS_INFLATION = "FILE_SAMPLING_WITH_PREVIOUS_INFLATION"
+
     """
     This approach strictly uses previous inflation and average record size to arrive
     at a resource estimate. It requires users to pass in previous inflation and average

deltacat/tests/compute/resource_estimation/test_delta.py

@@ -416,6 +416,29 @@ class TestEstimateResourcesRequiredToProcessDelta:
             == delta_without_manifest.meta.content_length
         )
 
+    def test_empty_delta_sampled_when_file_sampling_with_previous_inflation(
+        self, local_deltacat_storage_kwargs, delta_without_manifest: Delta
+    ):
+        params = EstimateResourcesParams.of(
+            resource_estimation_method=ResourceEstimationMethod.FILE_SAMPLING_WITH_PREVIOUS_INFLATION,
+            max_files_to_sample=2,
+        )
+
+        result = estimate_resources_required_to_process_delta(
+            delta=delta_without_manifest,
+            operation_type=OperationType.PYARROW_DOWNLOAD,
+            deltacat_storage=ds,
+            deltacat_storage_kwargs=local_deltacat_storage_kwargs,
+            estimate_resources_params=params,
+        )
+
+        assert delta_without_manifest.manifest is not None
+        assert result.memory_bytes is not None
+        assert (
+            result.statistics.on_disk_size_bytes
+            == delta_without_manifest.meta.content_length
+        )
+
     def test_delta_manifest_parquet_when_file_sampling(
         self, local_deltacat_storage_kwargs, parquet_delta_with_manifest: Delta
     ):
@@ -437,6 +460,27 @@ class TestEstimateResourcesRequiredToProcessDelta:
             == parquet_delta_with_manifest.meta.content_length
         )
 
+    def test_delta_manifest_parquet_when_file_sampling_with_previous_inflation(
+        self, local_deltacat_storage_kwargs, parquet_delta_with_manifest: Delta
+    ):
+        params = EstimateResourcesParams.of(
+            resource_estimation_method=ResourceEstimationMethod.FILE_SAMPLING_WITH_PREVIOUS_INFLATION,
+            max_files_to_sample=2,
+        )
+
+        result = estimate_resources_required_to_process_delta(
+            delta=parquet_delta_with_manifest,
+            operation_type=OperationType.PYARROW_DOWNLOAD,
+            deltacat_storage=ds,
+            deltacat_storage_kwargs=local_deltacat_storage_kwargs,
+            estimate_resources_params=params,
+        )
+        assert result.memory_bytes is not None
+        assert (
+            result.statistics.on_disk_size_bytes
+            == parquet_delta_with_manifest.meta.content_length
+        )
+
     def test_parquet_delta_when_file_sampling_and_arrow_size_zero(
         self,
         local_deltacat_storage_kwargs,
@@ -512,6 +556,28 @@ class TestEstimateResourcesRequiredToProcessDelta:
         )
         assert result is None
 
+    def test_delta_manifest_utsv_when_file_sampling_with_previous_inflation_zero_files_to_sample(
+        self, local_deltacat_storage_kwargs, utsv_delta_with_manifest: Delta
+    ):
+        previous_inflation = 7
+        params = EstimateResourcesParams.of(
+            resource_estimation_method=ResourceEstimationMethod.FILE_SAMPLING_WITH_PREVIOUS_INFLATION,
+            max_files_to_sample=None,
+            previous_inflation=previous_inflation,
+        )
+
+        result = estimate_resources_required_to_process_delta(
+            delta=utsv_delta_with_manifest,
+            operation_type=OperationType.PYARROW_DOWNLOAD,
+            deltacat_storage=ds,
+            deltacat_storage_kwargs=local_deltacat_storage_kwargs,
+            estimate_resources_params=params,
+        )
+        assert result is not None
+        assert result.memory_bytes == (
+            utsv_delta_with_manifest.meta.content_length * previous_inflation
+        )
+
     def test_empty_delta_when_default_v2(
         self, local_deltacat_storage_kwargs, delta_without_manifest: Delta
     ):

{deltacat-1.1.36.dist-info → deltacat-1.1.37.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.36
+Version: 1.1.37
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.36.dist-info → deltacat-1.1.37.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=9vJMHGceWew6atD_3VqKurlBJ3crD5mwAQIgSB1yjNY,1778
+deltacat/__init__.py,sha256=u00X92zHfZJzS08a-2kx3kCLcz40L-THm0HowDiBOiA,1778
 deltacat/constants.py,sha256=TUJLXUJ9xq1Ryil72yLkKR8EDH_Irp5wUg56QstbRNE,2181
 deltacat/exceptions.py,sha256=7sjk3BuMY5Oo-6OvAfHncZx_OcvtEL47BblWr2F7waE,12740
 deltacat/logs.py,sha256=EQSDin1deehzz5xlLV1_TrFJrO_IBZ9Ahp7MdL-4cK8,9363
@@ -76,7 +76,7 @@ deltacat/compute/compactor_v2/utils/dedupe.py,sha256=Jz1QbBOdZJwT8K1vD9q01eOn7hd
 deltacat/compute/compactor_v2/utils/delta.py,sha256=I7Yvda8NVbpKXG3nM2Ku1utvR2r2OpHvUMqUL2ja3aw,3626
 deltacat/compute/compactor_v2/utils/io.py,sha256=Xjs7_D-0xKSetvllIe4o96aM1elfdjt1Ii7YfsHPvZs,6108
 deltacat/compute/compactor_v2/utils/merge.py,sha256=fAzEYwQYH2ia8MLdEFdZFivWHpi6qZu8AyyEK0H0vwE,5363
-deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=SbQ97M1Cxld-zZik2QMSzlj20g6JlENaQx_0PhlCIP8,12034
+deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=Qsn0BQrlBWSLqu4srd-LJUX8BaVqG6Wo1oAros7LYWw,12677
 deltacat/compute/compactor_v2/utils/task_options.py,sha256=0GoB_DLkCN1q8CVKTlWlDYt55qnpTDIa9fPyXJwB-cU,13801
 deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
 deltacat/compute/merge_on_read/daft.py,sha256=1oC38u5ig_aTrq7EzyWBo8Ui54rb6yERYMk-vEFbpxM,1400
@@ -85,9 +85,9 @@ deltacat/compute/merge_on_read/model/merge_on_read_params.py,sha256=Q51znagh8PtL
 deltacat/compute/merge_on_read/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/merge_on_read/utils/delta.py,sha256=e4BtOHa5XPpUnR4r0HqBKjXckBsTI8qBwdUWwpJfkWQ,1367
 deltacat/compute/resource_estimation/__init__.py,sha256=4bfBXcq-VAt9JCmjvj3yAmn0lEHVGdGsUCCoMGxjEqA,799
-deltacat/compute/resource_estimation/delta.py,sha256=dN64jbUQ8OI1BTz4fYGbulJLWjKjdT-XvwDJNLM__Oo,10583
+deltacat/compute/resource_estimation/delta.py,sha256=zd1ivoA3EzdrjgJYYBXY3wrhwZDlt-Xoqke0e5xz6AY,10815
 deltacat/compute/resource_estimation/manifest.py,sha256=gSqOyIda-pYq3vRsKFq3IiZvwhV3mMqrWPtsmUH9dD8,13035
-deltacat/compute/resource_estimation/model.py,sha256=psyagFXdpLGt8DfDqy7c8DWiuXCacr0Swe5f0M7DdO4,5465
+deltacat/compute/resource_estimation/model.py,sha256=1svgVfhNIAyyVkHy-QXcOzO0UVigbVH8M7xyAlgvCbg,5741
 deltacat/compute/resource_estimation/parquet.py,sha256=5_apma4EKbKcm-nfV73-qN2nfnCeyhFW23ZHX3jz0Kw,3158
 deltacat/compute/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/stats/types.py,sha256=cp0lT8nITTKbnkc03OysRjXfcfXzQml9a4wqCnR6kqs,215
@@ -159,7 +159,7 @@ deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py,sha256=eoi
 deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py,sha256=aFb9rzT_EK9k8qAMHPtpqd5btyEmll1So1loDmZkotQ,1769
 deltacat/tests/compute/compactor_v2/utils/test_task_options.py,sha256=YDQKUKv3Vv8S1fe0YQmjHTrwnWSliqKHIWGu0fEdKnI,11478
 deltacat/tests/compute/resource_estimation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/tests/compute/resource_estimation/test_delta.py,sha256=HCL2oUnCqm0E26T3HLJjMhoAsHTJIWPYGwIKRgM_H7E,25712
+deltacat/tests/compute/resource_estimation/test_delta.py,sha256=vbqKwZOxrNtfbuXWz08nUvi_srR4y2aMQmUwLR2jDcs,28446
 deltacat/tests/compute/resource_estimation/test_manifest.py,sha256=yrMvqDjolExdRf6Vtg5XaKDuaKz9ok15PCZ7_aJOYrI,32893
 deltacat/tests/compute/resource_estimation/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -212,8 +212,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.36.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.36.dist-info/METADATA,sha256=wIZbEGHnJWq_TBKi0u463p4-PgG9R_0MApw7IIwmnRc,1733
-deltacat-1.1.36.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-deltacat-1.1.36.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.36.dist-info/RECORD,,
+deltacat-1.1.37.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.37.dist-info/METADATA,sha256=iHlaZ9sS-CrQby0kxCrOigl1ZGZKpniwf9LyYbagwzI,1733
+deltacat-1.1.37.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+deltacat-1.1.37.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.37.dist-info/RECORD,,