deltacat 1.1.36__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

deltacat/tests/catalog/test_default_catalog_impl.py

@@ -1,20 +1,23 @@
 import unittest
 import sqlite3
+import uuid
+
 import ray
 import os
 import deltacat.tests.local_deltacat_storage as ds
+from deltacat import Catalog
+from deltacat.catalog import CatalogProperties
 from deltacat.utils.common import current_time_ms
 from deltacat.tests.test_utils.pyarrow import (
     create_delta_from_csv_file,
     commit_delta_to_partition,
 )
 from deltacat.types.media import DistributedDatasetType, ContentType
-from deltacat.catalog import default_catalog_impl as dc
+import deltacat as dc
 
 
 class TestReadTable(unittest.TestCase):
     READ_TABLE_NAMESPACE = "catalog_read_table_namespace"
-    LOCAL_CATALOG_NAME = "local_catalog"
     DB_FILE_PATH = f"{current_time_ms()}.db"
     SAMPLE_FILE_PATH = "deltacat/tests/catalog/data/sample_table.csv"
 
@@ -31,6 +34,13 @@ class TestReadTable(unittest.TestCase):
         }
         cls.deltacat_storage_kwargs = {ds.DB_FILE_PATH_ARG: cls.DB_FILE_PATH}
 
+        cls.catalog_name = str(uuid.uuid4())
+        catalog_config = CatalogProperties(storage=ds)
+        dc.put_catalog(
+            cls.catalog_name,
+            catalog=Catalog.default(config=catalog_config),
+            ray_init_args={"ignore_reinit_error": True},
+        )
         super().setUpClass()
 
     @classmethod
@@ -49,13 +59,12 @@ class TestReadTable(unittest.TestCase):
             **self.kwargs,
         )
 
-        dc.initialize(ds=ds)
         df = dc.read_table(
             table=READ_TABLE_TABLE_NAME,
             namespace=self.READ_TABLE_NAMESPACE,
-            catalog=self.LOCAL_CATALOG_NAME,
+            catalog=self.catalog_name,
             distributed_dataset_type=DistributedDatasetType.DAFT,
-            deltacat_storage_kwargs=self.kwargs,
+            **self.kwargs,
         )
 
         # verify
@@ -81,14 +90,13 @@ class TestReadTable(unittest.TestCase):
         )
 
         # action
-        dc.initialize(ds=ds)
         df = dc.read_table(
             table=READ_TABLE_TABLE_NAME,
             namespace=self.READ_TABLE_NAMESPACE,
-            catalog=self.LOCAL_CATALOG_NAME,
+            catalog=self.catalog_name,
             distributed_dataset_type=DistributedDatasetType.DAFT,
             merge_on_read=False,
-            deltacat_storage_kwargs=self.kwargs,
+            **self.kwargs,
         )
 
         # verify

deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py

@@ -15,7 +15,7 @@ from deltacat.exceptions import ValidationError
 
 from deltacat.storage import (
     DeltaType,
-    DeleteParameters,
+    EntryParams,
 )
 
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
@@ -36,15 +36,15 @@ class MultipleRoundsTestCaseParams:
     Args:
         primary_keys: Set[str] - argument for the primary_keys parameter in compact_partition. Also needed for table/delta creation
         sort_keys: List[SortKey] - argument for the sort_keys parameter in compact_partition. Also needed for table/delta creation
-        partition_keys_param: List[PartitionKey] - argument for the partition_keys parameter. Needed for table/delta creation
-        partition_values_param: List[Optional[str]] - argument for the partition_valued parameter. Needed for table/delta creation
+        partition_keys: List[PartitionKey] - argument for the partition_keys parameter. Needed for table/delta creation
+        partition_values: List[Optional[str]] - argument for the partition_valued parameter. Needed for table/delta creation
         input_deltas: List[pa.Array] - argument required for delta creation during compact_partition test setup. Actual incoming deltas expressed as a PyArrow array (https://arrow.apache.org/docs/python/generated/pyarrow.array.html)
         expected_terminal_compact_partition_result: pa.Table - expected PyArrow table after compaction (i.e,. the state of the table after applying all row UPDATES/DELETES/INSERTS)
         expected_terminal_exception: BaseException - expected exception during compaction
         expected_terminal_exception_message: Optional[str] - expected exception message if present.
         do_create_placement_group: bool - toggles whether to create a placement group (https://docs.ray.io/en/latest/ray-core/scheduling/placement-group.html) or not
         records_per_compacted_file: int - argument for the records_per_compacted_file parameter in compact_partition
-        hash_bucket_count_param: int - argument for the hash_bucket_count parameter in compact_partition
+        hash_bucket_count: int - argument for the hash_bucket_count parameter in compact_partition
         read_kwargs_provider: Optional[ReadKwargsProvider] - argument for read_kwargs_provider parameter in compact_partition. If None then no ReadKwargsProvider is provided to compact_partition_params
         drop_duplicates: bool - argument for drop_duplicates parameter in compact_partition. Only recognized by compactor v2.
         skip_enabled_compact_partition_drivers: List[CompactorVersion] - skip whatever enabled_compact_partition_drivers are included in this list
@@ -57,7 +57,7 @@ class MultipleRoundsTestCaseParams:
     sort_keys: List[Optional[SortKey]]
     partition_keys: Optional[List[PartitionKey]]
     partition_values: List[Optional[str]]
-    input_deltas: Union[List[pa.Array], DeltaType, DeleteParameters]
+    input_deltas: Union[List[pa.Array], DeltaType]
     expected_terminal_compact_partition_result: pa.Table
     expected_terminal_exception: BaseException
     expected_terminal_exception_message: str
@@ -83,8 +83,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
     "1-multiple-rounds-sanity": MultipleRoundsTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
-            SortKey.of(key_name="sk_col_2"),
+            SortKey.of(key=["sk_col_1"]),
+            SortKey.of(key=["sk_col_2"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -177,8 +177,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
     "2-multiple-rounds-unique-values": MultipleRoundsTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
-            SortKey.of(key_name="sk_col_2"),
+            SortKey.of(key=["sk_col_1"]),
+            SortKey.of(key=["sk_col_2"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -270,8 +270,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
     "3-num-rounds-greater-than-deltas-count": MultipleRoundsTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
-            SortKey.of(key_name="sk_col_2"),
+            SortKey.of(key=["sk_col_1"]),
+            SortKey.of(key=["sk_col_2"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -364,8 +364,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
     "4-multiple-rounds-hb-count-equals-1": MultipleRoundsTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
-            SortKey.of(key_name="sk_col_2"),
+            SortKey.of(key=["sk_col_1"]),
+            SortKey.of(key=["sk_col_2"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -458,8 +458,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
     "5-multiple-rounds-only-supports-rebase": MultipleRoundsTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
-            SortKey.of(key_name="sk_col_2"),
+            SortKey.of(key=["sk_col_1"]),
+            SortKey.of(key=["sk_col_2"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -553,8 +553,8 @@ MULTIPLE_ROUNDS_TEST_CASES = {
     "6-multiple-rounds-test-pgm": MultipleRoundsTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
-            SortKey.of(key_name="sk_col_2"),
+            SortKey.of(key=["sk_col_1"]),
+            SortKey.of(key=["sk_col_2"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -690,7 +690,7 @@ MULTIPLE_ROUNDS_TEST_CASES = {
                     names=["pk_col_1", "col_1"],
                 ),
                 DeltaType.DELETE,
-                DeleteParameters.of(["pk_col_1", "col_1"]),
+                EntryParams.of(equality_field_locators=["pk_col_1", "col_1"]),
             ),
         ],
         rebase_expected_compact_partition_result=pa.Table.from_arrays(
@@ -758,7 +758,7 @@ MULTIPLE_ROUNDS_TEST_CASES = {
                     names=["pk_col_1", "col_1"],
                 ),
                 DeltaType.DELETE,
-                DeleteParameters.of(["pk_col_1", "col_1"]),
+                EntryParams.of(equality_field_locators=["pk_col_1", "col_1"]),
             ),
             (
                 pa.Table.from_arrays(
@@ -766,7 +766,7 @@ MULTIPLE_ROUNDS_TEST_CASES = {
                     names=["pk_col_1", "col_1"],
                 ),
                 DeltaType.DELETE,
-                DeleteParameters.of(["pk_col_1", "col_1"]),
+                EntryParams.of(equality_field_locators=["pk_col_1", "col_1"]),
             ),
             (
                 pa.Table.from_arrays(
@@ -897,7 +897,7 @@ MULTIPLE_ROUNDS_TEST_CASES = {
                     names=["pk_col_1", "col_1"],
                 ),
                 DeltaType.DELETE,
-                DeleteParameters.of(["pk_col_1", "col_1"]),
+                EntryParams.of(["pk_col_1", "col_1"]),
             ),
         ],
         rebase_expected_compact_partition_result=pa.Table.from_arrays(

deltacat/tests/compute/compact_partition_rebase_test_cases.py

@@ -41,8 +41,8 @@ REBASE_TEST_CASES = {
     "1-rebase-sanity": RebaseCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
-            SortKey.of(key_name="sk_col_2"),
+            SortKey.of(key=["sk_col_1"]),
+            SortKey.of(key=["sk_col_2"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -87,8 +87,8 @@ REBASE_TEST_CASES = {
     "2-rebase-with-null-pk": RebaseCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
-            SortKey.of(key_name="sk_col_2"),
+            SortKey.of(key=["sk_col_1"]),
+            SortKey.of(key=["sk_col_2"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -133,7 +133,7 @@ REBASE_TEST_CASES = {
     "3-rebase-with-null-two-pk": RebaseCompactionTestCaseParams(
         primary_keys={"pk_col_1", "pk_col_2"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
+            SortKey.of(key=["sk_col_1"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],
@@ -307,7 +307,7 @@ REBASE_TEST_CASES = {
     "7-rebase-drop-duplicates-false": RebaseCompactionTestCaseParams(
         primary_keys={"pk_col_1"},
         sort_keys=[
-            SortKey.of(key_name="sk_col_1"),
+            SortKey.of(key=["sk_col_1"]),
         ],
         partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
         partition_values=["1"],