deltacat 1.1.35__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/tests/compute/test_compact_partition_incremental.py

@@ -50,11 +50,6 @@ from deltacat.utils.placement import (
 )
 from deltacat import logs
 
-DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
-    "db_file_path",
-    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
-)
-
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
@@ -80,14 +75,6 @@ def mock_aws_credential():
     yield
 
 
-@pytest.fixture(autouse=True, scope="module")
-def cleanup_the_database_file_after_all_compaction_session_package_tests_complete():
-    # make sure the database file is deleted after all the compactor package tests are completed
-    yield
-    if os.path.exists(DATABASE_FILE_PATH_VALUE):
-        os.remove(DATABASE_FILE_PATH_VALUE)
-
-
 @pytest.fixture(scope="module")
 def s3_resource():
     with mock_s3():
@@ -108,32 +95,6 @@ FUNCTION scoped fixtures
 """
 
 
-@pytest.fixture(scope="function")
-def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
-    # see deltacat/tests/local_deltacat_storage/README.md for documentation
-    kwargs_for_local_deltacat_storage: Dict[str, Any] = {
-        DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
-    }
-    yield kwargs_for_local_deltacat_storage
-    if os.path.exists(DATABASE_FILE_PATH_VALUE):
-        os.remove(DATABASE_FILE_PATH_VALUE)
-
-
-@pytest.fixture(autouse=True, scope="function")
-def enable_bucketing_spec_validation(monkeypatch):
-    """
-    Enable the bucketing spec validation for all tests.
-    This will help catch hash bucket drift in testing.
-    """
-    import deltacat.compute.compactor_v2.steps.merge
-
-    monkeypatch.setattr(
-        deltacat.compute.compactor_v2.steps.merge,
-        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
-        "ASSERT",
-    )
-
-
 @pytest.mark.parametrize(
     [
         "test_name",
@@ -209,7 +170,7 @@ def enable_bucketing_spec_validation(monkeypatch):
 )
 def test_compact_partition_incremental(
     s3_resource: ServiceResource,
-    offer_local_deltacat_storage_kwargs: Dict[str, Any],
+    local_deltacat_storage_kwargs: Dict[str, Any],
     test_name: str,
     primary_keys: Set[str],
     sort_keys: Dict[str, str],
@@ -235,7 +196,7 @@ def test_compact_partition_incremental(
 ):
     import deltacat.tests.local_deltacat_storage as ds
 
-    ds_mock_kwargs: Dict[str, Any] = offer_local_deltacat_storage_kwargs
+    ds_mock_kwargs: Dict[str, Any] = local_deltacat_storage_kwargs
 
     # setup
     partition_keys = partition_keys_param
@@ -247,7 +208,6 @@ def test_compact_partition_incremental(
         source_table_name,
         source_table_version,
     ) = create_src_w_deltas_destination_plus_destination(
-        primary_keys,
         sort_keys,
         partition_keys,
         input_deltas,

deltacat/tests/compute/test_compact_partition_multiple_rounds.py

@@ -48,12 +48,6 @@ from deltacat.utils.placement import (
     PlacementGroupManager,
 )
 
-DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
-    "db_file_path",
-    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
-)
-
-
 """
 MODULE scoped fixtures
 """
@@ -76,13 +70,6 @@ def mock_aws_credential():
     yield
 
 
-@pytest.fixture(autouse=True, scope="module")
-def cleanup_the_database_file_after_all_compaction_session_package_tests_complete():
-    # make sure the database file is deleted after all the compactor package tests are completed
-    if os.path.exists(DATABASE_FILE_PATH_VALUE):
-        os.remove(DATABASE_FILE_PATH_VALUE)
-
-
 @pytest.fixture(scope="module")
 def s3_resource(mock_aws_credential):
     with mock_s3():
@@ -98,37 +85,6 @@ def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
     yield
 
 
-"""
-FUNCTION scoped fixtures
-"""
-
-
-@pytest.fixture(scope="function")
-def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
-    # see deltacat/tests/local_deltacat_storage/README.md for documentation
-    kwargs_for_local_deltacat_storage: Dict[str, Any] = {
-        DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
-    }
-    yield kwargs_for_local_deltacat_storage
-    if os.path.exists(DATABASE_FILE_PATH_VALUE):
-        os.remove(DATABASE_FILE_PATH_VALUE)
-
-
-@pytest.fixture(autouse=True, scope="function")
-def enable_bucketing_spec_validation(monkeypatch):
-    """
-    Enable the bucketing spec validation for all tests.
-    This will help catch hash bucket drift in testing.
-    """
-    import deltacat.compute.compactor_v2.steps.merge
-
-    monkeypatch.setattr(
-        deltacat.compute.compactor_v2.steps.merge,
-        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
-        "ASSERT",
-    )
-
-
 @pytest.mark.parametrize(
     [
         "test_name",
@@ -240,7 +196,6 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
         rebased_table_stream,
         _,
     ) = multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy(
-        primary_keys,
         sort_keys,
         partition_keys_param,
         input_deltas_param,
@@ -347,7 +302,11 @@ def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
         actual_rebase_compacted_table = pa.concat_tables(tables)
         # if no primary key is specified then sort by sort_key for consistent assertion
         sorting_cols: List[Any] = (
-            [(val, "ascending") for val in primary_keys] if primary_keys else sort_keys
+            [(val, "ascending") for val in primary_keys]
+            if primary_keys
+            else [pa_key for key in sort_keys for pa_key in key.arrow]
+            if sort_keys
+            else []
         )
         rebase_expected_compact_partition_result = (
             rebase_expected_compact_partition_result.combine_chunks().sort_by(

deltacat/tests/compute/test_compact_partition_params.py

@@ -23,7 +23,7 @@ class TestCompactPartitionParams(unittest.TestCase):
                         "tableVersion": "1",
                     },
                     "streamId": "foobar",
-                    "storageType": "fooType",
+                    "format": "fooType",
                 },
                 "partitionValues": [],
                 "partitionId": None,
@@ -47,7 +47,7 @@ class TestCompactPartitionParams(unittest.TestCase):
                         "table_version": "1",
                     },
                     "streamId": "foobar",
-                    "storageType": "fooType",
+                    "format": "fooType",
                 },
                 "partitionValues": [],
                 "partitionId": "79612ea39ac5493eae925abe60767d42",
@@ -67,7 +67,7 @@ class TestCompactPartitionParams(unittest.TestCase):
                         "tableVersion": "2",
                     },
                     "streamId": "foobar",
-                    "storageType": "fooType",
+                    "format": "fooType",
                 },
                 "partitionValues": [],
                 "partitionId": "79612ea39ac5493eae925abe60767d42",

deltacat/tests/compute/test_compact_partition_rebase.py

@@ -48,12 +48,6 @@ from deltacat.utils.placement import (
     PlacementGroupManager,
 )
 
-DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
-    "db_file_path",
-    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
-)
-
-
 """
 MODULE scoped fixtures
 """
@@ -76,13 +70,6 @@ def mock_aws_credential():
     yield
 
 
-@pytest.fixture(autouse=True, scope="module")
-def cleanup_the_database_file_after_all_compaction_session_package_tests_complete():
-    # make sure the database file is deleted after all the compactor package tests are completed
-    if os.path.exists(DATABASE_FILE_PATH_VALUE):
-        os.remove(DATABASE_FILE_PATH_VALUE)
-
-
 @pytest.fixture(scope="module")
 def s3_resource(mock_aws_credential):
     with mock_s3():
@@ -98,37 +85,6 @@ def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
     yield
 
 
-"""
-FUNCTION scoped fixtures
-"""
-
-
-@pytest.fixture(scope="function")
-def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
-    # see deltacat/tests/local_deltacat_storage/README.md for documentation
-    kwargs_for_local_deltacat_storage: Dict[str, Any] = {
-        DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
-    }
-    yield kwargs_for_local_deltacat_storage
-    if os.path.exists(DATABASE_FILE_PATH_VALUE):
-        os.remove(DATABASE_FILE_PATH_VALUE)
-
-
-@pytest.fixture(autouse=True, scope="function")
-def enable_bucketing_spec_validation(monkeypatch):
-    """
-    Enable the bucketing spec validation for all tests.
-    This will help catch hash bucket drift in testing.
-    """
-    import deltacat.compute.compactor_v2.steps.merge
-
-    monkeypatch.setattr(
-        deltacat.compute.compactor_v2.steps.merge,
-        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
-        "ASSERT",
-    )
-
-
 @pytest.mark.parametrize(
     [
         "test_name",
@@ -239,7 +195,6 @@ def test_compact_partition_rebase_same_source_and_destination(
         _,
         rebased_table_stream,
     ) = create_src_w_deltas_destination_rebase_w_deltas_strategy(
-        primary_keys,
         sort_keys,
         partition_keys,
         input_deltas_param,
@@ -351,7 +306,7 @@ def test_compact_partition_rebase_same_source_and_destination(
         if primary_keys:
             sorting_cols.extend([(val, "ascending") for val in primary_keys])
         if sort_keys:
-            sorting_cols.extend(sort_keys)
+            sorting_cols.extend([pa_key for key in sort_keys for pa_key in key.arrow])
 
         rebase_expected_compact_partition_result = (
             rebase_expected_compact_partition_result.combine_chunks().sort_by(

deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -52,12 +52,6 @@ from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
 )
 
-DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
-    "db_file_path",
-    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
-)
-
-
 """
 MODULE scoped fixtures
 """
@@ -80,13 +74,6 @@ def mock_aws_credential():
     yield
 
 
-@pytest.fixture(autouse=True, scope="module")
-def cleanup_the_database_file_after_all_compaction_session_package_tests_complete():
-    # make sure the database file is deleted after all the compactor package tests are completed
-    if os.path.exists(DATABASE_FILE_PATH_VALUE):
-        os.remove(DATABASE_FILE_PATH_VALUE)
-
-
 @pytest.fixture(scope="module")
 def s3_resource(mock_aws_credential):
     with mock_s3():
@@ -102,37 +89,6 @@ def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
     yield
 
 
-"""
-FUNCTION scoped fixtures
-"""
-
-
-@pytest.fixture(scope="function")
-def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
-    # see deltacat/tests/local_deltacat_storage/README.md for documentation
-    kwargs_for_local_deltacat_storage: Dict[str, Any] = {
-        DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
-    }
-    yield kwargs_for_local_deltacat_storage
-    if os.path.exists(DATABASE_FILE_PATH_VALUE):
-        os.remove(DATABASE_FILE_PATH_VALUE)
-
-
-@pytest.fixture(autouse=True, scope="function")
-def enable_bucketing_spec_validation(monkeypatch):
-    """
-    Enable the bucketing spec validation for all tests.
-    This will help catch hash bucket drift in testing.
-    """
-    import deltacat.compute.compactor_v2.steps.merge
-
-    monkeypatch.setattr(
-        deltacat.compute.compactor_v2.steps.merge,
-        "BUCKETING_SPEC_COMPLIANCE_PROFILE",
-        "ASSERT",
-    )
-
-
 @pytest.mark.parametrize(
     [
         "test_name",
@@ -244,7 +200,6 @@ def test_compact_partition_rebase_then_incremental(
         destination_table_stream,
         rebased_table_stream,
     ) = create_src_w_deltas_destination_rebase_w_deltas_strategy(
-        primary_keys,
         sort_keys,
         partition_keys,
         input_deltas_param,
@@ -308,7 +263,11 @@ def test_compact_partition_rebase_then_incremental(
     actual_rebase_compacted_table = pa.concat_tables(tables)
     # if no primary key is specified then sort by sort_key for consistent assertion
     sorting_cols: List[Any] = (
-        [(val, "ascending") for val in primary_keys] if primary_keys else sort_keys
+        [(val, "ascending") for val in primary_keys]
+        if primary_keys
+        else [pa_key for key in sort_keys for pa_key in key.arrow]
+        if sort_keys
+        else []
     )
     rebase_expected_compact_partition_result = (
         rebase_expected_compact_partition_result.combine_chunks().sort_by(sorting_cols)

deltacat/tests/compute/test_util_common.py

@@ -1,7 +1,7 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 from enum import Enum
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Dict, List, Optional
 import datetime as dt
 from boto3.resources.base import ServiceResource
 from datetime import timezone
@@ -27,11 +27,18 @@ from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
 )
 
-from deltacat.storage.model.partition import PartitionLocator
+from deltacat.storage.model.partition import (
+    PartitionLocator,
+    PartitionScheme,
+    PartitionKey as PartitionSchemeKey,
+)
 from deltacat.storage.model.stream import StreamLocator
 from deltacat.storage.model.table_version import TableVersionLocator
 from deltacat.storage.model.table import TableLocator
 from deltacat.storage.model.namespace import NamespaceLocator
+from deltacat.storage.model.sort_key import (
+    SortScheme,
+)
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 
 
@@ -77,7 +84,6 @@ def _create_table(
     namespace: str,
     table_name: str,
     table_version: str,
-    primary_keys: Set[str],
     sort_keys: Optional[List[Any]],
     partition_keys: Optional[List[PartitionKey]],
     ds_mock_kwargs: Optional[Dict[str, Any]],
@@ -86,13 +92,20 @@ def _create_table(
     from deltacat.types.media import ContentType
 
     ds.create_namespace(namespace, {}, **ds_mock_kwargs)
+    partition_scheme = (
+        PartitionScheme.of(
+            [PartitionSchemeKey.of(key.key_name) for key in partition_keys]
+        )
+        if partition_keys
+        else None
+    )
+    sort_scheme = SortScheme.of(sort_keys) if sort_keys else None
     ds.create_table_version(
         namespace,
         table_name,
         table_version,
-        primary_key_column_names=list(primary_keys),
-        sort_keys=sort_keys,
-        partition_keys=partition_keys,
+        sort_keys=sort_scheme,
+        partition_scheme=partition_scheme,
         supported_content_types=[ContentType.PARQUET],
         **ds_mock_kwargs,
     )
@@ -100,7 +113,6 @@ def _create_table(
 
 
 def create_src_table(
-    primary_keys: Set[str],
     sort_keys: Optional[List[Any]],
     partition_keys: Optional[List[PartitionKey]],
     ds_mock_kwargs: Optional[Dict[str, Any]],
@@ -112,7 +124,6 @@ def create_src_table(
         source_namespace,
         source_table_name,
         source_table_version,
-        primary_keys,
         sort_keys,
         partition_keys,
         ds_mock_kwargs,
@@ -120,7 +131,6 @@ def create_src_table(
 
 
 def create_destination_table(
-    primary_keys: Set[str],
     sort_keys: Optional[List[Any]],
     partition_keys: Optional[List[PartitionKey]],
     ds_mock_kwargs: Optional[Dict[str, Any]],
@@ -132,7 +142,6 @@ def create_destination_table(
         destination_namespace,
         destination_table_name,
         destination_table_version,
-        primary_keys,
         sort_keys,
         partition_keys,
         ds_mock_kwargs,
@@ -140,7 +149,6 @@ def create_destination_table(
 
 
 def create_rebase_table(
-    primary_keys: Set[str],
     sort_keys: Optional[List[Any]],
     partition_keys: Optional[List[PartitionKey]],
     ds_mock_kwargs: Optional[Dict[str, Any]],
@@ -152,7 +160,6 @@ def create_rebase_table(
         rebasing_namespace,
         rebasing_table_name,
         rebasing_table_version,
-        primary_keys,
         sort_keys,
         partition_keys,
         ds_mock_kwargs,

deltacat/tests/compute/test_util_create_table_deltas_repo.py

@@ -1,6 +1,6 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
-from typing import Any, Dict, List, Optional, Set, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 import pyarrow as pa
 
 from deltacat.tests.compute.test_util_common import (
@@ -38,7 +38,7 @@ def _add_deltas_to_partition(
             delta_data,
             partition,
             delta_type,
-            delete_parameters=delete_parameters,
+            entry_params=delete_parameters,
             **ds_mock_kwargs,
         )
         incremental_delta = ds.commit_delta(
@@ -87,7 +87,7 @@ def create_incremental_deltas_on_source_table(
                 incremental_data,
                 src_partition,
                 incremental_delta_type,
-                delete_parameters=incremental_delete_parameters,
+                entry_params=incremental_delete_parameters,
                 **ds_mock_kwargs,
             ),
             **ds_mock_kwargs,
@@ -113,7 +113,6 @@ def create_incremental_deltas_on_source_table(
 
 
 def create_src_w_deltas_destination_plus_destination(
-    primary_keys: Set[str],
     sort_keys: Optional[List[Any]],
     partition_keys: Optional[List[PartitionKey]],
     input_deltas: pa.Table,
@@ -125,7 +124,7 @@ def create_src_w_deltas_destination_plus_destination(
     import deltacat.tests.local_deltacat_storage as ds
 
     source_namespace, source_table_name, source_table_version = create_src_table(
-        primary_keys, sort_keys, partition_keys, ds_mock_kwargs
+        sort_keys, partition_keys, ds_mock_kwargs
     )
 
     source_table_stream: Stream = ds.get_stream(
@@ -158,9 +157,7 @@ def create_src_w_deltas_destination_plus_destination(
             destination_table_namespace,
             destination_table_name,
             destination_table_version,
-        ) = create_destination_table(
-            primary_keys, sort_keys, partition_keys, ds_mock_kwargs
-        )
+        ) = create_destination_table(sort_keys, partition_keys, ds_mock_kwargs)
     else:
         # not creating a table as in-place
         destination_table_namespace = source_namespace
@@ -184,7 +181,6 @@ def create_src_w_deltas_destination_plus_destination(
 
 
 def create_src_w_deltas_destination_rebase_w_deltas_strategy(
-    primary_keys: Set[str],
     sort_keys: Optional[List[Any]],
     partition_keys: Optional[List[PartitionKey]],
     input_deltas: pa.Table,
@@ -198,7 +194,7 @@ def create_src_w_deltas_destination_rebase_w_deltas_strategy(
 
     last_stream_position = current_time_ms()
     source_namespace, source_table_name, source_table_version = create_src_table(
-        primary_keys, sort_keys, partition_keys, ds_mock_kwargs
+        sort_keys, partition_keys, ds_mock_kwargs
     )
 
     source_table_stream: Stream = ds.get_stream(
@@ -230,15 +226,13 @@ def create_src_w_deltas_destination_rebase_w_deltas_strategy(
         destination_table_namespace,
         destination_table_name,
         destination_table_version,
-    ) = create_destination_table(
-        primary_keys, sort_keys, partition_keys, ds_mock_kwargs
-    )
+    ) = create_destination_table(sort_keys, partition_keys, ds_mock_kwargs)
     # create the rebase table
     (
         rebase_table_namespace,
         rebase_table_name,
         rebase_table_version,
-    ) = create_rebase_table(primary_keys, sort_keys, partition_keys, ds_mock_kwargs)
+    ) = create_rebase_table(sort_keys, partition_keys, ds_mock_kwargs)
     rebasing_table_stream: Stream = ds.get_stream(
         namespace=rebase_table_namespace,
         table_name=rebase_table_name,
@@ -280,7 +274,6 @@ def create_src_w_deltas_destination_rebase_w_deltas_strategy(
 
 
 def multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy(
-    primary_keys: Set[str],
     sort_keys: Optional[List[Any]],
     partition_keys: Optional[List[PartitionKey]],
     input_deltas: List[pa.Table],
@@ -291,7 +284,7 @@ def multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy(
     from deltacat.storage import Partition, Stream
 
     source_namespace, source_table_name, source_table_version = create_src_table(
-        primary_keys, sort_keys, partition_keys, ds_mock_kwargs
+        sort_keys, partition_keys, ds_mock_kwargs
     )
 
     source_table_stream: Stream = ds.get_stream(
@@ -316,7 +309,7 @@ def multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy(
             input_delta,
             staged_partition,
             input_delta_type,
-            delete_parameters=input_delta_parameters,
+            entry_params=input_delta_parameters,
             **ds_mock_kwargs,
         )
         ds.commit_delta(
@@ -336,15 +329,13 @@ def multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy(
         destination_table_namespace,
         destination_table_name,
         destination_table_version,
-    ) = create_destination_table(
-        primary_keys, sort_keys, partition_keys, ds_mock_kwargs
-    )
+    ) = create_destination_table(sort_keys, partition_keys, ds_mock_kwargs)
     # create the rebase table
     (
         rebase_table_namespace,
         rebase_table_name,
         rebase_table_version,
-    ) = create_rebase_table(primary_keys, sort_keys, partition_keys, ds_mock_kwargs)
+    ) = create_rebase_table(sort_keys, partition_keys, ds_mock_kwargs)
     rebasing_table_stream: Stream = ds.get_stream(
         namespace=rebase_table_namespace,
         table_name=rebase_table_name,
@@ -366,7 +357,7 @@ def multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy(
             input_delta,
             staged_partition,
             input_delta_type,
-            delete_parameters=input_delta_parameters,
+            entry_params=input_delta_parameters,
             **ds_mock_kwargs,
         )
         ds.commit_delta(