deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/compactor_v2/compaction_session.py

@@ -14,7 +14,6 @@ from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
     ExecutionCompactionResult,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
-from deltacat.compute.compactor.utils import round_completion_file as rcf
 from deltacat.compute.compactor import DeltaAnnotated
 from deltacat.compute.compactor_v2.deletes.delete_strategy import (
     DeleteStrategy,
@@ -27,9 +26,9 @@ from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
 from deltacat.storage import (
     Delta,
     DeltaLocator,
-    Manifest,
-    Partition,
+    PartitionLocator,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.compute.compactor.model.compact_partition_params import (
     CompactPartitionParams,
 )
@@ -37,13 +36,14 @@ from deltacat.utils.resources import (
     get_current_process_peak_memory_usage_in_bytes,
 )
 from deltacat.compute.compactor_v2.private.compaction_utils import (
+    _get_rci_source_partition_locator,
     _fetch_compaction_metadata,
     _build_uniform_deltas,
     _group_uniform_deltas,
     _stage_new_partition,
     _run_hash_and_merge,
     _process_merge_results,
-    _write_new_round_completion_file,
+    _create_round_completion_info,
     _commit_compaction_result,
 )
 from deltacat.utils.metrics import metrics
@@ -65,7 +65,7 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 @metrics(prefix=COMPACT_PARTITION_METRIC_PREFIX)
 @categorize_errors
-def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
+def compact_partition(params: CompactPartitionParams, **kwargs) -> None:
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
     ), "hash_bucket_count is a required arg for compactor v2"
@@ -85,7 +85,6 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
             **kwargs,
         )
         _commit_compaction_result(params, execute_compaction_result)
-        return execute_compaction_result.round_completion_file_s3_url
 
 
 def _execute_compaction(
@@ -100,12 +99,12 @@ def _execute_compaction(
         previous_compacted_delta_manifest,
         round_completion_info,
     ) = fetch_compaction_metadata_result
-    rcf_source_partition_locator: rcf.PartitionLocator = (
-        params.rebase_source_partition_locator or params.source_partition_locator
+    rci_source_partition_locator: PartitionLocator = _get_rci_source_partition_locator(
+        params
     )
 
-    base_audit_url: str = rcf_source_partition_locator.path(
-        f"s3://{params.compaction_artifact_s3_bucket}/compaction-audit"
+    base_audit_url: str = rci_source_partition_locator.path(
+        f"{params.compaction_artifact_path}/compaction-audit"
     )
     audit_url: str = f"{base_audit_url}.json"
     logger.info(f"Compaction audit will be written to {audit_url}")
@@ -140,9 +139,9 @@ def _execute_compaction(
     )
     if not input_deltas:
         logger.info("No input deltas found to compact.")
-        return ExecutionCompactionResult(None, None, None, False)
+        return ExecutionCompactionResult(None, None, False)
     build_uniform_deltas_result: tuple[
-        List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope], Partition
+        List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope]
     ] = _build_uniform_deltas(
         params, compaction_audit, input_deltas, delta_discovery_start
     )
@@ -203,13 +202,13 @@ def _execute_compaction(
 
     compaction_audit.save_round_completion_stats(mat_results)
 
-    compaction_result: ExecutionCompactionResult = _write_new_round_completion_file(
+    compaction_result: ExecutionCompactionResult = _create_round_completion_info(
         params,
         compaction_audit,
         compacted_partition,
         audit_url,
         hb_id_to_entry_indices_range,
-        rcf_source_partition_locator,
+        rci_source_partition_locator,
         new_compacted_delta_locator,
         pyarrow_write_result,
         round_completion_info,

deltacat/compute/compactor_v2/deletes/utils.py

@@ -49,7 +49,7 @@ def _aggregate_delete_deltas(input_deltas: List[Delta]) -> Dict[int, List[Delta]
     ] = [
         (is_delete, list(delete_delta_group))
         for (is_delete, _), delete_delta_group in itertools.groupby(
-            input_deltas, lambda d: (d.type is DeltaType.DELETE, d.delete_parameters)
+            input_deltas, lambda d: (d.type is DeltaType.DELETE, d.meta.entry_params)
         )
     ]
     for (
@@ -89,11 +89,11 @@ def _get_delete_file_envelopes(
         consecutive_delete_tables: List[pa.Table] = []
         for delete_delta in delete_delta_sequence:
             assert (
-                delete_delta.delete_parameters is not None
+                delete_delta.meta.entry_params is not None
             ), "Delete type deltas are required to have delete parameters defined"
             delete_columns: Optional[
                 List[str]
-            ] = delete_delta.delete_parameters.equality_column_names
+            ] = delete_delta.meta.entry_params.equality_field_locators
             assert len(delete_columns) > 0, "At least 1 delete column is required"
             # delete columns should exist in underlying table
             delete_dataset = params.deltacat_storage.download_delta(

deltacat/compute/compactor_v2/model/evaluate_compaction_result.py

@@ -13,7 +13,6 @@ from typing import Optional
 class ExecutionCompactionResult:
     new_compacted_partition: Optional[Partition]
     new_round_completion_info: Optional[RoundCompletionInfo]
-    round_completion_file_s3_url: Optional[str]
     is_inplace_compacted: bool
 
     def __iter__(self):

deltacat/compute/compactor_v2/model/hash_bucket_input.py

@@ -4,7 +4,7 @@ from typing import Dict, List, Optional, Any
 from deltacat.utils.metrics import MetricsConfig
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.io.object_store import IObjectStore
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.compute.compactor import DeltaAnnotated
 
 
@@ -15,12 +15,13 @@ class HashBucketInput(Dict):
         primary_keys: List[str],
         num_hash_buckets: int,
         num_hash_groups: int,
+        all_column_names: List[str],
         hb_task_index: Optional[int] = 0,
         enable_profiler: Optional[bool] = False,
         metrics_config: Optional[MetricsConfig] = None,
         read_kwargs_provider: Optional[ReadKwargsProvider] = None,
         object_store: Optional[IObjectStore] = None,
-        deltacat_storage=unimplemented_deltacat_storage,
+        deltacat_storage=metastore,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
     ) -> HashBucketInput:
@@ -31,6 +32,7 @@ class HashBucketInput(Dict):
         result["hb_task_index"] = hb_task_index
         result["num_hash_buckets"] = num_hash_buckets
         result["num_hash_groups"] = num_hash_groups
+        result["all_column_names"] = all_column_names
         result["enable_profiler"] = enable_profiler
         result["metrics_config"] = metrics_config
         result["read_kwargs_provider"] = read_kwargs_provider
@@ -61,6 +63,10 @@ class HashBucketInput(Dict):
     def num_hash_groups(self) -> int:
         return self["num_hash_groups"]
 
+    @property
+    def all_column_names(self) -> List[str]:
+        return self["all_column_names"]
+
     @property
     def enable_profiler(self) -> Optional[bool]:
         return self.get("enable_profiler")
@@ -78,7 +84,7 @@ class HashBucketInput(Dict):
         return self.get("object_store")
 
     @property
-    def deltacat_storage(self) -> unimplemented_deltacat_storage:
+    def deltacat_storage(self) -> metastore:
         return self.get("deltacat_storage")
 
     @property

deltacat/compute/compactor_v2/model/merge_file_group.py

@@ -16,7 +16,7 @@ from deltacat.compute.compactor_v2.utils.primary_key_index import (
     hash_group_index_to_hash_bucket_indices,
 )
 
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 
 from deltacat.io.object_store import IObjectStore
 
@@ -87,11 +87,13 @@ class LocalMergeFileGroupsProvider(MergeFileGroupsProvider):
     def __init__(
         self,
         uniform_deltas: List[DeltaAnnotated],
+        all_column_names: List[str],
         read_kwargs_provider: Optional[ReadKwargsProvider],
-        deltacat_storage=unimplemented_deltacat_storage,
+        deltacat_storage=metastore,
         deltacat_storage_kwargs: Optional[dict] = None,
     ):
         self._deltas = uniform_deltas
+        self._all_column_names = all_column_names
         self._read_kwargs_provider = read_kwargs_provider
         self._deltacat_storage = deltacat_storage
         self._deltacat_storage_kwargs = deltacat_storage_kwargs
@@ -110,6 +112,7 @@ class LocalMergeFileGroupsProvider(MergeFileGroupsProvider):
                 total_size_bytes,
             ) = read_delta_file_envelopes(
                 annotated_delta,
+                self._all_column_names,
                 self._read_kwargs_provider,
                 self._deltacat_storage,
                 self._deltacat_storage_kwargs,

deltacat/compute/compactor_v2/model/merge_input.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Dict, List, Optional, Any
+from typing import Dict, List, Optional, Any, Set
 
 from deltacat.compute.compactor_v2.model.merge_file_group import (
     MergeFileGroupsProvider,
@@ -12,9 +12,10 @@ from deltacat.utils.metrics import MetricsConfig
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.io.object_store import IObjectStore
 from deltacat.storage import (
+    Manifest,
     Partition,
     SortKey,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat.compute.compactor_v2.constants import (
     DROP_DUPLICATES,
@@ -32,23 +33,26 @@ class MergeInput(Dict):
         write_to_partition: Partition,
         compacted_file_content_type: ContentType,
         primary_keys: List[str],
+        all_column_names: List[str],
         drop_duplicates: Optional[bool] = DROP_DUPLICATES,
         sort_keys: Optional[List[SortKey]] = None,
         merge_task_index: Optional[int] = 0,
         max_records_per_output_file: Optional[int] = MAX_RECORDS_PER_COMPACTED_FILE,
         enable_profiler: Optional[bool] = False,
         metrics_config: Optional[MetricsConfig] = None,
-        s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+        table_writer_kwargs: Optional[Dict[str, Any]] = None,
         read_kwargs_provider: Optional[ReadKwargsProvider] = None,
         round_completion_info: Optional[RoundCompletionInfo] = None,
         object_store: Optional[IObjectStore] = None,
         delete_strategy: Optional[DeleteStrategy] = None,
-        delete_file_envelopes: Optional[List] = None,
-        deltacat_storage=unimplemented_deltacat_storage,
+        delete_file_envelopes: Optional[List[DeleteFileEnvelope]] = None,
+        deltacat_storage=metastore,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
         disable_copy_by_reference: Optional[bool] = None,
         hash_bucket_count: Optional[int] = None,
+        original_fields: Optional[Set[str]] = None,
+        compacted_manifest: Optional[Manifest] = None,
     ) -> MergeInput:
 
         result = MergeInput()
@@ -56,13 +60,14 @@ class MergeInput(Dict):
         result["write_to_partition"] = write_to_partition
         result["compacted_file_content_type"] = compacted_file_content_type
         result["primary_keys"] = primary_keys
+        result["all_column_names"] = all_column_names
         result["drop_duplicates"] = drop_duplicates
         result["sort_keys"] = sort_keys
         result["merge_task_index"] = merge_task_index
         result["max_records_per_output_file"] = max_records_per_output_file
         result["enable_profiler"] = enable_profiler
         result["metrics_config"] = metrics_config
-        result["s3_table_writer_kwargs"] = s3_table_writer_kwargs or {}
+        result["table_writer_kwargs"] = table_writer_kwargs or {}
         result["read_kwargs_provider"] = read_kwargs_provider
         result["round_completion_info"] = round_completion_info
         result["object_store"] = object_store
@@ -73,6 +78,8 @@ class MergeInput(Dict):
         result["memory_logs_enabled"] = memory_logs_enabled
         result["disable_copy_by_reference"] = disable_copy_by_reference
         result["hash_bucket_count"] = hash_bucket_count
+        result["original_fields"] = original_fields
+        result["compacted_manifest"] = compacted_manifest
         return result
 
     @property
@@ -91,6 +98,10 @@ class MergeInput(Dict):
     def primary_keys(self) -> List[str]:
         return self["primary_keys"]
 
+    @property
+    def all_column_names(self) -> List[str]:
+        return self["all_column_names"]
+
     @property
     def drop_duplicates(self) -> int:
         return self["drop_duplicates"]
@@ -116,8 +127,8 @@ class MergeInput(Dict):
         return self.get("metrics_config")
 
     @property
-    def s3_table_writer_kwargs(self) -> Optional[Dict[str, Any]]:
-        return self.get("s3_table_writer_kwargs")
+    def table_writer_kwargs(self) -> Optional[Dict[str, Any]]:
+        return self.get("table_writer_kwargs")
 
     @property
     def read_kwargs_provider(self) -> Optional[ReadKwargsProvider]:
@@ -132,7 +143,7 @@ class MergeInput(Dict):
         return self.get("object_store")
 
     @property
-    def deltacat_storage(self) -> unimplemented_deltacat_storage:
+    def deltacat_storage(self) -> metastore:
         return self["deltacat_storage"]
 
     @property
@@ -160,3 +171,11 @@ class MergeInput(Dict):
     @property
     def hash_bucket_count(self) -> int:
         return self["hash_bucket_count"]
+
+    @property
+    def original_fields(self) -> Optional[Set[str]]:
+        return self.get("original_fields")
+
+    @property
+    def compacted_manifest(self) -> Optional[Manifest]:
+        return self.get("compacted_manifest")