deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/compactor_v2/steps/merge.py

@@ -11,7 +11,7 @@ from deltacat.utils.pyarrow import MAX_INT_BYTES
 import deltacat.compute.compactor_v2.utils.merge as merge_utils
 from uuid import uuid4
 from deltacat import logs
-from typing import Callable, Iterator, List, Optional, Tuple
+from typing import Callable, Iterator, List, Optional, Tuple, Set
 from deltacat.compute.compactor_v2.model.merge_result import MergeResult
 from deltacat.compute.compactor_v2.model.merge_file_group import MergeFileGroup
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
@@ -38,10 +38,10 @@ from deltacat.storage import (
     Delta,
     DeltaLocator,
     DeltaType,
-    Manifest,
     Partition,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.compute.compactor_v2.utils.dedupe import drop_duplicates
 from deltacat.constants import BYTES_PER_GIBIBYTE
 from deltacat.compute.compactor_v2.constants import (
@@ -94,9 +94,12 @@ def _build_incremental_table(
     # sort by delta file stream position now instead of sorting every row later
     is_delete = False
     for df_envelope in df_envelopes:
-        assert (
-            df_envelope.delta_type != DeltaType.APPEND
-        ), "APPEND type deltas are not supported. Kindly use UPSERT or DELETE"
+        # Allow APPEND, UPSERT, and DELETE delta types
+        assert df_envelope.delta_type in (
+            DeltaType.APPEND,
+            DeltaType.UPSERT,
+            DeltaType.DELETE,
+        ), "Only APPEND, UPSERT, and DELETE delta types are supported"
         if df_envelope.delta_type == DeltaType.DELETE:
             is_delete = True
 
@@ -108,16 +111,35 @@ def _build_incremental_table(
             )
 
         hb_tables.append(table)
-    result = pa.concat_tables(hb_tables)
+    result = _concat_or_coerce_tables(hb_tables)
     return result
 
 
+def _concat_or_coerce_tables(all_tables: List[pa.Table]) -> pa.Table:
+    try:
+        return pa.concat_tables(all_tables)
+    except pa.ArrowInvalid:
+        # Fallback path: schema evolution needed - try PyArrow's built-in unification
+        if all_tables:
+            try:
+                return pa.concat_tables(
+                    all_tables, promote_options="permissive", unify_schemas=True
+                )
+            except (pa.ArrowInvalid, TypeError, pa.ArrowNotImplementedError):
+                # If PyArrow unification fails, re-raise the original error
+                raise
+        else:
+            # Empty table list - should not happen but handle gracefully
+            raise RuntimeError("Expected at least one table to merge, but found none.")
+
+
 def _merge_tables(
     table: pa.Table,
     primary_keys: List[str],
     can_drop_duplicates: bool,
     hb_index: int,
     num_buckets: int,
+    original_fields: Set[str],
     compacted_table: Optional[pa.Table] = None,
 ) -> pa.Table:
     """
@@ -159,7 +181,7 @@ def _merge_tables(
             all_tables[incremental_idx], DeltaType.DELETE
         )
         # we need not drop duplicates
-        return pa.concat_tables(all_tables)
+        return _concat_or_coerce_tables(all_tables)
 
     all_tables = generate_pk_hash_column(all_tables, primary_keys=primary_keys)
 
@@ -169,6 +191,12 @@ def _merge_tables(
         all_tables[incremental_idx], on=sc._PK_HASH_STRING_COLUMN_NAME
     )
 
+    # Always drop DELETE rows from incremental table
+    incremental_table = _drop_delta_type_rows(incremental_table, DeltaType.DELETE)
+
+    # Default to using incremental records as-is, override only if merging is needed
+    incremental_data = incremental_table
+
     if compacted_table:
         compacted_table = all_tables[0]
 
@@ -194,34 +222,100 @@ def _merge_tables(
                 incremental_pk_hash_str, pa.large_string()
             )
 
-        records_to_keep = pc.invert(
-            pc.is_in(
-                compacted_pk_hash_str,
-                incremental_pk_hash_str,
-            )
+        records_to_update = pc.is_in(
+            compacted_pk_hash_str,
+            incremental_pk_hash_str,
         )
 
+        records_to_keep = pc.invert(records_to_update)
+
+        # Keep records that don't have updates
         result_table_list.append(compacted_table.filter(records_to_keep))
 
-    incremental_table = _drop_delta_type_rows(incremental_table, DeltaType.DELETE)
-    result_table_list.append(incremental_table)
+        # Override default if merging is needed
+        if pc.sum(records_to_update).as_py() > 0:  # There are records to update
+            old_records_to_update = compacted_table.filter(records_to_update)
+            # Perform partial UPSERT: merge old and new records field by field
+            incremental_data = _merge_records_partially(
+                old_records=old_records_to_update,
+                new_records=incremental_table,
+                original_fields=original_fields,
+            )
+
+    # Add the determined incremental data
+    result_table_list.append(incremental_data)
 
-    final_table = pa.concat_tables(result_table_list)
+    final_table = _concat_or_coerce_tables(result_table_list)
     final_table = final_table.drop([sc._PK_HASH_STRING_COLUMN_NAME])
 
     return final_table
 
 
+def _merge_records_partially(
+    old_records: pa.Table, new_records: pa.Table, original_fields: Set[str]
+) -> pa.Table:
+    """
+    Merge records field by field for partial UPSERT behavior. Fills missing
+    fields in new_records with values from old_records.
+
+    Args:
+        old_records: Records from the compacted table that need updates
+        new_records: New records with potential partial field updates
+
+    Returns:
+        Table with merged records where missing fields preserve old values
+    """
+    # Get field sets (excluding hash column which is used for joining)
+    old_fields = set(old_records.column_names) - {sc._PK_HASH_STRING_COLUMN_NAME}
+    new_fields = set(new_records.column_names) - {sc._PK_HASH_STRING_COLUMN_NAME}
+
+    # Find fields that are missing from new_records but exist in old_records
+    missing_fields = old_fields - new_fields
+
+    # Find fields that were auto-added by schema coercion (missing from original user data)
+    # These should be treated as missing fields and filled from old_records
+    auto_added_null_fields = set()
+
+    # Use definitive information about which fields were originally provided
+    # Any field that exists in both tables but was NOT in the original user data
+    # should be treated as auto-added by schema coercion
+    for field_name in old_fields & new_fields:  # Fields that exist in both
+        if field_name not in original_fields:
+            auto_added_null_fields.add(field_name)
+
+    # Combine missing fields with auto-added null fields
+    fields_to_fill = missing_fields | auto_added_null_fields
+
+    # Start with new_records and add missing fields from old_records
+    result_columns = {}
+
+    # Copy all existing columns from new_records
+    for column_name in new_records.column_names:
+        result_columns[column_name] = new_records[column_name]
+
+    # Fill in missing/auto-added null fields with values from old_records
+    for field_name in fields_to_fill:
+        # For missing fields, use the old values entirely
+        result_columns[field_name] = old_records[field_name]
+
+    # Create the enhanced new_records table with all fields filled
+    enhanced_new_records = pa.table(result_columns)
+
+    # Now we can return the enhanced table - it has all the fields with proper values
+    # Missing fields are filled with old values, explicitly null fields remain null
+    return enhanced_new_records
+
+
 def _validate_bucketing_spec_compliance(
     table: pa.Table,
     num_buckets: int,
     hb_index: int,
     primary_keys: List[str],
-    rcf: RoundCompletionInfo = None,
+    rci: Optional[RoundCompletionInfo] = None,
     log_prefix=None,
 ) -> None:
-    if rcf is not None:
-        message_prefix = f"{log_prefix}{rcf.compacted_delta_locator.namespace}.{rcf.compacted_delta_locator.table_name}.{rcf.compacted_delta_locator.table_version}.{rcf.compacted_delta_locator.partition_id}.{rcf.compacted_delta_locator.partition_values}"
+    if rci is not None:
+        message_prefix = f"{log_prefix}{rci.compacted_delta_locator.namespace}.{rci.compacted_delta_locator.table_name}.{rci.compacted_delta_locator.table_version}.{rci.compacted_delta_locator.partition_id}.{rci.compacted_delta_locator.partition_values}"
     else:
         message_prefix = f"{log_prefix}"
     pki_table = generate_pk_hash_column(
@@ -251,14 +345,16 @@ def _validate_bucketing_spec_compliance(
 
 def _download_compacted_table(
     hb_index: int,
-    rcf: RoundCompletionInfo,
+    rci: RoundCompletionInfo,
     primary_keys: List[str],
+    all_column_names: List[str],
+    compacted_delta_manifest: Optional[Manifest] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage: metastore = metastore,
     deltacat_storage_kwargs: Optional[dict] = None,
 ) -> pa.Table:
     tables = []
-    hb_index_to_indices = rcf.hb_index_to_entry_range
+    hb_index_to_indices = rci.hb_index_to_entry_range
 
     if str(hb_index) not in hb_index_to_indices:
         return None
@@ -268,9 +364,16 @@ def _download_compacted_table(
     ), "indices should not be none and contains exactly two elements"
     for offset in range(indices[1] - indices[0]):
         table = deltacat_storage.download_delta_manifest_entry(
-            rcf.compacted_delta_locator,
+            Delta.of(
+                rci.compacted_delta_locator,
+                DeltaType.APPEND,
+                compacted_delta_manifest.meta,
+                None,
+                compacted_delta_manifest,
+            ),
             entry_index=(indices[0] + offset),
             file_reader_kwargs_provider=read_kwargs_provider,
+            all_column_names=all_column_names,
             **deltacat_storage_kwargs,
         )
 
@@ -291,10 +394,10 @@ def _download_compacted_table(
     if primary_keys and check_bucketing_spec:
         _validate_bucketing_spec_compliance(
             compacted_table,
-            rcf.hash_bucket_count,
+            rci.hash_bucket_count,
             hb_index,
             primary_keys,
-            rcf=rcf,
+            rci=rci,
             log_prefix=_EXISTING_VARIANT_LOG_PREFIX,
         )
     return compacted_table
@@ -304,15 +407,9 @@ def _copy_all_manifest_files_from_old_hash_buckets(
     hb_index_copy_by_reference: List[int],
     round_completion_info: RoundCompletionInfo,
     write_to_partition: Partition,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[dict] = None,
+    compacted_manifest: Optional[Manifest] = None,
 ) -> List[MaterializeResult]:
 
-    compacted_delta_locator = round_completion_info.compacted_delta_locator
-    manifest = deltacat_storage.get_delta_manifest(
-        compacted_delta_locator, **deltacat_storage_kwargs
-    )
-
     manifest_entry_referenced_list = []
     materialize_result_list = []
     hb_index_to_indices = round_completion_info.hb_index_to_entry_range
@@ -329,27 +426,27 @@ def _copy_all_manifest_files_from_old_hash_buckets(
         for offset in range(indices[1] - indices[0]):
             entry_index = indices[0] + offset
             assert entry_index < len(
-                manifest.entries
-            ), f"entry index: {entry_index} >= {len(manifest.entries)}"
-            manifest_entry = manifest.entries[entry_index]
+                compacted_manifest.entries
+            ), f"entry index: {entry_index} >= {len(compacted_manifest.entries)}"
+            manifest_entry = compacted_manifest.entries[entry_index]
             manifest_entry_referenced_list.append(manifest_entry)
 
-        manifest = Manifest.of(
+        compacted_manifest = Manifest.of(
             entries=manifest_entry_referenced_list, uuid=str(uuid4())
         )
         delta = Delta.of(
             locator=DeltaLocator.of(write_to_partition.locator),
-            delta_type=DeltaType.UPSERT,
-            meta=manifest.meta,
-            manifest=manifest,
+            delta_type=DeltaType.APPEND,  # Compaction always produces APPEND deltas
+            meta=compacted_manifest.meta,
+            manifest=compacted_manifest,
             previous_stream_position=write_to_partition.stream_position,
             properties={},
         )
         referenced_pyarrow_write_result = PyArrowWriteResult.of(
             len(manifest_entry_referenced_list),
-            manifest.meta.source_content_length,
-            manifest.meta.content_length,
-            manifest.meta.record_count,
+            compacted_manifest.meta.source_content_length,
+            compacted_manifest.meta.content_length,
+            compacted_manifest.meta.record_count,
         )
         materialize_result = MaterializeResult.of(
             delta=delta,
@@ -374,6 +471,7 @@ def _has_previous_compacted_table(input: MergeInput, hb_idx: int) -> bool:
     """
     return (
         input.round_completion_info
+        and input.compacted_manifest is not None
         and input.round_completion_info.hb_index_to_entry_range
         and input.round_completion_info.hb_index_to_entry_range.get(str(hb_idx))
         is not None
@@ -391,6 +489,7 @@ def _can_copy_by_reference(
         not has_delete
         and not merge_file_group.dfe_groups
         and input.round_completion_info is not None
+        and input.compacted_manifest is not None
     )
 
     if input.disable_copy_by_reference:
@@ -489,9 +588,9 @@ def _compact_tables(
         delete_file_envelopes + df_envelopes
     )
     assert all(
-        dfe.delta_type in (DeltaType.UPSERT, DeltaType.DELETE)
+        dfe.delta_type in (DeltaType.APPEND, DeltaType.UPSERT, DeltaType.DELETE)
         for dfe in reordered_all_dfes
-    ), "All reordered delta file envelopes must be of the UPSERT or DELETE"
+    ), "All reordered delta file envelopes must be of the APPEND, UPSERT or DELETE"
     table = compacted_table
     aggregated_incremental_len = 0
     aggregated_deduped_records = 0
@@ -499,7 +598,7 @@ def _compact_tables(
     for i, (delta_type, delta_type_sequence) in enumerate(
         _group_sequence_by_delta_type(reordered_all_dfes)
     ):
-        if delta_type is DeltaType.UPSERT:
+        if delta_type is DeltaType.UPSERT or delta_type is DeltaType.APPEND:
             (table, incremental_len, deduped_records, merge_time,) = _apply_upserts(
                 input=input,
                 dfe_list=delta_type_sequence,
@@ -540,8 +639,9 @@ def _apply_upserts(
     prev_table=None,
 ) -> Tuple[pa.Table, int, int, int]:
     assert all(
-        dfe.delta_type is DeltaType.UPSERT for dfe in dfe_list
-    ), "All incoming delta file envelopes must of the DeltaType.UPSERT"
+        dfe.delta_type is DeltaType.UPSERT or dfe.delta_type is DeltaType.APPEND
+        for dfe in dfe_list
+    ), "All incoming delta file envelopes must of the DeltaType.UPSERT or DeltaType.APPEND"
     logger.info(
         f"[Hash bucket index {hb_idx}] Reading dedupe input for "
         f"{len(dfe_list)} delta file envelope lists..."
@@ -556,16 +656,19 @@ def _apply_upserts(
         # on non event based sort key does not produce consistent
         # compaction results. E.g., compaction(delta1, delta2, delta3)
         # will not be equal to compaction(compaction(delta1, delta2), delta3).
-        table = table.sort_by(input.sort_keys)
+        table = table.sort_by(
+            [pa_key for key in input.sort_keys for pa_key in key.arrow]
+        )
     hb_table_record_count = len(table) + (len(prev_table) if prev_table else 0)
     table, merge_time = timed_invocation(
         func=_merge_tables,
         table=table,
         primary_keys=input.primary_keys,
         can_drop_duplicates=input.drop_duplicates,
-        compacted_table=prev_table,
         hb_index=hb_idx,
         num_buckets=input.hash_bucket_count,
+        original_fields=input.original_fields,
+        compacted_table=prev_table,
     )
     deduped_records = hb_table_record_count - len(table)
     return table, incremental_len, deduped_records, merge_time
@@ -582,8 +685,7 @@ def _copy_manifests_from_hash_bucketing(
                 hb_index_copy_by_reference_ids,
                 input.round_completion_info,
                 input.write_to_partition,
-                input.deltacat_storage,
-                input.deltacat_storage_kwargs,
+                input.compacted_manifest,
             )
         )
         logger.info(
@@ -623,12 +725,13 @@ def _timed_merge(input: MergeInput) -> MergeResult:
             ):
                 hb_index_copy_by_ref_ids.append(merge_file_group.hb_index)
                 continue
-
             if _has_previous_compacted_table(input, merge_file_group.hb_index):
                 compacted_table = _download_compacted_table(
                     hb_index=merge_file_group.hb_index,
-                    rcf=input.round_completion_info,
+                    rci=input.round_completion_info,
                     primary_keys=input.primary_keys,
+                    all_column_names=input.all_column_names,
+                    compacted_delta_manifest=input.compacted_manifest,
                     read_kwargs_provider=input.read_kwargs_provider,
                     deltacat_storage=input.deltacat_storage,
                     deltacat_storage_kwargs=input.deltacat_storage_kwargs,

deltacat/compute/compactor_v2/utils/content_type_params.py

@@ -1,6 +1,7 @@
 import logging
 import ray
 import functools
+from typing import List
 from deltacat.compute.compactor_v2.constants import (
     TASK_MAX_PARALLELISM,
     MAX_PARQUET_METADATA_SIZE,
@@ -11,10 +12,10 @@ from deltacat import logs
 from deltacat.storage import (
     Delta,
     ManifestEntry,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from typing import Dict, Optional, Any
-from deltacat.types.media import TableType
+from deltacat.types.media import DatasetType
 from deltacat.types.media import ContentType
 from deltacat.types.partial_download import PartialParquetParameters
 from deltacat.exceptions import RetryableError
@@ -74,7 +75,8 @@ class AppendContentTypeParamsCache:
 def _download_parquet_metadata_for_manifest_entry(
     delta: Delta,
     entry_index: int,
-    deltacat_storage: unimplemented_deltacat_storage,
+    all_column_names: List[str],
+    deltacat_storage: metastore,
     deltacat_storage_kwargs: Optional[Dict[Any, Any]] = {},
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> Dict[str, Any]:
@@ -86,11 +88,13 @@ def _download_parquet_metadata_for_manifest_entry(
             "'file_reader_kwargs_provider' is also present in deltacat_storage_kwargs. Removing to prevent multiple values for keyword argument"
         )
         deltacat_storage_kwargs.pop("file_reader_kwargs_provider")
+
     pq_file = deltacat_storage.download_delta_manifest_entry(
         delta,
         entry_index=entry_index,
-        table_type=TableType.PYARROW_PARQUET,
+        table_type=DatasetType.PYARROW_PARQUET,
         file_reader_kwargs_provider=file_reader_kwargs_provider,
+        all_column_names=all_column_names,
         **deltacat_storage_kwargs,
     )
 
@@ -104,9 +108,10 @@ def _download_parquet_metadata_for_manifest_entry(
 
 def append_content_type_params(
     delta: Delta,
+    all_column_names: List[str],
     task_max_parallelism: int = TASK_MAX_PARALLELISM,
     max_parquet_meta_size_bytes: Optional[int] = MAX_PARQUET_METADATA_SIZE,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage: metastore = metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> bool:
@@ -172,13 +177,19 @@ def append_content_type_params(
         max_parquet_meta_size_bytes=max_parquet_meta_size_bytes,
     )
 
+    # create a copy of deltacat_storage_kwargs without transaction key
+    deltacat_storage_kwargs_copy = {
+        k: v for k, v in deltacat_storage_kwargs.items() if k != "transaction"
+    }
+
     def input_provider(index, item) -> Dict:
         return {
             "file_reader_kwargs_provider": file_reader_kwargs_provider,
-            "deltacat_storage_kwargs": deltacat_storage_kwargs,
+            "deltacat_storage_kwargs": deltacat_storage_kwargs_copy,
             "deltacat_storage": deltacat_storage,
             "delta": delta,
             "entry_index": item,
+            "all_column_names": all_column_names,
         }
 
     logger.info(

deltacat/compute/compactor_v2/utils/delta.py

@@ -9,7 +9,7 @@ from deltacat.storage import (
     Delta,
 )
 from deltacat.storage.model.delta import DeltaType
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.types.media import StorageType
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat import logs
@@ -30,8 +30,9 @@ def contains_delete_deltas(deltas: List[Delta]) -> bool:
 
 def read_delta_file_envelopes(
     annotated_delta: DeltaAnnotated,
+    all_column_names: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage: metastore = metastore,
     deltacat_storage_kwargs: Optional[dict] = None,
 ) -> Tuple[Optional[List[DeltaFileEnvelope]], int, int]:
     tables = deltacat_storage.download_delta(
@@ -39,6 +40,7 @@ def read_delta_file_envelopes(
         max_parallelism=1,
         file_reader_kwargs_provider=read_kwargs_provider,
         storage_type=StorageType.LOCAL,
+        all_column_names=all_column_names,
         **deltacat_storage_kwargs,
     )
     annotations = annotated_delta.annotations
@@ -80,7 +82,7 @@ def read_delta_file_envelopes(
 def get_local_delta_file_envelopes(
     uniform_deltas: List[DeltaAnnotated],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[dict] = None,
 ) -> Tuple[List[DeltaFileEnvelope], int]:
     local_dfe_list = []

deltacat/compute/compactor_v2/utils/io.py

@@ -3,7 +3,7 @@ import functools
 from deltacat.storage import (
     PartitionLocator,
     Delta,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat import logs
 from deltacat.compute.compactor.utils import io as io_v1
@@ -38,7 +38,7 @@ def discover_deltas(
     rebase_source_partition_locator: Optional[PartitionLocator] = None,
     rebase_source_partition_high_watermark: Optional[int] = None,
     rcf_high_watermark: Optional[int] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[Delta]:
@@ -67,6 +67,11 @@ def discover_deltas(
         f"Length of input deltas from delta source table is {len(delta_source_incremental_deltas)}"
         f" from ({previous_compacted_high_watermark}, {last_stream_position_to_compact}]"
     )
+    logger.info(f"DEBUG: source_partition_locator = {source_partition_locator}")
+    logger.info(
+        f"DEBUG: source_partition_locator.partition_id = {getattr(source_partition_locator, 'partition_id', 'NO_PARTITION_ID')}"
+    )
+    logger.info(f"DEBUG: total input deltas found = {len(result)}")
 
     if rebase_source_partition_locator:
         previous_compacted_deltas = io_v1._discover_deltas(
@@ -93,7 +98,8 @@ def create_uniform_input_deltas(
     hash_bucket_count: int,
     compaction_audit: CompactionSessionAuditInfo,
     compact_partition_params: CompactPartitionParams,
-    deltacat_storage=unimplemented_deltacat_storage,
+    all_column_names: List[str],
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[DeltaAnnotated]:
 
@@ -113,6 +119,7 @@ def create_uniform_input_deltas(
             )
             append_content_type_params(
                 delta=delta,
+                all_column_names=all_column_names,
                 deltacat_storage=deltacat_storage,
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
                 task_max_parallelism=compact_partition_params.task_max_parallelism,

deltacat/compute/compactor_v2/utils/merge.py

@@ -23,6 +23,7 @@ from deltacat.types.tables import TABLE_CLASS_TO_SIZE_FUNC
 
 from deltacat.utils.performance import timed_invocation
 from deltacat.storage import (
+    DeltaType,
     Partition,
 )
 from deltacat.compute.compactor_v2.deletes.delete_strategy import (
@@ -47,13 +48,21 @@ def materialize(
         # TODO (pdames): compare performance to pandas-native materialize path
         df = compacted_table.to_pandas(split_blocks=True, self_destruct=True)
         compacted_table = df
+    # Extract schema from table_writer_kwargs to pass as direct parameter
+    # This ensures schema_id is properly set in the manifest
+    schema = None
+    if input.table_writer_kwargs and "schema" in input.table_writer_kwargs:
+        schema = input.table_writer_kwargs["schema"]
+
     delta, stage_delta_time = timed_invocation(
         input.deltacat_storage.stage_delta,
         compacted_table,
         input.write_to_partition,
+        delta_type=DeltaType.APPEND,  # Compaction always produces APPEND deltas
         max_records_per_entry=input.max_records_per_output_file,
         content_type=input.compacted_file_content_type,
-        s3_table_writer_kwargs=input.s3_table_writer_kwargs,
+        schema=schema,  # Pass schema as direct parameter for schema_id extraction
+        table_writer_kwargs=input.table_writer_kwargs,
         **input.deltacat_storage_kwargs,
     )
     compacted_table_size = TABLE_CLASS_TO_SIZE_FUNC[type(compacted_table)](
@@ -112,6 +121,7 @@ def generate_local_merge_input(
     return MergeInput.of(
         merge_file_groups_provider=LocalMergeFileGroupsProvider(
             annotated_deltas,
+            all_column_names=params.all_column_names,
             read_kwargs_provider=params.read_kwargs_provider,
             deltacat_storage=params.deltacat_storage,
             deltacat_storage_kwargs=params.deltacat_storage_kwargs,
@@ -119,12 +129,13 @@ def generate_local_merge_input(
         write_to_partition=compacted_partition,
         compacted_file_content_type=params.compacted_file_content_type,
         primary_keys=params.primary_keys,
+        all_column_names=params.all_column_names,
         sort_keys=params.sort_keys,
         drop_duplicates=params.drop_duplicates,
         max_records_per_output_file=params.records_per_compacted_file,
         enable_profiler=params.enable_profiler,
         metrics_config=params.metrics_config,
-        s3_table_writer_kwargs=params.s3_table_writer_kwargs,
+        table_writer_kwargs=params.table_writer_kwargs,
         read_kwargs_provider=params.read_kwargs_provider,
         round_completion_info=round_completion_info,
         object_store=params.object_store,
@@ -134,4 +145,5 @@ def generate_local_merge_input(
         delete_file_envelopes=delete_file_envelopes,
         disable_copy_by_reference=params.disable_copy_by_reference,
         hash_bucket_count=params.hash_bucket_count,
+        original_fields=params.original_fields,
     )

deltacat/compute/compactor_v2/utils/task_options.py

@@ -11,7 +11,7 @@ from deltacat.compute.compactor_v2.model.merge_file_group import (
 from deltacat.storage import (
     Manifest,
     ManifestEntry,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat.compute.compactor.model.delta_annotated import DeltaAnnotated
 from deltacat.compute.compactor.model.round_completion_info import RoundCompletionInfo
@@ -77,8 +77,6 @@ def _get_merge_task_options(
     round_completion_info: Optional[RoundCompletionInfo] = None,
     compacted_delta_manifest: Optional[Manifest] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
 ) -> Dict[str, Any]:
     if (
@@ -275,8 +273,6 @@ def merge_resource_options_provider(
     compacted_delta_manifest: Optional[Manifest] = None,
     ray_custom_resources: Optional[Dict] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
     **kwargs,
 ) -> Dict:
@@ -306,8 +302,6 @@ def merge_resource_options_provider(
         round_completion_info=round_completion_info,
         compacted_delta_manifest=compacted_delta_manifest,
         primary_keys=primary_keys,
-        deltacat_storage=deltacat_storage,
-        deltacat_storage_kwargs=deltacat_storage_kwargs,
         memory_logs_enabled=memory_logs_enabled,
         estimate_resources_params=estimate_resources_params,
     )
@@ -322,7 +316,7 @@ def local_merge_resource_options_provider(
     compacted_delta_manifest: Optional[Manifest] = None,
     ray_custom_resources: Optional[Dict] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
     **kwargs,
@@ -348,8 +342,6 @@ def local_merge_resource_options_provider(
         round_completion_info=round_completion_info,
         compacted_delta_manifest=compacted_delta_manifest,
         primary_keys=primary_keys,
-        deltacat_storage=deltacat_storage,
-        deltacat_storage_kwargs=deltacat_storage_kwargs,
         memory_logs_enabled=memory_logs_enabled,
         estimate_resources_params=estimate_resources_params,
     )