deltacat 1.1.36__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

deltacat/catalog/model/catalog.py

@@ -1,47 +1,129 @@
 # Allow self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from typing import Any, Dict, List, Optional
+import logging
+from types import ModuleType
 
+from typing import Any, Dict, List, Optional, Union
+from functools import partial
 import ray
 
-from deltacat.catalog import interface as catalog_interface
+from deltacat import logs
+from deltacat.annotations import ExperimentalAPI
+from deltacat.catalog.main import impl as DeltacatCatalog
+from deltacat.catalog.iceberg import impl as IcebergCatalog
+from deltacat.catalog import CatalogProperties
+from deltacat.catalog.iceberg import IcebergCatalogConfig
+from deltacat.constants import DEFAULT_CATALOG
 
-all_catalogs: Optional[Catalogs] = None
+all_catalogs: Optional[ray.actor.ActorHandle] = None
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 class Catalog:
-    def __init__(self, impl=catalog_interface, *args, **kwargs):
+    def __init__(self, impl: ModuleType = DeltacatCatalog, *args, **kwargs):
+        """
+        Constructor for a Catalog.
+
+        The args and kwargs here will be plumbed through to the catalog initialize function, and the results
+        are stored in Catalog.inner. Any state which is required (like: metastore root URI, pyiceberg native catalog)
+        MUST be returned by initialize.
+
+        Note: all initialization configuration MUST be pickle-able. When `Catalog` is pickled, _inner is excluded.
+        Instead, we only pass impl/args/kwargs, which are pickled and then _inner is re-constituted by calling __init__.
+        See `ray.util.register_serializer` in Catalogs class.
+        """
+        if not isinstance(self, Catalog):
+            # self may contain the tuple returned from __reduce__ (ray pickle bug?)
+            if callable(self[0]) and isinstance(self[1], tuple):
+                logger.info(f"Invoking {self[0]} with positional args: {self[1]}")
+                return self[0](*self[1])
+            else:
+                err_msg = f"Expected `self` to be {Catalog}, but found: {self}"
+                raise RuntimeError(err_msg)
+
         self._impl = impl
-        self._impl.initialize(*args, **kwargs)
+        self._inner = self._impl.initialize(*args, **kwargs)
+        self._args = args
+        self._kwargs = kwargs
+
+    @classmethod
+    @ExperimentalAPI
+    def iceberg(cls, config: IcebergCatalogConfig, *args, **kwargs):
+        """
+        !!! ICEBERG SUPPORT IS EXPERIMENTAL !!!
+
+        Factory method to construct a catalog from Iceberg catalog params
+
+        This method is just a wrapper around __init__ with stronger typing. You may still call __init__,
+        plumbing __params__ through as kwargs
+        """
+        return cls(impl=IcebergCatalog, *args, **{"config": config, **kwargs})
+
+    @classmethod
+    def default(cls, config: CatalogProperties, *args, **kwargs):
+        """
+        Factory method to construct a catalog with the default implementation
+
+        Uses CatalogProperties as configuration
+        """
+        return cls(impl=DeltacatCatalog, *args, **{"config": config, **kwargs})
 
     @property
     def impl(self):
         return self._impl
 
+    @property
+    def inner(self) -> Optional[Any]:
+        return self._inner
+
+    # support pickle, copy, deepcopy, etc.
+    def __reduce__(self):
+        # instantiated catalogs may fail to pickle, so exclude _inner
+        # (e.g. Iceberg catalog w/ unserializable SSLContext from boto3 client)
+        return partial(self.__class__, **self._kwargs), (self._impl, *self._args)
+
+    def __str__(self):
+        string_rep = f"{self.__class__.__name__}("
+        if self._args:
+            string_rep += f"args={self._args}, "
+        if self._kwargs:
+            string_rep += f"kwargs={self._kwargs}, "
+        if self._inner:
+            string_rep += f"inner={self._inner})"
+        return string_rep
+
+    def __repr__(self):
+        return self.__str__()
+
 
 @ray.remote
 class Catalogs:
     def __init__(
         self,
-        catalogs: Dict[str, Catalog],
-        default_catalog_name: str = None,
+        catalogs: Union[Catalog, Dict[str, Catalog]],
+        default: Optional[str] = None,
         *args,
         **kwargs,
     ):
-        if default_catalog_name and default_catalog_name not in catalogs:
+        if default and default not in catalogs:
             raise ValueError(
-                f"Catalog {default_catalog_name} not found "
-                f"in catalogs to register: {catalogs}"
+                f"Catalog {default} not found " f"in catalogs to register: {catalogs}"
             )
         if not catalogs:
             raise ValueError(
                 f"No catalogs given to register. "
                 f"Please specify one or more catalogs."
             )
+
+        # if user only provides single Catalog, override it to be a map with default key
+        if isinstance(catalogs, Catalog):
+            catalogs = {DEFAULT_CATALOG: catalogs}
+
         self.catalogs: Dict[str, Catalog] = catalogs
-        if default_catalog_name:
-            self.default_catalog = self.catalogs[default_catalog_name]
+        if default:
+            self.default_catalog = self.catalogs[default]
         elif len(catalogs) == 1:
             self.default_catalog = list(self.catalogs.values())[0]
         else:
@@ -53,8 +135,10 @@ class Catalogs:
     def names(self) -> List[str]:
         return list(self.catalogs.keys())
 
-    def put(self, name: str, catalog: Catalog) -> None:
+    def put(self, name: str, catalog: Catalog, set_default: bool = False) -> None:
         self.catalogs[name] = catalog
+        if set_default:
+            self.default_catalog = catalog
 
     def get(self, name) -> Catalog:
         return self.catalogs.get(name)
@@ -63,21 +147,144 @@ class Catalogs:
         return self.default_catalog
 
 
+def is_initialized(*args, **kwargs) -> bool:
+    """
+    Check if DeltaCAT is initialized
+    """
+    global all_catalogs
+
+    # If ray is not initialized, then Catalogs cannot be initialized
+    if not ray.is_initialized():
+        # Any existing actor reference stored in catalog_module must be stale - reset it
+        all_catalogs = None
+        return False
+
+    return all_catalogs is not None
+
+
 def init(
-    catalogs: Dict[str, Catalog],
-    default_catalog_name: str = None,
+    catalogs: Union[Dict[str, Catalog], Catalog],
+    default: Optional[str] = None,
     ray_init_args: Dict[str, Any] = None,
     *args,
+    force_reinitialize=False,
     **kwargs,
 ) -> None:
+    """
+    Initialize DeltaCAT catalogs.
 
-    if not ray.is_initialized():
+    :param catalogs: Either a single Catalog instance or a map of string to Catalog instance
+    :param default: The Catalog to use by default. If only one Catalog is provided, it will
+        be set as the default
+    :param ray_init_args: kwargs to pass to ray initialization
+    :param force_reinitialize: if True, force the reinitialization of Ray. If false, will do nothing if ray already initialized
+    """
+    global all_catalogs
+
+    if is_initialized() and not force_reinitialize:
+        logger.warning("DeltaCAT already initialized.")
+        return
+    else:
         if ray_init_args:
             ray.init(**ray_init_args)
         else:
-            ray.init(address="auto")
+            ray.init()
 
-    global all_catalogs
-    all_catalogs = Catalogs.remote(
-        catalogs=catalogs, default_catalog_name=default_catalog_name
+    # register custom serializer for catalogs since these may contain
+    # unserializable objects like boto3 clients with SSLContext
+    ray.util.register_serializer(
+        Catalog, serializer=Catalog.__reduce__, deserializer=Catalog.__init__
     )
+    all_catalogs = Catalogs.remote(catalogs=catalogs, default=default)
+
+
+def get_catalog(name: Optional[str] = None, **kwargs) -> Catalog:
+    """
+    Get a catalog by name, or the default catalog if no name is provided.
+
+    Args:
+        name: Name of catalog to retrieve (optional, uses default if not provided)
+
+    Returns:
+        The requested Catalog, or ValueError if it does not exist
+    """
+    global all_catalogs
+
+    if not all_catalogs:
+        raise ValueError(
+            "No catalogs available! Call "
+            "`deltacat.init(catalogs={...})` to register one or more "
+            "catalogs then retry."
+        )
+
+    if name is not None:
+        catalog = ray.get(all_catalogs.get.remote(name))
+        if not catalog:
+            available_catalogs = ray.get(all_catalogs.all.remote()).values()
+            raise ValueError(
+                f"Catalog '{name}' not found. Available catalogs: "
+                f"{available_catalogs}."
+            )
+        return catalog
+
+    else:
+        catalog = ray.get(all_catalogs.default.remote())
+        if not catalog:
+            available_catalogs = ray.get(all_catalogs.all.remote()).values()
+            raise ValueError(
+                f"Call to get_catalog without name set failed because there is no default Catalog set. Available catalogs: "
+                f"{available_catalogs}."
+            )
+        return catalog
+
+
+def put_catalog(
+    name: str,
+    catalog: Catalog = None,
+    *,
+    default: bool = False,
+    ray_init_args: Dict[str, Any] = None,
+    fail_if_exists: bool = False,
+    **kwargs,
+) -> None:
+    """
+    Add a named catalog to the global map of named catalogs. Initializes ray if not already initialized.
+
+    Args:
+        name: name of catalog
+        catalog: catalog instance to use, if provided
+        default:  Make this the default catalog if multiple catalogs are available.
+            ignored if this is the only catalog available, since it will always be the default catalog.
+        ray_init_args: ray initialization args (used only if ray not already initialized)
+        fail_if_exists: if True, raises KeyError if the catalog name already exists. Otherwise, overwrite catalog
+    """
+    global all_catalogs
+
+    # Initialize, if necessary
+    if not is_initialized():
+        # NOTE - since we are initializing with a single catalog, it will be set to the default
+        if not default:
+            logger.info(
+                f"Calling put_catalog with set_as_default=False, "
+                f"but still setting Catalog {catalog} as default since it is the only catalog."
+            )
+        init({name: catalog}, ray_init_args=ray_init_args)
+        return
+
+    # Fail if fail_if_exists and catalog already exists
+    if fail_if_exists:
+        catalog_already_exists = False
+        try:
+            get_catalog(name)
+            # Note - need to set state catalog_already_exists and throw ValueError later, or else it will be
+            # caught in the except block which is meant to catch the ValueError from get_catalog
+            catalog_already_exists = True
+        except ValueError:
+            pass
+        if catalog_already_exists:
+            raise ValueError(
+                f"Failed to put catalog {name} because it already exists and fail_if_exists={fail_if_exists}"
+            )
+
+    # Add the catalog (which may overwrite existing if fail_if_exists=False)
+    ray.get(all_catalogs.put.remote(name, catalog, default))

deltacat/catalog/model/properties.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+from typing import Optional, Any
+
+import pyarrow
+from deltacat.constants import DELTACAT_ROOT
+
+from deltacat.utils.filesystem import resolve_path_and_filesystem
+
+
+def get_catalog_properties(
+    *args,
+    catalog: Optional[CatalogProperties] = None,
+    inner: Optional[CatalogProperties] = None,
+    **kwargs,
+) -> CatalogProperties:
+    """
+    Helper function to fetch CatalogProperties instance. You are meant to call this by providing your functions
+    kwargs, OR to directly pass through CatalogProperty configuration keys like "root" in kwargs.
+
+    This will look for a CatalogProperty value in the kwargs "catalog" or "inner". If these are found, it returns
+    the CatalogProperty value under that kwarg. Otherwise, it will pass through kwargs to the CatalogProperties
+    constructor.
+    """
+    properties = catalog if catalog is not None else inner
+    if properties is not None and isinstance(properties, CatalogProperties):
+        return properties
+    elif properties is not None and not isinstance(properties, CatalogProperties):
+        raise ValueError(
+            f"Expected catalog properties of type {CatalogProperties.__name__} "
+            f"but found {type(properties)}."
+        )
+    else:
+        return CatalogProperties(**kwargs)
+
+
+class CatalogProperties:
+    """
+    DeltaCAT catalog properties used to deterministically resolve a durable
+    DeltaCAT catalog instance. Properties are set from system environment
+    variables unless explicit overrides are provided during initialization.
+
+    Catalog and storage APIs rely on the property catalog to retrieve durable state about the catalog they're
+    working against.
+
+    Attributes:
+        root (str): URI string The root path where catalog metadata and data
+            files are stored. Root is determined (in prededence order) by:
+            1. check kwargs for "root"
+            2. check env variable "DELTACAT_ROOT"
+            3. default to ${cwd}/.deltacat
+
+        filesystem: The filesystem implementation that should be used for
+            reading/writing files. If None, a filesystem will be inferred from
+            the catalog root path.
+
+        storage: Storage class implementation (overrides default filesystem storage impl)
+    """
+
+    def __init__(
+        self,
+        root: Optional[str] = None,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+        storage=None,
+        *args,
+        **kwargs,
+    ):
+        """
+        Initialize a CatalogProperties instance.
+
+        Args:
+            root: A single directory path that serves as the catalog root dir.
+            filesystem: The filesystem implementation that should be used for
+                reading these files. If None, a filesystem will be inferred.
+                If not None, the provided filesystem will still be validated
+                against the provided path to ensure compatibility.
+        """
+        # set root, using precedence rules described in pydoc
+        if root is None:
+            # Check environment variables
+            # This is set or defaulted in constants.py
+            root = DELTACAT_ROOT
+            if root is None:
+                raise ValueError(
+                    "Expected environment variable DELTACAT_ROOT to be set or defaulted"
+                )
+
+        resolved_root, resolved_filesystem = resolve_path_and_filesystem(
+            path=root,
+            filesystem=filesystem,
+        )
+        self._root = resolved_root
+        self._filesystem = resolved_filesystem
+        self._storage = storage
+
+    @property
+    def root(self) -> str:
+        return self._root
+
+    @property
+    def filesystem(self) -> Optional[pyarrow.fs.FileSystem]:
+        return self._filesystem
+
+    @property
+    def storage(self) -> Optional[Any]:
+        """
+        Return overridden storage impl, if any
+        """
+        return self._storage
+
+    def __str__(self):
+        return (
+            f"{self.__class__.__name__}(root={self.root}, filesystem={self.filesystem})"
+        )
+
+    def __repr__(self):
+        return self.__str__()

deltacat/catalog/model/table_definition.py

@@ -1,19 +1,30 @@
 # Allow self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+from typing import Optional, Any
+
 from deltacat.storage import Stream, Table, TableVersion
+from deltacat.storage.model.scan.push_down import Pushdown
+from deltacat.storage.model.scan.scan_plan import ScanPlan
+from deltacat.storage.util.scan_planner import ScanPlanner
 
 
 class TableDefinition(dict):
     @staticmethod
     def of(
-        table: Table, table_version: TableVersion, stream: Stream
+        table: Table,
+        table_version: TableVersion,
+        stream: Stream,
+        native_object: Optional[Any] = None,
+        scan_planner: Optional[ScanPlanner] = None,
     ) -> TableDefinition:
         return TableDefinition(
             {
                 "table": table,
                 "tableVersion": table_version,
                 "stream": stream,
+                "nativeObject": native_object,
+                "scan_planner": scan_planner,
             }
         )
 
@@ -28,3 +39,23 @@ class TableDefinition(dict):
     @property
     def stream(self) -> Stream:
         return self["stream"]
+
+    @property
+    def native_object(self) -> Optional[Any]:
+        return self.get("nativeObject")
+
+    @property
+    def scan_planner(self) -> Optional[ScanPlanner]:
+        return self.get("scan_planner")
+
+    def create_scan_plan(self, pushdown: Optional[Pushdown] = None) -> ScanPlan:
+        if not self.scan_planner:
+            raise RuntimeError(
+                f"ScanPlanner is not initialized for table '{self.table.table_name}' "
+                f"of namespace '{self.table.namespace}'"
+            )
+        return self.scan_planner.create_scan_plan(
+            table_name=self.table.table_name,
+            namespace=self.table.namespace,
+            pushdown=pushdown,
+        )

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -8,6 +8,7 @@ from typing import List, Union
 from deltacat.compute.compactor.model.hash_bucket_result import HashBucketResult
 from deltacat.compute.compactor.model.dedupe_result import DedupeResult
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
+from deltacat.compute.compactor_v2.model.merge_result import MergeResult
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.resources import ClusterUtilization, get_size_of_object_in_bytes
 from deltacat.compute.compactor import PyArrowWriteResult
@@ -670,13 +671,13 @@ class CompactionSessionAuditInfo(dict):
         self, output_size_bytes: float
     ) -> CompactionSessionAuditInfo:
         self["outputSizeBytes"] = output_size_bytes
-        return output_size_bytes
+        return self
 
     def set_output_size_pyarrow_bytes(
         self, output_size_pyarrow_bytes: float
     ) -> CompactionSessionAuditInfo:
         self["outputSizePyarrowBytes"] = output_size_pyarrow_bytes
-        return output_size_pyarrow_bytes
+        return self
 
     def set_total_cluster_memory_bytes(
         self, total_cluster_memory_bytes: float
@@ -787,7 +788,10 @@ class CompactionSessionAuditInfo(dict):
         self,
         step_name: str,
         task_results: Union[
-            List[HashBucketResult], List[DedupeResult], List[MaterializeResult]
+            List[HashBucketResult],
+            List[DedupeResult],
+            List[MaterializeResult],
+            List[MergeResult],
         ],
         task_results_retrieved_at: float,
         invoke_time_in_seconds: float,

deltacat/compute/compactor/model/delta_annotated.py

@@ -11,10 +11,10 @@ from deltacat import logs
 from deltacat.storage import (
     Delta,
     DeltaType,
-    Manifest,
     ManifestEntry,
     ManifestEntryList,
 )
+from deltacat.storage.model.manifest import Manifest
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -107,7 +107,7 @@ class DeltaAnnotated(Delta):
             assert len(src_da_annotations) == len(src_da_entries), (
                 f"Unexpected Error: Length of delta annotations "
                 f"({len(src_da_annotations)}) doesn't mach the length of "
-                f"delta manifest entries ({len(src_da_entries)}).",
+                f"delta manifest entries ({len(src_da_entries)})."
             )
             for i, src_entry in enumerate(src_da_entries):
                 # create a new da group if src and dest has different delta locator
@@ -161,7 +161,7 @@ class DeltaAnnotated(Delta):
         assert len(src_da_annotations) == len(src_da_entries), (
             f"Unexpected Error: Length of delta annotations "
             f"({len(src_da_annotations)}) doesn't mach the length of "
-            f"delta manifest entries ({len(src_da_entries)}).",
+            f"delta manifest entries ({len(src_da_entries)})."
         )
         src_da_entries_length = len(src_da_entries)
         equal_length = src_da_entries_length // pieces

deltacat/compute/compactor/model/delta_file_envelope.py

@@ -37,7 +37,9 @@ class DeltaFileEnvelope(dict):
                 pointing to a file from the uncompacted source table, False if
                 this Locator is pointing to a file in the compacted destination
                 table.
-            table_storage_strategy: The way the table object is stored in the delta file envelope. If None just stores the table normally
+            file_record_count: Record count in the delta file table.
+            table_storage_strategy: The way the table object is stored in the
+                delta file envelope. If None just stores the table normally
         Returns:
             A delta file envelope.
 

deltacat/compute/compactor/model/delta_file_locator.py

@@ -31,9 +31,11 @@ class DeltaFileLocator(Locator, tuple):
 
             file_index: Index of the file in the Delta Manifest.
 
+            file_record_count: Count of records in the Delta File.
+
         Returns:
             delta_file_locator: The Delta File Locator Tuple as
-            (is_source_delta, stream_position, file_index).
+            (is_src_delta, stream_position, file_index, file_record_count).
         """
         return DeltaFileLocator(
             (is_src_delta, stream_position, file_index, file_record_count)

deltacat/compute/compactor/model/round_completion_info.py

@@ -1,7 +1,7 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from typing import Tuple
+from typing import List, Tuple, Union
 from deltacat.storage import DeltaLocator, PartitionLocator
 from deltacat.compute.compactor.model.pyarrow_write_result import PyArrowWriteResult
 from typing import Any, Dict, Optional
@@ -34,7 +34,7 @@ class RoundCompletionInfo(dict):
 
     @staticmethod
     def of(
-        high_watermark: HighWatermark,
+        high_watermark: Union[HighWatermark, int],
         compacted_delta_locator: DeltaLocator,
         compacted_pyarrow_write_result: PyArrowWriteResult,
         sort_keys_bit_width: int,
@@ -66,7 +66,7 @@ class RoundCompletionInfo(dict):
         return rci
 
     @property
-    def high_watermark(self) -> HighWatermark:
+    def high_watermark(self) -> Union[HighWatermark, int]:
         val: Dict[str, Any] = self.get("highWatermark")
         if (
             val is not None
@@ -111,7 +111,7 @@ class RoundCompletionInfo(dict):
         return self["hashBucketCount"]
 
     @property
-    def hb_index_to_entry_range(self) -> Optional[Dict[int, Tuple[int, int]]]:
+    def hb_index_to_entry_range(self) -> Optional[Dict[str, Tuple[int, int]]]:
         """
         The start index is inclusive and end index is exclusive by default.
         """
@@ -130,5 +130,5 @@ class RoundCompletionInfo(dict):
         return self.get("inputAverageRecordSizeBytes")
 
     @staticmethod
-    def get_audit_bucket_name_and_key(compaction_audit_url: str) -> Tuple[str, str]:
+    def get_audit_bucket_name_and_key(compaction_audit_url: str) -> List[str]:
         return compaction_audit_url.replace("s3://", "").split("/", 1)

deltacat/compute/compactor/model/table_object_store.py

@@ -4,7 +4,7 @@ from ray.types import ObjectRef
 
 from typing import Any, Union
 
-from abc import ABC, abstractmethod, abstractproperty
+from abc import ABC, abstractmethod
 from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
 from deltacat.storage import (
     LocalTable,
@@ -15,7 +15,8 @@ LocalTableReference = Union[ObjectRef, LocalTable]
 
 
 class LocalTableStorageStrategy(ABC):
-    @abstractproperty
+    @property
+    @abstractmethod
     def object_store(cls) -> IObjectStore:
         pass
 

deltacat/compute/compactor/repartition_session.py

@@ -138,7 +138,7 @@ def repartition(
     )
     logger.info(f"Getting {len(repar_tasks_pending)} task results...")
     repar_results: List[RepartitionResult] = ray.get(repar_tasks_pending)
-    repar_results: List[Delta] = [rp.range_deltas for rp in repar_results]
+    repar_results: List[List[Delta]] = [rp.range_deltas for rp in repar_results]
     transposed = list(itertools.zip_longest(*repar_results, fillvalue=None))
     ordered_deltas: List[Delta] = [
         i for sublist in transposed for i in sublist if i is not None

deltacat/compute/compactor/steps/dedupe.py

@@ -15,7 +15,8 @@ from deltacat.compute.compactor import (
     DeltaFileEnvelope,
     DeltaFileLocator,
 )
-from deltacat.storage.model.sort_key import SortKey, SortOrder
+from deltacat.storage.model.sort_key import SortKey
+from deltacat.storage import SortOrder
 from deltacat.compute.compactor.model.dedupe_result import DedupeResult
 from deltacat.compute.compactor.utils import system_columns as sc
 from deltacat.utils.ray_utils.runtime import (
@@ -155,15 +156,21 @@ def _timed_dedupe(
                 sort_keys.extend(
                     [
                         SortKey.of(
-                            sc._PARTITION_STREAM_POSITION_COLUMN_NAME,
+                            [sc._PARTITION_STREAM_POSITION_COLUMN_NAME],
                             SortOrder.ASCENDING,
                         ),
                         SortKey.of(
-                            sc._ORDERED_FILE_IDX_COLUMN_NAME, SortOrder.ASCENDING
+                            [sc._ORDERED_FILE_IDX_COLUMN_NAME],
+                            SortOrder.ASCENDING,
                         ),
                     ]
                 )
-                table = table.take(pc.sort_indices(table, sort_keys=sort_keys))
+                table = table.take(
+                    pc.sort_indices(
+                        table,
+                        sort_keys=[pa_key for key in sort_keys for pa_key in key.arrow],
+                    )
+                )
 
             # drop duplicates by primary key hash column
             logger.info(