deltacat 0.1.18b12__tar.gz → 0.1.18b13__tar.gz

{deltacat-0.1.18b12 → deltacat-0.1.18b13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.1.18b12
+Version: 0.1.18b13
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-0.1.18b12 → deltacat-0.1.18b13}/deltacat/__init__.py

@@ -43,7 +43,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "0.1.18b12"
+__version__ = "0.1.18b13"
 
 
 __all__ = [

{deltacat-0.1.18b12 → deltacat-0.1.18b13}/deltacat/compute/compactor/repartition_session.py

@@ -144,7 +144,9 @@ def repartition(
     logger.info(f"repartition {repar_end - repar_start} seconds")
     logger.info(f"Got {len(ordered_deltas)} task results.")
     # ordered_deltas are ordered as [cold1, cold2, coldN, hot1, hot2, hotN]
-    merged_delta = Delta.merge_deltas(ordered_deltas)
+    merged_delta = Delta.merge_deltas(
+        ordered_deltas, stream_position=last_stream_position_to_compact
+    )
     compacted_delta = deltacat_storage.commit_delta(
         merged_delta, properties=kwargs.get("properties", {})
     )

{deltacat-0.1.18b12 → deltacat-0.1.18b13}/deltacat/compute/compactor/steps/repartition.py

@@ -2,6 +2,7 @@ import importlib
 import logging
 from contextlib import nullcontext
 import pyarrow.compute as pc
+from deltacat.constants import SIGNED_INT64_MIN_VALUE, SIGNED_INT64_MAX_VALUE
 import pyarrow as pa
 from typing import List, Optional
 from deltacat.types.media import StorageType, ContentType
@@ -93,7 +94,9 @@ def repartition_range(
     if not all(column in table.column_names for table in tables):
         raise ValueError(f"Column {column} does not exist in the table")
     partition_ranges.sort()
-    partition_ranges = [-float("Inf")] + partition_ranges + [float("Inf")]
+    partition_ranges = (
+        [SIGNED_INT64_MIN_VALUE] + partition_ranges + [SIGNED_INT64_MAX_VALUE]
+    )
     partitioned_tables_list = [[] for _ in range(len(partition_ranges) - 1)]
 
     total_record_count = 0

deltacat-0.1.18b13/deltacat/compute/compactor/utils/primary_key_index.py

@@ -0,0 +1,86 @@
+import logging
+from typing import List, Optional, Tuple
+
+import numpy as np
+import pyarrow as pa
+from ray.types import ObjectRef
+
+from deltacat import logs
+from deltacat.aws import s3u
+from deltacat.compute.compactor import (
+    PrimaryKeyIndexVersionLocator,
+)
+from deltacat.compute.compactor.utils import system_columns as sc
+from deltacat.io.object_store import IObjectStore
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def delete_primary_key_index_version(
+    s3_bucket: str, pki_version_locator: PrimaryKeyIndexVersionLocator
+) -> None:
+
+    logger.info(f"Deleting primary key index: {pki_version_locator}")
+    s3u.delete_files_by_prefix(
+        s3_bucket,
+        pki_version_locator.primary_key_index_version_root_path,
+    )
+    logger.info(f"Primary key index deleted: {pki_version_locator}")
+
+
+def group_record_indices_by_hash_bucket(
+    pki_table: pa.Table, num_buckets: int
+) -> np.ndarray:
+
+    hash_bucket_to_indices = np.empty([num_buckets], dtype="object")
+    record_index = 0
+    for digest in sc.pk_hash_column_np(pki_table):
+        hash_bucket = pk_digest_to_hash_bucket_index(digest, num_buckets)
+        if hash_bucket_to_indices[hash_bucket] is None:
+            hash_bucket_to_indices[hash_bucket] = []
+        hash_bucket_to_indices[hash_bucket].append(record_index)
+        record_index += 1
+    return hash_bucket_to_indices
+
+
+def group_hash_bucket_indices(
+    hash_bucket_object_groups: np.ndarray,
+    num_buckets: int,
+    num_groups: int,
+    object_store: Optional[IObjectStore] = None,
+) -> Tuple[np.ndarray, List[ObjectRef]]:
+    """
+    Groups all the ObjectRef that belongs to a particular hash bucket group and hash bucket index.
+    """
+
+    object_refs = []
+    hash_bucket_group_to_obj_id = np.empty([num_groups], dtype="object")
+
+    if hash_bucket_object_groups is None:
+        return hash_bucket_group_to_obj_id, object_refs
+
+    hb_group_to_object = np.empty([num_groups], dtype="object")
+    for hb_index, obj in enumerate(hash_bucket_object_groups):
+        if obj:
+            hb_group = hb_index % num_groups
+            if hb_group_to_object[hb_group] is None:
+                hb_group_to_object[hb_group] = np.empty([num_buckets], dtype="object")
+            hb_group_to_object[hb_group][hb_index] = obj
+
+    for hb_group, obj in enumerate(hb_group_to_object):
+        if obj is None:
+            continue
+        object_ref = object_store.put(obj)
+        object_refs.append(object_ref)
+        hash_bucket_group_to_obj_id[hb_group] = object_ref
+        del object_ref
+    return hash_bucket_group_to_obj_id, object_refs
+
+
+def pk_digest_to_hash_bucket_index(digest, num_buckets: int) -> int:
+    """
+    Deterministically get the hash bucket a particular digest belongs to
+    based on number of total hash buckets.
+    """
+
+    return int.from_bytes(digest, "big") % num_buckets

{deltacat-0.1.18b12 → deltacat-0.1.18b13}/deltacat/constants.py

@@ -36,6 +36,9 @@ BYTES_PER_GIBIBYTE = 2**30
 BYTES_PER_TEBIBYTE = 2**40
 BYTES_PER_PEBIBYTE = 2**50
 
+SIGNED_INT64_MIN_VALUE = -(2**63)
+SIGNED_INT64_MAX_VALUE = 2**63 - 1
+
 # Inflation multiplier from snappy-compressed parquet to pyarrow.
 # This should be kept larger than actual average inflation multipliers.
 # Note that this is a very rough guess since actual observed pyarrow
@@ -49,8 +52,4 @@ PYARROW_INFLATION_MULTIPLIER = 2.5
 # Inflation multiplier from snappy-compressed parquet to pyarrow for all columns.
 PYARROW_INFLATION_MULTIPLIER_ALL_COLUMNS = 6
 
-PRIMARY_KEY_INDEX_WRITE_BOTO3_CONFIG = {
-    "retries": {"max_attempts": 25, "mode": "standard"}
-}
-
 MEMORY_TO_HASH_BUCKET_COUNT_RATIO = 0.0512 * BYTES_PER_TEBIBYTE

{deltacat-0.1.18b12 → deltacat-0.1.18b13}/deltacat/utils/placement.py

@@ -229,8 +229,13 @@ class PlacementGroupManager:
 
     def get_current_node_resource_key(self) -> str:
         # on ec2: address="172.31.34.51:6379"
-        # on manta: address = "2600:1f10:4674:6815:aadb:2dc8:de61:bc8e:6379"
-        current_node_name = ray.experimental.internal_kv.global_gcs_client.address[:-5]
+        # on AWS Glue for Ray: address = "2600:1f10:4674:6815:aadb:2dc8:de61:bc8e:6379"
+        (
+            current_node_name,
+            _,
+        ) = ray.experimental.internal_kv.global_gcs_client.address.rsplit(
+            ":", 1
+        )  # using rsplit split on the last occurence of delimiter ":"
         for node in ray.nodes():
             if node["NodeName"] == current_node_name:
                 # Found the node.

{deltacat-0.1.18b12 → deltacat-0.1.18b13}/deltacat.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.1.18b12
+Version: 0.1.18b13
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-0.1.18b12 → deltacat-0.1.18b13}/deltacat.egg-info/SOURCES.txt

@@ -46,9 +46,6 @@ deltacat/compute/compactor/steps/dedupe.py
 deltacat/compute/compactor/steps/hash_bucket.py
 deltacat/compute/compactor/steps/materialize.py
 deltacat/compute/compactor/steps/repartition.py
-deltacat/compute/compactor/steps/rehash/__init__.py
-deltacat/compute/compactor/steps/rehash/rehash_bucket.py
-deltacat/compute/compactor/steps/rehash/rewrite_index.py
 deltacat/compute/compactor/utils/__init__.py
 deltacat/compute/compactor/utils/io.py
 deltacat/compute/compactor/utils/primary_key_index.py

deltacat-0.1.18b12/deltacat/compute/compactor/steps/rehash/rehash_bucket.py

@@ -1,57 +0,0 @@
-import logging
-from typing import List, Tuple
-
-import numpy as np
-import pyarrow as pa
-import ray
-from ray.types import ObjectRef
-
-from deltacat import logs
-from deltacat.compute.compactor import PrimaryKeyIndexVersionLocator
-from deltacat.compute.compactor.utils import primary_key_index as pki
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-
-def group_file_records_by_pk_hash_bucket(
-    pki_table: pa.Table, num_buckets: int
-) -> np.ndarray:
-    # generate the new table for each new hash bucket
-    hash_bucket_to_indices = pki.group_record_indices_by_hash_bucket(
-        pki_table,
-        num_buckets,
-    )
-    hash_bucket_to_table = np.empty([num_buckets], dtype="object")
-    for hash_bucket, indices in enumerate(hash_bucket_to_indices):
-        if indices:
-            hash_bucket_to_table[hash_bucket] = pki_table.take(indices)
-    return hash_bucket_to_table
-
-
-@ray.remote(num_cpus=1, num_returns=2)
-def rehash_bucket(
-    hash_bucket_index: int,
-    s3_bucket: str,
-    old_pki_version_locator: PrimaryKeyIndexVersionLocator,
-    num_buckets: int,
-    num_groups: int,
-) -> Tuple[np.ndarray, List[ObjectRef]]:
-
-    logger.info(f"Starting rehash bucket task...")
-    tables = pki.download_hash_bucket_entries(
-        s3_bucket,
-        hash_bucket_index,
-        old_pki_version_locator,
-    )
-    prior_pk_index_table = pa.concat_tables(tables)
-    hash_bucket_to_table = group_file_records_by_pk_hash_bucket(
-        prior_pk_index_table,
-        num_buckets,
-    )
-    hash_bucket_group_to_obj_id, object_refs = pki.group_hash_bucket_indices(
-        hash_bucket_to_table,
-        num_buckets,
-        num_groups,
-    )
-    logger.info(f"Finished rehash bucket task...")
-    return hash_bucket_group_to_obj_id, object_refs

deltacat-0.1.18b12/deltacat/compute/compactor/steps/rehash/rewrite_index.py

@@ -1,48 +0,0 @@
-import logging
-from collections import defaultdict
-from typing import Any, List, Tuple
-
-import pyarrow as pa
-import ray
-from ray import cloudpickle
-from ray.types import ObjectRef
-
-from deltacat import logs
-from deltacat.compute.compactor import PrimaryKeyIndexVersionLocator, PyArrowWriteResult
-from deltacat.compute.compactor.utils import primary_key_index as pki
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-
-@ray.remote(num_cpus=1, num_returns=2)
-def rewrite_index(
-    object_ids: List[Any],
-    s3_bucket: str,
-    new_primary_key_index_version_locator: PrimaryKeyIndexVersionLocator,
-    max_records_per_index_file: int,
-) -> Tuple[PyArrowWriteResult, List[ObjectRef]]:
-
-    logger.info(f"Starting rewrite primary key index task...")
-    object_refs = [cloudpickle.loads(obj_id_pkl) for obj_id_pkl in object_ids]
-    logger.info(f"Getting table groups object refs...")
-    table_groups_list = ray.get(object_refs)
-    logger.info(f"Got {len(table_groups_list)} table groups object refs...")
-    hb_index_to_tables = defaultdict(list)
-    for table_groups in table_groups_list:
-        for hb_index, table in enumerate(table_groups):
-            if table is not None:
-                hb_index_to_tables[hb_index].append(table)
-    logger.info(f"Running {len(hb_index_to_tables)} rewrite index rounds...")
-    pki_stats = []
-    for hb_index, tables in hb_index_to_tables.items():
-        table = pa.concat_tables(tables)
-        hb_pki_stats = pki.write_primary_key_index_files(
-            table,
-            new_primary_key_index_version_locator,
-            s3_bucket,
-            hb_index,
-            max_records_per_index_file,
-        )
-        pki_stats.append(hb_pki_stats)
-    logger.info(f"Finished rewrite primary key index task...")
-    return PyArrowWriteResult.union(pki_stats), object_refs

deltacat-0.1.18b12/deltacat/compute/compactor/utils/primary_key_index.py

@@ -1,307 +0,0 @@
-import json
-import logging
-from collections import defaultdict
-from typing import Any, Callable, Dict, List, Optional, Tuple
-
-import numpy as np
-import pyarrow as pa
-import ray
-import s3fs
-from ray.types import ObjectRef
-
-from deltacat import logs
-from deltacat.aws import s3u
-from deltacat.compute.compactor import (
-    PrimaryKeyIndexLocator,
-    PrimaryKeyIndexMeta,
-    PrimaryKeyIndexVersionLocator,
-    PrimaryKeyIndexVersionMeta,
-    PyArrowWriteResult,
-    RoundCompletionInfo,
-)
-from deltacat.compute.compactor.steps.rehash import rehash_bucket as rb
-from deltacat.compute.compactor.steps.rehash import rewrite_index as ri
-from deltacat.compute.compactor.utils import round_completion_file as rcf
-from deltacat.compute.compactor.utils import system_columns as sc
-from deltacat.constants import PRIMARY_KEY_INDEX_WRITE_BOTO3_CONFIG
-from deltacat.storage import Manifest, PartitionLocator
-from deltacat.types.media import ContentEncoding, ContentType
-from deltacat.types.tables import get_table_slicer, get_table_writer
-from deltacat.utils.common import ReadKwargsProvider
-from deltacat.utils.ray_utils.concurrency import invoke_parallel
-from deltacat.io.object_store import IObjectStore
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-
-def rehash(
-    options_provider: Callable[[int, Any], Dict[str, Any]],
-    s3_bucket: str,
-    source_partition_locator: PartitionLocator,
-    old_rci: RoundCompletionInfo,
-    new_hash_bucket_count: int,
-    hash_bucket_index_group_count: int,
-    records_per_primary_key_index_file: int,
-    delete_old_primary_key_index: bool,
-) -> RoundCompletionInfo:
-
-    logger.info(
-        f"Rehashing primary key index. Old round completion info: "
-        f"{old_rci}. New hash bucket count: {new_hash_bucket_count}"
-    )
-
-    # collect old primary key index information
-    old_pki_version_locator = old_rci.primary_key_index_version_locator
-    old_pkiv_meta = old_pki_version_locator.primary_key_index_version_meta
-    old_pki_meta = old_pkiv_meta.primary_key_index_meta
-    old_compacted_partition_locator = old_pki_meta.compacted_partition_locator
-    if old_pkiv_meta.hash_bucket_count == new_hash_bucket_count:
-        raise ValueError(
-            f"Primary key index rehash failed. Old hash bucket "
-            f"count ({new_hash_bucket_count}) is "
-            f"equal to new hash bucket count. Partition: "
-            f"{old_compacted_partition_locator}."
-        )
-
-    # generate a new unique primary key index version locator to rehash into
-    new_pki_meta = PrimaryKeyIndexMeta.of(
-        old_compacted_partition_locator,
-        old_pki_meta.primary_keys,
-        old_pki_meta.sort_keys,
-        old_pki_meta.primary_key_index_algorithm_version,
-    )
-    new_pki_locator = PrimaryKeyIndexLocator.of(new_pki_meta)
-    new_pki_version_meta = PrimaryKeyIndexVersionMeta.of(
-        new_pki_meta,
-        new_hash_bucket_count,
-    )
-    rehashed_pki_version_locator = PrimaryKeyIndexVersionLocator.generate(
-        new_pki_version_meta
-    )
-
-    # launch a rehash task for each bucket of the old primary key index version
-    old_hash_bucket_count = old_pkiv_meta.hash_bucket_count
-    hb_tasks_pending = invoke_parallel(
-        items=range(old_hash_bucket_count),
-        ray_task=rb.rehash_bucket,
-        max_parallelism=None,
-        options_provider=options_provider,
-        s3_bucket=s3_bucket,
-        old_pki_version_locator=old_pki_version_locator,
-        num_buckets=new_hash_bucket_count,
-        num_groups=hash_bucket_index_group_count,
-    )
-    logger.info(f"Getting {len(hb_tasks_pending)} rehash bucket results...")
-    hb_results = ray.get([t[0] for t in hb_tasks_pending])
-    logger.info(f"Got {len(hb_results)} rehash bucket results.")
-    all_hash_group_idx_to_obj_id = defaultdict(list)
-    for hash_group_idx_to_obj_id in hb_results:
-        for hash_group_index, object_id in enumerate(hash_group_idx_to_obj_id):
-            if object_id:
-                all_hash_group_idx_to_obj_id[hash_group_index].append(object_id)
-    hash_group_count = len(all_hash_group_idx_to_obj_id)
-    logger.info(f"Rehash bucket groups created: {hash_group_count}")
-
-    # write primary key index files for each rehashed output bucket
-    pki_stats_promises = invoke_parallel(
-        items=all_hash_group_idx_to_obj_id.values(),
-        ray_task=ri.rewrite_index,
-        max_parallelism=None,
-        options_provider=options_provider,
-        s3_bucket=s3_bucket,
-        new_primary_key_index_version_locator=rehashed_pki_version_locator,
-        max_records_per_index_file=records_per_primary_key_index_file,
-    )
-    logger.info(f"Getting {len(pki_stats_promises)} rewrite index results...")
-    pki_stats = ray.get([t[0] for t in pki_stats_promises])
-    logger.info(f"Got {len(pki_stats)} rewrite index results.")
-
-    round_completion_info = RoundCompletionInfo.of(
-        old_rci.high_watermark,
-        old_rci.compacted_delta_locator,
-        old_rci.compacted_pyarrow_write_result,
-        PyArrowWriteResult.union(pki_stats),
-        old_rci.sort_keys_bit_width,
-        rehashed_pki_version_locator,
-        old_rci.rebase_source_partition_locator,
-    )
-    rcf.write_round_completion_file(
-        s3_bucket,
-        source_partition_locator,
-        new_pki_locator.primary_key_index_root_path,
-        round_completion_info,
-    )
-    if delete_old_primary_key_index:
-        delete_primary_key_index_version(
-            s3_bucket,
-            old_pki_version_locator,
-        )
-    logger.info(
-        f"Rehashed primary key index. New round completion info: "
-        f"{round_completion_info}."
-    )
-    return round_completion_info
-
-
-def download_hash_bucket_entries(
-    s3_bucket: str,
-    hash_bucket_index: int,
-    primary_key_index_version_locator: PrimaryKeyIndexVersionLocator,
-    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
-) -> List[pa.Table]:
-
-    pk_index_manifest_s3_url = (
-        primary_key_index_version_locator.get_pkiv_hb_index_manifest_s3_url(
-            s3_bucket,
-            hash_bucket_index,
-        )
-    )
-    result = s3u.download(pk_index_manifest_s3_url, False)
-    logger.info(
-        f"Downloading primary key index hash bucket manifest entries: "
-        f"{pk_index_manifest_s3_url}. Primary key index version "
-        f"locator: {primary_key_index_version_locator}"
-    )
-    pk_index_manifest = Manifest(json.loads(result["Body"].read().decode("utf-8")))
-    tables = s3u.download_manifest_entries(
-        pk_index_manifest, file_reader_kwargs_provider=file_reader_kwargs_provider
-    )
-    if not tables:
-        logger.warning(
-            f"Primary key index manifest is empty at: "
-            f"{pk_index_manifest_s3_url}. Primary key index version "
-            f"locator: {primary_key_index_version_locator}"
-        )
-    return tables
-
-
-def delete_primary_key_index_version(
-    s3_bucket: str, pki_version_locator: PrimaryKeyIndexVersionLocator
-) -> None:
-
-    logger.info(f"Deleting primary key index: {pki_version_locator}")
-    s3u.delete_files_by_prefix(
-        s3_bucket,
-        pki_version_locator.primary_key_index_version_root_path,
-    )
-    logger.info(f"Primary key index deleted: {pki_version_locator}")
-
-
-def group_record_indices_by_hash_bucket(
-    pki_table: pa.Table, num_buckets: int
-) -> np.ndarray:
-
-    hash_bucket_to_indices = np.empty([num_buckets], dtype="object")
-    record_index = 0
-    for digest in sc.pk_hash_column_np(pki_table):
-        hash_bucket = pk_digest_to_hash_bucket_index(digest, num_buckets)
-        if hash_bucket_to_indices[hash_bucket] is None:
-            hash_bucket_to_indices[hash_bucket] = []
-        hash_bucket_to_indices[hash_bucket].append(record_index)
-        record_index += 1
-    return hash_bucket_to_indices
-
-
-def group_hash_bucket_indices(
-    hash_bucket_object_groups: np.ndarray,
-    num_buckets: int,
-    num_groups: int,
-    object_store: Optional[IObjectStore] = None,
-) -> Tuple[np.ndarray, List[ObjectRef]]:
-    """
-    Groups all the ObjectRef that belongs to a particular hash bucket group and hash bucket index.
-    """
-
-    object_refs = []
-    hash_bucket_group_to_obj_id = np.empty([num_groups], dtype="object")
-
-    if hash_bucket_object_groups is None:
-        return hash_bucket_group_to_obj_id, object_refs
-
-    hb_group_to_object = np.empty([num_groups], dtype="object")
-    for hb_index, obj in enumerate(hash_bucket_object_groups):
-        if obj:
-            hb_group = hb_index % num_groups
-            if hb_group_to_object[hb_group] is None:
-                hb_group_to_object[hb_group] = np.empty([num_buckets], dtype="object")
-            hb_group_to_object[hb_group][hb_index] = obj
-
-    for hb_group, obj in enumerate(hb_group_to_object):
-        if obj is None:
-            continue
-        object_ref = object_store.put(obj)
-        object_refs.append(object_ref)
-        hash_bucket_group_to_obj_id[hb_group] = object_ref
-        del object_ref
-    return hash_bucket_group_to_obj_id, object_refs
-
-
-def pk_digest_to_hash_bucket_index(digest, num_buckets: int) -> int:
-    """
-    Deterministically get the hash bucket a particular digest belongs to
-    based on number of total hash buckets.
-    """
-
-    return int.from_bytes(digest, "big") % num_buckets
-
-
-def write_primary_key_index_files(
-    table: pa.Table,
-    primary_key_index_version_locator: PrimaryKeyIndexVersionLocator,
-    s3_bucket: str,
-    hb_index: int,
-    records_per_index_file: int,
-) -> PyArrowWriteResult:
-    """
-    Writes primary key index files for the given hash bucket index out to the
-    specified S3 bucket at the path identified by the given primary key index
-    version locator. Output is written as 1 or more Parquet files with the
-    given maximum number of records per file.
-
-    TODO(raghumdani): Support writing primary key index to any data catalog
-    """
-    logger.info(
-        f"Writing primary key index files for hash bucket {hb_index}. "
-        f"Primary key index version locator: "
-        f"{primary_key_index_version_locator}."
-    )
-    s3_file_system = s3fs.S3FileSystem(
-        anon=False,
-        s3_additional_kwargs={
-            "ContentType": ContentType.PARQUET.value,
-            "ContentEncoding": ContentEncoding.IDENTITY.value,
-        },
-        config_kwargs=PRIMARY_KEY_INDEX_WRITE_BOTO3_CONFIG,
-    )
-    pkiv_hb_index_s3_url_base = (
-        primary_key_index_version_locator.get_pkiv_hb_index_s3_url_base(
-            s3_bucket, hb_index
-        )
-    )
-    manifest_entries = s3u.upload_sliced_table(
-        table,
-        pkiv_hb_index_s3_url_base,
-        s3_file_system,
-        records_per_index_file,
-        get_table_writer(table),
-        get_table_slicer(table),
-    )
-    manifest = Manifest.of(manifest_entries)
-    pkiv_hb_index_s3_manifest_s3_url = (
-        primary_key_index_version_locator.get_pkiv_hb_index_manifest_s3_url(
-            s3_bucket, hb_index
-        )
-    )
-    s3u.upload(pkiv_hb_index_s3_manifest_s3_url, str(json.dumps(manifest)))
-    result = PyArrowWriteResult.of(
-        len(manifest_entries),
-        table.nbytes,
-        manifest.meta.content_length,
-        len(table),
-    )
-    logger.info(
-        f"Wrote primary key index files for hash bucket {hb_index}. "
-        f"Primary key index version locator: "
-        f"{primary_key_index_version_locator}. Result: {result}"
-    )
-    return result