deltacat 0.2.7__tar.gz → 0.2.9__tar.gz

{deltacat-0.2.7 → deltacat-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.2.7
+Version: 0.2.9
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "0.2.7"
+__version__ = "0.2.9"
 
 
 __all__ = [

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/aws/s3u.py

@@ -383,6 +383,12 @@ def upload_table(
                 # s3fs may swallow S3 errors - we were probably throttled
                 raise RetryableError(f"Retry table upload to: {s3_url}") from e
             raise NonRetryableError(f"Failed table upload to: {s3_url}") from e
+        except BaseException as e:
+            logger.warn(
+                f"Upload has failed for {s3_url} and content_type={content_type}. Error: {e}",
+                exc_info=True,
+            )
+            raise e
     return manifest_entries
 
 

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor/compaction_session.py

@@ -52,7 +52,7 @@ from deltacat.compute.compactor.model.compaction_session_audit_info import (
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
-from deltacat.utils.resources import get_current_node_peak_memory_usage_in_bytes
+from deltacat.utils.resources import get_current_process_peak_memory_usage_in_bytes
 
 
 if importlib.util.find_spec("memray"):
@@ -293,7 +293,6 @@ def _execute_compaction_round(
             f"{node_resource_keys}"
         )
 
-    compaction_audit.set_cluster_cpu_max(cluster_cpus)
     # create a remote options provider to round-robin tasks across all nodes or allocated bundles
     logger.info(f"Setting round robin scheduling with node id:{node_resource_keys}")
     round_robin_opt_provider = functools.partial(
@@ -680,7 +679,7 @@ def _execute_compaction_round(
         [m.pyarrow_write_result for m in mat_results]
     )
 
-    session_peak_memory = get_current_node_peak_memory_usage_in_bytes()
+    session_peak_memory = get_current_process_peak_memory_usage_in_bytes()
     compaction_audit.set_peak_memory_used_bytes_by_compaction_session_process(
         session_peak_memory
     )

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -98,14 +98,6 @@ class CompactionSessionAuditInfo(dict):
         """
         return self.get("hashBucketCount")
 
-    @property
-    def cluster_cpu_max(self) -> float:
-        """
-        Total cluster cpu allocated for the compaction job. If it is autoscaling cluster,
-        max cpu at any time will be reported.
-        """
-        return self.get("clusterCpuMax")
-
     @property
     def compaction_time_in_seconds(self) -> float:
         """
@@ -423,35 +415,6 @@ class CompactionSessionAuditInfo(dict):
         """
         return self.get("hashBucketProcessedSizeBytes")
 
-    @property
-    def total_cpu_seconds(self) -> float:
-        """
-        Total number of vCPUs provisioned in the cluster weighted over time.
-        """
-        return self.get("totalCPUSeconds")
-
-    @property
-    def used_cpu_seconds(self) -> float:
-        """
-        Total used vCPU in the cluster weighted over time.
-        """
-        return self.get("usedCPUSeconds")
-
-    @property
-    def used_memory_gb_seconds(self) -> float:
-        """
-        The used memory in the cluster weighted over time. This
-        determines opportunities for better memory estimation.
-        """
-        return self.get("usedMemoryGBSeconds")
-
-    @property
-    def total_memory_gb_seconds(self) -> float:
-        """
-        Total memory in the cluster weighted over time in GB.
-        """
-        return self.get("totalMemoryGBSeconds")
-
     @property
     def pyarrow_version(self) -> str:
         """
@@ -510,10 +473,6 @@ class CompactionSessionAuditInfo(dict):
         self["hashBucketCount"] = hash_bucket_count
         return self
 
-    def set_cluster_cpu_max(self, cluster_cpu_max: float) -> CompactionSessionAuditInfo:
-        self["clusterCpuMax"] = cluster_cpu_max
-        return self
-
     def set_compaction_time_in_seconds(
         self, compaction_time_in_seconds: float
     ) -> CompactionSessionAuditInfo:
@@ -778,22 +737,6 @@ class CompactionSessionAuditInfo(dict):
         self["hashBucketProcessedSizeBytes"] = size
         return self
 
-    def set_total_cpu_seconds(self, value: float) -> CompactionSessionAuditInfo:
-        self["totalCPUSeconds"] = value
-        return self
-
-    def set_used_cpu_seconds(self, value: float) -> CompactionSessionAuditInfo:
-        self["usedCPUSeconds"] = value
-        return self
-
-    def set_used_memory_gb_seconds(self, value: float) -> CompactionSessionAuditInfo:
-        self["usedMemoryGBSeconds"] = value
-        return self
-
-    def set_total_memory_gb_seconds(self, value: float) -> CompactionSessionAuditInfo:
-        self["totalMemoryGBSeconds"] = value
-        return self
-
     def set_pyarrow_version(self, value: str) -> CompactionSessionAuditInfo:
         self["pyarrowVersion"] = value
         return self

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor/steps/dedupe.py

@@ -25,7 +25,7 @@ from deltacat.utils.ray_utils.runtime import (
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
 from deltacat.io.object_store import IObjectStore
-from deltacat.utils.resources import get_current_node_peak_memory_usage_in_bytes
+from deltacat.utils.resources import get_current_process_peak_memory_usage_in_bytes
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -228,7 +228,7 @@ def _timed_dedupe(
             f"{len(mat_bucket_to_dd_idx_obj_id)}"
         )
 
-        peak_memory_usage_bytes = get_current_node_peak_memory_usage_in_bytes()
+        peak_memory_usage_bytes = get_current_process_peak_memory_usage_in_bytes()
         return DedupeResult(
             mat_bucket_to_dd_idx_obj_id,
             np.int64(total_deduped_records),

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor/steps/hash_bucket.py

@@ -32,7 +32,7 @@ from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
 from deltacat.io.object_store import IObjectStore
-from deltacat.utils.resources import get_current_node_peak_memory_usage_in_bytes
+from deltacat.utils.resources import get_current_process_peak_memory_usage_in_bytes
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -228,7 +228,7 @@ def _timed_hash_bucket(
             delta_file_envelope_groups, num_buckets, num_groups, object_store
         )
 
-        peak_memory_usage_bytes = get_current_node_peak_memory_usage_in_bytes()
+        peak_memory_usage_bytes = get_current_process_peak_memory_usage_in_bytes()
         return HashBucketResult(
             hash_bucket_group_to_obj_id,
             np.int64(total_record_count),

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor/steps/materialize.py

@@ -44,7 +44,7 @@ from deltacat.utils.ray_utils.runtime import (
     get_current_ray_worker_id,
 )
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
-from deltacat.utils.resources import get_current_node_peak_memory_usage_in_bytes
+from deltacat.utils.resources import get_current_process_peak_memory_usage_in_bytes
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -314,7 +314,7 @@ def materialize(
             emit_metrics_time = latency
         logger.info(f"Materialize task ended in {end - start}s")
 
-        peak_memory_usage_bytes = get_current_node_peak_memory_usage_in_bytes()
+        peak_memory_usage_bytes = get_current_process_peak_memory_usage_in_bytes()
 
         # Merge all new deltas into one for this materialize bucket index
         merged_materialize_result = MaterializeResult.of(

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor_v2/compaction_session.py

@@ -41,13 +41,12 @@ from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
 )
 from deltacat.utils.resources import (
-    get_current_node_peak_memory_usage_in_bytes,
+    get_current_process_peak_memory_usage_in_bytes,
 )
 from deltacat.compute.compactor_v2.utils.task_options import (
     hash_bucket_resource_options_provider,
     merge_resource_options_provider,
 )
-from deltacat.utils.resources import ClusterUtilizationOverTimeRange
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 
 if importlib.util.find_spec("memray"):
@@ -65,10 +64,9 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
 
     with memray.Tracker(
         f"compaction_partition.bin"
-    ) if params.enable_profiler else nullcontext(), ClusterUtilizationOverTimeRange() as cluster_util:
+    ) if params.enable_profiler else nullcontext():
         (new_partition, new_rci, new_rcf_partition_locator,) = _execute_compaction(
             params,
-            cluster_util=cluster_util,
             **kwargs,
         )
 
@@ -469,7 +467,7 @@ def _execute_compaction(
         [m.pyarrow_write_result for m in mat_results]
     )
 
-    session_peak_memory = get_current_node_peak_memory_usage_in_bytes()
+    session_peak_memory = get_current_process_peak_memory_usage_in_bytes()
     compaction_audit.set_peak_memory_used_bytes_by_compaction_session_process(
         session_peak_memory
     )
@@ -478,17 +476,6 @@ def _execute_compaction(
         mat_results, telemetry_time_hb + telemetry_time_merge
     )
 
-    cluster_util: ClusterUtilizationOverTimeRange = kwargs.get("cluster_util")
-
-    if cluster_util:
-        compaction_audit.set_total_cpu_seconds(cluster_util.total_vcpu_seconds)
-        compaction_audit.set_used_cpu_seconds(cluster_util.used_vcpu_seconds)
-        compaction_audit.set_used_memory_gb_seconds(cluster_util.used_memory_gb_seconds)
-        compaction_audit.set_total_memory_gb_seconds(
-            cluster_util.total_memory_gb_seconds
-        )
-        compaction_audit.set_cluster_cpu_max(cluster_util.max_cpu)
-
     input_inflation = None
     input_average_record_size_bytes = None
     # Note: we only consider inflation for incremental delta

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor_v2/steps/hash_bucket.py

@@ -27,7 +27,11 @@ from deltacat.utils.ray_utils.runtime import (
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.metrics import emit_timer_metrics
-from deltacat.utils.resources import get_current_node_peak_memory_usage_in_bytes
+from deltacat.utils.resources import (
+    get_current_process_peak_memory_usage_in_bytes,
+    ProcessUtilizationOverTimeRange,
+)
+from deltacat.constants import BYTES_PER_GIBIBYTE
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -166,7 +170,10 @@ def _timed_hash_bucket(input: HashBucketInput):
             object_store=input.object_store,
         )
 
-        peak_memory_usage_bytes = get_current_node_peak_memory_usage_in_bytes()
+        peak_memory_usage_bytes = get_current_process_peak_memory_usage_in_bytes()
+        logger.info(
+            f"Peak memory usage in bytes after hash bucketing: {peak_memory_usage_bytes}"
+        )
         return HashBucketResult(
             hash_bucket_group_to_obj_id_tuple,
             np.int64(total_size_bytes),
@@ -179,28 +186,38 @@ def _timed_hash_bucket(input: HashBucketInput):
 
 @ray.remote
 def hash_bucket(input: HashBucketInput) -> HashBucketResult:
+    with ProcessUtilizationOverTimeRange() as process_util:
+        logger.info(f"Starting hash bucket task...")
 
-    logger.info(f"Starting hash bucket task...")
-    hash_bucket_result, duration = timed_invocation(
-        func=_timed_hash_bucket, input=input
-    )
+        # Log node peak memory utilization every 10 seconds
+        def log_peak_memory():
+            logger.debug(
+                f"Process peak memory utilization so far: {process_util.max_memory} bytes "
+                f"({process_util.max_memory/BYTES_PER_GIBIBYTE} GB)"
+            )
+
+        process_util.schedule_callback(log_peak_memory, 10)
 
-    emit_metrics_time = 0.0
-    if input.metrics_config:
-        emit_result, latency = timed_invocation(
-            func=emit_timer_metrics,
-            metrics_name="hash_bucket",
-            value=duration,
-            metrics_config=input.metrics_config,
+        hash_bucket_result, duration = timed_invocation(
+            func=_timed_hash_bucket, input=input
+        )
+
+        emit_metrics_time = 0.0
+        if input.metrics_config:
+            emit_result, latency = timed_invocation(
+                func=emit_timer_metrics,
+                metrics_name="hash_bucket",
+                value=duration,
+                metrics_config=input.metrics_config,
+            )
+            emit_metrics_time = latency
+
+        logger.info(f"Finished hash bucket task...")
+        return HashBucketResult(
+            hash_bucket_result[0],
+            hash_bucket_result[1],
+            hash_bucket_result[2],
+            hash_bucket_result[3],
+            np.double(emit_metrics_time),
+            hash_bucket_result[5],
         )
-        emit_metrics_time = latency
-
-    logger.info(f"Finished hash bucket task...")
-    return HashBucketResult(
-        hash_bucket_result[0],
-        hash_bucket_result[1],
-        hash_bucket_result[2],
-        hash_bucket_result[3],
-        np.double(emit_metrics_time),
-        hash_bucket_result[5],
-    )

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor_v2/steps/merge.py

@@ -30,7 +30,10 @@ from deltacat.compute.compactor.utils import system_columns as sc
 
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.metrics import emit_timer_metrics
-from deltacat.utils.resources import get_current_node_peak_memory_usage_in_bytes
+from deltacat.utils.resources import (
+    get_current_process_peak_memory_usage_in_bytes,
+    ProcessUtilizationOverTimeRange,
+)
 from deltacat.compute.compactor_v2.utils.primary_key_index import (
     generate_pk_hash_column,
     hash_group_index_to_hash_bucket_indices,
@@ -44,6 +47,7 @@ from deltacat.storage import (
     interface as unimplemented_deltacat_storage,
 )
 from deltacat.compute.compactor_v2.utils.dedupe import drop_duplicates
+from deltacat.constants import BYTES_PER_GIBIBYTE
 
 
 if importlib.util.find_spec("memray"):
@@ -436,7 +440,10 @@ def _timed_merge(input: MergeInput) -> MergeResult:
             f"{total_dfes_found} != {len(hb_index_to_delta_file_envelopes_list)}"
         )
 
-        peak_memory_usage_bytes = get_current_node_peak_memory_usage_in_bytes()
+        peak_memory_usage_bytes = get_current_process_peak_memory_usage_in_bytes()
+        logger.info(
+            f"Peak memory usage in bytes after merge: {peak_memory_usage_bytes}"
+        )
 
         return MergeResult(
             materialized_results,
@@ -449,25 +456,35 @@ def _timed_merge(input: MergeInput) -> MergeResult:
 
 @ray.remote
 def merge(input: MergeInput) -> MergeResult:
+    with ProcessUtilizationOverTimeRange() as process_util:
+        logger.info(f"Starting merge task...")
+
+        # Log node peak memory utilization every 10 seconds
+        def log_peak_memory():
+            logger.debug(
+                f"Process peak memory utilization so far: {process_util.max_memory} bytes "
+                f"({process_util.max_memory/BYTES_PER_GIBIBYTE} GB)"
+            )
+
+        process_util.schedule_callback(log_peak_memory, 10)
 
-    logger.info(f"Starting merge task...")
-    merge_result, duration = timed_invocation(func=_timed_merge, input=input)
+        merge_result, duration = timed_invocation(func=_timed_merge, input=input)
 
-    emit_metrics_time = 0.0
-    if input.metrics_config:
-        emit_result, latency = timed_invocation(
-            func=emit_timer_metrics,
-            metrics_name="merge",
-            value=duration,
-            metrics_config=input.metrics_config,
+        emit_metrics_time = 0.0
+        if input.metrics_config:
+            emit_result, latency = timed_invocation(
+                func=emit_timer_metrics,
+                metrics_name="merge",
+                value=duration,
+                metrics_config=input.metrics_config,
+            )
+            emit_metrics_time = latency
+
+        logger.info(f"Finished merge task...")
+        return MergeResult(
+            merge_result[0],
+            merge_result[1],
+            merge_result[2],
+            np.double(emit_metrics_time),
+            merge_result[4],
         )
-        emit_metrics_time = latency
-
-    logger.info(f"Finished merge task...")
-    return MergeResult(
-        merge_result[0],
-        merge_result[1],
-        merge_result[2],
-        np.double(emit_metrics_time),
-        merge_result[4],
-    )

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -17,6 +17,7 @@ from deltacat import logs
 from deltacat.compute.compactor.utils import system_columns as sc
 from deltacat.io.object_store import IObjectStore
 from deltacat.utils.performance import timed_invocation
+from deltacat.utils.pyarrow import sliced_string_cast
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -182,7 +183,7 @@ def generate_pk_hash_column(
     def _generate_pk_hash(table: pa.Table) -> pa.Array:
         pk_columns = []
         for pk_name in primary_keys:
-            pk_columns.append(pc.cast(table[pk_name], pa.string()))
+            pk_columns.append(sliced_string_cast(table[pk_name]))
 
         pk_columns.append(PK_DELIMITER)
         hash_column = pc.binary_join_element_wise(*pk_columns)

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/tests/compute/test_compact_partition_incremental.py

@@ -6,6 +6,8 @@ import boto3
 from typing import Any, Callable, Dict, List, Optional, Set
 from boto3.resources.base import ServiceResource
 import pyarrow as pa
+from pytest_benchmark.fixture import BenchmarkFixture
+
 from deltacat.tests.compute.test_util_common import (
     get_rcf,
 )
@@ -161,6 +163,7 @@ def test_compact_partition_incremental(
     read_kwargs_provider_param: Any,
     skip_enabled_compact_partition_drivers,
     compact_partition_func: Callable,
+    benchmark: BenchmarkFixture,
 ):
     import deltacat.tests.local_deltacat_storage as ds
     from deltacat.types.media import ContentType
@@ -235,8 +238,22 @@ def test_compact_partition_incremental(
             "sort_keys": sort_keys if sort_keys else None,
         }
     )
+
     # execute
-    rcf_file_s3_uri = compact_partition_func(compact_partition_params)
+    def _incremental_compaction_setup():
+        """
+        This callable runs right before invoking the benchmark target function (compaction).
+        This is needed as the benchmark module will invoke the target function multiple times
+        in a single test run, which can lead to non-idempotent behavior if RCFs are generated.
+
+        Returns: args, kwargs
+        """
+        setup_s3_resource.Bucket(TEST_S3_RCF_BUCKET_NAME).objects.all().delete()
+        return (compact_partition_params,), {}
+
+    rcf_file_s3_uri = benchmark.pedantic(
+        compact_partition_func, setup=_incremental_compaction_setup
+    )
     # validate
     round_completion_info = get_rcf(setup_s3_resource, rcf_file_s3_uri)
     compacted_delta_locator: DeltaLocator = (

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -5,6 +5,8 @@ import pytest
 import boto3
 from boto3.resources.base import ServiceResource
 import pyarrow as pa
+from pytest_benchmark.fixture import BenchmarkFixture
+
 from deltacat.tests.compute.test_util_constant import (
     BASE_TEST_SOURCE_NAMESPACE,
     BASE_TEST_SOURCE_TABLE_NAME,
@@ -182,6 +184,7 @@ def test_compact_partition_rebase_then_incremental(
     rebase_expected_compact_partition_result: pa.Table,
     skip_enabled_compact_partition_drivers,
     compact_partition_func: Callable,
+    benchmark: BenchmarkFixture,
 ):
     import deltacat.tests.local_deltacat_storage as ds
     from deltacat.types.media import ContentType
@@ -265,7 +268,7 @@ def test_compact_partition_rebase_then_incremental(
         }
     )
     # execute
-    rcf_file_s3_uri = compact_partition_func(compact_partition_params)
+    rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
     compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
         setup_s3_resource, rcf_file_s3_uri
     )

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/tests/utils/test_resources.py

@@ -49,3 +49,24 @@ class TestClusterUtilizationOverTimeRange(unittest.TestCase):
             self.assertIsNotNone(cu.total_memory_gb_seconds)
             self.assertIsNotNone(cu.used_memory_gb_seconds)
             self.assertIsNotNone(cu.max_cpu)
+
+
+class TestProcessUtilizationOverTimeRange(unittest.TestCase):
+    def test_sanity(self):
+        from deltacat.utils.resources import ProcessUtilizationOverTimeRange
+
+        with ProcessUtilizationOverTimeRange() as nu:
+            time.sleep(3)
+            self.assertIsNotNone(nu.max_memory)
+
+    def test_callback(self):
+        from deltacat.utils.resources import ProcessUtilizationOverTimeRange
+
+        with ProcessUtilizationOverTimeRange() as nu:
+
+            def test_callback():
+                nu.test_field_set = True
+
+            nu.schedule_callback(test_callback, 1)
+            time.sleep(3)
+            self.assertTrue(nu.test_field_set)

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/utils/daft.py

@@ -66,6 +66,8 @@ def daft_s3_file_to_table(
         )
     )
 
+    logger.debug(f"Preparing to read S3 object from {s3_url} into daft table")
+
     pa_table, latency = timed_invocation(
         read_parquet_into_pyarrow,
         path=s3_url,

{deltacat-0.2.7 → deltacat-0.2.9}/deltacat/utils/pyarrow.py

@@ -11,6 +11,8 @@ from pyarrow.parquet import ParquetFile
 from deltacat.exceptions import ValidationError
 
 import pyarrow as pa
+import numpy as np
+import pyarrow.compute as pc
 from fsspec import AbstractFileSystem
 from pyarrow import csv as pacsv
 from pyarrow import feather as paf
@@ -38,6 +40,7 @@ from deltacat.utils.arguments import (
     sanitize_kwargs_to_callable,
     sanitize_kwargs_by_supported_kwargs,
 )
+from functools import lru_cache
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -738,3 +741,69 @@ class RecordBatchTables:
         """
         self._remaining_tables.clear()
         self._remaining_record_count = 0
+
+
+@lru_cache(maxsize=1)
+def _int_max_string_len() -> int:
+    PA_UINT64_MAX_STR_BYTES = pc.binary_length(
+        pc.cast(pa.scalar(2**64 - 1, type=pa.uint64()), pa.string())
+    ).as_py()
+    PA_INT64_MAX_STR_BYTES = pc.binary_length(
+        pc.cast(pa.scalar(-(2**63), type=pa.int64()), pa.string())
+    ).as_py()
+    return max(PA_UINT64_MAX_STR_BYTES, PA_INT64_MAX_STR_BYTES)
+
+
+@lru_cache(maxsize=1)
+def _float_max_string_len() -> int:
+    PA_POS_FLOAT64_MAX_STR_BYTES = pc.binary_length(
+        pc.cast(pa.scalar(np.finfo(np.float64).max, type=pa.float64()), pa.string())
+    ).as_py()
+    PA_NEG_FLOAT64_MAX_STR_BYTES = pc.binary_length(
+        pc.cast(pa.scalar(np.finfo(np.float64).min, type=pa.float64()), pa.string())
+    ).as_py()
+    return max(PA_POS_FLOAT64_MAX_STR_BYTES, PA_NEG_FLOAT64_MAX_STR_BYTES)
+
+
+def _max_decimal128_string_len():
+    return 40  # "-" + 38 digits + decimal
+
+
+def _max_decimal256_string_len():
+    return 78  # "-" + 76 digits + decimal
+
+
+def sliced_string_cast(array: pa.ChunkedArray) -> pa.ChunkedArray:
+    """performs slicing of a pyarrow array prior casting to a string.
+    This prevents a pyarrow from allocating too large of an array causing a failure.
+    Issue: https://github.com/apache/arrow/issues/38835
+    TODO: deprecate this function when pyarrow performs proper ChunkedArray -> ChunkedArray casting
+    """
+    dtype = array.type
+    MAX_BYTES = 2147483646
+    max_str_len = None
+    if pa.types.is_integer(dtype):
+        max_str_len = _int_max_string_len()
+    elif pa.types.is_floating(dtype):
+        max_str_len = _float_max_string_len()
+    elif pa.types.is_decimal128(dtype):
+        max_str_len = _max_decimal128_string_len()
+    elif pa.types.is_decimal256(dtype):
+        max_str_len = _max_decimal256_string_len()
+
+    if max_str_len is not None:
+        max_elems_per_chunk = MAX_BYTES // (2 * max_str_len)  # safety factor of 2
+        all_chunks = []
+        for chunk in array.chunks:
+            if len(chunk) < max_elems_per_chunk:
+                all_chunks.append(chunk)
+            else:
+                curr_pos = 0
+                total_len = len(chunk)
+                while curr_pos < total_len:
+                    sliced = chunk.slice(curr_pos, max_elems_per_chunk)
+                    curr_pos += len(sliced)
+                    all_chunks.append(sliced)
+        array = pa.chunked_array(all_chunks, type=dtype)
+
+    return pc.cast(array, pa.string())