deltacat 0.1.18b17__py3-none-any.whl → 0.1.18b19__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "0.1.18b17"
+__version__ = "0.1.18b19"
 
 
 __all__ = [

deltacat/aws/clients.py

@@ -109,7 +109,7 @@ def block_until_instance_metadata_service_returns_success(
     url=INSTANCE_METADATA_SERVICE_IPV4_URI,
     retry_strategy=RetryIfRetryableHTTPStatusCode,
     wait_strategy=wait_fixed(2),  # wait 2 seconds before retrying,
-    stop_strategy=stop_after_delay(60 * 10),  # stop trying after 10 minutes
+    stop_strategy=stop_after_delay(60 * 30),  # stop trying after 30 minutes
 ) -> Optional[Response]:
     """Blocks until the instance metadata service returns a successful response.
 
@@ -151,10 +151,10 @@ def _resource(name: str, region: Optional[str], **kwargs) -> ServiceResource:
     boto3_session = _get_session_from_kwargs(kwargs)
 
     boto_config = Config(retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "adaptive"})
+    kwargs = {"config": boto_config, **kwargs}
     return boto3_session.resource(
         name,
         region,
-        config=boto_config,
         **kwargs,
     )
 
@@ -169,10 +169,10 @@ def _client(name: str, region: Optional[str], **kwargs) -> BaseClient:
         boto_config = Config(
             retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "adaptive"}
         )
+        kwargs = {"config": boto_config, **kwargs}
         return boto3_session.client(
             name,
             region,
-            config=boto_config,
             **kwargs,
         )
 

deltacat/aws/constants.py

@@ -1,6 +1,7 @@
 from typing import List
 
-from deltacat.utils.common import env_integer
+from deltacat.utils.common import env_integer, env_string
 
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 15)
 TIMEOUT_ERROR_CODES: List[str] = ["ReadTimeoutError", "ConnectTimeoutError"]
+AWS_REGION = env_string("AWS_REGION", "us-east-1")

deltacat/compute/compactor/model/compact_partition_params.py

@@ -90,6 +90,7 @@ class CompactPartitionParams(dict):
             "hash_group_count", result.hash_bucket_count
         )
         result.drop_duplicates = params.get("drop_duplicates", DROP_DUPLICATES)
+        result.ray_custom_resources = params.get("ray_custom_resources")
 
         if not importlib.util.find_spec("memray"):
             result.enable_profiler = False
@@ -288,6 +289,14 @@ class CompactPartitionParams(dict):
     def hash_group_count(self) -> int:
         return self["hash_group_count"]
 
+    @property
+    def ray_custom_resources(self) -> Dict:
+        return self["ray_custom_resources"]
+
+    @ray_custom_resources.setter
+    def ray_custom_resources(self, res) -> None:
+        self["ray_custom_resources"] = res
+
     @hash_group_count.setter
     def hash_group_count(self, count: int) -> None:
         self["hash_group_count"] = count

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -420,6 +420,21 @@ class CompactionSessionAuditInfo(dict):
         """
         return self.get("usedCPUSeconds")
 
+    @property
+    def used_memory_gb_seconds(self) -> float:
+        """
+        The used memory in the cluster weighted over time. This
+        determines opportunities for better memory estimation.
+        """
+        return self.get("usedMemoryGBSeconds")
+
+    @property
+    def total_memory_gb_seconds(self) -> float:
+        """
+        Total memory in the cluster weighted over time in GB.
+        """
+        return self.get("totalMemoryGBSeconds")
+
     @property
     def pyarrow_version(self) -> str:
         """
@@ -743,6 +758,14 @@ class CompactionSessionAuditInfo(dict):
         self["usedCPUSeconds"] = value
         return self
 
+    def set_used_memory_gb_seconds(self, value: float) -> CompactionSessionAuditInfo:
+        self["usedMemoryGBSeconds"] = value
+        return self
+
+    def set_total_memory_gb_seconds(self, value: float) -> CompactionSessionAuditInfo:
+        self["totalMemoryGBSeconds"] = value
+        return self
+
     def set_pyarrow_version(self, value: str) -> CompactionSessionAuditInfo:
         self["pyarrowVersion"] = value
         return self

deltacat/compute/compactor/model/delta_annotated.py

@@ -286,8 +286,15 @@ class DeltaAnnotated(Delta):
 
                         result.append(new_da)
                 else:
+                    logger.info(
+                        f"Split was not performed on delta with locator: {delta_annotated.locator} "
+                        "as partial parquet params was not found."
+                    )
                     return [delta_annotated]
 
+        if result:
+            return result
+
         logger.info(
             f"Split was not performed on the delta with locator: {delta_annotated.locator}"
         )

deltacat/compute/compactor_v2/compaction_session.py

@@ -217,6 +217,7 @@ def _execute_compaction(
         previous_inflation=params.previous_inflation,
         average_record_size_bytes=params.average_record_size_bytes,
         primary_keys=params.primary_keys,
+        ray_custom_resources=params.ray_custom_resources,
     )
 
     hb_start = time.monotonic()
@@ -337,6 +338,7 @@ def _execute_compaction(
         primary_keys=params.primary_keys,
         deltacat_storage=params.deltacat_storage,
         deltacat_storage_kwargs=params.deltacat_storage_kwargs,
+        ray_custom_resources=params.ray_custom_resources,
     )
 
     def merge_input_provider(index, item):
@@ -479,6 +481,10 @@ def _execute_compaction(
     if cluster_util:
         compaction_audit.set_total_cpu_seconds(cluster_util.total_vcpu_seconds)
         compaction_audit.set_used_cpu_seconds(cluster_util.used_vcpu_seconds)
+        compaction_audit.set_used_memory_gb_seconds(cluster_util.used_memory_gb_seconds)
+        compaction_audit.set_total_memory_gb_seconds(
+            cluster_util.total_memory_gb_seconds
+        )
 
     s3_utils.upload(
         compaction_audit.audit_url,

deltacat/compute/compactor_v2/constants.py

@@ -23,10 +23,10 @@ AVERAGE_RECORD_SIZE_BYTES = 1000
 # r5.8xlarge EC2 instances.
 TASK_MAX_PARALLELISM = 5367
 
-# The percentage of memory that needs to be estimated
+# The percentage of memory that needs to be allocated
 # as buffer. This value will ensure the job doesn't run out
 # of memory by considering buffer for uncertainities.
-TOTAL_MEMORY_BUFFER_PERCENTAGE = 20
+TOTAL_MEMORY_BUFFER_PERCENTAGE = 30
 
 # The total size of records that will be hash bucketed at once
 # Since, sorting is nlogn, we ensure that is not performed
@@ -35,3 +35,8 @@ MAX_SIZE_OF_RECORD_BATCH_IN_GIB = 2 * 1024 * 1024 * 1024
 
 # Whether to drop duplicates during merge.
 DROP_DUPLICATES = True
+
+# PARQUET to PYARROW inflation multiplier
+# This is the observed upper bound inflation for parquet
+# size in metadata to pyarrow table size.
+PARQUET_TO_PYARROW_INFLATION = 4

deltacat/compute/compactor_v2/steps/hash_bucket.py

@@ -109,7 +109,10 @@ def _group_file_records_by_pk_hash_bucket(
     if delta_file_envelopes is None:
         return None, 0, 0
 
-    logger.info(f"Read all delta file envelopes: {len(delta_file_envelopes)}")
+    logger.info(
+        f"Read all delta file envelopes: {len(delta_file_envelopes)} "
+        f"and total_size_bytes={total_size_bytes} and records={total_record_count}"
+    )
 
     # group the data by primary key hash value
     hb_to_delta_file_envelopes = np.empty([num_hash_buckets], dtype="object")

deltacat/compute/compactor_v2/utils/content_type_params.py

@@ -46,7 +46,7 @@ def append_content_type_params(
         pq_files = deltacat_storage.download_delta(
             delta,
             table_type=TableType.PYARROW_PARQUET,
-            storage_type=StorageType.DISTRIBUTED,
+            storage_type=StorageType.LOCAL,
             **deltacat_storage_kwargs,
         )
 

deltacat/compute/compactor_v2/utils/task_options.py

@@ -11,7 +11,10 @@ from deltacat.compute.compactor.model.round_completion_info import RoundCompleti
 from deltacat.compute.compactor_v2.utils.primary_key_index import (
     hash_group_index_to_hash_bucket_indices,
 )
-from deltacat.compute.compactor_v2.constants import TOTAL_MEMORY_BUFFER_PERCENTAGE
+from deltacat.compute.compactor_v2.constants import (
+    TOTAL_MEMORY_BUFFER_PERCENTAGE,
+    PARQUET_TO_PYARROW_INFLATION,
+)
 
 
 def _get_parquet_type_params_if_exist(
@@ -45,7 +48,19 @@ def _calculate_parquet_column_size(
             "Columns not found in the parquet data as "
             f"{columns_found} != {len(columns)}"
         )
-    return column_size
+    return column_size * PARQUET_TO_PYARROW_INFLATION
+
+
+def _get_task_options(
+    cpu: float, memory: float, ray_custom_resources: Optional[Dict] = None
+) -> Dict:
+
+    task_opts = {"num_cpus": cpu, "memory": memory}
+
+    if ray_custom_resources:
+        task_opts["resources"] = ray_custom_resources
+
+    return task_opts
 
 
 def estimate_manifest_entry_size_bytes(
@@ -57,7 +72,7 @@ def estimate_manifest_entry_size_bytes(
     type_params = _get_parquet_type_params_if_exist(entry=entry)
 
     if type_params:
-        return type_params.in_memory_size_bytes
+        return type_params.in_memory_size_bytes * PARQUET_TO_PYARROW_INFLATION
 
     return entry.meta.content_length * previous_inflation
 
@@ -103,6 +118,7 @@ def hash_bucket_resource_options_provider(
     previous_inflation: float,
     average_record_size_bytes: float,
     primary_keys: List[str] = None,
+    ray_custom_resources: Optional[Dict] = None,
     **kwargs,
 ) -> Dict:
     size_bytes = 0.0
@@ -141,7 +157,7 @@ def hash_bucket_resource_options_provider(
     # Consider buffer
     total_memory = total_memory * (1 + TOTAL_MEMORY_BUFFER_PERCENTAGE / 100.0)
 
-    return {"num_cpus": 0.01, "memory": total_memory}
+    return _get_task_options(0.01, total_memory, ray_custom_resources)
 
 
 def merge_resource_options_provider(
@@ -152,6 +168,7 @@ def merge_resource_options_provider(
     hash_group_num_rows: Dict[int, int],
     round_completion_info: Optional[RoundCompletionInfo] = None,
     compacted_delta_manifest: Optional[Manifest] = None,
+    ray_custom_resources: Optional[Dict] = None,
     primary_keys: Optional[List[str]] = None,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict] = {},
@@ -218,4 +235,4 @@ def merge_resource_options_provider(
 
     total_memory = total_memory * (1 + TOTAL_MEMORY_BUFFER_PERCENTAGE / 100.0)
 
-    return {"num_cpus": 0.01, "memory": total_memory}
+    return _get_task_options(0.01, total_memory, ray_custom_resources)

deltacat/compute/metastats/stats.py

@@ -8,6 +8,7 @@ from ray.types import ObjectRef
 from deltacat import logs
 from deltacat.aws import s3u as s3_utils
 from deltacat.aws.clients import client_cache
+from deltacat.aws.constants import AWS_REGION
 from deltacat.compute.compactor import DeltaAnnotated
 from deltacat.compute.metastats.utils.io import (
     cache_inflation_rate_data_for_delta_stats_ready,
@@ -94,7 +95,7 @@ def start_stats_collection(
 
 
 def _get_account_id() -> str:
-    client = client_cache("sts", None)
+    client = client_cache("sts", region_name=AWS_REGION)
     account_id = client.get_caller_identity()["Account"]
     return account_id
 

deltacat/io/memcached_object_store.py

@@ -29,6 +29,7 @@ class MemcachedObjectStore(IObjectStore):
         self.port = port
         self.storage_node_ips = storage_node_ips
         self.hasher = None
+        logger.info(f"The storage node IPs: {self.storage_node_ips}")
         super().__init__()
 
     def initialize_hasher(self):
@@ -129,9 +130,13 @@ class MemcachedObjectStore(IObjectStore):
         base_client = Client((ip_address, self.port))
         client = RetryingClient(
             base_client,
-            attempts=3,
-            retry_delay=0.01,
-            retry_for=[MemcacheUnexpectedCloseError, ConnectionResetError],
+            attempts=15,
+            retry_delay=1,
+            retry_for=[
+                MemcacheUnexpectedCloseError,
+                ConnectionResetError,
+                BrokenPipeError,
+            ],
         )
 
         self.client_cache[ip_address] = client

deltacat/tests/utils/test_pyarrow.py

@@ -1,10 +1,10 @@
 from unittest import TestCase
 from deltacat.utils.pyarrow import (
-    s3_parquet_file_to_table,
     s3_partial_parquet_file_to_table,
     pyarrow_read_csv,
     content_type_to_reader_kwargs,
     _add_column_kwargs,
+    s3_file_to_table,
     ReadKwargsProviderPyArrowSchemaOverride,
     RAISE_ON_EMPTY_CSV_KWARG,
 )
@@ -16,49 +16,8 @@ import pyarrow as pa
 PARQUET_FILE_PATH = "deltacat/tests/utils/data/test_file.parquet"
 EMPTY_UTSV_PATH = "deltacat/tests/utils/data/empty.csv"
 NON_EMPTY_VALID_UTSV_PATH = "deltacat/tests/utils/data/non_empty_valid.csv"
-
-
-class TestS3ParquetFileToTable(TestCase):
-    def test_s3_parquet_file_to_table_sanity(self):
-
-        result = s3_parquet_file_to_table(
-            PARQUET_FILE_PATH,
-            ContentType.PARQUET.value,
-            ContentEncoding.IDENTITY.value,
-            ["n_legs", "animal"],
-            ["n_legs"],
-        )
-
-        self.assertEqual(len(result), 6)
-        self.assertEqual(len(result.column_names), 1)
-        schema = result.schema
-        schema_index = schema.get_field_index("n_legs")
-        self.assertEqual(schema.field(schema_index).type, "int64")
-
-    def test_s3_parquet_file_to_table_when_schema_overridden(self):
-
-        schema = pa.schema(
-            [pa.field("animal", pa.string()), pa.field("n_legs", pa.string())]
-        )
-
-        pa_kwargs_provider = lambda content_type, kwargs: {"schema": schema}
-
-        result = s3_parquet_file_to_table(
-            PARQUET_FILE_PATH,
-            ContentType.PARQUET.value,
-            ContentEncoding.IDENTITY.value,
-            ["n_legs", "animal"],
-            pa_read_func_kwargs_provider=pa_kwargs_provider,
-        )
-
-        self.assertEqual(len(result), 6)
-        self.assertEqual(len(result.column_names), 2)
-
-        result_schema = result.schema
-        for index, field in enumerate(result_schema):
-            self.assertEqual(field.name, schema.field(index).name)
-
-        self.assertEqual(result.schema.field(1).type, "string")
+GZIP_COMPRESSED_FILE_UTSV_PATH = "deltacat/tests/utils/data/non_empty_compressed.gz"
+BZ2_COMPRESSED_FILE_UTSV_PATH = "deltacat/tests/utils/data/non_empty_compressed.bz2"
 
 
 class TestS3PartialParquetFileToTable(TestCase):
@@ -343,3 +302,131 @@ class TestReadCSV(TestCase):
                 EMPTY_UTSV_PATH, **{**kwargs, RAISE_ON_EMPTY_CSV_KWARG: True}
             ),
         )
+
+
+class TestS3FileToTable(TestCase):
+    def test_s3_file_to_table_identity_sanity(self):
+
+        schema = pa.schema(
+            [("is_active", pa.string()), ("ship_datetime_utc", pa.timestamp("us"))]
+        )
+
+        result = s3_file_to_table(
+            NON_EMPTY_VALID_UTSV_PATH,
+            ContentType.UNESCAPED_TSV.value,
+            ContentEncoding.IDENTITY.value,
+            ["is_active", "ship_datetime_utc"],
+            None,
+            pa_read_func_kwargs_provider=ReadKwargsProviderPyArrowSchemaOverride(
+                schema=schema
+            ),
+        )
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(len(result.column_names), 2)
+        result_schema = result.schema
+        for index, field in enumerate(result_schema):
+            self.assertEqual(field.name, schema.field(index).name)
+
+        self.assertEqual(result.schema.field(0).type, "string")
+
+    def test_s3_file_to_table_gzip_compressed_sanity(self):
+
+        schema = pa.schema(
+            [("is_active", pa.string()), ("ship_datetime_utc", pa.timestamp("us"))]
+        )
+
+        result = s3_file_to_table(
+            GZIP_COMPRESSED_FILE_UTSV_PATH,
+            ContentType.UNESCAPED_TSV.value,
+            ContentEncoding.GZIP.value,
+            ["is_active", "ship_datetime_utc"],
+            None,
+            pa_read_func_kwargs_provider=ReadKwargsProviderPyArrowSchemaOverride(
+                schema=schema
+            ),
+        )
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(len(result.column_names), 2)
+        result_schema = result.schema
+        for index, field in enumerate(result_schema):
+            self.assertEqual(field.name, schema.field(index).name)
+
+        self.assertEqual(result.schema.field(0).type, "string")
+
+    def test_s3_file_to_table_bz2_compressed_sanity(self):
+
+        schema = pa.schema(
+            [("is_active", pa.string()), ("ship_datetime_utc", pa.timestamp("us"))]
+        )
+
+        result = s3_file_to_table(
+            BZ2_COMPRESSED_FILE_UTSV_PATH,
+            ContentType.UNESCAPED_TSV.value,
+            ContentEncoding.BZIP2.value,
+            ["is_active", "ship_datetime_utc"],
+            None,
+            pa_read_func_kwargs_provider=ReadKwargsProviderPyArrowSchemaOverride(
+                schema=schema
+            ),
+        )
+
+        self.assertEqual(len(result), 3)
+        self.assertEqual(len(result.column_names), 2)
+        result_schema = result.schema
+        for index, field in enumerate(result_schema):
+            self.assertEqual(field.name, schema.field(index).name)
+
+        self.assertEqual(result.schema.field(0).type, "string")
+
+    def test_s3_file_to_table_when_parquet_sanity(self):
+
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            **kwargs,
+        }
+
+        result = s3_file_to_table(
+            PARQUET_FILE_PATH,
+            ContentType.PARQUET.value,
+            ContentEncoding.IDENTITY.value,
+            ["n_legs", "animal"],
+            ["n_legs"],
+            pa_read_func_kwargs_provider=pa_kwargs_provider,
+        )
+
+        self.assertEqual(len(result), 6)
+        self.assertEqual(len(result.column_names), 1)
+        schema = result.schema
+        schema_index = schema.get_field_index("n_legs")
+        self.assertEqual(schema.field(schema_index).type, "int64")
+
+    def test_s3_file_to_table_when_parquet_schema_overridden(self):
+
+        schema = pa.schema(
+            [pa.field("animal", pa.string()), pa.field("n_legs", pa.string())]
+        )
+
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "schema": schema,
+            "reader_type": "pyarrow",
+            **kwargs,
+        }
+
+        result = s3_file_to_table(
+            PARQUET_FILE_PATH,
+            ContentType.PARQUET.value,
+            ContentEncoding.IDENTITY.value,
+            ["n_legs", "animal"],
+            pa_read_func_kwargs_provider=pa_kwargs_provider,
+        )
+
+        self.assertEqual(len(result), 6)
+        self.assertEqual(len(result.column_names), 2)
+
+        result_schema = result.schema
+        for index, field in enumerate(result_schema):
+            self.assertEqual(field.name, schema.field(index).name)
+
+        self.assertEqual(result.schema.field(1).type, "string")

deltacat/tests/utils/test_resources.py

@@ -46,3 +46,5 @@ class TestClusterUtilizationOverTimeRange(unittest.TestCase):
             self.assertTrue(
                 cu.total_vcpu_seconds >= cu.used_vcpu_seconds
             )  # total is greater than used
+            self.assertIsNotNone(cu.total_memory_gb_seconds)
+            self.assertIsNotNone(cu.used_memory_gb_seconds)

deltacat/utils/pyarrow.py

@@ -41,6 +41,7 @@ from deltacat.utils.arguments import (
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 RAISE_ON_EMPTY_CSV_KWARG = "raise_on_empty_csv"
+READER_TYPE_KWARG = "reader_type"
 
 
 def _filter_schema_for_columns(schema: pa.Schema, columns: List[str]) -> pa.Schema:
@@ -173,9 +174,9 @@ def content_type_to_reader_kwargs(content_type: str) -> Dict[str, Any]:
 
 # TODO (pdames): add deflate and snappy
 ENCODING_TO_FILE_INIT: Dict[str, Callable] = {
-    ContentEncoding.GZIP.value: partial(gzip.GzipFile, mode="rb"),
-    ContentEncoding.BZIP2.value: partial(bz2.BZ2File, mode="rb"),
-    ContentEncoding.IDENTITY.value: lambda fileobj: fileobj,
+    ContentEncoding.GZIP.value: partial(gzip.open, mode="rb"),
+    ContentEncoding.BZIP2.value: partial(bz2.open, mode="rb"),
+    ContentEncoding.IDENTITY.value: lambda s3_file: s3_file,
 }
 
 
@@ -385,47 +386,6 @@ def s3_partial_parquet_file_to_table(
     return table
 
 
-def s3_parquet_file_to_table(
-    s3_url: str,
-    content_type: str,
-    content_encoding: str,
-    column_names: Optional[List[str]] = None,
-    include_columns: Optional[List[str]] = None,
-    pa_read_func_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    partial_file_download_params: Optional[PartialFileDownloadParams] = None,
-    **s3_client_kwargs,
-) -> pa.Table:
-
-    logger.debug(
-        f"Reading to Parquet table using read_table for {content_type} "
-        f"and encoding: {content_encoding}"
-    )
-
-    if s3_client_kwargs is None:
-        s3_client_kwargs = {}
-
-    kwargs = {}
-
-    if s3_url.startswith("s3://"):
-        s3_file_system = create_s3_file_system(s3_client_kwargs)
-        kwargs["filesystem"] = s3_file_system
-
-    _add_column_kwargs(
-        content_type=content_type,
-        column_names=column_names,
-        include_columns=include_columns,
-        kwargs=kwargs,
-    )
-
-    if pa_read_func_kwargs_provider:
-        kwargs = pa_read_func_kwargs_provider(content_type, kwargs)
-
-    table, latency = timed_invocation(papq.read_table, s3_url, **kwargs)
-
-    logger.debug(f"Successfully read the table from url={s3_url} in {latency}s")
-    return table
-
-
 def s3_file_to_table(
     s3_url: str,
     content_type: str,
@@ -437,13 +397,17 @@ def s3_file_to_table(
     **s3_client_kwargs,
 ) -> pa.Table:
 
-    from deltacat.aws import s3u as s3_utils
-
     logger.debug(
         f"Reading {s3_url} to PyArrow. Content type: {content_type}. "
         f"Encoding: {content_encoding}"
     )
 
+    kwargs = content_type_to_reader_kwargs(content_type)
+    _add_column_kwargs(content_type, column_names, include_columns, kwargs)
+
+    if pa_read_func_kwargs_provider is not None:
+        kwargs = pa_read_func_kwargs_provider(content_type, kwargs)
+
     if (
         content_type == ContentType.PARQUET.value
         and content_encoding == ContentEncoding.IDENTITY.value
@@ -452,47 +416,46 @@ def s3_file_to_table(
             f"Performing read using parquet reader for encoding={content_encoding} "
             f"and content_type={content_type}"
         )
-        kwargs = {}
-        if pa_read_func_kwargs_provider is not None:
-            kwargs = pa_read_func_kwargs_provider(content_type, kwargs)
 
-        if kwargs.get("reader_type", "daft") == "daft":
+        parquet_reader_func = None
+        if kwargs.get(READER_TYPE_KWARG, "daft") == "daft":
             parquet_reader_func = daft_s3_file_to_table
         elif partial_file_download_params and isinstance(
             partial_file_download_params, PartialParquetParameters
         ):
             parquet_reader_func = s3_partial_parquet_file_to_table
-        else:
-            parquet_reader_func = s3_parquet_file_to_table
-
-        return parquet_reader_func(
-            s3_url=s3_url,
-            content_type=content_type,
-            content_encoding=content_encoding,
-            column_names=column_names,
-            include_columns=include_columns,
-            pa_read_func_kwargs_provider=pa_read_func_kwargs_provider,
-            partial_file_download_params=partial_file_download_params,
-            **s3_client_kwargs,
-        )
 
-    s3_obj = s3_utils.get_object_at_url(s3_url, **s3_client_kwargs)
-    logger.debug(f"Read S3 object from {s3_url}: {s3_obj}")
-    pa_read_func = CONTENT_TYPE_TO_PA_READ_FUNC[content_type]
-    input_file_init = ENCODING_TO_FILE_INIT[content_encoding]
-    input_file = input_file_init(fileobj=io.BytesIO(s3_obj["Body"].read()))
+        if parquet_reader_func is not None:
+            return parquet_reader_func(
+                s3_url=s3_url,
+                content_type=content_type,
+                content_encoding=content_encoding,
+                column_names=column_names,
+                include_columns=include_columns,
+                pa_read_func_kwargs_provider=pa_read_func_kwargs_provider,
+                partial_file_download_params=partial_file_download_params,
+                **s3_client_kwargs,
+            )
 
-    args = [input_file]
-    kwargs = content_type_to_reader_kwargs(content_type)
-    _add_column_kwargs(content_type, column_names, include_columns, kwargs)
+        if READER_TYPE_KWARG in kwargs:
+            kwargs.pop(READER_TYPE_KWARG)
 
-    if pa_read_func_kwargs_provider:
-        kwargs = pa_read_func_kwargs_provider(content_type, kwargs)
+    filesystem = io
+    if s3_url.startswith("s3://"):
+        filesystem = create_s3_file_system(s3_client_kwargs)
 
-    logger.debug(f"Reading {s3_url} via {pa_read_func} with kwargs: {kwargs}")
-    table, latency = timed_invocation(pa_read_func, *args, **kwargs)
-    logger.debug(f"Time to read {s3_url} into PyArrow table: {latency}s")
-    return table
+    logger.debug(f"Read S3 object from {s3_url} using filesystem: {filesystem}")
+    input_file_init = ENCODING_TO_FILE_INIT[content_encoding]
+    pa_read_func = CONTENT_TYPE_TO_PA_READ_FUNC[content_type]
+
+    with filesystem.open(s3_url, "rb") as s3_file, input_file_init(
+        s3_file
+    ) as input_file:
+        args = [input_file]
+        logger.debug(f"Reading {s3_url} via {pa_read_func} with kwargs: {kwargs}")
+        table, latency = timed_invocation(pa_read_func, *args, **kwargs)
+        logger.debug(f"Time to read {s3_url} into PyArrow table: {latency}s")
+        return table
 
 
 def s3_file_to_parquet(

deltacat/utils/resources.py

@@ -15,6 +15,7 @@ from resource import getrusage, RUSAGE_SELF
 import platform
 import psutil
 import schedule
+from deltacat.constants import BYTES_PER_GIBIBYTE
 
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
@@ -73,9 +74,11 @@ class ClusterUtilizationOverTimeRange(AbstractContextManager):
     def __init__(self) -> None:
         self.total_vcpu_seconds = 0.0
         self.used_vcpu_seconds = 0.0
+        self.total_memory_gb_seconds = 0.0
+        self.used_memory_gb_seconds = 0.0
 
     def __enter__(self) -> Any:
-        schedule.every().second.do(self._update_vcpus)
+        schedule.every().second.do(self._update_resources)
         self.stop_run_schedules = self._run_schedule()
         return super().__enter__()
 
@@ -94,7 +97,7 @@ class ClusterUtilizationOverTimeRange(AbstractContextManager):
 
     # It is not truely parallel(due to GIL Ref: https://wiki.python.org/moin/GlobalInterpreterLock)
     # even if we are using threading library. However, it averages out and gives a very good approximation.
-    def _update_vcpus(self):
+    def _update_resources(self):
         cluster_resources = ray.cluster_resources()
         available_resources = ray.available_resources()
         if "CPU" not in cluster_resources:
@@ -104,14 +107,28 @@ class ClusterUtilizationOverTimeRange(AbstractContextManager):
             self.used_vcpu_seconds = self.used_vcpu_seconds + float(
                 str(cluster_resources["CPU"] - available_resources["CPU"])
             )
-            self.total_vcpu_seconds = self.total_vcpu_seconds + float(
-                str(cluster_resources["CPU"])
-            )
-        else:
-            self.total_vcpu_seconds = self.total_vcpu_seconds + float(
-                str(cluster_resources["CPU"])
+
+        self.total_vcpu_seconds = self.total_vcpu_seconds + float(
+            str(cluster_resources["CPU"])
+        )
+
+        if "memory" not in cluster_resources:
+            return
+
+        if "memory" in available_resources:
+            self.used_memory_gb_seconds = (
+                self.used_memory_gb_seconds
+                + float(
+                    str(cluster_resources["memory"] - available_resources["memory"])
+                )
+                / BYTES_PER_GIBIBYTE
             )
 
+        self.total_memory_gb_seconds = (
+            self.total_memory_gb_seconds
+            + float(str(cluster_resources["memory"])) / BYTES_PER_GIBIBYTE
+        )
+
     def _run_schedule(self, interval: Optional[float] = 1.0):
         cease_continuous_run = threading.Event()
 

{deltacat-0.1.18b17.dist-info → deltacat-0.1.18b19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.1.18b17
+Version: 0.1.18b19
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -26,7 +26,7 @@ Requires-Dist: tenacity ==8.1.0
 Requires-Dist: typing-extensions ==4.4.0
 Requires-Dist: pymemcache ==4.0.0
 Requires-Dist: redis ==4.6.0
-Requires-Dist: getdaft ==0.1.15
+Requires-Dist: getdaft ==0.1.16
 Requires-Dist: schedule ==1.2.0
 
 # DeltaCAT

{deltacat-0.1.18b17.dist-info → deltacat-0.1.18b19.dist-info}/RECORD

@@ -1,10 +1,10 @@
-deltacat/__init__.py,sha256=5vrP3GDwj9EZh1PiYbRr47nS2eLs5aa0VLg6xy90_-0,1781
+deltacat/__init__.py,sha256=0iDCTG5vkP_WA3KAWsoKCN1p_8ZONakFWjeoGeVspDU,1781
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=xqZf8CwysNYP2d39pf27OnXGStPREgBgIM-e2Tts-TI,199
 deltacat/logs.py,sha256=9XWuTBoWhhAF9rAL6t9veXmnAlJHsaqk0lTxteVPqyQ,5674
 deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/aws/clients.py,sha256=5TY2K8mCCZcjHEZVUiSu_rvZbJSwkOPHz0chnQ1UJrQ,6338
-deltacat/aws/constants.py,sha256=4OnwC1H12FPs2bpVN7tXkxn-DAEJS4MYRrFh5HKFv7s,204
+deltacat/aws/clients.py,sha256=1_h6SLlTWGuy9ZaUEtv96F34zfS4e8Rs94MFhLjSd7c,6376
+deltacat/aws/constants.py,sha256=01jrMFz3ZduGA0c7vz1wCHR7gBoNCMXXLi4JFYzZA9M,267
 deltacat/aws/s3u.py,sha256=mdJrX9z5O8kh00jUL0w8CYBxKAemVYs26sRDzwSonfg,18390
 deltacat/aws/redshift/__init__.py,sha256=fjuv3jWdPE8IgF4uSrL0YEqV3XUfqDULX3xV27ICceo,266
 deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -23,10 +23,10 @@ deltacat/compute/compactor/__init__.py,sha256=ivpOPve1yKi3Vz3tVgp-eeFMNEeUSf-dlR
 deltacat/compute/compactor/compaction_session.py,sha256=aHCkhjcJ3kgRcDDJ6snSgmPts7nLvtm_oGTqoxA3-68,27408
 deltacat/compute/compactor/repartition_session.py,sha256=f5BTTGNv365qSuTioL7QUuVm-px_l8-zz-OC_p7gXt4,7240
 deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor/model/compact_partition_params.py,sha256=qVm8ZCgsL7Taqq2xK6pADMYVI9NjW48qB7edPGM4sx4,13981
-deltacat/compute/compactor/model/compaction_session_audit_info.py,sha256=24yElKW5856O90wxgMha4WGiXtHPi5Y9k-dM2BINe8Q,30273
+deltacat/compute/compactor/model/compact_partition_params.py,sha256=ASPbmI9GMNG7Ho6XEJHyGDXbrHe-ytRvhjq_Bys2Oec,14283
+deltacat/compute/compactor/model/compaction_session_audit_info.py,sha256=GxAbBxEPJvCJoWqCJlojldYRQO635P915wy9XT9jgKE,31034
 deltacat/compute/compactor/model/dedupe_result.py,sha256=1OCV944qJdLQ_-8scisVKl45ej1eRv9OV539QYZtQ-U,292
-deltacat/compute/compactor/model/delta_annotated.py,sha256=3by7pj_LNk4yn87Q1CQ0Ck5YI9NOgREB5ZlFXFCsJwY,11989
+deltacat/compute/compactor/model/delta_annotated.py,sha256=NERB9rOtYg-xzBwvqGJ7_hBOzBC7g6X5M9-Cq5pbdH8,12258
 deltacat/compute/compactor/model/delta_file_envelope.py,sha256=et1KXJLwheEpzvy8vNjlYcgGavvwaNElZZYaCu7kyVA,2821
 deltacat/compute/compactor/model/delta_file_locator.py,sha256=AmhPGPDsmahVhp91rohJMx4ByumcIY5feqRLZTrNu4s,1905
 deltacat/compute/compactor/model/hash_bucket_result.py,sha256=71qGmaT1Mks-r3-aatjNbn2x3yWIgT8RmV0bRWe6pdA,275
@@ -47,25 +47,25 @@ deltacat/compute/compactor/utils/round_completion_file.py,sha256=DmZfHeAXlQn0DDd
 deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZRG2NqBlCw1X3_FBc,2397
 deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
 deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/compaction_session.py,sha256=YqkorTbVyWsJt1wfJP9vWn__0DU7FxGdXCFJqzqeEr4,18916
-deltacat/compute/compactor_v2/constants.py,sha256=aHpQrGL_Lm4apUpTtp2WqUWLEfQJvZefpAvu2m-TNbU,1296
+deltacat/compute/compactor_v2/compaction_session.py,sha256=rxIyLaOuoGEpZk-UdtGAPurJA7oB9X-Vkvr7hgWTlV0,19234
+deltacat/compute/compactor_v2/constants.py,sha256=skNkIOkvyfGm4z086ekln7niMwHmfMErhV9H0k8b1cc,1471
 deltacat/compute/compactor_v2/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/compactor_v2/model/hash_bucket_input.py,sha256=pgE2o8Z9-Dvs75C15LAkmfuJFFi5pRIuuxA9GGyDlLM,2631
 deltacat/compute/compactor_v2/model/hash_bucket_result.py,sha256=EsY9BPPywhmxlcLKn3kGWzAX4s4BTR2vYyPUB-wAEOc,309
 deltacat/compute/compactor_v2/model/merge_input.py,sha256=A-_Oq54sx1vrT-Ewv2_yKARdIh928yJvEuheCkw5tvQ,5049
 deltacat/compute/compactor_v2/model/merge_result.py,sha256=L53i9iL_XpzqBr7HETixD5v5qfLvitkGcjoML_hHfcs,368
 deltacat/compute/compactor_v2/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=_3bTqf67T8PvdmDmPLQrdcDA0UbW3W8PTR_Brzv0a2o,7276
+deltacat/compute/compactor_v2/steps/hash_bucket.py,sha256=NR-IDva6iB2YeNgxim_WsuZfEk5ooV8jAwzDJjdrsDE,7375
 deltacat/compute/compactor_v2/steps/merge.py,sha256=hgQiY2ui49HN-7ByIQlXVUCRbyrG7Jr61kohyGh6abY,17258
 deltacat/compute/compactor_v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=PY6xW1gCJJDHleau_eeGQbH9LXujxKKaeAJD5KLOcgA,2162
+deltacat/compute/compactor_v2/utils/content_type_params.py,sha256=rNKZisxGrLQOkwX8eHUQiFoTR1V-E66pMqWigtrs618,2156
 deltacat/compute/compactor_v2/utils/dedupe.py,sha256=A1xs5CU419h0nKv0B7R4tDkdgYAUIFQB_DWryRhpL98,1710
 deltacat/compute/compactor_v2/utils/io.py,sha256=4KV13VKwEtIzkwPJLJmEnp1dMOKHSxkEOQNQVbYrcwY,5177
 deltacat/compute/compactor_v2/utils/primary_key_index.py,sha256=NNF-h4zKRegVluAtXSDW4YRdOd4xJ6z_6uDaxfJyBiw,11122
-deltacat/compute/compactor_v2/utils/task_options.py,sha256=5QCxA84HdTshb0V2ycBsGxZjYyQ24Ibm7ycyajknHBU,7480
+deltacat/compute/compactor_v2/utils/task_options.py,sha256=jjN-NNkMQetbcFgJMhHqVBqNdWfXFrRglMaFYjTfeBo,7987
 deltacat/compute/metastats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/metastats/meta_stats.py,sha256=78hN3aN5wLHUFJsZXuv2JLeqA35HZ8mLUWJDMslMj5Q,18731
-deltacat/compute/metastats/stats.py,sha256=ftZs8ogLWKZgmrj5Dbrj27lp5wuQ6du4CNTAiQughZI,7299
+deltacat/compute/metastats/stats.py,sha256=8iUiSXOAjqiEeNP5RIb5gvhykBgpNHD5IKkB8zsPR0E,7363
 deltacat/compute/metastats/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/metastats/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/compute/metastats/model/partition_stats_dict.py,sha256=FbfoOxmTZfjRT7iHwc_96gHmB_r6iUvVM9BoTldD5mY,1123
@@ -91,7 +91,7 @@ deltacat/compute/stats/utils/manifest_stats_file.py,sha256=PtqW5Zc5e09HcfiAgvoZH
 deltacat/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/io/dataset.py,sha256=8w9sPVDpGnjjGVDWB39YSKWxq4zRv9VEfDtj7PYwjqM,3755
 deltacat/io/file_object_store.py,sha256=HCFeXu9cWXPXVk54MHel_nw3-wIuzhMt2RI6jKzjRYM,1346
-deltacat/io/memcached_object_store.py,sha256=Vi_Yd4F3tbgRSwMPGjQ0BW5YZSgQ3Fdnh6-ra8pl4RU,4956
+deltacat/io/memcached_object_store.py,sha256=eA0Ggk6BedFcyT_lbgScjRj2TSB2p45YwAs0B-qJEt0,5104
 deltacat/io/object_store.py,sha256=GX4pK-LY92s3uXRGcj8YsG2FFoiKfcJr2USIVz1ruGg,1380
 deltacat/io/ray_plasma_object_store.py,sha256=pupw7ulZY_EV5dERJDCCW_y_hzVx3Hl_uAvpQTNIh-E,705
 deltacat/io/read_api.py,sha256=BhkjL3xjY-fsa62AA9Yv20_88uTskn4_Bv2W6VmMXVA,7023
@@ -143,9 +143,9 @@ deltacat/tests/test_utils/pyarrow.py,sha256=EZk2Mtqiiu7Z79Lqm-hyHWbH6c-lbYnpvCn3
 deltacat/tests/test_utils/utils.py,sha256=a32qEwcSSd1lvRi0aJJ4ZLnc1ZyXmoQF_K95zaQRk2M,455
 deltacat/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/utils/test_daft.py,sha256=dfg4PYs6W4buBwj0FakTF2i7uFF6G4nj_48Dc8R11HQ,2852
-deltacat/tests/utils/test_pyarrow.py,sha256=IUNg_zO7ygUVyIJSX_rV2WalYyfxxt2wygJJF83Jm_0,12682
+deltacat/tests/utils/test_pyarrow.py,sha256=QspOrQRc6tLM52-taHYgw5v_49qKgvdgW7E4eNSv5Mk,15708
 deltacat/tests/utils/test_record_batch_tables.py,sha256=AkG1WyljQmjnl-AxhbFWyo5LnMIKRyLScfgC2B_ES-s,11321
-deltacat/tests/utils/test_resources.py,sha256=kMTIUmpx5-Y3ZkNPknM9Vj7Kx2OeG39Hjth8TCzoj5Y,1735
+deltacat/tests/utils/test_resources.py,sha256=NMiJl9wlKNnK-edBaftN2CWxDhFmR2AGsseNsg8n-zg,1856
 deltacat/tests/utils/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/types/media.py,sha256=RALwafQ0SwMyPUIcENhURk7Sor_2CIfEMztvFUnvZFQ,2227
@@ -160,8 +160,8 @@ deltacat/utils/numpy.py,sha256=ZiGREobTVT6IZXgPxkSUpLJFN2Hn8KEZcrqybLDXCIA,2027
 deltacat/utils/pandas.py,sha256=eGOpiZE1zLznTtuwoN80j4PBp1_bUV8SE4c951r0a3o,9561
 deltacat/utils/performance.py,sha256=7ZLaMkS1ehPSIhT5uOQVBHvjC70iKHzoFquFo-KL0PI,645
 deltacat/utils/placement.py,sha256=S80CwD1eEK47lQNr0xTmF9kq092-z6lTTmOOBv8cW_o,11723
-deltacat/utils/pyarrow.py,sha256=5XCkUg9GurBMGPP3BYk7o7R_bd51zHtK7sRePPby_Ds,27448
-deltacat/utils/resources.py,sha256=OExdoecfKmQQCKeYX5slaOGgdQCodBpxo5pUr7_rBYQ,5511
+deltacat/utils/pyarrow.py,sha256=Bm_B-gtI6enhlgDuzpSiaNUc20p1rHS-3Y6OdHySl0E,26352
+deltacat/utils/resources.py,sha256=2nrYzDm6BPedrpsWM_4NxCZZ4cubVYitO1Myev0B5W0,6044
 deltacat/utils/s3fs.py,sha256=PmUJ5Fm1WmD-_zp_M6yd9VbXvIoJuBeK6ApOdJJApLE,662
 deltacat/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/utils/ray_utils/collections.py,sha256=hj20s4D2RF2jZETU_44r6mFbsczA0JI_I_4kWKTmqes,1951
@@ -169,8 +169,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=MlpOHlKgJKSXzLsSR8mg4V_dWSVP_udrl
 deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=xOVkqL6o8qGsewGvzhMKxmCcqcFZDnNILuz5IGMgxSc,4991
-deltacat-0.1.18b17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-0.1.18b17.dist-info/METADATA,sha256=_XJqw_d08WCKFL_q8e2jLOPJi_6FjvFJlL_kdJ8_wv4,1740
-deltacat-0.1.18b17.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-deltacat-0.1.18b17.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-0.1.18b17.dist-info/RECORD,,
+deltacat-0.1.18b19.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-0.1.18b19.dist-info/METADATA,sha256=117_kv8NZ-J8NXxHG2F_Msgm_XvyjUJ_f2oEXJZ2PbU,1740
+deltacat-0.1.18b19.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+deltacat-0.1.18b19.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-0.1.18b19.dist-info/RECORD,,