deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/resource_estimation/delta.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, List
 from deltacat import logs
 from deltacat.storage import (
     Delta,
@@ -61,6 +61,7 @@ def _estimate_resources_required_to_process_delta_using_previous_inflation(
 def _estimate_resources_required_to_process_delta_using_type_params(
     delta: Delta,
     operation_type: OperationType,
+    all_column_names: List[str],
     estimate_resources_params: EstimateResourcesParams,
     deltacat_storage: unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Dict[str, Any],
@@ -93,11 +94,30 @@ def _estimate_resources_required_to_process_delta_using_type_params(
                 on_disk_size_bytes=delta.meta.content_length,
             ),
         )
-
+    file_reader_kwargs_provider = kwargs.get(
+        "file_reader_kwargs_provider"
+    ) or deltacat_storage_kwargs.get("file_reader_kwargs_provider")
+
+    """
+    NOTE: The file_reader_kwargs_provider parameter can be passed in two ways:
+    1. Nested within deltacat_storage_kwargs during resource estimation
+    2. As a top-level attribute of CompactPartitionsParams during compaction
+
+    This creates an inconsistent parameter path between resource estimation and compaction flows.
+    As a long-term solution, this should be unified to use a single consistent path (either always
+    nested in deltacat_storage_kwargs or always as a top-level parameter).
+
+    For now, this implementation handles the resource estimation case by:
+    1. First checking for file_reader_kwargs_provider as a direct kwarg
+    2. Falling back to deltacat_storage_kwargs if not found
+    This approach maintains backward compatibility by not modifying the DELTA_RESOURCE_ESTIMATION_FUNCTIONS signatures.
+    """
     appended = append_content_type_params(
         delta=delta,
+        all_column_names=all_column_names,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,
+        file_reader_kwargs_provider=file_reader_kwargs_provider,
     )
 
     if not appended:
@@ -152,6 +172,10 @@ def _estimate_resources_required_to_process_delta_using_file_sampling(
         operation_type == OperationType.PYARROW_DOWNLOAD
     ), "Number of rows can only be estimated for PYARROW_DOWNLOAD operation"
 
+    if not estimate_resources_params.max_files_to_sample:
+        # we cannot calculate if we cannot sample
+        return None
+
     if not delta.manifest:
         delta.manifest = deltacat_storage.get_delta_manifest(
             delta.locator,
@@ -168,10 +192,6 @@ def _estimate_resources_required_to_process_delta_using_file_sampling(
             ),
         )
 
-    if not estimate_resources_params.max_files_to_sample:
-        # we cannot calculate if we cannot sample
-        return None
-
     sampled_in_memory_size = 0.0
     sampled_on_disk_size = 0.0
     sampled_num_rows = 0
@@ -234,6 +254,10 @@ RESOURCE_ESTIMATION_METHOD_TO_DELTA_RESOURCE_ESTIMATION_FUNCTIONS = {
         _estimate_resources_required_to_process_delta_using_file_sampling,
         _estimate_resources_required_to_process_delta_using_previous_inflation,
     ],
+    ResourceEstimationMethod.FILE_SAMPLING_WITH_PREVIOUS_INFLATION: [
+        _estimate_resources_required_to_process_delta_using_file_sampling,
+        _estimate_resources_required_to_process_delta_using_previous_inflation,
+    ],
 }
 
 
@@ -267,10 +291,18 @@ def estimate_resources_required_to_process_delta(
         estimate_resources_params.resource_estimation_method
     )
 
+    all_column_names = deltacat_storage.get_table_version_column_names(
+        delta.locator.namespace,
+        delta.locator.table_name,
+        delta.locator.table_version,
+        **deltacat_storage_kwargs,
+    )
+
     for func in functions:
         resources = func(
             delta=delta,
             operation_type=operation_type,
+            all_column_names=all_column_names,
             estimate_resources_params=estimate_resources_params,
             deltacat_storage=deltacat_storage,
             deltacat_storage_kwargs=deltacat_storage_kwargs,

deltacat/compute/resource_estimation/model.py

@@ -23,6 +23,14 @@ class ResourceEstimationMethod(str, Enum):
     """
     DEFAULT_V2 = "DEFAULT_V2"
 
+    """
+    This approach combines file sampling estimation and inflation based methods
+    and runs them in the order specified below:
+    1. FILE_SAMPLING
+    2. PREVIOUS_INFLATION
+    """
+    FILE_SAMPLING_WITH_PREVIOUS_INFLATION = "FILE_SAMPLING_WITH_PREVIOUS_INFLATION"
+
     """
     This approach strictly uses previous inflation and average record size to arrive
     at a resource estimate. It requires users to pass in previous inflation and average

deltacat/constants.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+import botocore.exceptions
 
+from daft.exceptions import DaftTransientError
 from deltacat.utils.common import env_string, env_bool
-import os
+from deltacat.utils.common import env_integer
 
 # Environment variables
 DELTACAT_SYS_LOG_LEVEL = env_string("DELTACAT_SYS_LOG_LEVEL", "DEBUG")
@@ -40,14 +42,13 @@ DELTACAT_LOGGER_USE_SINGLE_HANDLER = env_bool(
 )
 DELTACAT_ROOT = env_string(
     "DELTACAT_ROOT",
-    os.path.join(os.getcwd(), ".deltacat"),
+    "",
 )
 
 # CLI Args
-METAFILE_FORMAT_KEY = "METAFILE_FORMAT"
 METAFILE_FORMAT_JSON = "json"
 METAFILE_FORMAT_MSGPACK = "msgpack"
-METAFILE_FORMAT = env_string(METAFILE_FORMAT_KEY, METAFILE_FORMAT_MSGPACK)
+METAFILE_FORMAT = env_string("METAFILE_FORMAT", METAFILE_FORMAT_MSGPACK)
 SUPPORTED_METAFILE_FORMATS = [METAFILE_FORMAT_JSON, METAFILE_FORMAT_MSGPACK]
 METAFILE_EXT = {
     "json": ".json",
@@ -93,14 +94,56 @@ REVISION_DIR_NAME: str = "rev"
 TXN_DIR_NAME: str = "txn"
 RUNNING_TXN_DIR_NAME: str = "running"
 FAILED_TXN_DIR_NAME: str = "failed"
+PAUSED_TXN_DIR_NAME: str = "paused"
 SUCCESS_TXN_DIR_NAME: str = "success"
+DATA_FILE_DIR_NAME: str = "data"
+REV_DIR_NAME: str = "rev"
 TXN_PART_SEPARATOR = "_"
+
 # Storage interface defaults
 # These defaults should be applied in catalog interface implementations
 # Storage interface implementations should be agnostic to defaults and require full information
-DEFAULT_CATALOG = "DEFAULT"
-DEFAULT_NAMESPACE = "DEFAULT"
+DEFAULT_CATALOG = "default"
+DEFAULT_NAMESPACE = "default"
 DEFAULT_TABLE_VERSION = "1"
 DEFAULT_STREAM_ID = "stream"
 DEFAULT_PARTITION_ID = "partition"
 DEFAULT_PARTITION_VALUES = ["default"]
+
+# Transaction Status constants
+SUCCESSFULLY_CLEANED = "cleaned"
+CURRENTLY_CLEANING = "cleaning"
+TIMEOUT_TXN = "timedout"
+
+# operation timeout constants
+OPERATION_TIMEOUTS = {
+    "create": 5,
+    "update": 3,
+    "delete": 4,
+    "read_siblings": 2,
+    "read_children": 2,
+    "read_latest": 3,
+    "read_exists": 1,
+}
+# Upload/Download Retry Defaults
+UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY = env_integer(
+    "UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY", 10 * 60
+)
+UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY = env_integer(
+    "UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY", 30 * 60
+)
+DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY = env_integer(
+    "DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY", 30 * 60
+)
+DEFAULT_FILE_READ_TIMEOUT_MS = env_integer(
+    "DEFAULT_FILE_READ_TIMEOUT_MS", 300_000
+)  # 5 mins
+RETRYABLE_TRANSIENT_ERRORS = (
+    OSError,
+    botocore.exceptions.ConnectionError,
+    botocore.exceptions.HTTPClientError,
+    botocore.exceptions.NoCredentialsError,
+    botocore.exceptions.ConnectTimeoutError,
+    botocore.exceptions.ReadTimeoutError,
+    DaftTransientError,
+)