deltacat 2.0.0b11__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/compute/janitor.py

@@ -0,0 +1,205 @@
+import time
+import os
+import posixpath
+import pyarrow.fs
+from pyarrow.fs import FileSelector, FileType
+from itertools import chain
+from deltacat.storage.model.transaction import Transaction
+from deltacat.utils.filesystem import resolve_path_and_filesystem
+from deltacat.constants import (
+    TXN_DIR_NAME,
+    RUNNING_TXN_DIR_NAME,
+    FAILED_TXN_DIR_NAME,
+    TXN_PART_SEPARATOR,
+)
+from deltacat.storage.model.types import TransactionState
+import logging
+from deltacat import logs
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def brute_force_search_matching_metafiles(
+    dirty_files_names, filesystem: pyarrow.fs.FileSystem, catalog_root
+):
+    txn_dir_name = TXN_DIR_NAME
+    # collect transaction ids of the files
+    transaction_ids = []
+    for dirty_file in dirty_files_names:
+        parts = dirty_file.split(TXN_PART_SEPARATOR)
+        if len(parts) < 2:
+            continue
+        transaction_ids.append(parts[1])
+
+    def recursive_search(path):
+        try:
+            selector = FileSelector(path, recursive=False)
+            entries = filesystem.get_file_info(selector)
+        except Exception as e:
+            logger.error(f"Error listing directory '{path}': {e}")
+            return
+
+        for entry in entries:
+            base_name = posixpath.basename(entry.path)
+            if entry.type == FileType.File:
+                for transaction_id in transaction_ids:
+                    # Look for transaction_id in the filename
+                    if transaction_id in base_name:
+                        try:
+                            filesystem.delete_file(entry.path)
+                            logger.debug(f"Deleted file: {entry.path}")
+                        except Exception as e:
+                            logger.error(f"Error deleting file '{entry.path}': {e}")
+
+            elif entry.type == FileType.Directory:
+                # Skip directories that match txn_dir_name
+                if posixpath.basename(entry.path) == txn_dir_name:
+                    logger.debug(f"Skipping directory: {entry.path}")
+                    continue
+                recursive_search(entry.path)
+
+    # Start recursive search from the catalog root
+    recursive_search(catalog_root)
+
+    # renaming to successful completion
+    for dirty_file in dirty_files_names:
+        failed_txn_log_dir = posixpath.join(
+            catalog_root, TXN_DIR_NAME, FAILED_TXN_DIR_NAME
+        )
+        old_log_path = posixpath.join(failed_txn_log_dir, dirty_file)
+
+        # new_filename = dirty_file.replace(TIMEOUT_TXN, SUCCESSFULLY_CLEANED)
+        new_log_path = posixpath.join(failed_txn_log_dir, dirty_file)
+        try:
+            filesystem.move(old_log_path, new_log_path)
+            logger.debug(f"Renamed file from {old_log_path} to {new_log_path}")
+        except Exception as e:
+            logger.error(f"Error renaming file '{old_log_path}': {e}")
+
+
+def janitor_delete_timed_out_transaction(catalog_root: str) -> None:
+    """
+    Traverse the running transactions directory and move transactions that have been
+    running longer than the threshold into the failed transactions directory.
+    """
+    catalog_root_normalized, filesystem = resolve_path_and_filesystem(catalog_root)
+
+    txn_log_dir = posixpath.join(catalog_root_normalized, TXN_DIR_NAME)
+    running_txn_log_dir = posixpath.join(txn_log_dir, RUNNING_TXN_DIR_NAME)
+    failed_txn_log_dir = posixpath.join(txn_log_dir, FAILED_TXN_DIR_NAME)
+
+    dirty_files = []
+
+    running_txn_file_selector = FileSelector(running_txn_log_dir, recursive=False)
+    running_txn_info_list = filesystem.get_file_info(running_txn_file_selector)
+
+    for running_txn_info in running_txn_info_list:
+        try:
+            filename = posixpath.basename(running_txn_info.path)
+            parts = filename.split(TXN_PART_SEPARATOR)
+            end_time_str = parts[-1]
+            end_time = float(end_time_str)
+            current_time = time.time_ns()
+            if end_time <= current_time:
+                src_path = running_txn_info.path
+                new_filename = f"{filename}"
+                dest_path = posixpath.join(failed_txn_log_dir, new_filename)
+
+                # Move the file using copy and delete
+                with filesystem.open_input_file(src_path) as src_file:
+                    contents = src_file.read()
+
+                with filesystem.open_output_stream(dest_path) as dest_file:
+                    dest_file.write(contents)
+                filesystem.delete_file(src_path)
+
+                dirty_files.append(new_filename)
+
+        except Exception as e:
+            logger.error(
+                f"Error cleaning failed transaction '{running_txn_info.path}': {e}"
+            )
+
+    # Pass catalog_root to the brute force search so it searches from the right place
+    brute_force_search_matching_metafiles(
+        dirty_files, filesystem, catalog_root_normalized
+    )
+
+
+def janitor_remove_files_in_failed(
+    catalog_root: str, filesystem: pyarrow.fs.FileSystem = None
+) -> None:
+    """
+    Cleans up metafiles and locator files associated with failed transactions.
+    """
+    if filesystem is None:
+        catalog_root_normalized, filesystem = resolve_path_and_filesystem(catalog_root)
+    else:
+        catalog_root_normalized, filesystem = resolve_path_and_filesystem(
+            catalog_root, filesystem
+        )
+
+    txn_log_dir = posixpath.join(catalog_root_normalized, TXN_DIR_NAME)
+    failed_txn_log_dir = posixpath.join(txn_log_dir, FAILED_TXN_DIR_NAME)
+    running_txn_log_dir = posixpath.join(txn_log_dir, RUNNING_TXN_DIR_NAME)
+    filesystem.create_dir(failed_txn_log_dir, recursive=True)
+
+    failed_txn_file_selector = FileSelector(failed_txn_log_dir, recursive=False)
+    failed_txn_info_list = filesystem.get_file_info(failed_txn_file_selector)
+
+    for failed_txn_info in failed_txn_info_list:
+        try:
+            txn = Transaction.read(failed_txn_info.path, filesystem)
+            failed_txn_basename = posixpath.basename(failed_txn_info.path)
+            should_process = True
+            try:
+                if txn.state(catalog_root_normalized) == TransactionState.PURGED:
+                    should_process = False
+            except Exception:
+                logger.error("Could not check attribute")
+            if should_process:
+                # Process if the file is marked as currently cleaning.
+                txnid = txn.id
+
+                if txn.state(catalog_root_normalized) == TransactionState.FAILED:
+
+                    txnid = txn.id
+
+                    operations = txn["operations"]
+                    known_write_paths = chain.from_iterable(
+                        (op["metafile_write_paths"] + op["locator_write_paths"])
+                        for op in operations
+                    )
+
+                    for write_path in known_write_paths:
+                        full_path = posixpath.join(catalog_root_normalized, write_path)
+                        try:
+                            filesystem.delete_file(full_path)
+                        except Exception as e:
+                            logger.error(f"Failed to delete file '{full_path}': {e}")
+
+                    new_filename = f"{txnid}"
+
+                    new_failed_txn_log_file_path = posixpath.join(
+                        failed_txn_log_dir, new_filename
+                    )
+                    running_txn_log_path = posixpath.join(
+                        running_txn_log_dir, new_filename
+                    )
+
+                    os.delete(running_txn_log_path)
+
+                    os.rename(failed_txn_info.path, new_failed_txn_log_file_path)
+                    logger.debug(
+                        f"Cleaned up failed transaction: {failed_txn_basename}"
+                    )
+
+        except Exception as e:
+            logger.error(
+                f"Could not read transaction '{failed_txn_info.path}', skipping: {e}"
+            )
+
+
+def janitor_job(catalog_root_dir: str) -> None:
+    janitor_delete_timed_out_transaction(catalog_root_dir)
+    janitor_remove_files_in_failed(catalog_root_dir)

deltacat/compute/jobs/client.py

@@ -21,11 +21,16 @@ def _run_cmd(cmd: str) -> None:
     assert exit_code == 0, f"`{cmd}` failed. Exit code: {exit_code}"
 
 
-def _ray_up(cluster_cfg: str, restart_only: bool = False) -> None:
+def _ray_up(
+    cluster_cfg: str, cluster_name_override: str = None, restart_only: bool = False
+) -> None:
     restart_flag = "--no-restart" if not restart_only else "--restart-only"
+    cluster_name_option = (
+        f"-n '{cluster_name_override}'" if cluster_name_override else ""
+    )
     print(f"Starting Ray cluster from '{cluster_cfg}'")
     _run_cmd(
-        f"ray up '{cluster_cfg}' -y --no-config-cache {restart_flag} --disable-usage-stats"
+        f"ray up '{cluster_cfg}' -y --no-config-cache {restart_flag} {cluster_name_option} --disable-usage-stats"
     )
     print(f"Started Ray cluster from '{cluster_cfg}'")
 
@@ -123,6 +128,7 @@ class DeltaCatJobClient(JobSubmissionClient):
         head_node_ip: str = None,
         dashboard_wait_time_seconds: int = 30,
         port: Union[int, str] = "8265",
+        cluster_name_override: str = None,
     ):
         job_submission_client_url = None
         try:
@@ -130,10 +136,12 @@ class DeltaCatJobClient(JobSubmissionClient):
             if cluster_cfg_file_path:
                 if launch_cluster:
                     if not _ray_cluster_running(cluster_cfg_file_path) or restart_ray:
-                        _ray_up(cluster_cfg_file_path)
+                        _ray_up(cluster_cfg_file_path, cluster_name_override)
                 elif restart_ray:
                     if _ray_cluster_running(cluster_cfg_file_path):
-                        _ray_up(cluster_cfg_file_path, restart_ray)
+                        _ray_up(
+                            cluster_cfg_file_path, restart_ray, cluster_name_override
+                        )
                     else:
                         raise RuntimeError(
                             f"Cannot Restart Ray: Ray Cluster for "
@@ -336,10 +344,11 @@ def local_job_client(*args, **kwargs) -> DeltaCatJobClient:
     Raises:
         RuntimeError: If a local Ray Job Server cannot be found.
     """
-    if not dc.is_initialized():
-        context = dc.init(*args, **kwargs)
-    else:
-        context = dc.init()
+    # force reinitialization to ensure that we can get the Ray context
+    kwargs["force"] = True
+    context = dc.init(*args, **kwargs)
+    if context is None:
+        raise RuntimeError("Failed to retrieve Ray context.")
     if context.dashboard_url:
         head_node_ip, port = context.dashboard_url.split(":")
     else:
@@ -366,6 +375,7 @@ def job_client(
     head_node_ip: str = None,
     dashboard_wait_time_seconds: int = 15,
     port: Union[str, int] = "8265",
+    cluster_name_override: str = None,
 ) -> DeltaCatJobClient:
     """
     Create a DeltaCAT Job Client that can be used to submit jobs to a remote
@@ -403,4 +413,5 @@ def job_client(
         head_node_ip=head_node_ip,
         dashboard_wait_time_seconds=dashboard_wait_time_seconds,
         port=port,
+        cluster_name_override=cluster_name_override,
     )

deltacat/compute/resource_estimation/delta.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, List
 from deltacat import logs
 from deltacat.storage import (
     Delta,
@@ -61,6 +61,7 @@ def _estimate_resources_required_to_process_delta_using_previous_inflation(
 def _estimate_resources_required_to_process_delta_using_type_params(
     delta: Delta,
     operation_type: OperationType,
+    all_column_names: List[str],
     estimate_resources_params: EstimateResourcesParams,
     deltacat_storage: unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Dict[str, Any],
@@ -93,11 +94,30 @@ def _estimate_resources_required_to_process_delta_using_type_params(
                 on_disk_size_bytes=delta.meta.content_length,
             ),
         )
-
+    file_reader_kwargs_provider = kwargs.get(
+        "file_reader_kwargs_provider"
+    ) or deltacat_storage_kwargs.get("file_reader_kwargs_provider")
+
+    """
+    NOTE: The file_reader_kwargs_provider parameter can be passed in two ways:
+    1. Nested within deltacat_storage_kwargs during resource estimation
+    2. As a top-level attribute of CompactPartitionsParams during compaction
+
+    This creates an inconsistent parameter path between resource estimation and compaction flows.
+    As a long-term solution, this should be unified to use a single consistent path (either always
+    nested in deltacat_storage_kwargs or always as a top-level parameter).
+
+    For now, this implementation handles the resource estimation case by:
+    1. First checking for file_reader_kwargs_provider as a direct kwarg
+    2. Falling back to deltacat_storage_kwargs if not found
+    This approach maintains backward compatibility by not modifying the DELTA_RESOURCE_ESTIMATION_FUNCTIONS signatures.
+    """
     appended = append_content_type_params(
         delta=delta,
+        all_column_names=all_column_names,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,
+        file_reader_kwargs_provider=file_reader_kwargs_provider,
     )
 
     if not appended:
@@ -152,6 +172,10 @@ def _estimate_resources_required_to_process_delta_using_file_sampling(
         operation_type == OperationType.PYARROW_DOWNLOAD
     ), "Number of rows can only be estimated for PYARROW_DOWNLOAD operation"
 
+    if not estimate_resources_params.max_files_to_sample:
+        # we cannot calculate if we cannot sample
+        return None
+
     if not delta.manifest:
         delta.manifest = deltacat_storage.get_delta_manifest(
             delta.locator,
@@ -168,10 +192,6 @@ def _estimate_resources_required_to_process_delta_using_file_sampling(
             ),
         )
 
-    if not estimate_resources_params.max_files_to_sample:
-        # we cannot calculate if we cannot sample
-        return None
-
     sampled_in_memory_size = 0.0
     sampled_on_disk_size = 0.0
     sampled_num_rows = 0
@@ -234,6 +254,10 @@ RESOURCE_ESTIMATION_METHOD_TO_DELTA_RESOURCE_ESTIMATION_FUNCTIONS = {
         _estimate_resources_required_to_process_delta_using_file_sampling,
         _estimate_resources_required_to_process_delta_using_previous_inflation,
     ],
+    ResourceEstimationMethod.FILE_SAMPLING_WITH_PREVIOUS_INFLATION: [
+        _estimate_resources_required_to_process_delta_using_file_sampling,
+        _estimate_resources_required_to_process_delta_using_previous_inflation,
+    ],
 }
 
 
@@ -267,10 +291,18 @@ def estimate_resources_required_to_process_delta(
         estimate_resources_params.resource_estimation_method
     )
 
+    all_column_names = deltacat_storage.get_table_version_column_names(
+        delta.locator.namespace,
+        delta.locator.table_name,
+        delta.locator.table_version,
+        **deltacat_storage_kwargs,
+    )
+
     for func in functions:
         resources = func(
             delta=delta,
             operation_type=operation_type,
+            all_column_names=all_column_names,
             estimate_resources_params=estimate_resources_params,
             deltacat_storage=deltacat_storage,
             deltacat_storage_kwargs=deltacat_storage_kwargs,

deltacat/compute/resource_estimation/model.py

@@ -23,6 +23,14 @@ class ResourceEstimationMethod(str, Enum):
     """
     DEFAULT_V2 = "DEFAULT_V2"
 
+    """
+    This approach combines file sampling estimation and inflation based methods
+    and runs them in the order specified below:
+    1. FILE_SAMPLING
+    2. PREVIOUS_INFLATION
+    """
+    FILE_SAMPLING_WITH_PREVIOUS_INFLATION = "FILE_SAMPLING_WITH_PREVIOUS_INFLATION"
+
     """
     This approach strictly uses previous inflation and average record size to arrive
     at a resource estimate. It requires users to pass in previous inflation and average

deltacat/constants.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+import botocore.exceptions
 
+from daft.exceptions import DaftTransientError
 from deltacat.utils.common import env_string, env_bool
+from deltacat.utils.common import env_integer
 
 # Environment variables
 DELTACAT_SYS_LOG_LEVEL = env_string("DELTACAT_SYS_LOG_LEVEL", "DEBUG")
@@ -91,7 +94,10 @@ REVISION_DIR_NAME: str = "rev"
 TXN_DIR_NAME: str = "txn"
 RUNNING_TXN_DIR_NAME: str = "running"
 FAILED_TXN_DIR_NAME: str = "failed"
+PAUSED_TXN_DIR_NAME: str = "paused"
 SUCCESS_TXN_DIR_NAME: str = "success"
+DATA_FILE_DIR_NAME: str = "data"
+REV_DIR_NAME: str = "rev"
 TXN_PART_SEPARATOR = "_"
 
 # Storage interface defaults
@@ -103,3 +109,41 @@ DEFAULT_TABLE_VERSION = "1"
 DEFAULT_STREAM_ID = "stream"
 DEFAULT_PARTITION_ID = "partition"
 DEFAULT_PARTITION_VALUES = ["default"]
+
+# Transaction Status constants
+SUCCESSFULLY_CLEANED = "cleaned"
+CURRENTLY_CLEANING = "cleaning"
+TIMEOUT_TXN = "timedout"
+
+# operation timeout constants
+OPERATION_TIMEOUTS = {
+    "create": 5,
+    "update": 3,
+    "delete": 4,
+    "read_siblings": 2,
+    "read_children": 2,
+    "read_latest": 3,
+    "read_exists": 1,
+}
+# Upload/Download Retry Defaults
+UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY = env_integer(
+    "UPLOAD_DOWNLOAD_RETRY_STOP_AFTER_DELAY", 10 * 60
+)
+UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY = env_integer(
+    "UPLOAD_SLICED_TABLE_RETRY_STOP_AFTER_DELAY", 30 * 60
+)
+DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY = env_integer(
+    "DOWNLOAD_MANIFEST_ENTRY_RETRY_STOP_AFTER_DELAY", 30 * 60
+)
+DEFAULT_FILE_READ_TIMEOUT_MS = env_integer(
+    "DEFAULT_FILE_READ_TIMEOUT_MS", 300_000
+)  # 5 mins
+RETRYABLE_TRANSIENT_ERRORS = (
+    OSError,
+    botocore.exceptions.ConnectionError,
+    botocore.exceptions.HTTPClientError,
+    botocore.exceptions.NoCredentialsError,
+    botocore.exceptions.ConnectTimeoutError,
+    botocore.exceptions.ReadTimeoutError,
+    DaftTransientError,
+)