deltacat 0.1.18b14__py3-none-any.whl → 0.1.18b16__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "0.1.18b14"
+__version__ = "0.1.18b16"
 
 
 __all__ = [

deltacat/aws/clients.py

@@ -1,6 +1,6 @@
 import logging
 from functools import lru_cache
-from typing import Optional
+from typing import Optional, FrozenSet
 from http import HTTPStatus
 
 import boto3
@@ -38,7 +38,7 @@ RETRYABLE_HTTP_STATUS_CODES = [
 ]
 
 
-class RetryIfRetrableHTTPStatusCode(retry_if_exception):
+class RetryIfRetryableHTTPStatusCode(retry_if_exception):
     """
     Retry strategy that retries if the exception is an ``HTTPError`` with
     a status code in the retryable errors list.
@@ -72,6 +72,7 @@ def retrying_get(
     retry_strategy,
     wait_strategy,
     stop_strategy,
+    short_circuit_on_status: FrozenSet[int] = {HTTPStatus.OK},
 ) -> Optional[Response]:
     """Retries a request to the given URL until it succeeds.
 
@@ -86,6 +87,9 @@ def retrying_get(
             failed after the maximum number of retries.
     """
     try:
+        resp = _get_url(url)
+        if resp.status_code in short_circuit_on_status:
+            return resp
         for attempt in Retrying(
             retry=retry_strategy(),
             wait=wait_strategy,
@@ -103,7 +107,7 @@ def retrying_get(
 
 def block_until_instance_metadata_service_returns_success(
     url=INSTANCE_METADATA_SERVICE_IPV4_URI,
-    retry_strategy=RetryIfRetrableHTTPStatusCode,
+    retry_strategy=RetryIfRetryableHTTPStatusCode,
     wait_strategy=wait_fixed(2),  # wait 2 seconds before retrying,
     stop_strategy=stop_after_delay(60 * 10),  # stop trying after 10 minutes
 ) -> Optional[Response]:
@@ -121,7 +125,14 @@ def block_until_instance_metadata_service_returns_success(
 
     https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
     """
-    return retrying_get(url, retry_strategy, wait_strategy, stop_strategy)
+    # We will get a 403 HTTP status code if running deltacat not in an EC2 instance. In that case we won't want to block.
+    return retrying_get(
+        url,
+        retry_strategy,
+        wait_strategy,
+        stop_strategy,
+        short_circuit_on_status={HTTPStatus.OK, HTTPStatus.FORBIDDEN},
+    )
 
 
 def _get_session_from_kwargs(input_kwargs):
@@ -139,7 +150,7 @@ def _get_session_from_kwargs(input_kwargs):
 def _resource(name: str, region: Optional[str], **kwargs) -> ServiceResource:
     boto3_session = _get_session_from_kwargs(kwargs)
 
-    boto_config = Config(retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "standard"})
+    boto_config = Config(retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "adaptive"})
     return boto3_session.resource(
         name,
         region,
@@ -156,7 +167,7 @@ def _client(name: str, region: Optional[str], **kwargs) -> BaseClient:
         # fall back for clients without an associated resource
         boto3_session = _get_session_from_kwargs(kwargs)
         boto_config = Config(
-            retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "standard"}
+            retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "adaptive"}
         )
         return boto3_session.client(
             name,

deltacat/aws/redshift/model/manifest.py

@@ -170,6 +170,10 @@ class ManifestMeta(dict):
     def content_type_parameters(self) -> Optional[List[Dict[str, str]]]:
         return self.get("content_type_parameters")
 
+    @content_type_parameters.setter
+    def content_type_parameters(self, params: List[Dict[str, str]]) -> None:
+        self["content_type_parameters"] = params
+
     @property
     def credentials(self) -> Optional[Dict[str, str]]:
         return self.get("credentials")

deltacat/aws/s3u.py

@@ -3,6 +3,8 @@ import multiprocessing
 from functools import partial
 from typing import Any, Callable, Dict, Generator, List, Optional, Union
 from uuid import uuid4
+from botocore.config import Config
+from deltacat.aws.constants import BOTO_MAX_RETRIES
 
 import pyarrow as pa
 import ray
@@ -39,6 +41,7 @@ from deltacat.types.tables import (
     TABLE_TYPE_TO_READER_FUNC,
     get_table_length,
 )
+from deltacat.types.partial_download import PartialFileDownloadParams
 from deltacat.utils.common import ReadKwargsProvider
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
@@ -197,6 +200,7 @@ def read_file(
     column_names: Optional[List[str]] = None,
     include_columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    partial_file_download_params: Optional[PartialFileDownloadParams] = None,
     **s3_client_kwargs,
 ) -> LocalTable:
 
@@ -209,6 +213,7 @@ def read_file(
             column_names,
             include_columns,
             file_reader_kwargs_provider,
+            partial_file_download_params,
             **s3_client_kwargs,
         )
         return table
@@ -217,6 +222,13 @@ def read_file(
             # Timeout error not caught by botocore
             raise RetryableError(f"Retry table download from: {s3_url}") from e
         raise NonRetryableError(f"Failed table download from: {s3_url}") from e
+    except BaseException as e:
+        logger.warn(
+            f"Read has failed for {s3_url} and content_type={content_type} "
+            f"and encoding={content_encoding}. Error: {e}",
+            exc_info=True,
+        )
+        raise e
 
 
 def upload_sliced_table(
@@ -385,14 +397,16 @@ def download_manifest_entry(
     content_encoding: Optional[ContentEncoding] = None,
 ) -> LocalTable:
 
+    conf = Config(retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "adaptive"})
     s3_client_kwargs = (
         {
             "aws_access_key_id": token_holder["accessKeyId"],
             "aws_secret_access_key": token_holder["secretAccessKey"],
             "aws_session_token": token_holder["sessionToken"],
+            "config": conf,
         }
         if token_holder
-        else {}
+        else {"config": conf}
     )
     if not content_type:
         content_type = manifest_entry.meta.content_type
@@ -409,6 +423,14 @@ def download_manifest_entry(
     s3_url = manifest_entry.uri
     if s3_url is None:
         s3_url = manifest_entry.url
+
+    partial_file_download_params = None
+    if manifest_entry.meta and manifest_entry.meta.content_type_parameters:
+        for type_params in manifest_entry.meta.content_type_parameters:
+            if isinstance(type_params, PartialFileDownloadParams):
+                partial_file_download_params = type_params
+                break
+
     # @retry decorator can't be pickled by Ray, so wrap download in Retrying
     retrying = Retrying(
         wait=wait_random_exponential(multiplier=1, max=60),
@@ -424,6 +446,7 @@ def download_manifest_entry(
         column_names,
         include_columns,
         file_reader_kwargs_provider,
+        partial_file_download_params,
         **s3_client_kwargs,
     )
     return table

deltacat/compute/compactor/compaction_session.py

@@ -60,6 +60,9 @@ if importlib.util.find_spec("memray"):
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
+DEFAULT_DEDUPE_MAX_PARALLELISM_RATIO_ARG: int = 1
+DEFAULT_PROPERTIES_ARG: Dict[str, Any] = {}
+
 
 def check_preconditions(
     source_partition_locator: PartitionLocator,
@@ -68,8 +71,11 @@ def check_preconditions(
     max_records_per_output_file: int,
     new_hash_bucket_count: Optional[int],
     deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
+    **kwargs,
 ) -> int:
-
+    if deltacat_storage_kwargs is None:
+        deltacat_storage_kwargs = {}
     assert (
         source_partition_locator.partition_values
         == destination_partition_locator.partition_values
@@ -88,6 +94,8 @@ def check_preconditions(
         source_partition_locator,
         sort_keys,
         deltacat_storage,
+        deltacat_storage_kwargs,
+        **kwargs,
     )
 
 
@@ -118,9 +126,11 @@ def compact_partition(
     object_store: Optional[IObjectStore] = RayPlasmaObjectStore(),
     s3_client_kwargs: Optional[Dict[str, Any]] = None,
     deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Optional[str]:
-
+    if deltacat_storage_kwargs is None:
+        deltacat_storage_kwargs = {}
     if not importlib.util.find_spec("memray"):
         logger.info(f"memray profiler not available, disabling all profiling")
         enable_profiler = False
@@ -162,6 +172,7 @@ def compact_partition(
             object_store,
             s3_client_kwargs,
             deltacat_storage,
+            deltacat_storage_kwargs,
             **kwargs,
         )
         if new_partition:
@@ -173,7 +184,9 @@ def compact_partition(
         round_completion_file_s3_url = None
         if partition:
             logger.info(f"Committing compacted partition to: {partition.locator}")
-            partition = deltacat_storage.commit_partition(partition)
+            partition = deltacat_storage.commit_partition(
+                partition, **deltacat_storage_kwargs
+            )
             logger.info(f"Committed compacted partition: {partition}")
 
             round_completion_file_s3_url = rcf.write_round_completion_file(
@@ -210,15 +223,16 @@ def _execute_compaction_round(
     object_store: Optional[IObjectStore],
     s3_client_kwargs: Optional[Dict[str, Any]],
     deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[Optional[Partition], Optional[RoundCompletionInfo], Optional[str]]:
-
+    if deltacat_storage_kwargs is None:
+        deltacat_storage_kwargs = {}
     rcf_source_partition_locator = (
         rebase_source_partition_locator
         if rebase_source_partition_locator
         else source_partition_locator
     )
-
     base_audit_url = rcf_source_partition_locator.path(
         f"s3://{compaction_artifact_s3_bucket}/compaction-audit"
     )
@@ -251,6 +265,8 @@ def _execute_compaction_round(
         records_per_compacted_file,
         hash_bucket_count,
         deltacat_storage,
+        deltacat_storage_kwargs,
+        **kwargs,
     )
 
     # sort primary keys to produce the same pk digest regardless of input order
@@ -330,7 +346,8 @@ def _execute_compaction_round(
         rebase_source_partition_locator,
         rebase_source_partition_high_watermark,
         deltacat_storage,
-        **list_deltas_kwargs,
+        deltacat_storage_kwargs,
+        list_deltas_kwargs,
     )
 
     delta_discovery_end = time.monotonic()
@@ -363,6 +380,8 @@ def _execute_compaction_round(
             compaction_audit,
             hash_bucket_count,
             deltacat_storage=deltacat_storage,
+            deltacat_storage_kwargs=deltacat_storage_kwargs,
+            **kwargs,
         )
         if input_deltas_stats is None
         else io.limit_input_deltas(
@@ -373,6 +392,8 @@ def _execute_compaction_round(
             compaction_audit=compaction_audit,
             input_deltas_stats=input_deltas_stats,
             deltacat_storage=deltacat_storage,
+            deltacat_storage_kwargs=deltacat_storage_kwargs,
+            **kwargs,
         )
     )
 
@@ -400,9 +421,7 @@ def _execute_compaction_round(
         raise AssertionError(
             "Multiple rounds are not supported. Please increase the cluster size and run again."
         )
-
     hb_start = time.monotonic()
-
     hb_tasks_pending = invoke_parallel(
         items=uniform_deltas,
         ray_task=hb.hash_bucket,
@@ -418,8 +437,9 @@ def _execute_compaction_round(
         read_kwargs_provider=read_kwargs_provider,
         object_store=object_store,
         deltacat_storage=deltacat_storage,
+        deltacat_storage_kwargs=deltacat_storage_kwargs,
+        **kwargs,
     )
-
     hb_invoke_end = time.monotonic()
 
     logger.info(f"Getting {len(hb_tasks_pending)} hash bucket results...")
@@ -457,7 +477,6 @@ def _execute_compaction_round(
     )
 
     compaction_audit.set_input_records(total_hb_record_count.item())
-
     # TODO (pdames): when resources are freed during the last round of hash
     #  bucketing, start running dedupe tasks that read existing dedupe
     #  output from S3 then wait for hash bucketing to finish before continuing
@@ -468,13 +487,14 @@ def _execute_compaction_round(
         compacted_stream_locator.namespace,
         compacted_stream_locator.table_name,
         compacted_stream_locator.table_version,
+        **deltacat_storage_kwargs,
     )
     partition = deltacat_storage.stage_partition(
         stream,
         destination_partition_locator.partition_values,
+        **deltacat_storage_kwargs,
     )
     new_compacted_partition_locator = partition.locator
-
     # parallel step 2:
     # discover records with duplicate primary keys in each hash bucket, and
     # identify the index of records to keep or drop based on sort keys
@@ -483,7 +503,10 @@ def _execute_compaction_round(
 
     dedupe_start = time.monotonic()
     dd_max_parallelism = int(
-        max_parallelism * kwargs.get("dd_max_parallelism_ratio", 1)
+        max_parallelism
+        * kwargs.get(
+            "dd_max_parallelism_ratio", DEFAULT_DEDUPE_MAX_PARALLELISM_RATIO_ARG
+        )
     )
     logger.info(
         f"dd max_parallelism is set to {dd_max_parallelism}, max_parallelism is {max_parallelism}"
@@ -527,7 +550,6 @@ def _execute_compaction_round(
     )
 
     compaction_audit.set_records_deduped(total_dd_record_count.item())
-
     all_mat_buckets_to_obj_id = defaultdict(list)
     for dd_result in dd_results:
         for (
@@ -541,7 +563,6 @@ def _execute_compaction_round(
     logger.info(f"Materialize buckets created: " f"{len(all_mat_buckets_to_obj_id)}")
 
     compaction_audit.set_materialize_buckets(len(all_mat_buckets_to_obj_id))
-
     # TODO(pdames): when resources are freed during the last round of deduping
     #  start running materialize tasks that read materialization source file
     #  tables from S3 then wait for deduping to finish before continuing
@@ -562,7 +583,6 @@ def _execute_compaction_round(
     )
 
     materialize_start = time.monotonic()
-
     mat_tasks_pending = invoke_parallel(
         items=all_mat_buckets_to_obj_id.items(),
         ray_task=mat.materialize,
@@ -585,6 +605,7 @@ def _execute_compaction_round(
         s3_table_writer_kwargs=s3_table_writer_kwargs,
         object_store=object_store,
         deltacat_storage=deltacat_storage,
+        deltacat_storage_kwargs=deltacat_storage_kwargs,
     )
 
     materialize_invoke_end = time.monotonic()
@@ -630,7 +651,9 @@ def _execute_compaction_round(
         f" {record_info_msg}"
     )
     compacted_delta = deltacat_storage.commit_delta(
-        merged_delta, properties=kwargs.get("properties", {})
+        merged_delta,
+        properties=kwargs.get("properties", DEFAULT_PROPERTIES_ARG),
+        **deltacat_storage_kwargs,
     )
     logger.info(f"Committed compacted delta: {compacted_delta}")
 
@@ -692,10 +715,11 @@ def _execute_compaction_round(
 
 def compact_partition_from_request(
     compact_partition_params: CompactPartitionParams,
+    *compact_partition_pos_args,
 ) -> Optional[str]:
     """
     Wrapper for compact_partition that allows for the compact_partition parameters to be
-    passed in as a custom dictionary-like CompactPartitionParams object.
+    passed in as a custom dictionary-like CompactPartitionParams object along with any compact_partition positional arguments.
     :param compact_partition_params:
     """
-    return compact_partition(**compact_partition_params)
+    return compact_partition(*compact_partition_pos_args, **compact_partition_params)