deltacat 0.1.18b13__tar.gz → 0.1.18b15__tar.gz

{deltacat-0.1.18b13 → deltacat-0.1.18b15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.1.18b13
+Version: 0.1.18b15
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -18,12 +18,24 @@ Description: # DeltaCAT
         change-data-capture, data consistency checks, and table repair.
         
         ## Getting Started
-        ---
+        
         ### Install
+        
         ```
         pip install deltacat
         ```
         
+        ### Running Tests
+        
+        ```
+        pip3 install virtualenv
+        virtualenv test_env
+        source test_env/bin/activate
+        pip3 install -r requirements.txt
+        
+        pytest
+        ```
+        
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers

{deltacat-0.1.18b13 → deltacat-0.1.18b15}/README.md

@@ -11,8 +11,20 @@ for common table management tasks, including petabyte-scale
 change-data-capture, data consistency checks, and table repair.
 
 ## Getting Started
----
+
 ### Install
+
 ```
 pip install deltacat
 ```
+
+### Running Tests
+
+```
+pip3 install virtualenv
+virtualenv test_env
+source test_env/bin/activate
+pip3 install -r requirements.txt
+
+pytest
+```

{deltacat-0.1.18b13 → deltacat-0.1.18b15}/deltacat/__init__.py

@@ -28,7 +28,6 @@ from deltacat.catalog.model.catalog import (  # noqa: F401
     init,
 )
 from deltacat.catalog.model.table_definition import TableDefinition
-from deltacat.compute.compactor import SortKey, SortOrder
 from deltacat.storage import (
     DistributedDataset,
     LifecycleState,
@@ -37,13 +36,15 @@ from deltacat.storage import (
     LocalTable,
     Namespace,
     SchemaConsistencyType,
+    SortKey,
+    SortOrder,
 )
 from deltacat.types.media import ContentEncoding, ContentType, TableType
 from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "0.1.18b13"
+__version__ = "0.1.18b15"
 
 
 __all__ = [

deltacat-0.1.18b15/deltacat/aws/clients.py

@@ -0,0 +1,189 @@
+import logging
+from functools import lru_cache
+from typing import Optional, FrozenSet
+from http import HTTPStatus
+
+import boto3
+from boto3.exceptions import ResourceNotExistsError
+from boto3.resources.base import ServiceResource
+from botocore.client import BaseClient
+from botocore.config import Config
+from requests.adapters import Response
+from tenacity import (
+    RetryError,
+    Retrying,
+    wait_fixed,
+    retry_if_exception,
+    stop_after_delay,
+)
+
+from deltacat import logs
+from deltacat.aws.constants import BOTO_MAX_RETRIES
+import requests
+
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+BOTO3_PROFILE_NAME_KWARG_KEY = "boto3_profile_name"
+INSTANCE_METADATA_SERVICE_IPV4_URI = "http://169.254.169.254/latest/meta-data/"  # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
+RETRYABLE_HTTP_STATUS_CODES = [
+    # 429
+    HTTPStatus.TOO_MANY_REQUESTS,
+    # 5xx
+    HTTPStatus.INTERNAL_SERVER_ERROR,
+    HTTPStatus.NOT_IMPLEMENTED,
+    HTTPStatus.BAD_GATEWAY,
+    HTTPStatus.SERVICE_UNAVAILABLE,
+    HTTPStatus.GATEWAY_TIMEOUT,
+]
+
+
+class RetryIfRetryableHTTPStatusCode(retry_if_exception):
+    """
+    Retry strategy that retries if the exception is an ``HTTPError`` with
+    a status code in the retryable errors list.
+    """
+
+    def __init__(self):
+        def is_retryable_error(exception):
+            return (
+                isinstance(exception, requests.exceptions.HTTPError)
+                and exception.response.status_code in RETRYABLE_HTTP_STATUS_CODES
+            )
+
+        super().__init__(predicate=is_retryable_error)
+
+
+def _log_attempt_number(retry_state):
+    """return the result of the last call attempt"""
+    logger.warning(f"Retrying: {retry_state.attempt_number}...")
+
+
+def _get_url(url: str, get_url_kwargs=None):
+    if get_url_kwargs is None:
+        get_url_kwargs = {}
+    resp = requests.get(url, **get_url_kwargs)
+    resp.raise_for_status()
+    return resp
+
+
+def retrying_get(
+    url: str,
+    retry_strategy,
+    wait_strategy,
+    stop_strategy,
+    short_circuit_on_status: FrozenSet[int] = {HTTPStatus.OK},
+) -> Optional[Response]:
+    """Retries a request to the given URL until it succeeds.
+
+    Args:
+        retry_strategy (Callable): A function that returns a retry strategy.
+        wait_strategy (Callable): A function that returns a wait strategy.
+        stop_strategy (Callable): A function that returns a stop strategy.
+        url (str): The URL to retry.
+
+    Returns:
+        Optional[Response]: The response from the URL, or None if the request
+            failed after the maximum number of retries.
+    """
+    try:
+        resp = _get_url(url)
+        if resp.status_code in short_circuit_on_status:
+            return resp
+        for attempt in Retrying(
+            retry=retry_strategy(),
+            wait=wait_strategy,
+            stop=stop_strategy,
+            after=_log_attempt_number,
+        ):
+            with attempt:
+                resp = _get_url(url)
+                return resp
+    except RetryError as re:
+        logger.error(f"Failed to retry URL: {url} - {re}")
+    logger.info(f"Unable to get from URL: {url}")
+    return None
+
+
+def block_until_instance_metadata_service_returns_success(
+    url=INSTANCE_METADATA_SERVICE_IPV4_URI,
+    retry_strategy=RetryIfRetryableHTTPStatusCode,
+    wait_strategy=wait_fixed(2),  # wait 2 seconds before retrying,
+    stop_strategy=stop_after_delay(60 * 10),  # stop trying after 10 minutes
+) -> Optional[Response]:
+    """Blocks until the instance metadata service returns a successful response.
+
+    Args:
+        retry_strategy (Callable): A function that returns a retry strategy.
+        wait_strategy (Callable): A function that returns a wait strategy.
+        stop_strategy (Callable): A function that returns a stop strategy.
+        url (str): The URL of the instance metadata service.
+
+    Returns:
+        Optional[Response]: The response from the instance metadata service,
+            or None if the request failed after the maximum number of retries.
+
+    https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
+    """
+    # We will get a 403 HTTP status code if running deltacat not in an EC2 instance. In that case we won't want to block.
+    return retrying_get(
+        url,
+        retry_strategy,
+        wait_strategy,
+        stop_strategy,
+        short_circuit_on_status={HTTPStatus.OK, HTTPStatus.FORBIDDEN},
+    )
+
+
+def _get_session_from_kwargs(input_kwargs):
+    block_until_instance_metadata_service_returns_success()
+    if input_kwargs.get(BOTO3_PROFILE_NAME_KWARG_KEY) is not None:
+        boto3_session = boto3.Session(
+            profile_name=input_kwargs.get(BOTO3_PROFILE_NAME_KWARG_KEY)
+        )
+        input_kwargs.pop(BOTO3_PROFILE_NAME_KWARG_KEY)
+        return boto3_session
+    else:
+        return boto3.Session()
+
+
+def _resource(name: str, region: Optional[str], **kwargs) -> ServiceResource:
+    boto3_session = _get_session_from_kwargs(kwargs)
+
+    boto_config = Config(retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "adaptive"})
+    return boto3_session.resource(
+        name,
+        region,
+        config=boto_config,
+        **kwargs,
+    )
+
+
+def _client(name: str, region: Optional[str], **kwargs) -> BaseClient:
+    try:
+        # try to re-use a client from the resource cache first
+        return resource_cache(name, region, **kwargs).meta.client
+    except ResourceNotExistsError:
+        # fall back for clients without an associated resource
+        boto3_session = _get_session_from_kwargs(kwargs)
+        boto_config = Config(
+            retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "adaptive"}
+        )
+        return boto3_session.client(
+            name,
+            region,
+            config=boto_config,
+            **kwargs,
+        )
+
+
+def resource_cache(name: str, region: Optional[str], **kwargs) -> ServiceResource:
+    # we don't use the @lru_cache decorator because Ray can't pickle it
+    cached_function = lru_cache()(_resource)
+    return cached_function(name, region, **kwargs)
+
+
+def client_cache(name: str, region: Optional[str], **kwargs) -> BaseClient:
+    # we don't use the @lru_cache decorator because Ray can't pickle it
+    cached_function = lru_cache()(_client)
+    return cached_function(name, region, **kwargs)

{deltacat-0.1.18b13 → deltacat-0.1.18b15}/deltacat/aws/redshift/model/manifest.py

@@ -170,6 +170,10 @@ class ManifestMeta(dict):
     def content_type_parameters(self) -> Optional[List[Dict[str, str]]]:
         return self.get("content_type_parameters")
 
+    @content_type_parameters.setter
+    def content_type_parameters(self, params: List[Dict[str, str]]) -> None:
+        self["content_type_parameters"] = params
+
     @property
     def credentials(self) -> Optional[Dict[str, str]]:
         return self.get("credentials")

{deltacat-0.1.18b13 → deltacat-0.1.18b15}/deltacat/aws/s3u.py

@@ -3,6 +3,8 @@ import multiprocessing
 from functools import partial
 from typing import Any, Callable, Dict, Generator, List, Optional, Union
 from uuid import uuid4
+from botocore.config import Config
+from deltacat.aws.constants import BOTO_MAX_RETRIES
 
 import pyarrow as pa
 import ray
@@ -39,6 +41,7 @@ from deltacat.types.tables import (
     TABLE_TYPE_TO_READER_FUNC,
     get_table_length,
 )
+from deltacat.types.partial_download import PartialFileDownloadParams
 from deltacat.utils.common import ReadKwargsProvider
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
@@ -197,6 +200,7 @@ def read_file(
     column_names: Optional[List[str]] = None,
     include_columns: Optional[List[str]] = None,
     file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
+    partial_file_download_params: Optional[PartialFileDownloadParams] = None,
     **s3_client_kwargs,
 ) -> LocalTable:
 
@@ -209,6 +213,7 @@ def read_file(
             column_names,
             include_columns,
             file_reader_kwargs_provider,
+            partial_file_download_params,
             **s3_client_kwargs,
         )
         return table
@@ -217,6 +222,13 @@ def read_file(
             # Timeout error not caught by botocore
             raise RetryableError(f"Retry table download from: {s3_url}") from e
         raise NonRetryableError(f"Failed table download from: {s3_url}") from e
+    except BaseException as e:
+        logger.warn(
+            f"Read has failed for {s3_url} and content_type={content_type} "
+            f"and encoding={content_encoding}. Error: {e}",
+            exc_info=True,
+        )
+        raise e
 
 
 def upload_sliced_table(
@@ -385,14 +397,16 @@ def download_manifest_entry(
     content_encoding: Optional[ContentEncoding] = None,
 ) -> LocalTable:
 
+    conf = Config(retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "adaptive"})
     s3_client_kwargs = (
         {
             "aws_access_key_id": token_holder["accessKeyId"],
             "aws_secret_access_key": token_holder["secretAccessKey"],
             "aws_session_token": token_holder["sessionToken"],
+            "config": conf,
         }
         if token_holder
-        else {}
+        else {"config": conf}
     )
     if not content_type:
         content_type = manifest_entry.meta.content_type
@@ -409,6 +423,14 @@ def download_manifest_entry(
     s3_url = manifest_entry.uri
     if s3_url is None:
         s3_url = manifest_entry.url
+
+    partial_file_download_params = None
+    if manifest_entry.meta and manifest_entry.meta.content_type_parameters:
+        for type_params in manifest_entry.meta.content_type_parameters:
+            if isinstance(type_params, PartialFileDownloadParams):
+                partial_file_download_params = type_params
+                break
+
     # @retry decorator can't be pickled by Ray, so wrap download in Retrying
     retrying = Retrying(
         wait=wait_random_exponential(multiplier=1, max=60),
@@ -424,6 +446,7 @@ def download_manifest_entry(
         column_names,
         include_columns,
         file_reader_kwargs_provider,
+        partial_file_download_params,
         **s3_client_kwargs,
     )
     return table

deltacat-0.1.18b15/deltacat/benchmarking/benchmark_parquet_reads.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import pytest
+
+
+# Benchmarks for retrieving a single column in the Parquet file
+SINGLE_COLUMN_BENCHMARKS = {
+    "mvp": ("s3://daft-public-data/test_fixtures/parquet-dev/mvp.parquet", ["a"]),
+    "TPCH-lineitems-200MB-2RG": (
+        "s3://daft-public-data/test_fixtures/parquet-dev/daft_200MB_lineitem_chunk.RG-2.parquet",
+        ["L_ORDERKEY"],
+    ),
+}
+
+# Benchmarks for retrieving all columns in the Parquet file
+ALL_COLUMN_BENCHMARKS = {
+    "mvp": ("s3://daft-public-data/test_fixtures/parquet-dev/mvp.parquet", None),
+    "TPCH-lineitems-200MB-2RG": (
+        "s3://daft-public-data/test_fixtures/parquet-dev/daft_200MB_lineitem_chunk.RG-2.parquet",
+        None,
+    ),
+}
+
+
+@pytest.mark.benchmark(group="num_rowgroups_single_column")
+@pytest.mark.parametrize(
+    ["name", "path", "columns"],
+    [
+        (name, path, columns)
+        for name, (path, columns) in SINGLE_COLUMN_BENCHMARKS.items()
+    ],
+    ids=[name for name in SINGLE_COLUMN_BENCHMARKS],
+)
+def test_read_parquet_num_rowgroups_single_column(
+    name, path, columns, read_fn, benchmark
+):
+    data = benchmark(read_fn, path, columns=columns)
+    if columns is not None:
+        assert data.column_names == columns
+
+
+@pytest.mark.benchmark(group="num_rowgroups_all_columns")
+@pytest.mark.parametrize(
+    ["name", "path", "columns"],
+    [(name, path, columns) for name, (path, columns) in ALL_COLUMN_BENCHMARKS.items()],
+    ids=[name for name in ALL_COLUMN_BENCHMARKS],
+)
+def test_read_parquet_num_rowgroups_all_columns(
+    name, path, columns, read_fn, benchmark
+):
+    data = benchmark(read_fn, path, columns=columns)
+    if columns is not None:
+        assert data.column_names == columns

deltacat-0.1.18b15/deltacat/benchmarking/conftest.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import pyarrow as pa
+import pyarrow.fs as pafs
+import pyarrow.parquet as papq
+import pytest
+
+from deltacat.utils.pyarrow import s3_file_to_table
+from deltacat.types.media import (
+    ContentEncoding,
+    ContentType,
+)
+
+
+def pyarrow_read(path: str, columns: list[str] | None = None) -> pa.Table:
+    assert path.startswith(
+        "s3://"
+    ), f"Expected file path to start with 's3://', but got {path}."
+    fs = pafs.S3FileSystem()
+    path = path.replace("s3://", "")
+    return papq.read_table(path, columns=columns, filesystem=fs)
+
+
+def deltacat_read(path: str, columns: list[str] | None = None) -> pa.Table:
+    assert path.startswith("s3://")
+    return s3_file_to_table(
+        path,
+        content_type=ContentType.PARQUET,
+        content_encoding=ContentEncoding.IDENTITY,
+        column_names=None,  # Parquet files are schemaful
+        include_columns=columns,
+    )
+
+
+def daft_table_read(path: str, columns: list[str] | None = None) -> pa.Table:
+    try:
+        import daft
+    except ImportError:
+        raise ImportError(
+            "Daft not installed. Install Daft using pip to run these benchmarks: `pip install getdaft`"
+        )
+
+    tbl = daft.table.Table.read_parquet(path, columns=columns)
+    return tbl.to_arrow()
+
+
+@pytest.fixture(
+    params=[
+        daft_table_read,
+        pyarrow_read,
+        deltacat_read,
+    ],
+    ids=[
+        "daft_table",
+        "pyarrow",
+        "deltacat",
+    ],
+)
+def read_fn(request):
+    """Fixture which returns the function to read a PyArrow table from a path"""
+    return request.param

{deltacat-0.1.18b13 → deltacat-0.1.18b15}/deltacat/catalog/delegate.py

@@ -5,7 +5,7 @@ import ray
 
 from deltacat.catalog.model.catalog import Catalog, all_catalogs
 from deltacat.catalog.model.table_definition import TableDefinition
-from deltacat.compute.compactor.model.sort_key import SortKey
+from deltacat.storage.model.sort_key import SortKey
 from deltacat.storage.model.list_result import ListResult
 from deltacat.storage.model.namespace import Namespace
 from deltacat.storage.model.types import (

{deltacat-0.1.18b13 → deltacat-0.1.18b15}/deltacat/catalog/interface.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Set, Union
 import pyarrow as pa
 
 from deltacat.catalog.model.table_definition import TableDefinition
-from deltacat.compute.compactor.model.sort_key import SortKey
+from deltacat.storage.model.sort_key import SortKey
 from deltacat.storage.model.list_result import ListResult
 from deltacat.storage.model.namespace import Namespace
 from deltacat.storage.model.types import (

{deltacat-0.1.18b13 → deltacat-0.1.18b15}/deltacat/compute/compactor/__init__.py

@@ -13,7 +13,6 @@ from deltacat.compute.compactor.model.round_completion_info import (
     RoundCompletionInfo,
     HighWatermark,
 )
-from deltacat.compute.compactor.model.sort_key import SortKey, SortOrder
 
 __all__ = [
     "DeltaAnnotated",
@@ -27,6 +26,4 @@ __all__ = [
     "PyArrowWriteResult",
     "RoundCompletionInfo",
     "HighWatermark",
-    "SortKey",
-    "SortOrder",
 ]