mountaineer-cloud 0.2.0.dev1__tar.gz → 0.2.1__tar.gz

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: mountaineer-cloud
-Version: 0.2.0.dev1
+Version: 0.2.1
 Author-email: Pierce Freeman <pierce@freeman.vc>
 Requires-Python: >=3.11
 Requires-Dist: aioboto3>=14.0.0
@@ -71,6 +71,14 @@ async def get_asset_contents(
     aws: AWSCore = Depends(AWSDependencies.get_aws_core),
 ) -> bytes:
     return await asset.file_url.get_content(aws)
+
+
+async def delete_asset(
+    asset: Asset,
+    aws: AWSCore = Depends(AWSDependencies.get_aws_core),
+) -> None:
+    # Removes the remote object and resets `asset.file_url` to an empty pointer
+    await asset.file_url.delete(aws)
 ```
 
 If you later move that same model or endpoint to another provider, the primitive API stays the same. Thanks to the magic of typehinting, we'll proactively flag errors if you're trying to use a backend provider that doesn't support the functionality you expect. The main thing that changes is the core type:
@@ -79,7 +87,23 @@ If you later move that same model or endpoint to another provider, the primitive
 - `CloudFile[CloudflareCore]`
 - `CloudFile[DigitalOceanCore]`
 
-CloudFile has support for writing and reading in one fell-swoop, in addition to streaming reads that are more efficient for large files stored remotely.
+CloudFile has support for writing, reading, and deleting in one fell-swoop, in addition to streaming reads that are more efficient for large files stored remotely.
+
+Storage failures are raised as native Python exceptions where one fits, so remote files can be handled like local ones. Conditions without a native equivalent get their own types, which subclass botocore's `ClientError` so existing handlers still catch them. The original provider error is always available as `__cause__`.
+
+| Condition | Raised |
+|-----------|--------|
+| Object does not exist | `FileNotFoundError` |
+| Access denied | `PermissionError` |
+| Bucket does not exist | `StorageBucketNotFoundError` |
+| Request throttled | `StorageRateLimitError` |
+
+```python
+try:
+    contents = await asset.file_url.get_content(aws)
+except FileNotFoundError:
+    contents = None
+```
 
 ### Email Primitive
 

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/README.md

@@ -57,6 +57,14 @@ async def get_asset_contents(
     aws: AWSCore = Depends(AWSDependencies.get_aws_core),
 ) -> bytes:
     return await asset.file_url.get_content(aws)
+
+
+async def delete_asset(
+    asset: Asset,
+    aws: AWSCore = Depends(AWSDependencies.get_aws_core),
+) -> None:
+    # Removes the remote object and resets `asset.file_url` to an empty pointer
+    await asset.file_url.delete(aws)
 ```
 
 If you later move that same model or endpoint to another provider, the primitive API stays the same. Thanks to the magic of typehinting, we'll proactively flag errors if you're trying to use a backend provider that doesn't support the functionality you expect. The main thing that changes is the core type:
@@ -65,7 +73,23 @@ If you later move that same model or endpoint to another provider, the primitive
 - `CloudFile[CloudflareCore]`
 - `CloudFile[DigitalOceanCore]`
 
-CloudFile has support for writing and reading in one fell-swoop, in addition to streaming reads that are more efficient for large files stored remotely.
+CloudFile has support for writing, reading, and deleting in one fell-swoop, in addition to streaming reads that are more efficient for large files stored remotely.
+
+Storage failures are raised as native Python exceptions where one fits, so remote files can be handled like local ones. Conditions without a native equivalent get their own types, which subclass botocore's `ClientError` so existing handlers still catch them. The original provider error is always available as `__cause__`.
+
+| Condition | Raised |
+|-----------|--------|
+| Object does not exist | `FileNotFoundError` |
+| Access denied | `PermissionError` |
+| Bucket does not exist | `StorageBucketNotFoundError` |
+| Request throttled | `StorageRateLimitError` |
+
+```python
+try:
+    contents = await asset.file_url.get_content(aws)
+except FileNotFoundError:
+    contents = None
+```
 
 ### Email Primitive
 

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/mountaineer_cloud/__init__.py

@@ -29,3 +29,8 @@ from .providers.resend import (
     ResendCore as ResendCore,
     ResendDependencies as ResendDependencies,
 )
+from .providers_common.storage_errors import (
+    StorageBucketNotFoundError as StorageBucketNotFoundError,
+    StorageClientError as StorageClientError,
+    StorageRateLimitError as StorageRateLimitError,
+)

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/mountaineer_cloud/__tests__/aws/test_s3.py

@@ -6,6 +6,7 @@ from unittest.mock import patch
 import aioboto3
 import pytest
 import pytest_asyncio
+from botocore.exceptions import ClientError
 from iceaxe import Field as IceaxeField, TableBase
 from iceaxe.schemas.db_memory_serializer import DatabaseHandler
 from iceaxe.schemas.db_stubs import DBColumn
@@ -24,6 +25,11 @@ from mountaineer_cloud.primitives.storage import (
 from mountaineer_cloud.providers.aws import AWSConfig, AWSCore
 from mountaineer_cloud.providers.aws.dependencies import build_aws_core
 from mountaineer_cloud.providers_common.s3_compat import S3CompatibleMetadataBase
+from mountaineer_cloud.providers_common.storage_errors import (
+    StorageBucketNotFoundError,
+    StorageRateLimitError,
+    translate_storage_error,
+)
 from mountaineer_cloud.test_utilities import MockAWS
 from mountaineer_cloud.test_utilities.fixtures import get_mock_aws
 
@@ -278,3 +284,169 @@ async def test_cloudfile_in_iceaxe_model(
     )
     assert await stored["Body"].read() == b"test data"
     assert await asset.file_url.get_content(aws_core) == b"test data"
+
+
+@pytest.mark.asyncio
+async def test_s3_delete(mock_aws: MockAWS, aws_core: AWSCore):
+    metadata = S3Metadata(bucket="mountaineer-test", prefix="test-prefix")
+
+    await mock_aws.mock_s3.put_object(
+        Bucket="mountaineer-test",
+        Key="test-prefix/test-uuid",
+        Body=b"test data",
+    )
+
+    await aws_core.storage_delete(
+        path="s3://mountaineer-test/test-prefix/test-uuid",
+        metadata=metadata,
+    )
+
+    listing = await mock_aws.mock_s3.list_objects_v2(
+        Bucket="mountaineer-test", Prefix="test-prefix/"
+    )
+    assert listing.get("KeyCount", 0) == 0
+
+
+@pytest.mark.asyncio
+async def test_s3_delete_requires_path(aws_core: AWSCore):
+    metadata = S3Metadata(bucket="mountaineer-test", prefix="test-prefix")
+
+    with pytest.raises(ValueError):
+        await aws_core.storage_delete(path=None, metadata=metadata)
+
+
+@pytest.mark.asyncio
+async def test_cloudfile_delete_in_iceaxe_model(
+    mock_aws: MockAWS,
+    aws_core: AWSCore,
+):
+    asset = ExampleAWSAsset(id=1, file_url=CloudFile(""))
+    assert asset.file_url is not None
+
+    with patch(
+        "mountaineer_cloud.providers_common.s3_compat.uuid4",
+        return_value="test-uuid",
+    ):
+        await asset.file_url.put_content(aws_core, b"test data")
+
+    assert asset.file_url == "s3://mountaineer-test/test-prefix/test-uuid"
+
+    cleared = await asset.file_url.delete(aws_core)
+
+    assert cleared == ""
+    assert asset.file_url == ""
+    assert isinstance(asset.file_url, CloudFile)
+    assert asset.file_url._cloud_definition is not None
+
+    listing = await mock_aws.mock_s3.list_objects_v2(
+        Bucket="mountaineer-test", Prefix="test-prefix/"
+    )
+    assert listing.get("KeyCount", 0) == 0
+
+
+@pytest.mark.asyncio
+async def test_s3_download_missing_key_raises_file_not_found(aws_core: AWSCore):
+    metadata = S3Metadata(bucket="mountaineer-test", prefix="test-prefix")
+    path = "s3://mountaineer-test/test-prefix/missing"
+
+    with pytest.raises(FileNotFoundError) as exc_info:
+        async with aws_core.storage_read(path=path, metadata=metadata):
+            pass
+
+    assert exc_info.value.filename == path
+    assert isinstance(exc_info.value.__cause__, ClientError)
+
+
+@pytest.mark.asyncio
+async def test_cloudfile_get_content_missing_key_raises_file_not_found(
+    aws_core: AWSCore,
+):
+    asset = ExampleAWSAsset(
+        id=1,
+        file_url=CloudFile("s3://mountaineer-test/test-prefix/missing"),
+    )
+    assert asset.file_url is not None
+
+    with pytest.raises(FileNotFoundError):
+        await asset.file_url.get_content(aws_core)
+
+
+@pytest.mark.asyncio
+async def test_s3_download_missing_bucket_raises_bucket_not_found(
+    aws_core: AWSCore,
+):
+    metadata = S3Metadata(bucket="missing-bucket", prefix="test-prefix")
+    path = "s3://missing-bucket/test-prefix/test-uuid"
+
+    with pytest.raises(StorageBucketNotFoundError) as exc_info:
+        async with aws_core.storage_read(path=path, metadata=metadata):
+            pass
+
+    # Storage-specific errors remain catchable as the provider's ClientError
+    assert isinstance(exc_info.value, ClientError)
+    assert exc_info.value.path == path
+
+
+@pytest.mark.asyncio
+async def test_s3_read_does_not_translate_caller_errors(
+    mock_aws: MockAWS, aws_core: AWSCore
+):
+    metadata = S3Metadata(bucket="mountaineer-test", prefix="test-prefix")
+    await mock_aws.mock_s3.put_object(
+        Bucket="mountaineer-test",
+        Key="test-prefix/test-uuid",
+        Body=b"test data",
+    )
+    caller_error = ClientError({"Error": {"Code": "NoSuchKey"}}, "Other")
+
+    with pytest.raises(ClientError) as exc_info:
+        async with aws_core.storage_read(
+            path="s3://mountaineer-test/test-prefix/test-uuid",
+            metadata=metadata,
+        ):
+            raise caller_error
+
+    assert exc_info.value is caller_error
+
+
+@pytest.mark.parametrize(
+    "error_response,expected_type",
+    [
+        ({"Error": {"Code": "NoSuchKey"}}, FileNotFoundError),
+        (
+            {"Error": {"Code": "404"}, "ResponseMetadata": {"HTTPStatusCode": 404}},
+            FileNotFoundError,
+        ),
+        ({"Error": {"Code": "AccessDenied"}}, PermissionError),
+        (
+            {"Error": {"Code": "403"}, "ResponseMetadata": {"HTTPStatusCode": 403}},
+            PermissionError,
+        ),
+        (
+            {
+                "Error": {"Code": "NoSuchBucket"},
+                "ResponseMetadata": {"HTTPStatusCode": 404},
+            },
+            StorageBucketNotFoundError,
+        ),
+        ({"Error": {"Code": "SlowDown"}}, StorageRateLimitError),
+        (
+            {"Error": {"Code": "Unknown"}, "ResponseMetadata": {"HTTPStatusCode": 429}},
+            StorageRateLimitError,
+        ),
+    ],
+)
+def test_translate_storage_error(
+    error_response: dict[str, Any], expected_type: type[Exception]
+):
+    exc = ClientError(cast(Any, error_response), "GetObject")
+    translated = translate_storage_error(exc, path="s3://bucket/key")
+    assert type(translated) is expected_type
+
+
+def test_translate_storage_error_passes_through_unknown_codes():
+    exc = ClientError(
+        cast(Any, {"Error": {"Code": "InternalError"}}),
+        "GetObject",
+    )
+    assert translate_storage_error(exc, path="s3://bucket/key") is None

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/mountaineer_cloud/primitives/storage.py

@@ -156,6 +156,20 @@ class CloudFile(str, CloudValueBase[TStorageCore]):
         async with self.get_contents(core) as file:
             return file.read()
 
+    async def delete(self, core: TStorageCore) -> "CloudFile[TStorageCore]":
+        """
+        Delete the remote object and clear this pointer. When bound to a model,
+        the owning field is reset to an empty `CloudFile` so the stale path is
+        no longer referenced.
+        """
+        definition = self._require_definition()
+        await core.storage_delete(
+            path=str(self) or None,
+            metadata=definition.storage_metadata,
+        )
+
+        return self._apply_new_value("")
+
     #
     # Coerce from the string base class that's actually stored within the database
     # level varchar column into the CloudFile primitive.

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/mountaineer_cloud/providers_common/__init__.py

@@ -16,3 +16,8 @@ from .storage import (
     StorageMetadata as StorageMetadata,
     StorageProviderCore as StorageProviderCore,
 )
+from .storage_errors import (
+    StorageBucketNotFoundError as StorageBucketNotFoundError,
+    StorageClientError as StorageClientError,
+    StorageRateLimitError as StorageRateLimitError,
+)

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/mountaineer_cloud/providers_common/s3_compat.py

@@ -1,6 +1,6 @@
 import gzip
 from abc import ABC
-from collections.abc import AsyncGenerator, Awaitable, Callable
+from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator
 from contextlib import asynccontextmanager, contextmanager
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
@@ -23,6 +23,9 @@ from mountaineer_cloud.providers_common.storage import (
     StorageMetadata,
     StorageProviderCore,
 )
+from mountaineer_cloud.providers_common.storage_errors import (
+    translate_storage_error,
+)
 
 if TYPE_CHECKING:
     from types_aiobotocore_s3.client import S3Client
@@ -208,28 +211,27 @@ class S3CompatibleStorageCore(StorageProviderCore[TConfig], Generic[TConfig], AB
         s3_parsed = urlparse(path)
 
         async with self.get_storage_client() as s3:
-            try:
-                raw_contents = await s3.get_object(
-                    Bucket=s3_parsed.netloc,
-                    Key=s3_parsed.path.strip("/"),
-                )
-                with self._get_output_io(s3_metadata) as output_file:
+            with self._get_output_io(s3_metadata) as output_file:
+                with _translate_client_errors(path, action="accessing"):
+                    raw_contents = await s3.get_object(
+                        Bucket=s3_parsed.netloc,
+                        Key=s3_parsed.path.strip("/"),
+                    )
                     buffer_size = 24 * 1024
                     while True:
                         chunk = await raw_contents["Body"].read(buffer_size)
                         if not chunk:
                             break
                         output_file.write(chunk)
-                    output_file.seek(0)
-                    with self._unwrap_compressed_file(
-                        output_file,
-                        s3_metadata,
-                    ) as decompressed_file:
-                        yield decompressed_file
+                output_file.seek(0)
 
-            except ClientError as e:
-                error(f"Error encountered when accessing {path}: {e}")
-                raise
+                # Errors raised by the caller while consuming the file are
+                # theirs, so the yield sits outside of the translation scope.
+                with self._unwrap_compressed_file(
+                    output_file,
+                    s3_metadata,
+                ) as decompressed_file:
+                    yield decompressed_file
 
     async def storage_write(
         self,
@@ -290,7 +292,7 @@ class S3CompatibleStorageCore(StorageProviderCore[TConfig], Generic[TConfig], AB
         s3_parsed = urlparse(s3_metadata_path)
 
         async with self.get_storage_client() as s3:
-            try:
+            with _translate_client_errors(s3_metadata_path, action="accessing"):
                 if content_type:
                     await s3.upload_fileobj(
                         Bucket=s3_parsed.netloc,
@@ -304,12 +306,29 @@ class S3CompatibleStorageCore(StorageProviderCore[TConfig], Generic[TConfig], AB
                         Key=s3_parsed.path.strip("/"),
                         Fileobj=payload,
                     )
-            except ClientError as e:
-                error(f"Error encountered when accessing {s3_metadata_path}: {e}")
-                raise
 
         return s3_metadata_path
 
+    async def storage_delete(
+        self,
+        *,
+        path: str | None,
+        metadata: StorageMetadata,
+    ) -> None:
+        del metadata
+
+        if not path:
+            raise ValueError("S3 object not found")
+
+        s3_parsed = urlparse(path)
+
+        async with self.get_storage_client() as s3:
+            with _translate_client_errors(path, action="deleting"):
+                await s3.delete_object(
+                    Bucket=s3_parsed.netloc,
+                    Key=s3_parsed.path.strip("/"),
+                )
+
     @contextmanager
     def _wrap_compressed_file(
         self,
@@ -390,6 +409,22 @@ class S3CompatibleStorageCore(StorageProviderCore[TConfig], Generic[TConfig], AB
             )
 
 
+@contextmanager
+def _translate_client_errors(path: str, *, action: str) -> Iterator[None]:
+    """
+    Re-raise provider `ClientError`s as the typed errors defined in
+    `storage_errors`, keeping the original exception as the cause.
+    """
+    try:
+        yield
+    except ClientError as e:
+        error(f"Error encountered when {action} {path}: {e}")
+        translated = translate_storage_error(e, path=path)
+        if translated is None:
+            raise
+        raise translated from e
+
+
 def _coerce_s3_metadata(metadata: StorageMetadata) -> S3CompatibleMetadataBase:
     if isinstance(metadata, S3CompatibleMetadataBase):
         return metadata

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/mountaineer_cloud/providers_common/storage.py

@@ -68,3 +68,14 @@ class StorageProviderCore(ProviderCore[TConfig], Generic[TConfig], ABC):
         """
         Persist payload data and return the resulting storage path.
         """
+
+    @abstractmethod
+    async def storage_delete(
+        self,
+        *,
+        path: str | None,
+        metadata: StorageMetadata,
+    ) -> None:
+        """
+        Remove the payload stored at `path`.
+        """

mountaineer_cloud-0.2.1/mountaineer_cloud/providers_common/storage_errors.py

@@ -0,0 +1,98 @@
+"""
+Typed errors raised by storage providers.
+
+Where a failure has a native Python equivalent we raise that type directly, so
+callers can handle remote objects the same way they would local files:
+
+- Missing objects raise `FileNotFoundError`.
+- Denied access raises `PermissionError`.
+
+Failures without a native equivalent raise subclasses of botocore's
+`ClientError`, so existing `except ClientError` handlers keep catching them
+while new code can match on the specific condition.
+
+In every case the original `ClientError` from the provider is preserved as the
+`__cause__` of the raised exception.
+"""
+
+import errno
+import os
+from typing import Any
+
+from botocore.exceptions import ClientError
+
+# Error codes are matched before HTTP status codes, since some conditions share
+# a status (S3 returns 404 for both a missing key and a missing bucket).
+NOT_FOUND_ERROR_CODES = frozenset({"NoSuchKey", "NotFound", "404"})
+PERMISSION_ERROR_CODES = frozenset(
+    {"AccessDenied", "AllAccessDisabled", "Forbidden", "403"}
+)
+BUCKET_NOT_FOUND_ERROR_CODES = frozenset({"NoSuchBucket"})
+RATE_LIMIT_ERROR_CODES = frozenset(
+    {
+        "SlowDown",
+        "Throttling",
+        "ThrottlingException",
+        "RequestThrottled",
+        "TooManyRequests",
+        "TooManyRequestsException",
+    }
+)
+
+
+class StorageClientError(ClientError):
+    """
+    Base class for storage failures that have no native Python equivalent.
+    """
+
+    def __init__(
+        self,
+        error_response: Any,
+        operation_name: str,
+        *,
+        path: str | None = None,
+    ):
+        super().__init__(error_response, operation_name)
+        self.path = path
+
+
+class StorageBucketNotFoundError(StorageClientError):
+    """
+    The bucket referenced by the storage path does not exist.
+    """
+
+
+class StorageRateLimitError(StorageClientError):
+    """
+    The provider throttled the request. Retrying after a backoff may succeed.
+    """
+
+
+def translate_storage_error(exc: ClientError, *, path: str) -> Exception | None:
+    """
+    Map a provider `ClientError` onto a native or storage-specific exception.
+
+    Returns `None` when the error has no more specific type, in which case the
+    caller should re-raise the original `ClientError`.
+    """
+    error = exc.response.get("Error", {})
+    code = str(error.get("Code", ""))
+    status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
+
+    if code in BUCKET_NOT_FOUND_ERROR_CODES:
+        return StorageBucketNotFoundError(exc.response, exc.operation_name, path=path)
+    if code in RATE_LIMIT_ERROR_CODES:
+        return StorageRateLimitError(exc.response, exc.operation_name, path=path)
+    if code in NOT_FOUND_ERROR_CODES:
+        return FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
+    if code in PERMISSION_ERROR_CODES:
+        return PermissionError(errno.EACCES, os.strerror(errno.EACCES), path)
+
+    if status == 404:
+        return FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
+    if status == 403:
+        return PermissionError(errno.EACCES, os.strerror(errno.EACCES), path)
+    if status == 429:
+        return StorageRateLimitError(exc.response, exc.operation_name, path=path)
+
+    return None

{mountaineer_cloud-0.2.0.dev1 → mountaineer_cloud-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mountaineer-cloud"
-version = "0.2.0.dev1"
+version = "0.2.1"
 description = ""
 readme = "README.md"
 requires-python = ">=3.11"