planar 0.5.0__py3-none-any.whl

planar/files/storage/local_directory.py

@@ -0,0 +1,188 @@
+import uuid
+from pathlib import Path
+from typing import AsyncGenerator
+
+import aiofiles
+import aiofiles.os
+
+from planar.logging import get_logger
+
+from .base import Storage
+
+logger = get_logger(__name__)
+
+
+class LocalDirectoryStorage(Storage):
+    """Stores files and mime types in separate subdirectories on local disk."""
+
+    BLOB_SUBDIR = "blob"
+    MIME_SUBDIR = "mime"
+
+    def __init__(self, storage_dir: str | Path):
+        """
+        Initializes LocalDirectoryStorage.
+
+        Args:
+            storage_dir: The root directory where 'blob' and 'mime' subdirs will reside.
+                         It will be created if it doesn't exist.
+        """
+        self.base_dir = Path(storage_dir).resolve()
+        self.blob_dir = self.base_dir / self.BLOB_SUBDIR
+        self.mime_dir = self.base_dir / self.MIME_SUBDIR
+        self.blob_dir.mkdir(parents=True, exist_ok=True)
+        self.mime_dir.mkdir(parents=True, exist_ok=True)
+
+    def _get_path(self, ref: str, subdir: str) -> Path:
+        """Constructs the full path for a given storage reference in a specific subdir."""
+        try:
+            # Validate ref is a UUID string
+            ref_uuid = str(uuid.UUID(ref))
+        except ValueError:
+            raise ValueError(f"Invalid storage reference format: {ref}")
+
+        if subdir == self.BLOB_SUBDIR:
+            return self.blob_dir / ref_uuid
+        elif subdir == self.MIME_SUBDIR:
+            return self.mime_dir / ref_uuid
+        else:
+            raise ValueError(f"Invalid subdir specified: {subdir}")
+
+    async def put(
+        self, stream: AsyncGenerator[bytes, None], mime_type: str | None = None
+    ) -> str:
+        """
+        Stores a stream to a local file and its mime type in separate files.
+
+        The storage reference returned is the unique filename (UUID).
+        """
+        ref = str(uuid.uuid4())
+        blob_path = self._get_path(ref, self.BLOB_SUBDIR)
+        mime_path = self._get_path(ref, self.MIME_SUBDIR)
+
+        try:
+            # Write blob data
+            async with aiofiles.open(blob_path, mode="wb") as f:
+                async for chunk in stream:
+                    await f.write(chunk)
+
+            # Write mime type if provided
+            if mime_type:
+                async with aiofiles.open(mime_path, mode="w", encoding="utf-8") as f:
+                    await f.write(mime_type)
+
+            return ref
+        except Exception as e:
+            logger.exception("error during put operation", ref=ref)
+            # Attempt to clean up potentially partially written files
+            if await aiofiles.os.path.exists(blob_path):
+                try:
+                    await aiofiles.os.remove(blob_path)
+                except OSError as e2:
+                    logger.warning(
+                        "failed to cleanup blob file",
+                        path=str(blob_path),
+                        os_error=str(e2),
+                    )
+            if await aiofiles.os.path.exists(mime_path):
+                try:
+                    await aiofiles.os.remove(mime_path)
+                except OSError as e2:
+                    logger.warning(
+                        "failed to cleanup mime file",
+                        path=str(mime_path),
+                        os_error=str(e2),
+                    )
+            raise IOError(f"Failed to store file or mime type for ref {ref}") from e
+
+    async def get(self, ref: str) -> tuple[AsyncGenerator[bytes, None], str | None]:
+        """
+        Retrieves a stream and its mime type from local files.
+        """
+        blob_path = self._get_path(ref, self.BLOB_SUBDIR)
+        mime_path = self._get_path(ref, self.MIME_SUBDIR)
+
+        if not await aiofiles.os.path.isfile(blob_path):
+            raise FileNotFoundError(f"Storage reference blob not found: {ref}")
+
+        # Read mime type first
+        mime_type: str | None = None
+        if await aiofiles.os.path.isfile(mime_path):
+            try:
+                async with aiofiles.open(mime_path, mode="r", encoding="utf-8") as f:
+                    mime_type = (await f.read()).strip()
+            except Exception:
+                logger.exception(
+                    "failed to read mime type file",
+                    path=str(mime_path),
+                    ref=ref,
+                )
+                # Proceed without mime type if reading fails
+
+        async def _stream():
+            try:
+                async with aiofiles.open(blob_path, mode="rb") as f:
+                    while True:
+                        chunk = await f.read(0xFFFF)  # Read in 64k chunks
+                        if not chunk:
+                            break
+                        yield chunk
+            except Exception as e:
+                logger.exception(
+                    "error reading blob file", path=str(blob_path), ref=ref
+                )
+                raise IOError(f"Failed to read blob file for ref {ref}") from e
+
+        return _stream(), mime_type
+
+    async def delete(self, ref: str) -> None:
+        """
+        Deletes the blob file and its corresponding mime type file.
+        """
+        blob_path = self._get_path(ref, self.BLOB_SUBDIR)
+        mime_path = self._get_path(ref, self.MIME_SUBDIR)
+        deleted_blob = False
+        deleted_mime = False
+
+        # Delete blob file
+        if await aiofiles.os.path.isfile(blob_path):
+            try:
+                await aiofiles.os.remove(blob_path)
+                deleted_blob = True
+            except Exception as e:
+                raise IOError(f"Failed to delete blob file {blob_path}: {e}") from e
+        else:
+            # If blob doesn't exist, we consider it 'deleted' in terms of state
+            deleted_blob = True
+
+        # Delete mime file if it exists
+        if await aiofiles.os.path.isfile(mime_path):
+            try:
+                await aiofiles.os.remove(mime_path)
+                deleted_mime = True
+            except Exception as e:
+                # Log warning but don't raise if blob was successfully deleted or didn't exist
+                logger.exception(
+                    "failed to delete mime file",
+                    path=str(mime_path),
+                )
+                if not deleted_blob:  # Re-raise if blob deletion also failed
+                    raise IOError(
+                        f"Failed to delete mime file {mime_path} after blob deletion failure: {e}"
+                    ) from e
+
+        # Raise FileNotFoundError only if neither file existed initially
+        if (
+            not await aiofiles.os.path.exists(blob_path)
+            and not await aiofiles.os.path.exists(mime_path)
+            and not deleted_blob
+            and not deleted_mime
+        ):
+            # Check existence again to handle race conditions, though unlikely here
+            # If we get here, it means the initial check passed but deletion failed somehow,
+            # OR the files never existed. We treat the latter as FileNotFoundError.
+            # This logic might need refinement based on desired atomicity guarantees.
+            # For now, if the blob path doesn't exist after trying to delete, assume success or prior non-existence.
+            pass  # Or raise FileNotFoundError if strict check is needed: raise FileNotFoundError(f"Storage reference not found: {ref}")
+
+    async def external_url(self, ref: str) -> str | None:
+        return None

planar/files/storage/s3.py

@@ -0,0 +1,220 @@
+import asyncio
+import io
+import uuid
+from typing import Any, AsyncGenerator, Dict, Optional, Tuple
+
+import boto3
+from botocore.client import Config as BotoConfig
+from botocore.exceptions import ClientError
+
+from planar.logging import get_logger
+
+from .base import Storage
+
+logger = get_logger(__name__)
+
+
+class S3Storage(Storage):
+    """Stores files and mime types in an S3-compatible bucket using boto3."""
+
+    def __init__(
+        self,
+        bucket_name: str,
+        region: str,
+        endpoint_url: Optional[str] = None,
+        access_key_id: Optional[str] = None,
+        secret_access_key: Optional[str] = None,
+        session_token: Optional[str] = None,  # For temporary credentials
+        boto_config: Optional[Dict[str, Any]] = None,  # Additional boto3 client config
+        presigned_url_ttl: int = 3600,
+    ):
+        """
+        Initializes S3Storage.
+
+        Args:
+            bucket_name: The name of the S3 bucket.
+            endpoint_url: The S3 endpoint URL (e.g., 'https://s3.amazonaws.com' or custom).
+            access_key_id: AWS Access Key ID.
+            secret_access_key: AWS Secret Access Key.
+            region: The AWS region of the bucket.
+            session_token: AWS Session Token (for temporary credentials).
+            boto_config: Additional configuration options for boto3 client.
+            presigned_url_ttl: Time in seconds for which the presigned URL is valid.
+        """
+        self.bucket_name = bucket_name
+        self.endpoint_url = (
+            endpoint_url  # Boto3 generally prefers endpoint_url without trailing slash
+        )
+        self.access_key_id = access_key_id
+        self.secret_access_key = secret_access_key
+        self.region = region
+        self.session_token = session_token
+        self.presigned_url_ttl = presigned_url_ttl
+
+        # Initialize boto3 S3 client
+        # Using s3v4 signature is often necessary for S3 compatible services like MinIO/LocalStack
+        config_options = {"signature_version": "s3v4"}
+        if boto_config:
+            config_options.update(boto_config)
+        config = BotoConfig(**config_options)
+        self.s3_client = boto3.client(
+            "s3",
+            endpoint_url=self.endpoint_url,
+            aws_access_key_id=self.access_key_id,
+            aws_secret_access_key=self.secret_access_key,
+            aws_session_token=self.session_token,
+            region_name=self.region,
+            config=config,
+        )
+
+    async def _get_object_url(self, ref: str) -> str | None:
+        """Generates a presigned URL for a given object reference."""
+        try:
+            # generate_presigned_url is synchronous, so we run it in a thread
+            url = await asyncio.to_thread(
+                self.s3_client.generate_presigned_url,
+                "get_object",
+                Params={"Bucket": self.bucket_name, "Key": ref},
+                ExpiresIn=self.presigned_url_ttl,
+            )
+            return url
+        except ClientError:
+            logger.exception(
+                "failed to generate presigned url",
+                ref=ref,
+                bucket_name=self.bucket_name,
+            )
+            # Returning None is a safe default if URL generation fails.
+            return None
+
+    async def put(
+        self, stream: AsyncGenerator[bytes, None], mime_type: str | None = None
+    ) -> str:
+        """
+        Stores a stream and optional mime type to an S3 object with a unique name.
+
+        The storage reference returned is the unique object key (UUID).
+        The mime_type is stored as the Content-Type metadata.
+        """
+        ref = str(uuid.uuid4())
+
+        # Collect data from async generator into bytes
+        data_bytes_list = []
+        async for chunk in stream:
+            data_bytes_list.append(chunk)
+        data_bytes = b"".join(data_bytes_list)
+
+        extra_args = {}
+        if mime_type:
+            extra_args["ContentType"] = mime_type
+
+        try:
+            await asyncio.to_thread(
+                self.s3_client.put_object,
+                Bucket=self.bucket_name,
+                Key=ref,
+                Body=data_bytes,
+                **extra_args,
+            )
+            return ref
+        except ClientError as e:
+            logger.exception(
+                "failed s3 put object",
+                ref=ref,
+                bucket_name=self.bucket_name,
+                error_response=e.response,
+            )
+            raise IOError(f"Failed to upload to S3 object {ref}. Error: {e}") from e
+        except Exception as e:
+            logger.exception(
+                "an unexpected error occurred during s3 upload",
+                ref=ref,
+            )
+            raise IOError(f"An error occurred during S3 upload for {ref}") from e
+
+    async def get(self, ref: str) -> Tuple[AsyncGenerator[bytes, None], str | None]:
+        """
+        Retrieves a stream and its mime type (Content-Type) from an S3 object
+        using its storage reference (object key).
+        """
+        try:
+            response = await asyncio.to_thread(
+                self.s3_client.get_object, Bucket=self.bucket_name, Key=ref
+            )
+
+            streaming_body = response["Body"]
+            mime_type = response.get("ContentType")
+
+            async def _stream_wrapper(body):
+                try:
+                    while True:
+                        # Read a chunk in the executor
+                        chunk = await asyncio.to_thread(
+                            body.read, io.DEFAULT_BUFFER_SIZE
+                        )
+                        if not chunk:
+                            break
+                        yield chunk
+                finally:
+                    # Ensure the boto3 stream is closed
+                    await asyncio.to_thread(body.close)
+
+            return _stream_wrapper(streaming_body), mime_type
+
+        except ClientError as e:
+            if e.response.get("Error", {}).get("Code") == "NoSuchKey":
+                logger.warning(
+                    "s3 object not found", ref=ref, bucket_name=self.bucket_name
+                )
+                raise FileNotFoundError(f"S3 object not found: {ref}") from e
+            else:
+                logger.exception(
+                    "failed s3 get object",
+                    ref=ref,
+                    bucket_name=self.bucket_name,
+                    error_response=e.response,
+                )
+                raise IOError(
+                    f"Failed to download from S3 object {ref}. Error: {e}"
+                ) from e
+        except Exception as e:
+            logger.exception(
+                "an unexpected error occurred during s3 download",
+                ref=ref,
+            )
+            raise IOError(f"An error occurred during S3 download for {ref}") from e
+
+    async def delete(self, ref: str) -> None:
+        """
+        Deletes an object from the S3 bucket using its storage reference (object key).
+        Boto3's delete_object is idempotent and does not error if the object is not found.
+        """
+        try:
+            await asyncio.to_thread(
+                self.s3_client.delete_object, Bucket=self.bucket_name, Key=ref
+            )
+        except ClientError as e:
+            # delete_object is generally idempotent. Log and raise if it's not a 'not found' scenario
+            # (though boto3 usually handles 'NoSuchKey' gracefully for delete).
+            logger.exception(
+                "failed s3 delete object",
+                ref=ref,
+                bucket_name=self.bucket_name,
+                error_response=e.response,
+            )
+            raise IOError(f"Failed to delete S3 object {ref}. Error: {e}") from e
+        except Exception as e:
+            logger.exception(
+                "an unexpected error occurred during s3 delete",
+                ref=ref,
+            )
+            raise IOError(f"An error occurred during S3 delete for {ref}") from e
+
+    async def external_url(self, ref: str) -> str | None:
+        """
+        Returns a presigned URL to access the S3 object.
+
+        The URL is temporary and its validity is determined by the `presigned_url_ttl`
+        parameter set during initialization.
+        """
+        return await self._get_object_url(ref)

planar/files/storage/test_local_directory.py

@@ -0,0 +1,162 @@
+import asyncio
+import uuid
+from pathlib import Path
+
+import aiofiles.os
+import pytest
+
+from planar.files.storage.local_directory import LocalDirectoryStorage
+
+
+@pytest.fixture
+async def storage(tmp_path: Path) -> LocalDirectoryStorage:
+    """Provides an instance of LocalDirectoryStorage using a temporary directory."""
+    storage_instance = LocalDirectoryStorage(tmp_path)
+    # Ensure subdirectories exist (though constructor should handle this)
+    await aiofiles.os.makedirs(storage_instance.blob_dir, exist_ok=True)
+    await aiofiles.os.makedirs(storage_instance.mime_dir, exist_ok=True)
+    return storage_instance
+
+
+async def test_put_get_bytes(storage: LocalDirectoryStorage):
+    """Test storing and retrieving raw bytes."""
+    test_data = b"some binary data \x00\xff"
+    mime_type = "application/octet-stream"
+
+    ref = await storage.put_bytes(test_data, mime_type=mime_type)
+    assert isinstance(ref, str)
+    try:
+        uuid.UUID(ref)  # Check if ref is a valid UUID string
+    except ValueError:
+        pytest.fail(f"Returned ref '{ref}' is not a valid UUID string")
+
+    retrieved_data, retrieved_mime = await storage.get_bytes(ref)
+
+    assert retrieved_data == test_data
+    assert retrieved_mime == mime_type
+
+    # Check underlying files exist
+    blob_path = storage._get_path(ref, storage.BLOB_SUBDIR)
+    mime_path = storage._get_path(ref, storage.MIME_SUBDIR)
+    assert await aiofiles.os.path.exists(blob_path)
+    assert await aiofiles.os.path.exists(mime_path)
+
+
+async def test_put_get_string(storage: LocalDirectoryStorage):
+    """Test storing and retrieving a string."""
+    test_string = "Hello, world! This is a test string with Unicode: éàçü."
+    mime_type = "text/plain"
+    encoding = "utf-16"
+
+    # Store with explicit encoding and mime type
+    ref = await storage.put_string(test_string, encoding=encoding, mime_type=mime_type)
+    expected_mime_type = f"{mime_type}; charset={encoding}"
+
+    retrieved_string, retrieved_mime = await storage.get_string(ref, encoding=encoding)
+
+    assert retrieved_string == test_string
+    assert retrieved_mime == expected_mime_type
+
+    # Test default encoding (utf-8)
+    ref_utf8 = await storage.put_string(test_string, mime_type="text/html")
+    expected_mime_utf8 = "text/html; charset=utf-8"
+    retrieved_string_utf8, retrieved_mime_utf8 = await storage.get_string(ref_utf8)
+    assert retrieved_string_utf8 == test_string
+    assert retrieved_mime_utf8 == expected_mime_utf8
+
+
+async def test_put_get_stream(storage: LocalDirectoryStorage):
+    """Test storing data from an async generator stream."""
+    test_chunks = [b"chunk1 ", b"chunk2 ", b"chunk3"]
+    full_data = b"".join(test_chunks)
+    mime_type = "image/png"
+
+    async def _test_stream():
+        for chunk in test_chunks:
+            yield chunk
+            await asyncio.sleep(0.01)  # Simulate async work
+
+    ref = await storage.put(_test_stream(), mime_type=mime_type)
+
+    stream, retrieved_mime = await storage.get(ref)
+    retrieved_data = b""
+    async for chunk in stream:
+        retrieved_data += chunk
+
+    assert retrieved_data == full_data
+    assert retrieved_mime == mime_type
+
+
+async def test_put_no_mime_type(storage: LocalDirectoryStorage):
+    """Test storing data without providing a mime type."""
+    test_data = b"data without mime"
+
+    ref = await storage.put_bytes(test_data)
+    retrieved_data, retrieved_mime = await storage.get_bytes(ref)
+
+    assert retrieved_data == test_data
+    assert retrieved_mime is None
+
+    # Check only blob file exists
+    blob_path = storage._get_path(ref, storage.BLOB_SUBDIR)
+    mime_path = storage._get_path(ref, storage.MIME_SUBDIR)
+    assert await aiofiles.os.path.exists(blob_path)
+    assert not await aiofiles.os.path.exists(mime_path)
+
+
+async def test_delete(storage: LocalDirectoryStorage):
+    """Test deleting stored data."""
+    ref = await storage.put_bytes(b"to be deleted", mime_type="text/plain")
+
+    blob_path = storage._get_path(ref, storage.BLOB_SUBDIR)
+    mime_path = storage._get_path(ref, storage.MIME_SUBDIR)
+
+    # Verify files exist before delete
+    assert await aiofiles.os.path.exists(blob_path)
+    assert await aiofiles.os.path.exists(mime_path)
+
+    await storage.delete(ref)
+
+    # Verify files are gone after delete
+    assert not await aiofiles.os.path.exists(blob_path)
+    assert not await aiofiles.os.path.exists(mime_path)
+
+    # Try getting deleted ref
+    with pytest.raises(FileNotFoundError):
+        await storage.get(ref)
+
+
+async def test_get_non_existent(storage: LocalDirectoryStorage):
+    """Test getting a reference that does not exist."""
+    non_existent_ref = str(uuid.uuid4())
+    with pytest.raises(FileNotFoundError):
+        await storage.get(non_existent_ref)
+
+
+async def test_delete_non_existent(storage: LocalDirectoryStorage):
+    """Test deleting a reference that does not exist (should not raise error)."""
+    non_existent_ref = str(uuid.uuid4())
+    try:
+        await storage.delete(non_existent_ref)
+    except Exception as e:
+        pytest.fail(f"Deleting non-existent ref raised an exception: {e}")
+
+
+async def test_invalid_ref_format(storage: LocalDirectoryStorage):
+    """Test operations with an invalid storage reference format."""
+    invalid_ref = "not-a-uuid"
+    with pytest.raises(ValueError):
+        await storage.get(invalid_ref)
+
+    with pytest.raises(ValueError):
+        await storage.delete(invalid_ref)
+
+    with pytest.raises(ValueError):
+        storage._get_path(invalid_ref, storage.BLOB_SUBDIR)
+
+
+async def test_external_url(storage: LocalDirectoryStorage):
+    """Test that external_url returns None for local storage."""
+    ref = await storage.put_bytes(b"some data")
+    url = await storage.external_url(ref)
+    assert url is None