elaunira-r2index 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

elaunira/r2index/{async_uploader.py → async_storage.py}

@@ -1,4 +1,4 @@
-"""Asynchronous R2 uploader using aioboto3."""
+"""Asynchronous R2 storage operations using aioboto3."""
 
 from collections.abc import Callable
 from pathlib import Path
@@ -6,20 +6,16 @@ from pathlib import Path
 import aioboto3
 from aiobotocore.config import AioConfig
 
-from .exceptions import UploadError
-from .uploader import R2Config
+from .exceptions import DownloadError, UploadError
+from .storage import R2Config, R2TransferConfig
 
-# 100MB threshold and part size for multipart uploads
-MULTIPART_THRESHOLD = 100 * 1024 * 1024
-MULTIPART_PART_SIZE = 100 * 1024 * 1024
 
-
-class AsyncR2Uploader:
-    """Asynchronous R2 uploader using aioboto3."""
+class AsyncR2Storage:
+    """Asynchronous R2 storage client using aioboto3."""
 
     def __init__(self, config: R2Config) -> None:
         """
-        Initialize the async R2 uploader.
+        Initialize the async R2 storage client.
 
         Args:
             config: R2 configuration with credentials and endpoint.
@@ -30,20 +26,24 @@ class AsyncR2Uploader:
     async def upload_file(
         self,
         file_path: str | Path,
+        bucket: str,
         object_key: str,
         content_type: str | None = None,
         progress_callback: Callable[[int], None] | None = None,
+        transfer_config: R2TransferConfig | None = None,
     ) -> str:
         """
         Upload a file to R2 asynchronously.
 
-        Uses multipart upload for files larger than 100MB.
+        Uses multipart upload for files larger than the configured threshold.
 
         Args:
             file_path: Path to the file to upload.
+            bucket: The R2 bucket name.
             object_key: The key (path) to store the object under in R2.
             content_type: Optional content type for the object.
             progress_callback: Optional callback called with bytes uploaded so far.
+            transfer_config: Optional transfer configuration for multipart/threading.
 
         Returns:
             The object key of the uploaded file.
@@ -56,8 +56,9 @@ class AsyncR2Uploader:
         if not file_path.exists():
             raise UploadError(f"File not found: {file_path}")
 
-        transfer_config = AioConfig(
-            max_pool_connections=10,
+        tc = transfer_config or R2TransferConfig()
+        aio_config = AioConfig(
+            max_pool_connections=tc.max_concurrency,
         )
 
         extra_args = {}
@@ -71,7 +72,7 @@ class AsyncR2Uploader:
                 aws_secret_access_key=self.config.secret_access_key,
                 endpoint_url=self.config.endpoint_url,
                 region_name=self.config.region,
-                config=transfer_config,
+                config=aio_config,
             ) as client:
                 callback = None
                 if progress_callback:
@@ -79,7 +80,7 @@ class AsyncR2Uploader:
 
                 await client.upload_file(
                     str(file_path),
-                    self.config.bucket,
+                    bucket,
                     object_key,
                     ExtraArgs=extra_args if extra_args else None,
                     Callback=callback,
@@ -89,11 +90,12 @@ class AsyncR2Uploader:
 
         return object_key
 
-    async def delete_object(self, object_key: str) -> None:
+    async def delete_object(self, bucket: str, object_key: str) -> None:
         """
         Delete an object from R2 asynchronously.
 
         Args:
+            bucket: The R2 bucket name.
             object_key: The key of the object to delete.
 
         Raises:
@@ -107,15 +109,16 @@ class AsyncR2Uploader:
                 endpoint_url=self.config.endpoint_url,
                 region_name=self.config.region,
             ) as client:
-                await client.delete_object(Bucket=self.config.bucket, Key=object_key)
+                await client.delete_object(Bucket=bucket, Key=object_key)
         except Exception as e:
             raise UploadError(f"Failed to delete object from R2: {e}") from e
 
-    async def object_exists(self, object_key: str) -> bool:
+    async def object_exists(self, bucket: str, object_key: str) -> bool:
         """
         Check if an object exists in R2 asynchronously.
 
         Args:
+            bucket: The R2 bucket name.
             object_key: The key of the object to check.
 
         Returns:
@@ -129,13 +132,71 @@ class AsyncR2Uploader:
                 endpoint_url=self.config.endpoint_url,
                 region_name=self.config.region,
             ) as client:
-                await client.head_object(Bucket=self.config.bucket, Key=object_key)
+                await client.head_object(Bucket=bucket, Key=object_key)
                 return True
         except client.exceptions.ClientError as e:
             if e.response["Error"]["Code"] == "404":
                 return False
             raise UploadError(f"Failed to check object existence: {e}") from e
 
+    async def download_file(
+        self,
+        bucket: str,
+        object_key: str,
+        file_path: str | Path,
+        progress_callback: Callable[[int], None] | None = None,
+        transfer_config: R2TransferConfig | None = None,
+    ) -> Path:
+        """
+        Download a file from R2 asynchronously.
+
+        Args:
+            bucket: The R2 bucket name.
+            object_key: The key (path) of the object in R2.
+            file_path: Local path where the file will be saved.
+            progress_callback: Optional callback called with bytes downloaded so far.
+            transfer_config: Optional transfer configuration for multipart/threading.
+
+        Returns:
+            The path to the downloaded file.
+
+        Raises:
+            DownloadError: If the download fails.
+        """
+        file_path = Path(file_path)
+
+        # Ensure parent directory exists
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        tc = transfer_config or R2TransferConfig()
+        aio_config = AioConfig(
+            max_pool_connections=tc.max_concurrency,
+        )
+
+        try:
+            async with self._session.client(
+                "s3",
+                aws_access_key_id=self.config.access_key_id,
+                aws_secret_access_key=self.config.secret_access_key,
+                endpoint_url=self.config.endpoint_url,
+                region_name=self.config.region,
+                config=aio_config,
+            ) as client:
+                callback = None
+                if progress_callback:
+                    callback = _AsyncProgressCallback(progress_callback)
+
+                await client.download_file(
+                    bucket,
+                    object_key,
+                    str(file_path),
+                    Callback=callback,
+                )
+        except Exception as e:
+            raise DownloadError(f"Failed to download file from R2: {e}") from e
+
+        return file_path
+
 
 class _AsyncProgressCallback:
     """Wrapper to track cumulative progress for aioboto3 callback."""

elaunira/r2index/client.py

@@ -31,7 +31,48 @@ from .models import (
     TimeseriesResponse,
     UserAgentsResponse,
 )
-from .uploader import R2Config, R2Uploader
+from . import __version__
+from .storage import R2Config, R2Storage, R2TransferConfig
+
+CHECKIP_URL = "https://checkip.amazonaws.com"
+DEFAULT_USER_AGENT = f"elaunira-r2index/{__version__}"
+
+
+def _parse_object_id(object_id: str, bucket: str) -> RemoteTuple:
+    """
+    Parse an object_id into remote_path, remote_version, and remote_filename.
+
+    Format: /path/to/object/version/filename.ext
+    - remote_filename: last component (filename.ext)
+    - remote_version: second-to-last component (version)
+    - remote_path: everything before that (/path/to/object)
+
+    Args:
+        object_id: Full object path like /releases/myapp/v1/myapp.zip
+        bucket: The S3/R2 bucket name.
+
+    Returns:
+        RemoteTuple with parsed components including bucket.
+
+    Raises:
+        ValueError: If object_id doesn't have enough components.
+    """
+    parts = object_id.strip("/").split("/")
+    if len(parts) < 3:
+        raise ValueError(
+            f"object_id must have at least 3 components (path/version/filename), got: {object_id}"
+        )
+
+    remote_filename = parts[-1]
+    remote_version = parts[-2]
+    remote_path = "/" + "/".join(parts[:-2])
+
+    return RemoteTuple(
+        bucket=bucket,
+        remote_path=remote_path,
+        remote_filename=remote_filename,
+        remote_version=remote_version,
+    )
 
 
 class R2IndexClient:
@@ -39,29 +80,42 @@ class R2IndexClient:
 
     def __init__(
         self,
-        api_url: str,
-        api_token: str,
-        r2_config: R2Config | None = None,
+        index_api_url: str,
+        index_api_token: str,
+        r2_access_key_id: str | None = None,
+        r2_secret_access_key: str | None = None,
+        r2_endpoint_url: str | None = None,
         timeout: float = 30.0,
     ) -> None:
         """
         Initialize the R2Index client.
 
         Args:
-            api_url: Base URL of the r2index API.
-            api_token: Bearer token for authentication.
-            r2_config: Optional R2 configuration for upload operations.
+            index_api_url: Base URL of the r2index API.
+            index_api_token: Bearer token for authentication.
+            r2_access_key_id: R2 access key ID for storage operations.
+            r2_secret_access_key: R2 secret access key for storage operations.
+            r2_endpoint_url: R2 endpoint URL for storage operations.
             timeout: Request timeout in seconds.
         """
-        self.api_url = api_url.rstrip("/")
-        self._token = api_token
+        self.api_url = index_api_url.rstrip("/")
+        self._token = index_api_token
         self._timeout = timeout
-        self._r2_config = r2_config
-        self._uploader: R2Uploader | None = None
+        self._storage: R2Storage | None = None
+
+        # Build R2 config if credentials provided
+        if r2_access_key_id and r2_secret_access_key and r2_endpoint_url:
+            self._r2_config: R2Config | None = R2Config(
+                access_key_id=r2_access_key_id,
+                secret_access_key=r2_secret_access_key,
+                endpoint_url=r2_endpoint_url,
+            )
+        else:
+            self._r2_config = None
 
         self._client = httpx.Client(
             base_url=self.api_url,
-            headers={"Authorization": f"Bearer {api_token}"},
+            headers={"Authorization": f"Bearer {index_api_token}"},
             timeout=timeout,
         )
 
@@ -75,13 +129,13 @@ class R2IndexClient:
         """Close the HTTP client."""
         self._client.close()
 
-    def _get_uploader(self) -> R2Uploader:
+    def _get_storage(self) -> R2Storage:
         """Get or create the R2 uploader."""
         if self._r2_config is None:
             raise R2IndexError("R2 configuration required for upload operations")
-        if self._uploader is None:
-            self._uploader = R2Uploader(self._r2_config)
-        return self._uploader
+        if self._storage is None:
+            self._storage = R2Storage(self._r2_config)
+        return self._storage
 
     def _handle_response(self, response: httpx.Response) -> Any:
         """Handle API response and raise appropriate exceptions."""
@@ -108,8 +162,9 @@ class R2IndexClient:
 
     # File Operations
 
-    def list_files(
+    def list(
         self,
+        bucket: str | None = None,
         category: str | None = None,
         entity: str | None = None,
         tags: list[str] | None = None,
@@ -120,6 +175,7 @@ class R2IndexClient:
         List files with optional filters.
 
         Args:
+            bucket: Filter by bucket.
             category: Filter by category.
             entity: Filter by entity.
             tags: Filter by tags.
@@ -130,6 +186,8 @@ class R2IndexClient:
             FileListResponse with files and pagination info.
         """
         params: dict[str, Any] = {}
+        if bucket:
+            params["bucket"] = bucket
         if category:
             params["category"] = category
         if entity:
@@ -145,7 +203,7 @@ class R2IndexClient:
         data = self._handle_response(response)
         return FileListResponse.model_validate(data)
 
-    def create_file(self, data: FileCreateRequest) -> FileRecord:
+    def create(self, data: FileCreateRequest) -> FileRecord:
         """
         Create or upsert a file record.
 
@@ -159,7 +217,7 @@ class R2IndexClient:
         result = self._handle_response(response)
         return FileRecord.model_validate(result)
 
-    def get_file(self, file_id: str) -> FileRecord:
+    def get(self, file_id: str) -> FileRecord:
         """
         Get a file by ID.
 
@@ -176,7 +234,7 @@ class R2IndexClient:
         data = self._handle_response(response)
         return FileRecord.model_validate(data)
 
-    def update_file(self, file_id: str, data: FileUpdateRequest) -> FileRecord:
+    def update(self, file_id: str, data: FileUpdateRequest) -> FileRecord:
         """
         Update a file record.
 
@@ -194,7 +252,7 @@ class R2IndexClient:
         result = self._handle_response(response)
         return FileRecord.model_validate(result)
 
-    def delete_file(self, file_id: str) -> None:
+    def delete(self, file_id: str) -> None:
         """
         Delete a file by ID.
 
@@ -207,17 +265,18 @@ class R2IndexClient:
         response = self._client.delete(f"/files/{file_id}")
         self._handle_response(response)
 
-    def delete_file_by_tuple(self, remote_tuple: RemoteTuple) -> None:
+    def delete_by_tuple(self, remote_tuple: RemoteTuple) -> None:
         """
         Delete a file by remote tuple.
 
         Args:
-            remote_tuple: The remote path, filename, and version.
+            remote_tuple: The bucket, remote path, filename, and version.
 
         Raises:
             NotFoundError: If the file is not found.
         """
         params = {
+            "bucket": remote_tuple.bucket,
             "remotePath": remote_tuple.remote_path,
             "remoteFilename": remote_tuple.remote_filename,
             "remoteVersion": remote_tuple.remote_version,
@@ -225,8 +284,32 @@ class R2IndexClient:
         response = self._client.delete("/files", params=params)
         self._handle_response(response)
 
-    def get_index(
+    def get_by_tuple(self, remote_tuple: RemoteTuple) -> FileRecord:
+        """
+        Get a file by remote tuple.
+
+        Args:
+            remote_tuple: The bucket, remote path, filename, and version.
+
+        Returns:
+            The FileRecord.
+
+        Raises:
+            NotFoundError: If the file is not found.
+        """
+        params = {
+            "bucket": remote_tuple.bucket,
+            "remotePath": remote_tuple.remote_path,
+            "remoteFilename": remote_tuple.remote_filename,
+            "remoteVersion": remote_tuple.remote_version,
+        }
+        response = self._client.get("/files/by-tuple", params=params)
+        data = self._handle_response(response)
+        return FileRecord.model_validate(data)
+
+    def index(
         self,
+        bucket: str | None = None,
         category: str | None = None,
         entity: str | None = None,
         tags: list[str] | None = None,
@@ -235,6 +318,7 @@ class R2IndexClient:
         Get file index (lightweight listing).
 
         Args:
+            bucket: Filter by bucket.
             category: Filter by category.
             entity: Filter by entity.
             tags: Filter by tags.
@@ -243,6 +327,8 @@ class R2IndexClient:
             List of IndexEntry objects.
         """
         params: dict[str, Any] = {}
+        if bucket:
+            params["bucket"] = bucket
         if category:
             params["category"] = category
         if entity:
@@ -423,9 +509,10 @@ class R2IndexClient:
 
     # High-Level Pipeline
 
-    def upload_and_register(
+    def upload(
         self,
-        file_path: str | Path,
+        bucket: str,
+        local_path: str | Path,
         category: str,
         entity: str,
         remote_path: str,
@@ -446,7 +533,8 @@ class R2IndexClient:
         3. Register with r2index API
 
         Args:
-            file_path: Path to the file to upload.
+            bucket: The S3/R2 bucket name.
+            local_path: Local path to the file to upload.
             category: File category.
             entity: File entity.
             remote_path: Remote path in R2 (e.g., "/data/files").
@@ -465,18 +553,19 @@ class R2IndexClient:
             R2IndexError: If R2 config is not provided.
             UploadError: If upload fails.
         """
-        file_path = Path(file_path)
-        uploader = self._get_uploader()
+        local_path = Path(local_path)
+        uploader = self._get_storage()
 
         # Step 1: Compute checksums
-        checksums = compute_checksums(file_path)
+        checksums = compute_checksums(local_path)
 
         # Step 2: Build R2 object key
         object_key = f"{remote_path.strip('/')}/{remote_filename}"
 
         # Step 3: Upload to R2
         uploader.upload_file(
-            file_path,
+            local_path,
+            bucket,
             object_key,
             content_type=content_type,
             progress_callback=progress_callback,
@@ -484,6 +573,7 @@ class R2IndexClient:
 
         # Step 4: Register with API
         create_request = FileCreateRequest(
+            bucket=bucket,
             category=category,
             entity=entity,
             remote_path=remote_path,
@@ -499,4 +589,85 @@ class R2IndexClient:
             sha512=checksums.sha512,
         )
 
-        return self.create_file(create_request)
+        return self.create(create_request)
+
+    def _get_public_ip(self) -> str:
+        """Fetch public IP address from checkip.amazonaws.com."""
+        response = httpx.get(CHECKIP_URL, timeout=10.0)
+        return response.text.strip()
+
+    def download(
+        self,
+        bucket: str,
+        object_id: str,
+        destination: str | Path,
+        ip_address: str | None = None,
+        user_agent: str | None = None,
+        progress_callback: Callable[[int], None] | None = None,
+        transfer_config: R2TransferConfig | None = None,
+    ) -> tuple[Path, FileRecord]:
+        """
+        Download a file from R2 and record the download in the index.
+
+        This is a convenience method that performs:
+        1. Parse object_id into remote_path, remote_version, remote_filename
+        2. Fetch file record from the API using these components
+        3. Download the file from R2
+        4. Record the download in the index for analytics
+
+        Args:
+            bucket: The S3/R2 bucket name.
+            object_id: Full S3 object path in format: /path/to/object/version/filename
+                Example: /releases/myapp/v1/myapp.zip
+                - remote_path: /releases/myapp
+                - remote_version: v1
+                - remote_filename: myapp.zip
+            destination: Local path where the file will be saved.
+            ip_address: IP address of the downloader. If not provided, fetched
+                from checkip.amazonaws.com.
+            user_agent: User agent string. Defaults to "elaunira-r2index/<version>".
+            progress_callback: Optional callback for download progress.
+            transfer_config: Optional transfer configuration for multipart/threading.
+
+        Returns:
+            A tuple of (downloaded file path, file record).
+
+        Raises:
+            R2IndexError: If R2 config is not provided.
+            ValueError: If object_id format is invalid.
+            NotFoundError: If the file is not found in the index.
+            DownloadError: If download fails.
+        """
+        storage = self._get_storage()
+
+        # Resolve defaults
+        if ip_address is None:
+            ip_address = self._get_public_ip()
+        if user_agent is None:
+            user_agent = DEFAULT_USER_AGENT
+
+        # Step 1: Parse object_id into components
+        remote_tuple = _parse_object_id(object_id, bucket)
+
+        # Step 2: Get file record by tuple
+        file_record = self.get_by_tuple(remote_tuple)
+
+        # Step 3: Build R2 object key and download
+        object_key = object_id.strip("/")
+        downloaded_path = storage.download_file(
+            bucket,
+            object_key,
+            destination,
+            progress_callback=progress_callback,
+            transfer_config=transfer_config,
+        )
+
+        # Step 4: Record the download
+        download_request = DownloadRecordRequest(
+            file_id=file_record.id,
+            ip_address=ip_address,
+            user_agent=user_agent,
+        )
+        self.record_download(download_request)
+
+        return downloaded_path, file_record

elaunira/r2index/exceptions.py

@@ -38,3 +38,9 @@ class UploadError(R2IndexError):
     """Raised for R2 upload failures."""
 
     pass
+
+
+class DownloadError(R2IndexError):
+    """Raised for R2 download failures."""
+
+    pass

elaunira/r2index/models.py

@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field
 class RemoteTuple(BaseModel):
     """Remote file identifier tuple."""
 
+    bucket: str
     remote_path: str
     remote_filename: str
     remote_version: str
@@ -17,6 +18,7 @@ class RemoteTuple(BaseModel):
 class FileCreateRequest(BaseModel):
     """Request payload for creating/upserting a file record."""
 
+    bucket: str
     category: str
     entity: str
     remote_path: str
@@ -35,6 +37,7 @@ class FileCreateRequest(BaseModel):
 class FileUpdateRequest(BaseModel):
     """Request payload for updating a file record."""
 
+    bucket: str | None = None
     category: str | None = None
     entity: str | None = None
     remote_path: str | None = None
@@ -54,6 +57,7 @@ class FileRecord(BaseModel):
     """File record as returned by the API."""
 
     id: str
+    bucket: str
     category: str
     entity: str
     remote_path: str
@@ -86,6 +90,7 @@ class IndexEntry(BaseModel):
     """Single entry in the index response."""
 
     id: str
+    bucket: str
     category: str
     entity: str
     remote_path: str