fallbacks3 0.1.0__py3-none-any.whl

fallbacks3/__init__.py

@@ -0,0 +1,7 @@
+"""fallbacks3 - A Python library for S3-compatible storage with fallback support."""
+
+from fallbacks3.storage import Storage
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__", "Storage"]

fallbacks3/config.py

@@ -0,0 +1,11 @@
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Config(BaseSettings):
+    providers: str
+    fallback_provider: str
+
+    model_config = SettingsConfigDict(case_sensitive=False)
+
+
+config = Config()

fallbacks3/provider/__init__.py

@@ -0,0 +1,3 @@
+from fallbacks3.provider.provider import S3Provider
+
+__all__ = ["S3Provider"]

fallbacks3/provider/config_credentials.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel, ConfigDict
+
+
+class ProviderCredentials(BaseModel):
+    """Parsed provider URI in the format: <provider>://<access_key>:<secret_key>@<endpoint>."""
+
+    provider_scheme: str
+    access_key: str
+    secret_key: str
+    endpoint: str
+
+    model_config = ConfigDict(frozen=True)

fallbacks3/provider/provider.py

@@ -0,0 +1,81 @@
+from boto3 import client
+from botocore.client import Config as BotoConfig
+from mypy_boto3_s3.client import S3Client
+
+
+class S3Provider:
+    """S3-compatible storage provider."""
+
+    def __init__(
+        self,
+        provider_scheme: str,
+        endpoint: str,
+        access_key_id: str,
+        secret_access_key: str,
+    ):
+        """Initialize S3 provider with credentials.
+
+        Args:
+            provider_scheme: URI scheme for the provider (e.g., 'ps3', 'r2')
+            endpoint: S3 endpoint URL with scheme (e.g., 'https://ps3.palver.com')
+            access_key_id: Access key ID
+            secret_access_key: Secret access key
+        """
+        self.provider_scheme: str = provider_scheme
+        self.endpoint: str = endpoint
+        self.client: S3Client = client(
+            "s3",
+            config=BotoConfig(signature_version="s3v4"),
+            endpoint_url=endpoint,
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+        )
+
+    def upload_file(self, file_name: str, file_path: str, bucket: str) -> str:
+        """Upload a file to S3 bucket.
+
+        Args:
+            file_name: Name/key for the file in the bucket
+            file_path: Local path to the file to upload
+            bucket: Bucket name
+
+        Returns:
+            Full URI of the uploaded file in format:
+            <provider_scheme>://<bucket>/<file_name>
+        """
+        self.client.upload_file(file_path, bucket, file_name)
+        return f"{self.provider_scheme}://{bucket}/{file_name}"
+
+    def download_file(self, file_name: str, local_path: str, bucket: str) -> str:
+        """Download a file from S3 bucket.
+
+        Args:
+            file_name: Name/key of the file in the bucket
+            local_path: Local path where the file will be saved
+            bucket: Bucket name
+
+        Returns:
+            Local path where the file was saved
+        """
+        self.client.download_file(bucket, file_name, local_path)
+        return local_path
+
+    def generate_signed_url(
+        self, file_name: str, bucket: str, expiration: int = 60
+    ) -> str:
+        """Generate a presigned URL for temporary access to a file.
+
+        Args:
+            file_name: Name/key of the file in the bucket
+            bucket: Bucket name
+            expiration: Time in seconds for the URL to remain valid (default: 3600)
+
+        Returns:
+            Presigned URL string
+        """
+        url: str = self.client.generate_presigned_url(
+            "get_object",
+            Params={"Bucket": bucket, "Key": file_name},
+            ExpiresIn=expiration,
+        )
+        return url

fallbacks3/storage.py

@@ -0,0 +1,178 @@
+from fallbacks3.config import config
+from fallbacks3.provider import S3Provider
+from fallbacks3.utils.uri import build_provider_credentials, parse_uri
+from typing import Dict, Tuple
+from urllib.parse import ParseResult
+
+
+class Storage:
+    """S3-compatible storage client with multi-provider support and automatic fallback."""
+
+    def __init__(
+        self,
+        providers: str = config.providers,
+        fallback_provider: str = config.fallback_provider,
+    ):
+        """Initialize storage with multiple providers.
+
+        Args:
+            providers: Comma-separated provider URIs in format:
+                <provider>://<access_key>:<secret_key>@<endpoint>
+                Example: "ps3://key:secret@ps3.palver.com,s4://a:b@s4.mega.com"
+                Defaults to PROVIDERS environment variable.
+            fallback_provider: Provider scheme to use for fallback (e.g., "r2").
+                Must be one of the providers in the providers string.
+                Defaults to FALLBACK_PROVIDER environment variable.
+
+        Raises:
+            ValueError: If fallback_provider not found in providers
+        """
+        self.providers: Dict[str, S3Provider] = {}
+
+        # Parse and initialize all providers
+        for provider in providers.split(","):
+            stripped_uri: str = provider.strip()
+            if stripped_uri:
+                creds = build_provider_credentials(uri=stripped_uri)
+                self.providers[creds.provider_scheme] = S3Provider(
+                    provider_scheme=creds.provider_scheme,
+                    endpoint=creds.endpoint,
+                    access_key_id=creds.access_key,
+                    secret_access_key=creds.secret_key,
+                )
+
+        self.fallback_provider: str = fallback_provider
+
+        if fallback_provider not in self.providers:
+            raise ValueError(
+                f"Fallback provider '{fallback_provider}' not found in providers"
+            )
+
+    def upload_file(self, remote_file_uri: str, local_file_path: str) -> str:
+        """Upload a file with automatic fallback on failure.
+
+        Args:
+            remote_file_uri: Remote file URI in format <provider>://<bucket>/<file_path>
+                Example: "ps3://palver-whatsapp/file.mp4"
+            local_file_path: Local path to the file to upload
+
+        Returns:
+            URI of uploaded file in format: <provider>://<bucket>/<file_path>
+
+        Raises:
+            ValueError: If provider is not configured
+            Exception: If upload fails on both primary and fallback providers
+        """
+        parsed_uri, provider = self._resolve_uri(uri=remote_file_uri)
+
+        try:
+            return provider.upload_file(
+                file_name=parsed_uri.path.lstrip("/"),
+                file_path=local_file_path,
+                bucket=parsed_uri.netloc,
+            )
+        except Exception as e:
+            # Try fallback if different from primary
+            if self.fallback_provider != parsed_uri.scheme:
+                try:
+                    fallback: S3Provider = self.providers[self.fallback_provider]
+                    # Use same bucket and key with fallback provider
+                    return fallback.upload_file(
+                        file_name=parsed_uri.path.lstrip("/"),
+                        file_path=local_file_path,
+                        bucket=parsed_uri.netloc,
+                    )
+                except Exception as fallback_error:
+                    raise Exception(
+                        f"Both primary and fallback providers failed. "
+                        f"Primary error: {e}. Fallback error: {fallback_error}"
+                    )
+            raise
+
+    def download_file(self, remote_file_uri: str, local_file_path: str) -> str:
+        """Download a file from storage.
+
+        Args:
+            remote_file_uri: Remote file URI in format <provider>://<bucket>/<file_path>
+                Example: "ps3://palver-whatsapp/file.mp4"
+            local_file_path: Local path where file will be saved
+
+        Returns:
+            Local path where file was saved
+
+        Raises:
+            ValueError: If provider is not configured
+        """
+        parsed_uri, provider = self._resolve_uri(uri=remote_file_uri)
+
+        return provider.download_file(
+            file_name=parsed_uri.path.lstrip("/"),
+            local_path=local_file_path,
+            bucket=parsed_uri.netloc,
+        )
+
+    def generate_signed_url(self, remote_file_uri: str, expiration: int = 60) -> str:
+        """Generate a presigned URL for temporary access.
+
+        Args:
+            remote_file_uri: Remote file URI in format <provider>://<bucket>/<file_path>
+                Example: "ps3://palver-whatsapp/file.mp4"
+            expiration: Time in seconds for URL to remain valid (default: 60)
+
+        Returns:
+            Presigned URL string
+
+        Raises:
+            ValueError: If provider is not configured
+        """
+        parsed_uri, provider = self._resolve_uri(uri=remote_file_uri)
+
+        return provider.generate_signed_url(
+            file_name=parsed_uri.path.lstrip("/"),
+            bucket=parsed_uri.netloc,
+            expiration=expiration,
+        )
+
+    def _resolve_uri(self, uri: str) -> Tuple[ParseResult, S3Provider]:
+        """Parse and validate URI, then return parsed URI and provider.
+
+        Args:
+            uri: Remote file URI in format <provider>://<bucket>/<file_path>
+
+        Returns:
+            Tuple of (parsed_uri, provider)
+
+        Raises:
+            ValueError: If URI format is invalid or provider not configured
+        """
+        parsed_uri: ParseResult = parse_uri(uri=uri)
+        self._validate_parsed_uri(parsed_uri=parsed_uri, remote_file_uri=uri)
+        provider: S3Provider = self.providers[parsed_uri.scheme]
+        return parsed_uri, provider
+
+    def _validate_parsed_uri(
+        self, parsed_uri: ParseResult, remote_file_uri: str
+    ) -> None:
+        """Validate parsed URI format and provider configuration.
+
+        Args:
+            parsed_uri: ParseResult from urlparse
+            remote_file_uri: Original URI string for error messages
+
+        Raises:
+            ValueError: If URI format is invalid or provider not configured
+        """
+        if (
+            not parsed_uri.scheme
+            or not parsed_uri.netloc
+            or not parsed_uri.path.lstrip("/")
+        ):
+            raise ValueError(
+                f"Invalid media path format: {remote_file_uri}. "
+                f"Expected format: <provider>://<bucket>/<file_path>"
+            )
+
+        if parsed_uri.scheme not in self.providers:
+            raise ValueError(f"Provider '{parsed_uri.scheme}' not configured")
+
+        return

fallbacks3/utils/uri.py

@@ -0,0 +1,51 @@
+from fallbacks3.provider.config_credentials import ProviderCredentials
+from urllib.parse import urlparse, ParseResult
+
+
+def parse_uri(uri: str) -> ParseResult:
+    """
+    Parse provider URI format: <provider_name>://<access_key>:<secret_key>@<endpoint>
+
+    Example: ps3://key:secret@ps3.palver.com
+
+    Args:
+        uri: Provider URI string
+
+    Returns:
+        ParseResult with the following fields:
+            - scheme: Provider name (e.g., "ps3", "s4", "r2")
+            - username: Access key
+            - password: Secret key
+            - hostname: Endpoint hostname
+            - port: Endpoint port (None if not specified)
+    """
+    return urlparse(uri)
+
+
+def build_provider_credentials(uri: str) -> ProviderCredentials:
+    """
+    Build provider credentials from URI.
+
+    Args:
+        uri: Provider URI string in format <provider_name>://<access_key>:<secret_key>@<endpoint>
+
+    Returns:
+        ProviderCredentials with parsed fields
+    """
+    uri_parsed: ParseResult = parse_uri(uri=uri)
+
+    scheme: str = (
+        "http" if uri_parsed.hostname in ("localhost", "127.0.0.1") else "https"
+    )
+    endpoint = (
+        f"{scheme}://{uri_parsed.hostname}"
+        if not uri_parsed.port
+        else f"{scheme}://{uri_parsed.hostname}:{uri_parsed.port}"
+    )
+
+    return ProviderCredentials(
+        provider_scheme=uri_parsed.scheme,
+        access_key=uri_parsed.username,
+        secret_key=uri_parsed.password,
+        endpoint=endpoint,
+    )

fallbacks3-0.1.0.dist-info/METADATA

@@ -0,0 +1,172 @@
+Metadata-Version: 2.4
+Name: fallbacks3
+Version: 0.1.0
+Summary: S3-compatible storage library with automatic fallback support
+Author-email: Mila de Oliveira <mila.oliveira@palver.com.br>
+Project-URL: Homepage, https://github.com/palverdata/fallbacks3
+Project-URL: Repository, https://github.com/palverdata/fallbacks3
+Project-URL: Issues, https://github.com/palverdata/fallbacks3/issues
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: pydantic-settings>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: black>=23.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: build>=1.0.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Requires-Dist: boto3-stubs[s3]>=1.34.0; extra == "dev"
+
+# fallbacks3
+
+S3-compatible storage library with automatic fallback for the upload process.
+
+
+## Installation
+
+```bash
+pip install fallbacks3
+```
+
+For development:
+
+```bash
+pip install -e .
+```
+
+
+## Configuration
+
+You must configure a comma-separated list of providers, as well as the scheme of the fallback provider, with environment variables named `PROVIDERS` and `FALLBACK_PROVIDER`.
+
+```bash
+# Comma-separated list of provider URIs
+PROVIDERS="ps3://access_key:secret_key@ps3.palver.com,r2://access_key:secret_key@account.r2.cloudflarestorage.com"
+
+# Fallback provider scheme (must be one of the providers in PROVIDERS)
+FALLBACK_PROVIDER="r2"
+```
+
+
+### Provider URI Format
+
+Provider URIs follow the format:
+```
+<provider_scheme>://<access_key>:<secret_key>@<endpoint>
+```
+
+Examples: `r2://key:secret@account.r2.cloudflarestorage.com`, `local://test:testkey@localhost:9000`
+
+
+### Remote File URI Format
+
+File URIs should specify the exact location from a provider bucket:
+
+```
+<provider>://<bucket>/<file_path>
+```
+
+Examples: `ps3://palver-whatsapp/audio.mp3`, `s3://my-bucket/documents/report.pdf`
+
+## Usage
+
+```python
+from fallbacks3 import Storage
+
+# Initialize storage using environment variables (PROVIDERS and FALLBACK_PROVIDER)
+storage = Storage()
+
+# Upload a local file to the provided remote path
+stored_file_uri = storage.upload_file(
+    remote_file_uri="ps3://palver-whatsapp/audio.mp3",
+    local_file_path="/local/path/audio.mp3"
+)
+print(uri)  # "ps3://palver-whatsapp/audio.mp3"
+
+# Download a file
+downloaded_file_path = storage.download_file(
+    remote_file_uri="ps3://palver-whatsapp/audio.mp3",
+    local_file_path="/local/path/audio.mp3"
+)
+print(local_path)  # "/local/path/audio.mp3"
+
+# Generate a signed URL for temporary access (default: 60 seconds)
+signed_url = storage.generate_signed_url(
+    remote_file_uri="ps3://palver-whatsapp/audio.mp3",
+    expiration=3600
+)
+print(signed_url)  # "https://ps3.palver.com/palver-whatsapp/audio.mp3?signature=..."
+```
+
+### Upload with Automatic Fallback
+
+If the primary provider fails for the upload method, the library automatically retries with the fallback provider (using the same bucket and file path), as to ensure files are not lost.
+
+```python
+# If ps3 fails, automatically falls back to r2
+stored_file_path = storage.upload_file(
+    remote_file_uri="ps3://palver-whatsapp/audio.mp3",
+    local_file_path="/local/path/audio.mp3"
+)
+# Returns "r2://palver-whatsapp/audio.mp3" if ps3 failed
+```
+
+Raises an exception if both primary and fallback providers fail.
+
+### Download and Signed URLs
+
+Download and signed URL generation use the provider specified in the remote file URI.
+
+
+## Development
+
+Install development dependencies:
+
+```bash
+pip install -e ".[dev]"
+```
+
+You can also develop with `uv` if available.
+
+```bash
+uv pip install -e ".[dev]"
+```
+
+Run tests:
+
+```bash
+pytest --cov=fallbacks3
+
+# alternatively, with uv:
+uv run pytest --cov=fallbacks3
+```
+
+## Publishing to PyPI
+
+The project is set up to be automatically uploaded to PyPI when a new tag is pushed. For a successful push, all tests must pass and the project must pass the `check` and `format` requirements.
+
+### One-time setup
+
+Before publishing for the first time:
+
+1. **Configure Trusted Publishing on PyPI**:
+   - Go to https://pypi.org/manage/account/publishing/
+   - Add a new pending publisher:
+     - **PyPI Project Name**: `fallbacks3`
+     - **Owner**: `palverdata`
+     - **Repository name**: `fallbacks3`
+     - **Workflow name**: `publish.yaml`
+     - **Environment name**: `pypi`
+
+You must have access to the `pypi` GitHub environment or be approved by required reviewers (`milasd` or `giancarlopro`) to publish.
+
+### Publishing a new version
+
+1. Update version in `pyproject.toml`
+2. Create and push a tag:
+   ```bash
+   git tag v[new version] # eg.: git tag v0.1.0
+   git push origin v[new version] # eg.: git push origin v0.1.0
+   ```

fallbacks3-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+fallbacks3/__init__.py,sha256=e8KpdMS8mWubH6QFn2_TGZgrGZtpZjil43vxzO-j_YM,186
+fallbacks3/config.py,sha256=BIlekhzOkEVdqSmETpS1Y4Y8FpcJdwrdjnISnXVCevo,220
+fallbacks3/storage.py,sha256=RicumVKzyJgsvpDL1u4yp2NiI1a47DLHNSVKkWwT_q0,6709
+fallbacks3/provider/__init__.py,sha256=lFphOB3APdNzwQoJP_GgakL0ELXduBViFqoBBb3AR_0,78
+fallbacks3/provider/config_credentials.py,sha256=cLvA4GVC9JYoJMBi17lbeibBGhf6exD7nCg-j0aEMXg,307
+fallbacks3/provider/provider.py,sha256=V49uJht5m2o3qjOKVh4tjgUfpVDDEKiTkJcsbUhxNqY,2644
+fallbacks3/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fallbacks3/utils/uri.py,sha256=8ur7zPs-ZdZBb-rZA9oG0_fo7CzMoOp9HnlAVMcV2sI,1475
+fallbacks3-0.1.0.dist-info/METADATA,sha256=Z70wTuSDu8GSXUOLg5TE8XOsP_E-dYv8IW52GoAPo9A,4751
+fallbacks3-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+fallbacks3-0.1.0.dist-info/top_level.txt,sha256=JsC-ee_dPAtcp_-4LvPVZk6KUAtrj8HQNWQzbxF9myU,11
+fallbacks3-0.1.0.dist-info/RECORD,,

fallbacks3-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fallbacks3-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+fallbacks3