svc-infra 0.1.600__py3-none-any.whl → 0.1.664__py3-none-any.whl

svc_infra/storage/easy.py

@@ -0,0 +1,182 @@
+"""
+Easy storage backend builder with auto-detection.
+
+Simplifies storage backend initialization with sensible defaults.
+"""
+
+import logging
+import os
+from typing import Optional
+
+from .backends import LocalBackend, MemoryBackend, S3Backend
+from .base import StorageBackend
+from .settings import StorageSettings
+
+logger = logging.getLogger(__name__)
+
+
+def easy_storage(
+    backend: Optional[str] = None,
+    **kwargs,
+) -> StorageBackend:
+    """
+    Create a storage backend with auto-detection or explicit selection.
+
+    This is the recommended way to initialize storage in most applications.
+    It handles environment-based configuration and provides sensible defaults.
+
+    Args:
+        backend: Explicit backend type ("local", "s3", "gcs", "cloudinary", "memory")
+                If None, auto-detects from environment variables
+        **kwargs: Backend-specific configuration overrides
+
+    Returns:
+        Initialized storage backend
+
+    Auto-Detection Order:
+        1. Explicit backend parameter
+        2. STORAGE_BACKEND environment variable
+        3. Railway volume (RAILWAY_VOLUME_MOUNT_PATH) → LocalBackend
+        4. S3 credentials (AWS_ACCESS_KEY_ID or STORAGE_S3_BUCKET) → S3Backend
+        5. GCS credentials (GOOGLE_APPLICATION_CREDENTIALS) → GCSBackend
+        6. Cloudinary credentials (CLOUDINARY_URL) → CloudinaryBackend
+        7. Default: MemoryBackend (with warning)
+
+    Examples:
+        >>> # Auto-detect backend from environment
+        >>> storage = easy_storage()
+        >>>
+        >>> # Explicit local backend
+        >>> storage = easy_storage(
+        ...     backend="local",
+        ...     base_path="/data/uploads"
+        ... )
+        >>>
+        >>> # Explicit S3 backend
+        >>> storage = easy_storage(
+        ...     backend="s3",
+        ...     bucket="my-uploads",
+        ...     region="us-west-2"
+        ... )
+        >>>
+        >>> # DigitalOcean Spaces
+        >>> storage = easy_storage(
+        ...     backend="s3",
+        ...     bucket="my-uploads",
+        ...     region="nyc3",
+        ...     endpoint="https://nyc3.digitaloceanspaces.com"
+        ... )
+
+    Environment Variables:
+        See StorageSettings for full list of environment variables.
+
+    Raises:
+        ValueError: If backend type is unsupported or configuration is invalid
+        ImportError: If required backend dependencies are not installed
+
+    Note:
+        For production deployments, it's recommended to set STORAGE_BACKEND
+        explicitly to avoid unexpected auto-detection behavior.
+    """
+    # Load settings
+    settings = StorageSettings()
+
+    # Determine backend type
+    backend_type = backend or settings.detect_backend()
+
+    logger.info(f"Initializing {backend_type} storage backend")
+
+    # Create backend instance
+    if backend_type == "memory":
+        # Memory backend
+        if backend_type == settings.detect_backend() and not backend:
+            logger.warning(
+                "Using MemoryBackend (in-memory storage). "
+                "Data will be lost on restart. "
+                "Set STORAGE_BACKEND environment variable for production."
+            )
+
+        max_size = kwargs.get("max_size", 100_000_000)
+        return MemoryBackend(max_size=max_size)
+
+    elif backend_type == "local":
+        # Local filesystem backend
+        base_path = kwargs.get("base_path") or settings.storage_base_path
+
+        # Check for Railway volume
+        railway_volume = os.getenv("RAILWAY_VOLUME_MOUNT_PATH")
+        if railway_volume and not kwargs.get("base_path"):
+            base_path = railway_volume
+            logger.info(f"Detected Railway volume at {base_path}")
+
+        base_url = kwargs.get("base_url") or settings.storage_base_url
+        signing_secret = kwargs.get("signing_secret") or settings.storage_signing_secret
+
+        return LocalBackend(
+            base_path=base_path,
+            base_url=base_url,
+            signing_secret=signing_secret,
+        )
+
+    elif backend_type == "s3":
+        # S3-compatible backend
+        bucket = kwargs.get("bucket") or settings.storage_s3_bucket
+        if not bucket:
+            raise ValueError(
+                "S3 bucket is required. "
+                "Set STORAGE_S3_BUCKET environment variable or pass bucket parameter."
+            )
+
+        region = kwargs.get("region") or settings.storage_s3_region
+        endpoint = kwargs.get("endpoint") or settings.storage_s3_endpoint
+
+        # Get credentials with fallback
+        access_key = kwargs.get("access_key")
+        secret_key = kwargs.get("secret_key")
+
+        if not access_key or not secret_key:
+            access_key_from_settings, secret_key_from_settings = settings.get_s3_credentials()
+            access_key = access_key or access_key_from_settings
+            secret_key = secret_key or secret_key_from_settings
+
+        # Log provider detection
+        if endpoint:
+            if "digitalocean" in endpoint:
+                logger.info("Detected DigitalOcean Spaces")
+            elif "wasabi" in endpoint:
+                logger.info("Detected Wasabi")
+            elif "backblaze" in endpoint:
+                logger.info("Detected Backblaze B2")
+            else:
+                logger.info(f"Using custom S3 endpoint: {endpoint}")
+        else:
+            logger.info("Using AWS S3")
+
+        return S3Backend(
+            bucket=bucket,
+            region=region,
+            endpoint=endpoint,
+            access_key=access_key,
+            secret_key=secret_key,
+        )
+
+    elif backend_type == "gcs":
+        # Google Cloud Storage backend
+        raise NotImplementedError(
+            "GCS backend not yet implemented. " "Use 'local' or 's3' backend for now."
+        )
+
+    elif backend_type == "cloudinary":
+        # Cloudinary backend
+        raise NotImplementedError(
+            "Cloudinary backend not yet implemented. " "Use 'local' or 's3' backend for now."
+        )
+
+    else:
+        raise ValueError(
+            f"Unsupported storage backend: {backend_type}. "
+            f"Supported: local, s3, memory (gcs, cloudinary coming soon)"
+        )
+
+
+__all__ = ["easy_storage"]

svc_infra/storage/settings.py

@@ -0,0 +1,192 @@
+"""
+Storage configuration and settings.
+
+Handles environment-based configuration and auto-detection of storage backends.
+"""
+
+import os
+from typing import Literal, Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class StorageSettings(BaseSettings):
+    """
+    Storage system configuration.
+
+    Supports multiple backends with auto-detection from environment variables.
+
+    Environment Variables:
+        STORAGE_BACKEND: Explicit backend selection ("local", "s3", "gcs", "cloudinary", "memory")
+
+        Local Backend:
+            STORAGE_BASE_PATH: Base directory for file storage (default: /data/uploads)
+            STORAGE_BASE_URL: Base URL for file serving (default: http://localhost:8000/files)
+            STORAGE_SIGNING_SECRET: Secret key for URL signing (auto-generated if not set)
+
+        S3 Backend:
+            STORAGE_S3_BUCKET: S3 bucket name (required for S3)
+            STORAGE_S3_REGION: AWS region (default: us-east-1)
+            STORAGE_S3_ENDPOINT: Custom endpoint for S3-compatible services (optional)
+            STORAGE_S3_ACCESS_KEY: AWS access key (optional, uses AWS_ACCESS_KEY_ID if not set)
+            STORAGE_S3_SECRET_KEY: AWS secret key (optional, uses AWS_SECRET_ACCESS_KEY if not set)
+
+        GCS Backend:
+            STORAGE_GCS_BUCKET: GCS bucket name (required for GCS)
+            STORAGE_GCS_PROJECT: GCP project ID (optional)
+            STORAGE_GCS_CREDENTIALS_PATH: Path to service account JSON (optional)
+
+        Cloudinary Backend:
+            STORAGE_CLOUDINARY_CLOUD_NAME: Cloudinary cloud name (required)
+            STORAGE_CLOUDINARY_API_KEY: Cloudinary API key (required)
+            STORAGE_CLOUDINARY_API_SECRET: Cloudinary API secret (required)
+
+    Example:
+        >>> # Auto-detect backend from environment
+        >>> settings = StorageSettings()
+        >>> backend = settings.detect_backend()
+        >>>
+        >>> # Explicit backend selection
+        >>> settings = StorageSettings(storage_backend="s3")
+    """
+
+    # Backend selection
+    storage_backend: Optional[Literal["local", "s3", "gcs", "cloudinary", "memory"]] = Field(
+        default=None,
+        description="Storage backend type (auto-detected if not set)",
+    )
+
+    # Local backend settings
+    storage_base_path: str = Field(
+        default="/data/uploads",
+        description="Base directory for local file storage",
+    )
+    storage_base_url: str = Field(
+        default="http://localhost:8000/files",
+        description="Base URL for serving files",
+    )
+    storage_signing_secret: Optional[str] = Field(
+        default=None,
+        description="Secret key for URL signing (auto-generated if not set)",
+    )
+
+    # S3 backend settings
+    storage_s3_bucket: Optional[str] = Field(
+        default=None,
+        description="S3 bucket name",
+    )
+    storage_s3_region: str = Field(
+        default="us-east-1",
+        description="AWS region",
+    )
+    storage_s3_endpoint: Optional[str] = Field(
+        default=None,
+        description="Custom S3 endpoint (for DigitalOcean Spaces, Wasabi, etc.)",
+    )
+    storage_s3_access_key: Optional[str] = Field(
+        default=None,
+        description="S3 access key (falls back to AWS_ACCESS_KEY_ID)",
+    )
+    storage_s3_secret_key: Optional[str] = Field(
+        default=None,
+        description="S3 secret key (falls back to AWS_SECRET_ACCESS_KEY)",
+    )
+
+    # GCS backend settings
+    storage_gcs_bucket: Optional[str] = Field(
+        default=None,
+        description="Google Cloud Storage bucket name",
+    )
+    storage_gcs_project: Optional[str] = Field(
+        default=None,
+        description="GCP project ID",
+    )
+    storage_gcs_credentials_path: Optional[str] = Field(
+        default=None,
+        description="Path to GCP service account JSON",
+    )
+
+    # Cloudinary backend settings
+    storage_cloudinary_cloud_name: Optional[str] = Field(
+        default=None,
+        description="Cloudinary cloud name",
+    )
+    storage_cloudinary_api_key: Optional[str] = Field(
+        default=None,
+        description="Cloudinary API key",
+    )
+    storage_cloudinary_api_secret: Optional[str] = Field(
+        default=None,
+        description="Cloudinary API secret",
+    )
+
+    model_config = {
+        "env_file": ".env",
+        "case_sensitive": False,
+    }
+
+    def detect_backend(self) -> str:
+        """
+        Auto-detect storage backend from environment.
+
+        Detection order:
+            1. Explicit STORAGE_BACKEND setting
+            2. Railway volume (RAILWAY_VOLUME_MOUNT_PATH) → local
+            3. S3 credentials (AWS_ACCESS_KEY_ID or STORAGE_S3_BUCKET) → s3
+            4. GCS credentials (GOOGLE_APPLICATION_CREDENTIALS or STORAGE_GCS_BUCKET) → gcs
+            5. Cloudinary credentials (CLOUDINARY_URL or STORAGE_CLOUDINARY_CLOUD_NAME) → cloudinary
+            6. Default → memory (with warning)
+
+        Returns:
+            Backend type string
+
+        Example:
+            >>> settings = StorageSettings()
+            >>> backend_type = settings.detect_backend()
+            >>> print(f"Using {backend_type} backend")
+        """
+        # Explicit setting takes precedence
+        if self.storage_backend:
+            return self.storage_backend
+
+        # Check for Railway volume
+        railway_volume = os.getenv("RAILWAY_VOLUME_MOUNT_PATH")
+        if railway_volume:
+            return "local"
+
+        # Check for S3
+        has_s3_key = os.getenv("AWS_ACCESS_KEY_ID") or self.storage_s3_access_key
+        if has_s3_key or self.storage_s3_bucket:
+            return "s3"
+
+        # Check for GCS
+        has_gcs_creds = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
+        if has_gcs_creds or self.storage_gcs_bucket:
+            return "gcs"
+
+        # Check for Cloudinary
+        has_cloudinary = os.getenv("CLOUDINARY_URL")
+        if has_cloudinary or self.storage_cloudinary_cloud_name:
+            return "cloudinary"
+
+        # Default to memory (for development/testing)
+        return "memory"
+
+    def get_s3_credentials(self) -> tuple[Optional[str], Optional[str]]:
+        """
+        Get S3 credentials with fallback to AWS environment variables.
+
+        Returns:
+            Tuple of (access_key, secret_key)
+
+        Example:
+            >>> settings = StorageSettings()
+            >>> access_key, secret_key = settings.get_s3_credentials()
+        """
+        access_key = self.storage_s3_access_key or os.getenv("AWS_ACCESS_KEY_ID")
+        secret_key = self.storage_s3_secret_key or os.getenv("AWS_SECRET_ACCESS_KEY")
+        return access_key, secret_key
+
+
+__all__ = ["StorageSettings"]

svc_infra/webhooks/service.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from typing import Dict, List
-
 from uuid import uuid4
 
 from svc_infra.db.outbox import OutboxStore
@@ -22,7 +21,16 @@ class InMemoryWebhookSubscriptions:
         self._subs: Dict[str, List[WebhookSubscription]] = {}
 
     def add(self, topic: str, url: str, secret: str) -> None:
-        self._subs.setdefault(topic, []).append(WebhookSubscription(topic, url, secret))
+        # Upsert semantics per (topic, url): if a subscription already exists
+        # for this topic and URL, rotate its secret instead of adding a new row.
+        # This mirrors typical real-world secret rotation flows where the
+        # endpoint remains the same but the signing secret changes.
+        lst = self._subs.setdefault(topic, [])
+        for sub in lst:
+            if sub.url == url:
+                sub.secret = secret
+                return
+        lst.append(WebhookSubscription(topic, url, secret))
 
     def get_for_topic(self, topic: str) -> List[WebhookSubscription]:
         return list(self._subs.get(topic, []))

{svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: svc-infra
-Version: 0.1.600
+Version: 0.1.664
 Summary: Infrastructure for building and deploying prod-ready services
 License: MIT
 Keywords: fastapi,sqlalchemy,alembic,auth,infra,async,pydantic
@@ -24,10 +24,13 @@ Provides-Extra: mysql
 Provides-Extra: pg
 Provides-Extra: pg2
 Provides-Extra: redshift
+Provides-Extra: s3
 Provides-Extra: snowflake
 Provides-Extra: sqlite
 Requires-Dist: adyen (>=13.4.0,<14.0.0)
 Requires-Dist: ai-infra (>=0.1.63,<0.2.0)
+Requires-Dist: aioboto3 (>=13.0.0,<14.0.0) ; extra == "s3"
+Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
 Requires-Dist: aiosqlite (>=0.20.0,<0.21.0) ; extra == "sqlite"
 Requires-Dist: alembic (>=1.13.2,<2.0.0)
 Requires-Dist: asyncpg (>=0.30.0,<0.31.0) ; extra == "pg"
@@ -77,22 +80,49 @@ Description-Content-Type: text/markdown
 # svc-infra
 
 [![PyPI](https://img.shields.io/pypi/v/svc-infra.svg)](https://pypi.org/project/svc-infra/)
-[![Docs](https://img.shields.io/badge/docs-reference-blue)](docs/)
+[![Docs](https://img.shields.io/badge/docs-reference-blue)](.)
 
 svc-infra packages the shared building blocks we use to ship production FastAPI services fast—HTTP APIs with secure auth, durable persistence, background execution, cache, observability, and webhook plumbing that all share the same batteries-included defaults.
 
 ## Helper index
 
-| Helper | What it covers | Guide |
+| Area | What it covers | Guide |
 | --- | --- | --- |
-| API | FastAPI bootstrap, envelopes, middleware, docs wiring | [FastAPI guide](docs/api.md) |
-| Auth | Sessions, OAuth/OIDC, MFA, SMTP delivery | [Auth settings](docs/auth.md) |
-| Database | SQL + Mongo wiring, Alembic helpers, inbox/outbox patterns | [Database guide](docs/database.md) |
-| Jobs | JobQueue, scheduler, CLI worker | [Jobs quickstart](docs/jobs.md) |
-| Cache | cashews decorators, namespace management, TTL helpers | [Cache guide](docs/cache.md) |
-| Observability | Prometheus middleware, Grafana automation, OTEL hooks | [Observability guide](docs/observability.md) |
-| Webhooks | Subscription store, signing, retry worker | [Webhooks framework](docs/webhooks.md) |
-| Security | Password policy, lockout, signed cookies, headers | [Security hardening](docs/security.md) |
+| Getting Started | Overview and entry points | [This page](src/svc_infra/docs/getting-started.md) |
+| Environment | Feature switches and env vars | [Environment](src/svc_infra/docs/environment.md) |
+| API | FastAPI bootstrap, middleware, docs wiring | [API guide](src/svc_infra/docs/api.md) |
+| Auth | Sessions, OAuth/OIDC, MFA, SMTP delivery | [Auth](src/svc_infra/docs/auth.md) |
+| Security | Password policy, lockout, signed cookies, headers | [Security](src/svc_infra/docs/security.md) |
+| Database | SQL + Mongo wiring, Alembic helpers, inbox/outbox patterns | [Database](src/svc_infra/docs/database.md) |
+| Storage | File storage with S3, local, memory backends | [Storage](src/svc_infra/docs/storage.md) |
+| Tenancy | Multi-tenant boundaries and helpers | [Tenancy](src/svc_infra/docs/tenancy.md) |
+| Idempotency | Idempotent endpoints and middleware | [Idempotency](src/svc_infra/docs/idempotency.md) |
+| Rate Limiting | Middleware, dependency limiter, headers | [Rate limiting](src/svc_infra/docs/rate-limiting.md) |
+| Cache | cashews decorators, namespace management, TTL helpers | [Cache](src/svc_infra/docs/cache.md) |
+| Jobs | JobQueue, scheduler, CLI worker | [Jobs](src/svc_infra/docs/jobs.md) |
+| Observability | Prometheus, Grafana, OpenTelemetry | [Observability](src/svc_infra/docs/observability.md) |
+| Ops | Probes, breakers, SLOs & dashboards | [Ops](src/svc_infra/docs/ops.md) |
+| Webhooks | Subscription store, signing, retry worker | [Webhooks](src/svc_infra/docs/webhooks.md) |
+| CLI | Command groups for sql/mongo/obs/docs/dx/sdk/jobs | [CLI](src/svc_infra/docs/cli.md) |
+| Docs & SDKs | Publishing docs, generating SDKs | [Docs & SDKs](src/svc_infra/docs/docs-and-sdks.md) |
+| Acceptance | Acceptance harness and flows | [Acceptance](src/svc_infra/docs/acceptance.md), [Matrix](src/svc_infra/docs/acceptance-matrix.md) |
+| Contributing | Dev setup and quality gates | [Contributing](src/svc_infra/docs/contributing.md) |
+| Repo Review | Checklist for releasing/PRs | [Repo review](src/svc_infra/docs/repo-review.md) |
+| Data Lifecycle | Fixtures, retention, erasure, backups | [Data lifecycle](src/svc_infra/docs/data-lifecycle.md) |
+
+## Quick Start with Template Example
+
+See **ALL** svc-infra features working together in a complete example:
+
+```bash
+# One-time setup (from repo root)
+make setup-template    # Scaffolds models, runs migrations
+
+# Run the example server
+make run-template      # Starts at http://localhost:8001
+```
+
+See [`examples/README.md`](examples/README.md) for full documentation and manual setup options.
 
 ## Minimal FastAPI bootstrap
 
@@ -120,9 +150,10 @@ async def handle_webhook(payload = Depends(require_signature(lambda: ["current",
 - **API** – toggle logging/observability and docs exposure with `ENABLE_LOGGING`, `LOG_LEVEL`, `LOG_FORMAT`, `ENABLE_OBS`, `METRICS_PATH`, `OBS_SKIP_PATHS`, and `CORS_ALLOW_ORIGINS`. 【F:src/svc_infra/api/fastapi/ease.py†L67-L111】【F:src/svc_infra/api/fastapi/setup.py†L47-L88】
 - **Auth** – configure JWT secrets, SMTP, cookies, and policy using the `AUTH_…` settings family (e.g., `AUTH_JWT__SECRET`, `AUTH_SMTP_HOST`, `AUTH_SESSION_COOKIE_SECURE`). 【F:src/svc_infra/api/fastapi/auth/settings.py†L23-L91】
 - **Database** – set connection URLs or components via `SQL_URL`/`SQL_URL_FILE`, `DB_DIALECT`, `DB_HOST`, `DB_USER`, `DB_PASSWORD`, plus Mongo knobs like `MONGO_URL`, `MONGO_DB`, and `MONGO_URL_FILE`. 【F:src/svc_infra/api/fastapi/db/sql/add.py†L55-L114】【F:src/svc_infra/db/sql/utils.py†L85-L206】【F:src/svc_infra/db/nosql/mongo/settings.py†L9-L13】【F:src/svc_infra/db/nosql/utils.py†L56-L113】
-- **Jobs** – choose the queue backend with `JOBS_DRIVER` and provide Redis via `REDIS_URL`; interval schedules can be declared with `JOBS_SCHEDULE_JSON`. 【F:src/svc_infra/jobs/easy.py†L11-L27】【F:docs/jobs.md†L11-L48】
+- **Storage** – choose backend with `STORAGE_BACKEND` (local, s3, memory) and configure with `STORAGE_S3_BUCKET`, `STORAGE_S3_REGION`, `STORAGE_BASE_PATH`, or auto-detect from `RAILWAY_VOLUME_MOUNT_PATH` / AWS credentials. 【F:src/svc_infra/storage/settings.py】【F:src/svc_infra/docs/storage.md】
+- **Jobs** – choose the queue backend with `JOBS_DRIVER` and provide Redis via `REDIS_URL`; interval schedules can be declared with `JOBS_SCHEDULE_JSON`. 【F:src/svc_infra/jobs/easy.py†L11-L27】【F:src/svc_infra/docs/jobs.md†L11-L48】
 - **Cache** – namespace keys and lifetimes through `CACHE_PREFIX`, `CACHE_VERSION`, and TTL overrides `CACHE_TTL_DEFAULT`, `CACHE_TTL_SHORT`, `CACHE_TTL_LONG`. 【F:src/svc_infra/cache/README.md†L20-L173】【F:src/svc_infra/cache/ttl.py†L26-L55】
 - **Observability** – turn metrics on/off or adjust scrape paths with `ENABLE_OBS`, `METRICS_PATH`, `OBS_SKIP_PATHS`, and Prometheus/Grafana flags like `SVC_INFRA_DISABLE_PROMETHEUS`, `SVC_INFRA_RATE_WINDOW`, `SVC_INFRA_DASHBOARD_REFRESH`, `SVC_INFRA_DASHBOARD_RANGE`. 【F:src/svc_infra/api/fastapi/ease.py†L67-L111】【F:src/svc_infra/obs/metrics/asgi.py†L49-L206】【F:src/svc_infra/obs/cloud_dash.py†L85-L108】
-- **Webhooks** – reuse the jobs envs (`JOBS_DRIVER`, `REDIS_URL`) for the delivery worker and queue configuration. 【F:docs/webhooks.md†L32-L53】
-- **Security** – enforce password policy, MFA, and rotation with auth prefixes such as `AUTH_PASSWORD_MIN_LENGTH`, `AUTH_PASSWORD_REQUIRE_SYMBOL`, `AUTH_JWT__SECRET`, and `AUTH_JWT__OLD_SECRETS`. 【F:docs/security.md†L24-L70】
+- **Webhooks** – reuse the jobs envs (`JOBS_DRIVER`, `REDIS_URL`) for the delivery worker and queue configuration. 【F:src/svc_infra/docs/webhooks.md†L32-L53】
+- **Security** – enforce password policy, MFA, and rotation with auth prefixes such as `AUTH_PASSWORD_MIN_LENGTH`, `AUTH_PASSWORD_REQUIRE_SYMBOL`, `AUTH_JWT__SECRET`, and `AUTH_JWT__OLD_SECRETS`. 【F:src/svc_infra/docs/security.md†L24-L70】