svc-infra 0.1.640__py3-none-any.whl → 0.1.664__py3-none-any.whl

svc_infra/db/sql/templates/setup/env_sync.py.tmpl

@@ -6,11 +6,10 @@ from typing import List, Tuple
 import sys, pathlib, importlib, pkgutil, traceback
 
 from alembic import context
+from sqlalchemy import MetaData
 from sqlalchemy.engine import make_url, URL
 
 from svc_infra.db.sql.utils import (
-    _coerce_sync_driver,
-    _ensure_ssl_default,
     get_database_url_from_env,
     build_engine,
 )
@@ -103,7 +102,6 @@ if not effective_url:
 
 u = make_url(effective_url)
 u = _coerce_sync_driver(u)
-u = _ensure_ssl_default(u)
 config.set_main_option("sqlalchemy.url", u.render_as_string(hide_password=False))
 
 
@@ -142,14 +140,16 @@ def _collect_metadata() -> list[object]:
 
     def _maybe_add(obj: object) -> None:
         md = getattr(obj, "metadata", None) or obj
-        if hasattr(md, "tables") and getattr(md, "tables"):
+        # Strict check: must be actual MetaData instance
+        if isinstance(md, MetaData) and md.tables:
             found.append(md)
 
     def _scan_module_objects(mod: object) -> None:
         try:
             for val in vars(mod).values():
                 md = getattr(val, "metadata", None) or None
-                if md is not None and hasattr(md, "tables") and getattr(md, "tables"):
+                # Only add if it's a SQLAlchemy MetaData object (has tables dict, not a callable/generator)
+                if md is not None and hasattr(md, "tables") and isinstance(getattr(md, "tables", None), dict):
                     found.append(md)
         except Exception:
             pass
@@ -246,6 +246,21 @@ def _collect_metadata() -> list[object]:
     except Exception:
         _note("ModelBase import", False, traceback.format_exc())
 
+    # Core security models (AuthSession, RefreshToken, etc.)
+    try:
+        import svc_infra.security.models  # noqa: F401
+        _note("svc_infra.security.models", True, None)
+    except Exception:
+        _note("svc_infra.security.models", False, traceback.format_exc())
+
+    # OAuth models (opt-in via environment variable)
+    if os.getenv("ALEMBIC_ENABLE_OAUTH", "").lower() in {"1", "true", "yes"}:
+        try:
+            import svc_infra.security.oauth_models  # noqa: F401
+            _note("svc_infra.security.oauth_models", True, None)
+        except Exception:
+            _note("svc_infra.security.oauth_models", False, traceback.format_exc())
+
     # Optional: autobind API key model
     try:
         from svc_infra.db.sql.apikey import try_autobind_apikey_model

svc_infra/docs/acceptance-matrix.md

@@ -69,3 +69,20 @@ This document maps Acceptance scenarios (A-IDs) to endpoints, CLIs, fixtures, an
 - A10-01 DB migrate/rollback/seed
 - A10-02 Jobs runner consumes a sample job
 - A10-03 SDK smoke-import and /ping
+
+## A22. Storage System
+- A22-01 Local backend file upload and retrieval
+  - Endpoints: POST /_storage/upload, GET /_storage/download/{filename}
+  - Assertions: Upload returns URL, download returns matching content
+- A22-02 S3 backend operations (memory backend in acceptance)
+  - Endpoints: POST /_storage/upload, GET /_storage/download/{filename}
+  - Assertions: Upload succeeds, download returns correct content
+- A22-03 Storage backend auto-detection
+  - Endpoints: GET /_storage/backend-info, POST /_storage/upload
+  - Assertions: Backend detected (MemoryBackend), app.state.storage configured
+- A22-04 File deletion and cleanup
+  - Endpoints: POST /_storage/upload, DELETE /_storage/files/{filename}, GET /_storage/download/{filename}
+  - Assertions: Upload succeeds, delete returns 204, subsequent GET returns 404
+- A22-05 Metadata and listing
+  - Endpoints: POST /_storage/upload, GET /_storage/list, GET /_storage/metadata/{filename}
+  - Assertions: Metadata stored and retrievable, list returns correct keys, prefix filtering works

svc_infra/docs/adr/0012-generic-file-storage.md

@@ -0,0 +1,498 @@
+# ADR 0012: Generic File Storage System
+
+## Status
+
+Proposed — Design phase (2025-11-17)
+
+## Context
+
+svc-infra currently lacks a generic file storage abstraction. Applications built on svc-infra need to store user-uploaded files (profile pictures, documents, media, attachments) with backend flexibility. Current state:
+
+- **svc-infra**: No file storage system exists (only commented references to "use svc-infra storage")
+- **fin-infra**: Documents module uses in-memory placeholder dictionaries (`_documents`, `_file_storage`)
+- **Applications**: No standard way to store files, forcing each app to implement custom solutions
+
+### Requirements
+
+1. **Backend Agnostic**: Work with ANY storage provider without code changes
+   - Local filesystem (Railway persistent volumes, Render, development)
+   - S3-compatible (AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio)
+   - Google Cloud Storage
+   - Cloudinary (image optimization)
+   - In-memory (testing)
+
+2. **Security First**:
+   - Signed URLs with expiration by default
+   - No exposure of raw file paths
+   - Tenant isolation via key prefixes
+   - Metadata validation
+
+3. **Production Ready**:
+   - Async operations (non-blocking)
+   - Connection pooling
+   - Retry logic for transient failures
+   - Health checks
+   - Observability (metrics, logging)
+
+4. **Developer Experience**:
+   - One-line integration via `add_storage(app)`
+   - Auto-detection from environment variables
+   - Type-safe with Pydantic settings
+   - Clear error messages
+
+5. **Separation of Concerns**:
+   - **svc-infra/storage/**: Generic file storage infrastructure (reusable)
+   - **Domain packages** (fin-infra, etc.): Domain-specific features built ON TOP
+   - Example: fin-infra documents keep OCR/AI analysis, delegate storage to svc-infra
+
+## Decisions
+
+### 1. Abstract Storage Backend Interface
+
+Define a protocol-based interface that all backends must implement:
+
+```python
+from typing import Protocol, Optional
+
+class StorageBackend(Protocol):
+    """Abstract storage backend interface."""
+
+    async def put(
+        self,
+        key: str,
+        data: bytes,
+        content_type: str,
+        metadata: Optional[dict] = None,
+    ) -> str:
+        """Store file and return public/signed URL."""
+        ...
+
+    async def get(self, key: str) -> bytes:
+        """Retrieve file content."""
+        ...
+
+    async def delete(self, key: str) -> bool:
+        """Remove file. Returns True if deleted, False if not found."""
+        ...
+
+    async def exists(self, key: str) -> bool:
+        """Check if file exists."""
+        ...
+
+    async def get_url(
+        self,
+        key: str,
+        expires_in: int = 3600,
+        download: bool = False,
+    ) -> str:
+        """Generate signed/public URL."""
+        ...
+
+    async def list_keys(
+        self,
+        prefix: str = "",
+        limit: int = 100,
+    ) -> list[str]:
+        """List stored file keys with optional prefix filter."""
+        ...
+
+    async def get_metadata(self, key: str) -> dict:
+        """Get file metadata (size, content_type, custom metadata)."""
+        ...
+```
+
+**Rationale**: Protocol-based design (PEP 544) allows structural subtyping without inheritance, making it easy to add new backends without modifying existing code.
+
+### 2. Exception Hierarchy
+
+```python
+class StorageError(Exception):
+    """Base exception for storage operations."""
+    pass
+
+class FileNotFoundError(StorageError):
+    """Raised when file does not exist."""
+    pass
+
+class PermissionDeniedError(StorageError):
+    """Raised when lacking permissions for operation."""
+    pass
+
+class QuotaExceededError(StorageError):
+    """Raised when storage quota is exceeded."""
+    pass
+
+class InvalidKeyError(StorageError):
+    """Raised when key format is invalid."""
+    pass
+```
+
+**Rationale**: Specific exceptions enable fine-grained error handling and better user feedback.
+
+### 3. Backend Implementations
+
+#### LocalBackend (Filesystem)
+
+Use cases: Railway persistent volumes, Render disks, local development
+
+```python
+class LocalBackend:
+    def __init__(
+        self,
+        base_path: str = "/data/uploads",
+        base_url: str = "http://localhost:8000/files",
+        signing_secret: Optional[str] = None,
+    ):
+        self.base_path = Path(base_path)
+        self.base_url = base_url
+        self.signing_secret = signing_secret or secrets.token_urlsafe(32)
+```
+
+Features:
+- Async file I/O using `aiofiles`
+- HMAC-based URL signing with expiration
+- Metadata stored as JSON sidecar files (`{key}.meta.json`)
+- Atomic writes using temp files
+- Automatic directory creation
+
+**Railway Integration**: Detects `RAILWAY_VOLUME_MOUNT_PATH` and uses it as base_path
+
+#### S3Backend (S3-Compatible)
+
+Use cases: AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio
+
+```python
+class S3Backend:
+    def __init__(
+        self,
+        bucket: str,
+        region: str = "us-east-1",
+        endpoint: Optional[str] = None,
+        access_key: Optional[str] = None,
+        secret_key: Optional[str] = None,
+    ):
+        self.bucket = bucket
+        self.region = region
+        self.endpoint = endpoint  # For DigitalOcean, Wasabi, etc.
+```
+
+Features:
+- Uses `aioboto3` for async operations
+- Custom endpoint support for S3-compatible services
+- Presigned URLs with configurable expiration
+- Multipart upload for large files (>5MB)
+- Connection pooling
+- Metadata stored in S3 object metadata
+- Retry logic with exponential backoff
+
+**DigitalOcean Spaces Example**:
+```python
+S3Backend(
+    bucket="my-uploads",
+    region="nyc3",
+    endpoint="https://nyc3.digitaloceanspaces.com",
+)
+```
+
+#### MemoryBackend (In-Memory)
+
+Use cases: Unit tests, development, temporary storage
+
+```python
+class MemoryBackend:
+    def __init__(self, max_size: int = 100_000_000):  # 100MB default
+        self._storage: dict[str, bytes] = {}
+        self._metadata: dict[str, dict] = {}
+        self.max_size = max_size
+```
+
+Features:
+- Dictionary-based storage
+- Thread-safe operations using `asyncio.Lock`
+- TTL support for automatic expiration
+- Size limits
+- No persistence across restarts
+
+#### GCSBackend (Google Cloud Storage) - Optional for v1
+
+Use cases: Google Cloud Platform deployments
+
+```python
+class GCSBackend:
+    def __init__(
+        self,
+        bucket: str,
+        project: Optional[str] = None,
+        credentials_path: Optional[str] = None,
+    ):
+        self.bucket = bucket
+        self.project = project
+```
+
+Features:
+- Uses `google-cloud-storage` SDK
+- Async wrapper around sync SDK
+- Signed URLs with service account
+- Metadata in GCS object metadata
+
+**Defer to fast-follow** if time constrained for v1.
+
+#### CloudinaryBackend (Image Optimization) - Optional for v1
+
+Use cases: Image-heavy applications, automatic optimization
+
+```python
+class CloudinaryBackend:
+    def __init__(
+        self,
+        cloud_name: str,
+        api_key: str,
+        api_secret: str,
+    ):
+        self.cloud_name = cloud_name
+```
+
+Features:
+- Uses `cloudinary` SDK
+- Automatic image optimization
+- URL transformations (resize, crop, format)
+- CDN delivery
+
+**Defer to fast-follow** if time constrained for v1.
+
+### 4. Configuration and Auto-Detection
+
+Use Pydantic settings with environment variables:
+
+```python
+class StorageSettings(BaseSettings):
+    # Backend selection
+    storage_backend: Optional[str] = None  # "local", "s3", "gcs", "cloudinary", "memory"
+
+    # Local backend
+    storage_base_path: str = "/data/uploads"
+    storage_base_url: str = "http://localhost:8000/files"
+    storage_signing_secret: Optional[str] = None
+
+    # S3 backend
+    storage_s3_bucket: Optional[str] = None
+    storage_s3_region: str = "us-east-1"
+    storage_s3_endpoint: Optional[str] = None
+    storage_s3_access_key: Optional[str] = None
+    storage_s3_secret_key: Optional[str] = None
+
+    # GCS backend
+    storage_gcs_bucket: Optional[str] = None
+    storage_gcs_project: Optional[str] = None
+    storage_gcs_credentials_path: Optional[str] = None
+
+    # Cloudinary backend
+    storage_cloudinary_cloud_name: Optional[str] = None
+    storage_cloudinary_api_key: Optional[str] = None
+    storage_cloudinary_api_secret: Optional[str] = None
+
+    class Config:
+        env_file = ".env"
+```
+
+**Auto-Detection Logic** (when `storage_backend` not explicitly set):
+
+1. Check for Railway: `RAILWAY_VOLUME_MOUNT_PATH` exists → LocalBackend
+2. Check for S3: `AWS_ACCESS_KEY_ID` or `storage_s3_bucket` set → S3Backend
+3. Check for GCS: `GOOGLE_APPLICATION_CREDENTIALS` or `storage_gcs_bucket` set → GCSBackend
+4. Check for Cloudinary: `CLOUDINARY_URL` set → CloudinaryBackend
+5. Default: MemoryBackend (with warning log)
+
+**Rationale**: Zero-config for common platforms (Railway, AWS) while allowing explicit override.
+
+### 5. FastAPI Integration Pattern
+
+```python
+from svc_infra.storage import add_storage, easy_storage
+
+# Option 1: Auto-detect backend
+storage = easy_storage()
+add_storage(app, storage)
+
+# Option 2: Explicit backend
+storage = easy_storage(backend="s3", bucket="my-uploads")
+add_storage(app, storage)
+
+# Option 3: Custom backend instance
+from svc_infra.storage.backends import S3Backend
+storage = S3Backend(bucket="my-uploads", region="us-west-2")
+add_storage(app, storage)
+```
+
+The `add_storage` helper:
+- Stores backend in `app.state.storage`
+- Registers startup hook for connection testing
+- Registers shutdown hook for cleanup
+- Adds health check endpoint (`/_health/storage`)
+- Optionally mounts file serving route (`/files/{path:path}`)
+
+Dependency injection in routes:
+
+```python
+from svc_infra.storage import get_storage
+from fastapi import Depends, UploadFile
+
+@router.post("/avatar")
+async def upload_avatar(
+    file: UploadFile,
+    storage: StorageBackend = Depends(get_storage),
+):
+    content = await file.read()
+    url = await storage.put(
+        key=f"avatars/{user_id}/{file.filename}",
+        data=content,
+        content_type=file.content_type,
+        metadata={"user_id": user_id},
+    )
+    return {"url": url}
+```
+
+### 6. Key Naming Conventions
+
+Support tenant-scoped and resource-scoped keys:
+
+```
+{tenant_id}/{resource_type}/{resource_id}/{filename}
+```
+
+Examples:
+- `tenant_123/avatars/user_456/profile.jpg`
+- `tenant_123/documents/doc_789/invoice.pdf`
+- `public/logos/company-logo.png` (no tenant isolation)
+
+**Validation**: Keys must be:
+- Relative paths (no leading `/`)
+- No `..` path traversal
+- Max 1024 characters
+- Safe characters: `a-zA-Z0-9._-/`
+
+### 7. Security Considerations
+
+1. **Signed URLs by Default**: All URLs should have expiration (default 1 hour)
+2. **No Raw Path Exposure**: Never expose filesystem paths or S3 keys directly
+3. **Content-Type Validation**: Validate MIME types before storage
+4. **Size Limits**: Enforce max file size (default 10MB, configurable)
+5. **Virus Scanning**: Hook for integration (ClamAV, VirusTotal) - defer to fast-follow
+6. **Tenant Isolation**: Enforce key prefixes, prevent cross-tenant access
+7. **Rate Limiting**: Apply to upload endpoints
+
+### 8. Observability
+
+Metrics (Prometheus):
+```
+storage_operations_total{backend, operation, status}
+storage_operation_duration_seconds{backend, operation}
+storage_file_size_bytes{backend}
+storage_errors_total{backend, error_type}
+```
+
+Logging:
+- Info: File uploaded/deleted with key and size
+- Error: Operation failures with backend and error details
+- Debug: URL generation, metadata operations
+
+### 9. Migration Path
+
+For existing applications using placeholder storage (like fin-infra documents):
+
+1. **Add svc-infra storage dependency**
+2. **Configure backend via environment variables**
+3. **Replace direct storage calls with svc-infra storage**
+4. **Keep domain-specific logic** (OCR, analysis, retention policies)
+5. **Test with MemoryBackend first**, then switch to production backend
+6. **Migrate existing files** (one-time script to copy from old storage to new)
+
+Example refactor for fin-infra documents:
+
+```python
+# Before (in-memory)
+_file_storage: Dict[str, bytes] = {}
+
+def download_document(doc_id: str) -> bytes:
+    return _file_storage[doc_id]
+
+# After (svc-infra storage)
+from svc_infra.storage import get_storage
+
+async def download_document(doc_id: str, storage: StorageBackend) -> bytes:
+    return await storage.get(f"documents/{doc_id}")
+```
+
+### 10. Testing Strategy
+
+- **Unit Tests**: Mock storage backends, test each implementation separately
+- **Integration Tests**: Real backends (S3, GCS) with test credentials (marked for CI skip)
+- **Acceptance Tests**: End-to-end scenarios (upload → retrieve → delete)
+- **Property Tests**: Key validation, URL signing, metadata preservation
+- **Performance Tests**: Large file uploads, concurrent operations
+
+## Alternatives Considered
+
+### 1. Use Third-Party Library (e.g., fsspec, cloudpathlib)
+
+**Rejected**: These libraries are sync-first and don't integrate well with FastAPI's async patterns. Custom implementation gives us full control over async operations, error handling, and FastAPI integration.
+
+### 2. Build Only S3 Backend
+
+**Rejected**: Many users deploy on Railway or Render with persistent volumes (no S3). Supporting local filesystem from day one makes svc-infra accessible to all deployment scenarios.
+
+### 3. Store Files in Database (PostgreSQL BYTEA)
+
+**Rejected**: Database storage doesn't scale well for large files. BYTEA columns increase backup size, slow down queries, and don't support features like CDN delivery or image transformations.
+
+### 4. Use APF Payments for File Storage
+
+**Rejected**: APF Payments is for payment processing, not file storage. Mixing concerns would violate separation of concerns principle.
+
+## Consequences
+
+### Positive
+
+1. **Reusability**: All applications built on svc-infra get file storage for free
+2. **Flexibility**: Switch backends without code changes (local → S3 → GCS)
+3. **Railway-Friendly**: Works perfectly with Railway persistent volumes
+4. **Testing**: MemoryBackend makes unit tests fast and isolated
+5. **Security**: Signed URLs and validation baked in from day one
+6. **Observability**: Built-in metrics and logging for production monitoring
+
+### Negative
+
+1. **Maintenance Burden**: Need to maintain multiple backend implementations
+2. **Dependency Management**: Each backend adds dependencies (boto3, google-cloud-storage, etc.)
+3. **Testing Complexity**: Need credentials for integration tests
+4. **Learning Curve**: Users need to understand backend configuration
+
+### Neutral
+
+1. **API Surface**: Adds new public APIs to svc-infra
+2. **Documentation**: Requires comprehensive docs for each backend
+3. **Migration**: Existing apps need to migrate from placeholder storage
+
+## Implementation Plan
+
+See `.github/PLAN.md` Section 22 for detailed implementation checklist.
+
+**Priority**: MUST-HAVE for v1 (foundational infrastructure)
+
+**Timeline Estimate**: ~15 days (3 weeks, 1 developer)
+
+**Dependencies**: None (can be built in parallel)
+
+## References
+
+- [AWS S3 API](https://docs.aws.amazon.com/s3/)
+- [DigitalOcean Spaces](https://docs.digitalocean.com/products/spaces/)
+- [Railway Volumes](https://docs.railway.app/reference/volumes)
+- [Google Cloud Storage](https://cloud.google.com/storage/docs)
+- [Cloudinary API](https://cloudinary.com/documentation)
+- [PEP 544: Protocols](https://peps.python.org/pep-0544/)
+- [FastAPI Dependency Injection](https://fastapi.tiangolo.com/tutorial/dependencies/)
+
+## Revision History
+
+- 2025-11-17: Initial draft (Research & Design phase)

svc_infra/docs/api.md

@@ -57,3 +57,130 @@ export ENABLE_OBS=true
 export METRICS_PATH=/metrics
 export CORS_ALLOW_ORIGINS=https://app.example.com,https://admin.example.com
 ```
+
+## Integration Helpers
+
+svc-infra provides one-line `add_*` helpers to integrate common functionality into your FastAPI application. Each helper follows the same pattern: wire dependencies, register lifecycle hooks, and expose via `app.state` for dependency injection.
+
+### Storage (`add_storage`)
+
+Add file storage backend with auto-detection or explicit configuration.
+
+```python
+from fastapi import FastAPI, Depends, UploadFile
+from svc_infra.storage import add_storage, get_storage, StorageBackend
+
+app = FastAPI()
+
+# Auto-detect backend from environment (Railway, S3, etc.)
+storage = add_storage(app)
+
+# Or explicit backend
+from svc_infra.storage import easy_storage
+backend = easy_storage(backend="s3", bucket="my-uploads")
+storage = add_storage(app, backend)
+
+# With file serving for LocalBackend
+backend = easy_storage(backend="local")
+storage = add_storage(app, backend, serve_files=True)
+
+# Use in routes via dependency injection
+@app.post("/upload")
+async def upload_file(
+    file: UploadFile,
+    storage: StorageBackend = Depends(get_storage),
+):
+    content = await file.read()
+    url = await storage.put(
+        key=f"uploads/{file.filename}",
+        data=content,
+        content_type=file.content_type or "application/octet-stream",
+        metadata={"user": "current_user"}
+    )
+    return {"url": url}
+```
+
+**Environment variables**:
+- `STORAGE_BACKEND`: Backend type (`local`, `s3`, `memory`) or auto-detect
+- `STORAGE_S3_BUCKET`, `STORAGE_S3_REGION`: S3 configuration
+- `STORAGE_BASE_PATH`: Local backend directory (default: `/data/uploads`)
+- Auto-detects Railway volumes via `RAILWAY_VOLUME_MOUNT_PATH`
+
+**See**: [Storage Guide](storage.md) for comprehensive documentation.
+
+### Database (`add_sql_db`)
+
+Wire SQLAlchemy connection with health checks and lifecycle management.
+
+```python
+from svc_infra.api.fastapi.db.sql.add import add_sql_db
+
+app = FastAPI()
+add_sql_db(app)  # Reads SQL_URL or DB_* environment variables
+```
+
+**See**: [Database Guide](database.md)
+
+### Auth (`add_auth_users`)
+
+Wire FastAPI Users with sessions, OAuth, MFA, and API keys.
+
+```python
+from svc_infra.api.fastapi.auth.add import add_auth_users
+
+app = FastAPI()
+add_auth_users(app, User, UserCreate, UserRead, UserUpdate)
+```
+
+**See**: [Auth Guide](auth.md)
+
+### Observability (`add_observability`)
+
+Add Prometheus metrics, request tracking, and health endpoints.
+
+```python
+from svc_infra.obs.add import add_observability
+
+app = FastAPI()
+add_observability(app)  # Honors ENABLE_OBS, METRICS_PATH environment variables
+```
+
+**See**: [Observability Guide](observability.md)
+
+### Webhooks (`add_webhooks`)
+
+Wire webhook producer and verification middleware.
+
+```python
+from svc_infra.webhooks.add import add_webhooks
+
+app = FastAPI()
+add_webhooks(app)  # Mounts /_webhooks routes and verification middleware
+```
+
+**See**: [Webhooks Guide](webhooks.md)
+
+### Jobs (`easy_jobs`)
+
+Initialize job queue and scheduler.
+
+```python
+from svc_infra.jobs.easy import easy_jobs
+
+queue, scheduler = easy_jobs()  # Reads JOBS_DRIVER, REDIS_URL
+```
+
+**See**: [Jobs Guide](jobs.md)
+
+### Pattern
+
+All integration helpers follow this pattern:
+
+1. **Accept app instance**: `add_*(app, ...)`
+2. **Auto-configure from environment**: Read from env vars with sensible defaults
+3. **Store in app.state**: Make available via `app.state.storage`, `app.state.db`, etc.
+4. **Provide dependency**: Export `get_*` function for route injection
+5. **Register lifecycle hooks**: Handle startup/shutdown (connection pools, cleanup)
+6. **Add health checks**: Integrate with `/healthz` endpoints
+
+This enables one-line integration with zero configuration in most cases, while supporting explicit overrides when needed.