svc-infra 0.1.600__py3-none-any.whl → 0.1.664__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of svc-infra might be problematic. Click here for more details.
- svc_infra/api/fastapi/admin/__init__.py +3 -0
- svc_infra/api/fastapi/admin/add.py +231 -0
- svc_infra/api/fastapi/apf_payments/setup.py +0 -2
- svc_infra/api/fastapi/auth/add.py +0 -4
- svc_infra/api/fastapi/auth/routers/oauth_router.py +19 -4
- svc_infra/api/fastapi/billing/router.py +64 -0
- svc_infra/api/fastapi/billing/setup.py +19 -0
- svc_infra/api/fastapi/cache/add.py +9 -5
- svc_infra/api/fastapi/db/nosql/mongo/add.py +33 -27
- svc_infra/api/fastapi/db/sql/add.py +40 -18
- svc_infra/api/fastapi/db/sql/crud_router.py +176 -14
- svc_infra/api/fastapi/db/sql/session.py +16 -0
- svc_infra/api/fastapi/dependencies/ratelimit.py +57 -7
- svc_infra/api/fastapi/docs/add.py +160 -0
- svc_infra/api/fastapi/docs/landing.py +1 -1
- svc_infra/api/fastapi/docs/scoped.py +41 -6
- svc_infra/api/fastapi/middleware/errors/handlers.py +45 -7
- svc_infra/api/fastapi/middleware/graceful_shutdown.py +87 -0
- svc_infra/api/fastapi/middleware/ratelimit.py +59 -1
- svc_infra/api/fastapi/middleware/ratelimit_store.py +12 -6
- svc_infra/api/fastapi/middleware/timeout.py +148 -0
- svc_infra/api/fastapi/openapi/mutators.py +114 -0
- svc_infra/api/fastapi/ops/add.py +73 -0
- svc_infra/api/fastapi/pagination.py +3 -1
- svc_infra/api/fastapi/routers/ping.py +1 -0
- svc_infra/api/fastapi/setup.py +21 -13
- svc_infra/api/fastapi/tenancy/add.py +19 -0
- svc_infra/api/fastapi/tenancy/context.py +112 -0
- svc_infra/api/fastapi/versioned.py +101 -0
- svc_infra/app/README.md +5 -5
- svc_infra/billing/__init__.py +23 -0
- svc_infra/billing/async_service.py +147 -0
- svc_infra/billing/jobs.py +230 -0
- svc_infra/billing/models.py +131 -0
- svc_infra/billing/quotas.py +101 -0
- svc_infra/billing/schemas.py +33 -0
- svc_infra/billing/service.py +115 -0
- svc_infra/bundled_docs/README.md +5 -0
- svc_infra/bundled_docs/__init__.py +1 -0
- svc_infra/bundled_docs/getting-started.md +6 -0
- svc_infra/cache/__init__.py +4 -0
- svc_infra/cache/add.py +158 -0
- svc_infra/cache/backend.py +5 -2
- svc_infra/cache/decorators.py +19 -1
- svc_infra/cache/keys.py +24 -4
- svc_infra/cli/__init__.py +28 -8
- svc_infra/cli/cmds/__init__.py +8 -0
- svc_infra/cli/cmds/db/nosql/mongo/mongo_cmds.py +4 -3
- svc_infra/cli/cmds/db/nosql/mongo/mongo_scaffold_cmds.py +4 -4
- svc_infra/cli/cmds/db/sql/alembic_cmds.py +80 -11
- svc_infra/cli/cmds/db/sql/sql_export_cmds.py +80 -0
- svc_infra/cli/cmds/db/sql/sql_scaffold_cmds.py +3 -3
- svc_infra/cli/cmds/docs/docs_cmds.py +140 -0
- svc_infra/cli/cmds/dx/__init__.py +12 -0
- svc_infra/cli/cmds/dx/dx_cmds.py +99 -0
- svc_infra/cli/cmds/help.py +4 -0
- svc_infra/cli/cmds/obs/obs_cmds.py +4 -3
- svc_infra/cli/cmds/sdk/__init__.py +0 -0
- svc_infra/cli/cmds/sdk/sdk_cmds.py +102 -0
- svc_infra/data/add.py +61 -0
- svc_infra/data/backup.py +53 -0
- svc_infra/data/erasure.py +45 -0
- svc_infra/data/fixtures.py +40 -0
- svc_infra/data/retention.py +55 -0
- svc_infra/db/nosql/mongo/README.md +13 -13
- svc_infra/db/sql/repository.py +51 -11
- svc_infra/db/sql/resource.py +5 -0
- svc_infra/db/sql/templates/models_schemas/auth/models.py.tmpl +7 -56
- svc_infra/db/sql/templates/setup/env_async.py.tmpl +34 -12
- svc_infra/db/sql/templates/setup/env_sync.py.tmpl +29 -7
- svc_infra/db/sql/tenant.py +79 -0
- svc_infra/db/sql/utils.py +18 -4
- svc_infra/docs/acceptance-matrix.md +88 -0
- svc_infra/docs/acceptance.md +44 -0
- svc_infra/docs/admin.md +425 -0
- svc_infra/docs/adr/0002-background-jobs-and-scheduling.md +40 -0
- svc_infra/docs/adr/0003-webhooks-framework.md +24 -0
- svc_infra/docs/adr/0004-tenancy-model.md +42 -0
- svc_infra/docs/adr/0005-data-lifecycle.md +86 -0
- svc_infra/docs/adr/0006-ops-slos-and-metrics.md +47 -0
- svc_infra/docs/adr/0007-docs-and-sdks.md +83 -0
- svc_infra/docs/adr/0008-billing-primitives.md +143 -0
- svc_infra/docs/adr/0009-acceptance-harness.md +40 -0
- svc_infra/docs/adr/0010-timeouts-and-resource-limits.md +54 -0
- svc_infra/docs/adr/0011-admin-scope-and-impersonation.md +73 -0
- svc_infra/docs/adr/0012-generic-file-storage.md +498 -0
- svc_infra/docs/api.md +186 -0
- svc_infra/docs/auth.md +11 -0
- svc_infra/docs/billing.md +190 -0
- svc_infra/docs/cache.md +76 -0
- svc_infra/docs/cli.md +74 -0
- svc_infra/docs/contributing.md +34 -0
- svc_infra/docs/data-lifecycle.md +52 -0
- svc_infra/docs/database.md +14 -0
- svc_infra/docs/docs-and-sdks.md +62 -0
- svc_infra/docs/environment.md +114 -0
- svc_infra/docs/getting-started.md +63 -0
- svc_infra/docs/idempotency.md +111 -0
- svc_infra/docs/jobs.md +67 -0
- svc_infra/docs/observability.md +16 -0
- svc_infra/docs/ops.md +37 -0
- svc_infra/docs/rate-limiting.md +125 -0
- svc_infra/docs/repo-review.md +48 -0
- svc_infra/docs/security.md +176 -0
- svc_infra/docs/storage.md +982 -0
- svc_infra/docs/tenancy.md +35 -0
- svc_infra/docs/timeouts-and-resource-limits.md +147 -0
- svc_infra/docs/versioned-integrations.md +146 -0
- svc_infra/docs/webhooks.md +112 -0
- svc_infra/dx/add.py +63 -0
- svc_infra/dx/changelog.py +74 -0
- svc_infra/dx/checks.py +67 -0
- svc_infra/http/__init__.py +13 -0
- svc_infra/http/client.py +72 -0
- svc_infra/jobs/builtins/webhook_delivery.py +14 -2
- svc_infra/jobs/queue.py +9 -1
- svc_infra/jobs/runner.py +75 -0
- svc_infra/jobs/worker.py +17 -1
- svc_infra/mcp/svc_infra_mcp.py +85 -28
- svc_infra/obs/add.py +54 -7
- svc_infra/obs/grafana/dashboards/http-overview.json +45 -0
- svc_infra/security/headers.py +15 -2
- svc_infra/security/hibp.py +6 -2
- svc_infra/security/models.py +27 -7
- svc_infra/security/oauth_models.py +59 -0
- svc_infra/security/permissions.py +1 -0
- svc_infra/storage/__init__.py +93 -0
- svc_infra/storage/add.py +250 -0
- svc_infra/storage/backends/__init__.py +11 -0
- svc_infra/storage/backends/local.py +331 -0
- svc_infra/storage/backends/memory.py +214 -0
- svc_infra/storage/backends/s3.py +329 -0
- svc_infra/storage/base.py +239 -0
- svc_infra/storage/easy.py +182 -0
- svc_infra/storage/settings.py +192 -0
- svc_infra/webhooks/service.py +10 -2
- {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/METADATA +45 -14
- {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/RECORD +140 -52
- {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/WHEEL +0 -0
- {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,982 @@
|
|
|
1
|
+
# Storage System
|
|
2
|
+
|
|
3
|
+
`svc_infra.storage` provides a backend-agnostic file storage abstraction with support for multiple providers (local filesystem, S3-compatible services, Google Cloud Storage, Cloudinary, and in-memory storage). The system enables applications to store and retrieve files without coupling to a specific storage provider, making it easy to switch backends or support multiple environments.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
The storage system provides:
|
|
8
|
+
|
|
9
|
+
- **Backend abstraction**: Write code once, deploy to any storage provider
|
|
10
|
+
- **Multiple backends**: Local filesystem, S3-compatible (AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio), Google Cloud Storage (coming soon), Cloudinary (coming soon), in-memory (testing)
|
|
11
|
+
- **Signed URLs**: Secure, time-limited access to files without exposing raw paths
|
|
12
|
+
- **Metadata support**: Attach custom metadata (user_id, tenant_id, tags) to stored files
|
|
13
|
+
- **Key validation**: Automatic validation of storage keys to prevent path traversal and other attacks
|
|
14
|
+
- **FastAPI integration**: One-line setup with dependency injection
|
|
15
|
+
- **Health checks**: Built-in storage backend health monitoring
|
|
16
|
+
- **Auto-detection**: Automatically detect and configure backend from environment variables
|
|
17
|
+
|
|
18
|
+
## Architecture
|
|
19
|
+
|
|
20
|
+
All storage backends implement the `StorageBackend` protocol with these core operations:
|
|
21
|
+
|
|
22
|
+
- `put(key, data, content_type, metadata)` → Store file and return URL
|
|
23
|
+
- `get(key)` → Retrieve file content
|
|
24
|
+
- `delete(key)` → Remove file
|
|
25
|
+
- `exists(key)` → Check if file exists
|
|
26
|
+
- `get_url(key, expires_in, download)` → Generate signed/public URL
|
|
27
|
+
- `list_keys(prefix, limit)` → List stored files
|
|
28
|
+
- `get_metadata(key)` → Get file metadata
|
|
29
|
+
|
|
30
|
+
This abstraction enables:
|
|
31
|
+
- Switching storage providers without code changes
|
|
32
|
+
- Testing with in-memory backend
|
|
33
|
+
- Multi-region/multi-provider deployments
|
|
34
|
+
- Provider-specific features (S3 presigned URLs, Cloudinary transformations)
|
|
35
|
+
|
|
36
|
+
## Quick Start
|
|
37
|
+
|
|
38
|
+
### Installation
|
|
39
|
+
|
|
40
|
+
Storage dependencies are included in svc-infra. For S3 support, ensure `aioboto3` is installed:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
poetry add svc-infra
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### One-Line Integration
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from fastapi import FastAPI
|
|
50
|
+
from svc_infra.storage import add_storage
|
|
51
|
+
|
|
52
|
+
app = FastAPI()
|
|
53
|
+
|
|
54
|
+
# Auto-detect backend from environment
|
|
55
|
+
storage = add_storage(app)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Using Storage in Routes
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from fastapi import APIRouter, Depends, UploadFile
|
|
62
|
+
from svc_infra.storage import get_storage, StorageBackend
|
|
63
|
+
|
|
64
|
+
router = APIRouter()
|
|
65
|
+
|
|
66
|
+
@router.post("/upload")
|
|
67
|
+
async def upload_file(
|
|
68
|
+
file: UploadFile,
|
|
69
|
+
storage: StorageBackend = Depends(get_storage),
|
|
70
|
+
):
|
|
71
|
+
"""Upload a file and return its URL."""
|
|
72
|
+
content = await file.read()
|
|
73
|
+
|
|
74
|
+
url = await storage.put(
|
|
75
|
+
key=f"uploads/{file.filename}",
|
|
76
|
+
data=content,
|
|
77
|
+
content_type=file.content_type or "application/octet-stream",
|
|
78
|
+
metadata={"uploader": "user_123", "timestamp": "2025-11-18"}
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
return {"url": url, "filename": file.filename}
|
|
82
|
+
|
|
83
|
+
@router.get("/download/{filename}")
|
|
84
|
+
async def download_file(
|
|
85
|
+
filename: str,
|
|
86
|
+
storage: StorageBackend = Depends(get_storage),
|
|
87
|
+
):
|
|
88
|
+
"""Download a file by filename."""
|
|
89
|
+
key = f"uploads/{filename}"
|
|
90
|
+
|
|
91
|
+
try:
|
|
92
|
+
content = await storage.get(key)
|
|
93
|
+
return Response(content=content, media_type="application/octet-stream")
|
|
94
|
+
except FileNotFoundError:
|
|
95
|
+
raise HTTPException(status_code=404, detail="File not found")
|
|
96
|
+
|
|
97
|
+
@router.delete("/files/{filename}")
|
|
98
|
+
async def delete_file(
|
|
99
|
+
filename: str,
|
|
100
|
+
storage: StorageBackend = Depends(get_storage),
|
|
101
|
+
):
|
|
102
|
+
"""Delete a file."""
|
|
103
|
+
key = f"uploads/{filename}"
|
|
104
|
+
await storage.delete(key)
|
|
105
|
+
return {"status": "deleted"}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Configuration
|
|
109
|
+
|
|
110
|
+
### Environment Variables
|
|
111
|
+
|
|
112
|
+
#### Backend Selection
|
|
113
|
+
|
|
114
|
+
- `STORAGE_BACKEND`: Explicit backend type (`local`, `s3`, `gcs`, `cloudinary`, `memory`)
|
|
115
|
+
- If not set, auto-detection is used (see Auto-Detection section)
|
|
116
|
+
|
|
117
|
+
#### Local Backend
|
|
118
|
+
|
|
119
|
+
For Railway persistent volumes, Render disks, or local development:
|
|
120
|
+
|
|
121
|
+
- `STORAGE_BASE_PATH`: Directory for files (default: `/data/uploads`)
|
|
122
|
+
- `STORAGE_BASE_URL`: URL for file serving (default: `http://localhost:8000/files`)
|
|
123
|
+
- `STORAGE_URL_SECRET`: Secret for signing URLs (auto-generated if not set)
|
|
124
|
+
- `STORAGE_URL_EXPIRATION`: Default URL expiration in seconds (default: `3600`)
|
|
125
|
+
|
|
126
|
+
#### S3 Backend
|
|
127
|
+
|
|
128
|
+
For AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio, or any S3-compatible service:
|
|
129
|
+
|
|
130
|
+
- `STORAGE_S3_BUCKET`: Bucket name (required)
|
|
131
|
+
- `STORAGE_S3_REGION`: AWS region (default: `us-east-1`)
|
|
132
|
+
- `STORAGE_S3_ENDPOINT`: Custom endpoint URL for S3-compatible services
|
|
133
|
+
- `STORAGE_S3_ACCESS_KEY`: Access key (falls back to `AWS_ACCESS_KEY_ID`)
|
|
134
|
+
- `STORAGE_S3_SECRET_KEY`: Secret key (falls back to `AWS_SECRET_ACCESS_KEY`)
|
|
135
|
+
|
|
136
|
+
#### GCS Backend (Coming Soon)
|
|
137
|
+
|
|
138
|
+
For Google Cloud Storage:
|
|
139
|
+
|
|
140
|
+
- `STORAGE_GCS_BUCKET`: Bucket name
|
|
141
|
+
- `STORAGE_GCS_PROJECT`: GCP project ID
|
|
142
|
+
- `GOOGLE_APPLICATION_CREDENTIALS`: Path to service account JSON
|
|
143
|
+
|
|
144
|
+
#### Cloudinary Backend (Coming Soon)
|
|
145
|
+
|
|
146
|
+
For image optimization and transformations:
|
|
147
|
+
|
|
148
|
+
- `CLOUDINARY_URL`: Cloudinary connection URL
|
|
149
|
+
- `STORAGE_CLOUDINARY_CLOUD_NAME`: Cloud name
|
|
150
|
+
- `STORAGE_CLOUDINARY_API_KEY`: API key
|
|
151
|
+
- `STORAGE_CLOUDINARY_API_SECRET`: API secret
|
|
152
|
+
|
|
153
|
+
### Auto-Detection
|
|
154
|
+
|
|
155
|
+
When `STORAGE_BACKEND` is not set, the system auto-detects the backend in this order:
|
|
156
|
+
|
|
157
|
+
1. **Railway Volume**: If `RAILWAY_VOLUME_MOUNT_PATH` exists → `LocalBackend`
|
|
158
|
+
2. **S3 Credentials**: If `AWS_ACCESS_KEY_ID` or `STORAGE_S3_BUCKET` exists → `S3Backend`
|
|
159
|
+
3. **GCS Credentials**: If `GOOGLE_APPLICATION_CREDENTIALS` exists → `GCSBackend` (coming soon)
|
|
160
|
+
4. **Cloudinary**: If `CLOUDINARY_URL` exists → `CloudinaryBackend` (coming soon)
|
|
161
|
+
5. **Default**: `MemoryBackend` (with warning about data loss)
|
|
162
|
+
|
|
163
|
+
**Production Recommendation**: Always set `STORAGE_BACKEND` explicitly to avoid unexpected behavior.
|
|
164
|
+
|
|
165
|
+
## Backend Comparison
|
|
166
|
+
|
|
167
|
+
### When to Use Each Backend
|
|
168
|
+
|
|
169
|
+
| Backend | Best For | Pros | Cons |
|
|
170
|
+
|---------|----------|------|------|
|
|
171
|
+
| **LocalBackend** | Railway, Render, small deployments, development | Simple, no external dependencies, fast | Not scalable across multiple servers, requires persistent volumes |
|
|
172
|
+
| **S3Backend** | Production deployments, multi-region, CDN integration | Highly scalable, durable, integrates with CloudFront/CDN | Requires AWS account or S3-compatible service, potential egress costs |
|
|
173
|
+
| **GCSBackend** | GCP-native deployments | GCP integration, global CDN | Requires GCP account |
|
|
174
|
+
| **CloudinaryBackend** | Image-heavy applications | Automatic image optimization, transformations, CDN | Additional service cost, image-focused |
|
|
175
|
+
| **MemoryBackend** | Testing, CI/CD | Fast, no setup | Data lost on restart, limited by RAM |
|
|
176
|
+
|
|
177
|
+
### Provider-Specific Notes
|
|
178
|
+
|
|
179
|
+
#### Railway Persistent Volumes
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
# Railway automatically sets this variable
|
|
183
|
+
RAILWAY_VOLUME_MOUNT_PATH=/data
|
|
184
|
+
|
|
185
|
+
# Storage auto-detects and uses LocalBackend
|
|
186
|
+
STORAGE_BASE_PATH=/data/uploads
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
#### AWS S3
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
STORAGE_BACKEND=s3
|
|
193
|
+
STORAGE_S3_BUCKET=my-app-uploads
|
|
194
|
+
STORAGE_S3_REGION=us-east-1
|
|
195
|
+
AWS_ACCESS_KEY_ID=AKIA...
|
|
196
|
+
AWS_SECRET_ACCESS_KEY=...
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
#### DigitalOcean Spaces
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
STORAGE_BACKEND=s3
|
|
203
|
+
STORAGE_S3_BUCKET=my-app-uploads
|
|
204
|
+
STORAGE_S3_REGION=nyc3
|
|
205
|
+
STORAGE_S3_ENDPOINT=https://nyc3.digitaloceanspaces.com
|
|
206
|
+
STORAGE_S3_ACCESS_KEY=...
|
|
207
|
+
STORAGE_S3_SECRET_KEY=...
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
#### Wasabi
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
STORAGE_BACKEND=s3
|
|
214
|
+
STORAGE_S3_BUCKET=my-app-uploads
|
|
215
|
+
STORAGE_S3_REGION=us-east-1
|
|
216
|
+
STORAGE_S3_ENDPOINT=https://s3.wasabisys.com
|
|
217
|
+
STORAGE_S3_ACCESS_KEY=...
|
|
218
|
+
STORAGE_S3_SECRET_KEY=...
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
#### Backblaze B2
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
STORAGE_BACKEND=s3
|
|
225
|
+
STORAGE_S3_BUCKET=my-app-uploads
|
|
226
|
+
STORAGE_S3_REGION=us-west-000
|
|
227
|
+
STORAGE_S3_ENDPOINT=https://s3.us-west-000.backblazeb2.com
|
|
228
|
+
STORAGE_S3_ACCESS_KEY=...
|
|
229
|
+
STORAGE_S3_SECRET_KEY=...
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
#### Minio (Self-Hosted)
|
|
233
|
+
|
|
234
|
+
```bash
|
|
235
|
+
STORAGE_BACKEND=s3
|
|
236
|
+
STORAGE_S3_BUCKET=my-app-uploads
|
|
237
|
+
STORAGE_S3_REGION=us-east-1
|
|
238
|
+
STORAGE_S3_ENDPOINT=https://minio.example.com
|
|
239
|
+
STORAGE_S3_ACCESS_KEY=minioadmin
|
|
240
|
+
STORAGE_S3_SECRET_KEY=minioadmin
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
## Examples
|
|
244
|
+
|
|
245
|
+
### Profile Picture Upload
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
|
249
|
+
from svc_infra.storage import get_storage, StorageBackend
|
|
250
|
+
from PIL import Image
|
|
251
|
+
import io
|
|
252
|
+
|
|
253
|
+
router = APIRouter()
|
|
254
|
+
|
|
255
|
+
MAX_SIZE = 2 * 1024 * 1024 # 2MB
|
|
256
|
+
ALLOWED_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
|
|
257
|
+
|
|
258
|
+
@router.post("/users/me/avatar")
|
|
259
|
+
async def upload_avatar(
|
|
260
|
+
file: UploadFile = File(...),
|
|
261
|
+
storage: StorageBackend = Depends(get_storage),
|
|
262
|
+
current_user=Depends(get_current_user), # Your auth dependency
|
|
263
|
+
):
|
|
264
|
+
"""Upload user profile picture."""
|
|
265
|
+
# Validate content type
|
|
266
|
+
if file.content_type not in ALLOWED_TYPES:
|
|
267
|
+
raise HTTPException(
|
|
268
|
+
status_code=415,
|
|
269
|
+
detail=f"Unsupported file type. Allowed: {ALLOWED_TYPES}"
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
# Read and validate size
|
|
273
|
+
content = await file.read()
|
|
274
|
+
if len(content) > MAX_SIZE:
|
|
275
|
+
raise HTTPException(
|
|
276
|
+
status_code=413,
|
|
277
|
+
detail=f"File too large. Max size: {MAX_SIZE} bytes"
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
# Validate image and resize
|
|
281
|
+
try:
|
|
282
|
+
image = Image.open(io.BytesIO(content))
|
|
283
|
+
image.verify() # Verify it's a valid image
|
|
284
|
+
|
|
285
|
+
# Reopen and resize
|
|
286
|
+
image = Image.open(io.BytesIO(content))
|
|
287
|
+
image.thumbnail((200, 200))
|
|
288
|
+
|
|
289
|
+
# Save to bytes
|
|
290
|
+
output = io.BytesIO()
|
|
291
|
+
image.save(output, format=image.format)
|
|
292
|
+
resized_content = output.getvalue()
|
|
293
|
+
except Exception as e:
|
|
294
|
+
raise HTTPException(status_code=400, detail="Invalid image file")
|
|
295
|
+
|
|
296
|
+
# Store with user-specific key
|
|
297
|
+
key = f"avatars/{current_user.id}/profile.{file.filename.split('.')[-1]}"
|
|
298
|
+
|
|
299
|
+
url = await storage.put(
|
|
300
|
+
key=key,
|
|
301
|
+
data=resized_content,
|
|
302
|
+
content_type=file.content_type,
|
|
303
|
+
metadata={
|
|
304
|
+
"user_id": str(current_user.id),
|
|
305
|
+
"original_filename": file.filename,
|
|
306
|
+
"uploaded_at": datetime.utcnow().isoformat(),
|
|
307
|
+
}
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
# Update user record with new avatar URL
|
|
311
|
+
# await update_user_avatar(current_user.id, url)
|
|
312
|
+
|
|
313
|
+
return {"avatar_url": url}
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
### Document Storage with Metadata
|
|
317
|
+
|
|
318
|
+
```python
|
|
319
|
+
from fastapi import APIRouter, Depends, UploadFile, Query
|
|
320
|
+
from svc_infra.storage import get_storage, StorageBackend
|
|
321
|
+
from typing import List, Optional
|
|
322
|
+
|
|
323
|
+
router = APIRouter()
|
|
324
|
+
|
|
325
|
+
@router.post("/documents/upload")
|
|
326
|
+
async def upload_document(
|
|
327
|
+
file: UploadFile,
|
|
328
|
+
tags: List[str] = Query(default=[]),
|
|
329
|
+
category: str = Query(default="general"),
|
|
330
|
+
storage: StorageBackend = Depends(get_storage),
|
|
331
|
+
current_user=Depends(get_current_user),
|
|
332
|
+
):
|
|
333
|
+
"""Upload a document with metadata."""
|
|
334
|
+
content = await file.read()
|
|
335
|
+
|
|
336
|
+
# Generate unique key
|
|
337
|
+
file_id = uuid4()
|
|
338
|
+
key = f"documents/{current_user.id}/{category}/{file_id}/{file.filename}"
|
|
339
|
+
|
|
340
|
+
url = await storage.put(
|
|
341
|
+
key=key,
|
|
342
|
+
data=content,
|
|
343
|
+
content_type=file.content_type or "application/octet-stream",
|
|
344
|
+
metadata={
|
|
345
|
+
"user_id": str(current_user.id),
|
|
346
|
+
"document_id": str(file_id),
|
|
347
|
+
"category": category,
|
|
348
|
+
"tags": ",".join(tags),
|
|
349
|
+
"original_filename": file.filename,
|
|
350
|
+
"size": len(content),
|
|
351
|
+
"uploaded_at": datetime.utcnow().isoformat(),
|
|
352
|
+
}
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
# Store document record in database
|
|
356
|
+
# document = await create_document_record(...)
|
|
357
|
+
|
|
358
|
+
return {
|
|
359
|
+
"document_id": str(file_id),
|
|
360
|
+
"url": url,
|
|
361
|
+
"filename": file.filename,
|
|
362
|
+
"size": len(content),
|
|
363
|
+
"category": category,
|
|
364
|
+
"tags": tags,
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
@router.get("/documents")
|
|
368
|
+
async def list_documents(
|
|
369
|
+
category: Optional[str] = None,
|
|
370
|
+
storage: StorageBackend = Depends(get_storage),
|
|
371
|
+
current_user=Depends(get_current_user),
|
|
372
|
+
):
|
|
373
|
+
"""List user's documents."""
|
|
374
|
+
prefix = f"documents/{current_user.id}/"
|
|
375
|
+
if category:
|
|
376
|
+
prefix += f"{category}/"
|
|
377
|
+
|
|
378
|
+
keys = await storage.list_keys(prefix=prefix, limit=100)
|
|
379
|
+
|
|
380
|
+
# Get metadata for each file
|
|
381
|
+
documents = []
|
|
382
|
+
for key in keys:
|
|
383
|
+
metadata = await storage.get_metadata(key)
|
|
384
|
+
documents.append({
|
|
385
|
+
"key": key,
|
|
386
|
+
"filename": metadata.get("original_filename"),
|
|
387
|
+
"size": metadata.get("size"),
|
|
388
|
+
"category": metadata.get("category"),
|
|
389
|
+
"uploaded_at": metadata.get("uploaded_at"),
|
|
390
|
+
})
|
|
391
|
+
|
|
392
|
+
return {"documents": documents}
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
### Tenant-Scoped File Storage
|
|
396
|
+
|
|
397
|
+
```python
|
|
398
|
+
from fastapi import APIRouter, Depends, UploadFile
|
|
399
|
+
from svc_infra.storage import get_storage, StorageBackend
|
|
400
|
+
from svc_infra.tenancy import require_tenant_id
|
|
401
|
+
|
|
402
|
+
router = APIRouter()
|
|
403
|
+
|
|
404
|
+
@router.post("/tenant-files/upload")
|
|
405
|
+
async def upload_tenant_file(
|
|
406
|
+
file: UploadFile,
|
|
407
|
+
storage: StorageBackend = Depends(get_storage),
|
|
408
|
+
tenant_id: str = Depends(require_tenant_id),
|
|
409
|
+
current_user=Depends(get_current_user),
|
|
410
|
+
):
|
|
411
|
+
"""Upload a file scoped to current tenant."""
|
|
412
|
+
content = await file.read()
|
|
413
|
+
|
|
414
|
+
# Tenant-scoped key ensures isolation
|
|
415
|
+
key = f"tenants/{tenant_id}/files/{uuid4()}/{file.filename}"
|
|
416
|
+
|
|
417
|
+
url = await storage.put(
|
|
418
|
+
key=key,
|
|
419
|
+
data=content,
|
|
420
|
+
content_type=file.content_type or "application/octet-stream",
|
|
421
|
+
metadata={
|
|
422
|
+
"tenant_id": tenant_id,
|
|
423
|
+
"user_id": str(current_user.id),
|
|
424
|
+
"filename": file.filename,
|
|
425
|
+
"uploaded_at": datetime.utcnow().isoformat(),
|
|
426
|
+
}
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
return {"url": url, "filename": file.filename}
|
|
430
|
+
|
|
431
|
+
@router.get("/tenant-files")
|
|
432
|
+
async def list_tenant_files(
|
|
433
|
+
storage: StorageBackend = Depends(get_storage),
|
|
434
|
+
tenant_id: str = Depends(require_tenant_id),
|
|
435
|
+
):
|
|
436
|
+
"""List files for current tenant only."""
|
|
437
|
+
# Prefix ensures tenant isolation
|
|
438
|
+
prefix = f"tenants/{tenant_id}/files/"
|
|
439
|
+
|
|
440
|
+
keys = await storage.list_keys(prefix=prefix, limit=100)
|
|
441
|
+
|
|
442
|
+
return {"files": keys, "count": len(keys)}
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
### Signed URL Generation
|
|
446
|
+
|
|
447
|
+
```python
|
|
448
|
+
from fastapi import APIRouter, Depends, HTTPException
|
|
449
|
+
from svc_infra.storage import get_storage, StorageBackend
|
|
450
|
+
|
|
451
|
+
router = APIRouter()
|
|
452
|
+
|
|
453
|
+
@router.get("/files/{file_id}/download-url")
|
|
454
|
+
async def get_download_url(
|
|
455
|
+
file_id: str,
|
|
456
|
+
expires_in: int = Query(default=3600, ge=60, le=86400), # 1 min to 24 hours
|
|
457
|
+
download: bool = Query(default=True),
|
|
458
|
+
storage: StorageBackend = Depends(get_storage),
|
|
459
|
+
current_user=Depends(get_current_user),
|
|
460
|
+
):
|
|
461
|
+
"""Generate a signed URL for file download."""
|
|
462
|
+
# Verify user owns the file
|
|
463
|
+
# file = await get_file_record(file_id)
|
|
464
|
+
# if file.user_id != current_user.id:
|
|
465
|
+
# raise HTTPException(status_code=403, detail="Access denied")
|
|
466
|
+
|
|
467
|
+
key = f"uploads/{file_id}/document.pdf"
|
|
468
|
+
|
|
469
|
+
# Check file exists
|
|
470
|
+
if not await storage.exists(key):
|
|
471
|
+
raise HTTPException(status_code=404, detail="File not found")
|
|
472
|
+
|
|
473
|
+
# Generate signed URL
|
|
474
|
+
url = await storage.get_url(
|
|
475
|
+
key=key,
|
|
476
|
+
expires_in=expires_in,
|
|
477
|
+
download=download # If True, browser downloads instead of displaying
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
return {
|
|
481
|
+
"url": url,
|
|
482
|
+
"expires_in": expires_in,
|
|
483
|
+
"expires_at": (datetime.utcnow() + timedelta(seconds=expires_in)).isoformat()
|
|
484
|
+
}
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
### Large File Uploads with Progress
|
|
488
|
+
|
|
489
|
+
```python
|
|
490
|
+
from fastapi import APIRouter, Depends, UploadFile, BackgroundTasks
|
|
491
|
+
from svc_infra.storage import get_storage, StorageBackend
|
|
492
|
+
|
|
493
|
+
router = APIRouter()
|
|
494
|
+
|
|
495
|
+
@router.post("/large-files/upload")
|
|
496
|
+
async def upload_large_file(
|
|
497
|
+
file: UploadFile,
|
|
498
|
+
background_tasks: BackgroundTasks,
|
|
499
|
+
storage: StorageBackend = Depends(get_storage),
|
|
500
|
+
current_user=Depends(get_current_user),
|
|
501
|
+
):
|
|
502
|
+
"""Upload large file with background processing."""
|
|
503
|
+
# For very large files, consider chunked uploads
|
|
504
|
+
# This is a simple example that reads entire file
|
|
505
|
+
|
|
506
|
+
file_id = uuid4()
|
|
507
|
+
key = f"large-files/{current_user.id}/{file_id}/{file.filename}"
|
|
508
|
+
|
|
509
|
+
# Read file in chunks
|
|
510
|
+
chunks = []
|
|
511
|
+
while chunk := await file.read(1024 * 1024): # 1MB chunks
|
|
512
|
+
chunks.append(chunk)
|
|
513
|
+
|
|
514
|
+
content = b"".join(chunks)
|
|
515
|
+
|
|
516
|
+
# Store file
|
|
517
|
+
url = await storage.put(
|
|
518
|
+
key=key,
|
|
519
|
+
data=content,
|
|
520
|
+
content_type=file.content_type or "application/octet-stream",
|
|
521
|
+
metadata={
|
|
522
|
+
"user_id": str(current_user.id),
|
|
523
|
+
"file_id": str(file_id),
|
|
524
|
+
"size": len(content),
|
|
525
|
+
"uploaded_at": datetime.utcnow().isoformat(),
|
|
526
|
+
}
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
# Background task for post-processing (virus scan, thumbnail generation, etc.)
|
|
530
|
+
# background_tasks.add_task(process_file, file_id, key)
|
|
531
|
+
|
|
532
|
+
return {
|
|
533
|
+
"file_id": str(file_id),
|
|
534
|
+
"url": url,
|
|
535
|
+
"size": len(content),
|
|
536
|
+
"status": "uploaded"
|
|
537
|
+
}
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
## Production Recommendations
|
|
541
|
+
|
|
542
|
+
### Railway Deployments
|
|
543
|
+
|
|
544
|
+
Railway persistent volumes are ideal for simple deployments:
|
|
545
|
+
|
|
546
|
+
```bash
|
|
547
|
+
# Railway automatically mounts volume
|
|
548
|
+
RAILWAY_VOLUME_MOUNT_PATH=/data
|
|
549
|
+
|
|
550
|
+
# Storage auto-detects LocalBackend
|
|
551
|
+
# No additional configuration needed
|
|
552
|
+
```
|
|
553
|
+
|
|
554
|
+
**Pros**:
|
|
555
|
+
- Simple setup, no external services
|
|
556
|
+
- Cost-effective for small/medium apps
|
|
557
|
+
- Fast local access
|
|
558
|
+
|
|
559
|
+
**Cons**:
|
|
560
|
+
- Single server only (not suitable for horizontal scaling)
|
|
561
|
+
- Manual backups required
|
|
562
|
+
- Volume size limits
|
|
563
|
+
|
|
564
|
+
### AWS Deployments
|
|
565
|
+
|
|
566
|
+
S3 is recommended for production:
|
|
567
|
+
|
|
568
|
+
```bash
|
|
569
|
+
STORAGE_BACKEND=s3
|
|
570
|
+
STORAGE_S3_BUCKET=myapp-uploads-prod
|
|
571
|
+
STORAGE_S3_REGION=us-east-1
|
|
572
|
+
AWS_ACCESS_KEY_ID=AKIA...
|
|
573
|
+
AWS_SECRET_ACCESS_KEY=...
|
|
574
|
+
```
|
|
575
|
+
|
|
576
|
+
**Additional recommendations**:
|
|
577
|
+
- Enable versioning for backup/recovery
|
|
578
|
+
- Configure lifecycle policies to archive old files to Glacier
|
|
579
|
+
- Use CloudFront CDN for global distribution
|
|
580
|
+
- Enable server-side encryption (SSE-S3 or SSE-KMS)
|
|
581
|
+
- Set up bucket policies for least-privilege access
|
|
582
|
+
|
|
583
|
+
### DigitalOcean Deployments
|
|
584
|
+
|
|
585
|
+
DigitalOcean Spaces (S3-compatible) offers simple pricing:
|
|
586
|
+
|
|
587
|
+
```bash
|
|
588
|
+
STORAGE_BACKEND=s3
|
|
589
|
+
STORAGE_S3_BUCKET=myapp-uploads
|
|
590
|
+
STORAGE_S3_REGION=nyc3
|
|
591
|
+
STORAGE_S3_ENDPOINT=https://nyc3.digitaloceanspaces.com
|
|
592
|
+
STORAGE_S3_ACCESS_KEY=...
|
|
593
|
+
STORAGE_S3_SECRET_KEY=...
|
|
594
|
+
```
|
|
595
|
+
|
|
596
|
+
**Pros**:
|
|
597
|
+
- Predictable pricing ($5/250GB)
|
|
598
|
+
- Built-in CDN
|
|
599
|
+
- S3-compatible API
|
|
600
|
+
|
|
601
|
+
### GCP Deployments
|
|
602
|
+
|
|
603
|
+
Google Cloud Storage for GCP-native apps:
|
|
604
|
+
|
|
605
|
+
```bash
|
|
606
|
+
STORAGE_BACKEND=gcs
|
|
607
|
+
STORAGE_GCS_BUCKET=myapp-uploads
|
|
608
|
+
STORAGE_GCS_PROJECT=my-gcp-project
|
|
609
|
+
GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
|
|
610
|
+
```
|
|
611
|
+
|
|
612
|
+
(Coming soon)
|
|
613
|
+
|
|
614
|
+
### Image-Heavy Applications
|
|
615
|
+
|
|
616
|
+
Consider Cloudinary for automatic optimization:
|
|
617
|
+
|
|
618
|
+
```bash
|
|
619
|
+
STORAGE_BACKEND=cloudinary
|
|
620
|
+
CLOUDINARY_URL=cloudinary://...
|
|
621
|
+
```
|
|
622
|
+
|
|
623
|
+
**Features**:
|
|
624
|
+
- Automatic image optimization
|
|
625
|
+
- On-the-fly transformations (resize, crop, format)
|
|
626
|
+
- Global CDN
|
|
627
|
+
- Video support
|
|
628
|
+
|
|
629
|
+
(Coming soon)
|
|
630
|
+
|
|
631
|
+
## Security Considerations
|
|
632
|
+
|
|
633
|
+
### Never Expose Raw File Paths
|
|
634
|
+
|
|
635
|
+
❌ **Bad**:
|
|
636
|
+
```python
|
|
637
|
+
# Don't return raw storage keys or file paths
|
|
638
|
+
return {"path": "/data/uploads/secret-document.pdf"}
|
|
639
|
+
```
|
|
640
|
+
|
|
641
|
+
✅ **Good**:
|
|
642
|
+
```python
|
|
643
|
+
# Return signed URLs with expiration
|
|
644
|
+
url = await storage.get_url(key, expires_in=3600)
|
|
645
|
+
return {"url": url}
|
|
646
|
+
```
|
|
647
|
+
|
|
648
|
+
### Always Use Signed URLs with Expiration
|
|
649
|
+
|
|
650
|
+
```python
|
|
651
|
+
# Short expiration for sensitive documents
|
|
652
|
+
url = await storage.get_url(key, expires_in=300) # 5 minutes
|
|
653
|
+
|
|
654
|
+
# Longer expiration for public assets
|
|
655
|
+
url = await storage.get_url(key, expires_in=86400) # 24 hours
|
|
656
|
+
```
|
|
657
|
+
|
|
658
|
+
### Validate File Types and Sizes Before Upload
|
|
659
|
+
|
|
660
|
+
```python
|
|
661
|
+
MAX_SIZE = 10 * 1024 * 1024 # 10MB
|
|
662
|
+
ALLOWED_TYPES = {"image/jpeg", "image/png", "application/pdf"}
|
|
663
|
+
|
|
664
|
+
if file.content_type not in ALLOWED_TYPES:
|
|
665
|
+
raise HTTPException(status_code=415, detail="Unsupported file type")
|
|
666
|
+
|
|
667
|
+
content = await file.read()
|
|
668
|
+
if len(content) > MAX_SIZE:
|
|
669
|
+
raise HTTPException(status_code=413, detail="File too large")
|
|
670
|
+
```
|
|
671
|
+
|
|
672
|
+
### Scan for Viruses
|
|
673
|
+
|
|
674
|
+
Integration with ClamAV or similar (coming in future version):
|
|
675
|
+
|
|
676
|
+
```python
|
|
677
|
+
# Future API
|
|
678
|
+
from svc_infra.storage.scanners import scan_file
|
|
679
|
+
|
|
680
|
+
result = await scan_file(content)
|
|
681
|
+
if result.is_infected:
|
|
682
|
+
raise HTTPException(status_code=400, detail="File contains malware")
|
|
683
|
+
```
|
|
684
|
+
|
|
685
|
+
### Implement Tenant Isolation via Key Prefixes
|
|
686
|
+
|
|
687
|
+
```python
|
|
688
|
+
# Always scope keys by tenant
|
|
689
|
+
key = f"tenants/{tenant_id}/documents/{file_id}"
|
|
690
|
+
|
|
691
|
+
# Verify access before operations
|
|
692
|
+
if not await verify_tenant_access(current_user, tenant_id):
|
|
693
|
+
raise HTTPException(status_code=403, detail="Access denied")
|
|
694
|
+
```
|
|
695
|
+
|
|
696
|
+
### Use IAM Policies for Least-Privilege Access
|
|
697
|
+
|
|
698
|
+
For S3/GCS, create service accounts with minimal permissions:
|
|
699
|
+
|
|
700
|
+
```json
|
|
701
|
+
{
|
|
702
|
+
"Version": "2012-10-17",
|
|
703
|
+
"Statement": [
|
|
704
|
+
{
|
|
705
|
+
"Effect": "Allow",
|
|
706
|
+
"Action": [
|
|
707
|
+
"s3:PutObject",
|
|
708
|
+
"s3:GetObject",
|
|
709
|
+
"s3:DeleteObject",
|
|
710
|
+
"s3:ListBucket"
|
|
711
|
+
],
|
|
712
|
+
"Resource": [
|
|
713
|
+
"arn:aws:s3:::myapp-uploads-prod",
|
|
714
|
+
"arn:aws:s3:::myapp-uploads-prod/*"
|
|
715
|
+
]
|
|
716
|
+
}
|
|
717
|
+
]
|
|
718
|
+
}
|
|
719
|
+
```
|
|
720
|
+
|
|
721
|
+
### Enable Encryption at Rest
|
|
722
|
+
|
|
723
|
+
For S3:
|
|
724
|
+
```bash
|
|
725
|
+
# Enable default encryption in bucket settings
|
|
726
|
+
aws s3api put-bucket-encryption \
|
|
727
|
+
--bucket myapp-uploads-prod \
|
|
728
|
+
--server-side-encryption-configuration '{
|
|
729
|
+
"Rules": [{
|
|
730
|
+
"ApplyServerSideEncryptionByDefault": {
|
|
731
|
+
"SSEAlgorithm": "AES256"
|
|
732
|
+
}
|
|
733
|
+
}]
|
|
734
|
+
}'
|
|
735
|
+
```
|
|
736
|
+
|
|
737
|
+
## Troubleshooting
|
|
738
|
+
|
|
739
|
+
### Error: "Storage not configured"
|
|
740
|
+
|
|
741
|
+
**Cause**: `add_storage()` was not called or `get_storage()` dependency used without configuration.
|
|
742
|
+
|
|
743
|
+
**Solution**:
|
|
744
|
+
```python
|
|
745
|
+
from svc_infra.storage import add_storage
|
|
746
|
+
|
|
747
|
+
app = FastAPI()
|
|
748
|
+
storage = add_storage(app) # Add this line
|
|
749
|
+
```
|
|
750
|
+
|
|
751
|
+
### Error: "No module named 'aioboto3'"
|
|
752
|
+
|
|
753
|
+
**Cause**: S3Backend requires `aioboto3` dependency.
|
|
754
|
+
|
|
755
|
+
**Solution**:
|
|
756
|
+
```bash
|
|
757
|
+
poetry add aioboto3
|
|
758
|
+
```
|
|
759
|
+
|
|
760
|
+
### Error: "Access Denied" (S3)
|
|
761
|
+
|
|
762
|
+
**Cause**: Invalid credentials or insufficient IAM permissions.
|
|
763
|
+
|
|
764
|
+
**Solution**:
|
|
765
|
+
- Verify `STORAGE_S3_ACCESS_KEY` and `STORAGE_S3_SECRET_KEY`
|
|
766
|
+
- Check IAM policy allows required S3 actions
|
|
767
|
+
- Verify bucket name and region are correct
|
|
768
|
+
|
|
769
|
+
### Error: "Bucket does not exist"
|
|
770
|
+
|
|
771
|
+
**Cause**: S3 bucket not created or wrong bucket name.
|
|
772
|
+
|
|
773
|
+
**Solution**:
|
|
774
|
+
```bash
|
|
775
|
+
# Create bucket
|
|
776
|
+
aws s3 mb s3://myapp-uploads-prod --region us-east-1
|
|
777
|
+
|
|
778
|
+
# Or via S3 console
|
|
779
|
+
```
|
|
780
|
+
|
|
781
|
+
### Files Not Persisting (LocalBackend)
|
|
782
|
+
|
|
783
|
+
**Cause**: Using in-memory filesystem or container without persistent volume.
|
|
784
|
+
|
|
785
|
+
**Solution**:
|
|
786
|
+
- Railway: Ensure persistent volume is mounted
|
|
787
|
+
- Docker: Mount volume: `docker run -v /data/uploads:/data/uploads ...`
|
|
788
|
+
- Render: Use persistent disks feature
|
|
789
|
+
|
|
790
|
+
### URLs Expire Too Quickly
|
|
791
|
+
|
|
792
|
+
**Cause**: Default expiration is 1 hour.
|
|
793
|
+
|
|
794
|
+
**Solution**:
|
|
795
|
+
```python
|
|
796
|
+
# Increase expiration
|
|
797
|
+
url = await storage.get_url(key, expires_in=86400) # 24 hours
|
|
798
|
+
|
|
799
|
+
# Or set default in environment
|
|
800
|
+
STORAGE_URL_EXPIRATION=86400
|
|
801
|
+
```
|
|
802
|
+
|
|
803
|
+
### Large File Uploads Fail
|
|
804
|
+
|
|
805
|
+
**Cause**: Request timeout or size limits.
|
|
806
|
+
|
|
807
|
+
**Solution**:
|
|
808
|
+
```python
|
|
809
|
+
# Increase timeouts in uvicorn
|
|
810
|
+
uvicorn main:app --timeout-keep-alive 300
|
|
811
|
+
|
|
812
|
+
# Or chunk uploads for very large files (>100MB)
|
|
813
|
+
```
|
|
814
|
+
|
|
815
|
+
## API Reference
|
|
816
|
+
|
|
817
|
+
### Core Functions
|
|
818
|
+
|
|
819
|
+
#### `add_storage(app, backend, serve_files, file_route_prefix)`
|
|
820
|
+
|
|
821
|
+
Integrate storage backend with FastAPI application.
|
|
822
|
+
|
|
823
|
+
**Parameters**:
|
|
824
|
+
- `app: FastAPI` - Application instance
|
|
825
|
+
- `backend: Optional[StorageBackend]` - Storage backend (auto-detected if None)
|
|
826
|
+
- `serve_files: bool` - Mount file serving route (LocalBackend only, default: False)
|
|
827
|
+
- `file_route_prefix: str` - URL prefix for files (default: "/files")
|
|
828
|
+
|
|
829
|
+
**Returns**: `StorageBackend` instance
|
|
830
|
+
|
|
831
|
+
**Example**:
|
|
832
|
+
```python
|
|
833
|
+
storage = add_storage(app)
|
|
834
|
+
```
|
|
835
|
+
|
|
836
|
+
#### `easy_storage(backend, **kwargs)`
|
|
837
|
+
|
|
838
|
+
Create storage backend with auto-detection.
|
|
839
|
+
|
|
840
|
+
**Parameters**:
|
|
841
|
+
- `backend: Optional[str]` - Backend type ("local", "s3", "memory") or None for auto-detect
|
|
842
|
+
- `**kwargs` - Backend-specific configuration
|
|
843
|
+
|
|
844
|
+
**Returns**: `StorageBackend` instance
|
|
845
|
+
|
|
846
|
+
**Example**:
|
|
847
|
+
```python
|
|
848
|
+
storage = easy_storage(backend="s3", bucket="uploads", region="us-east-1")
|
|
849
|
+
```
|
|
850
|
+
|
|
851
|
+
#### `get_storage(request)`
|
|
852
|
+
|
|
853
|
+
FastAPI dependency to inject storage backend.
|
|
854
|
+
|
|
855
|
+
**Parameters**:
|
|
856
|
+
- `request: Request` - FastAPI request
|
|
857
|
+
|
|
858
|
+
**Returns**: `StorageBackend` from app.state.storage
|
|
859
|
+
|
|
860
|
+
**Example**:
|
|
861
|
+
```python
|
|
862
|
+
async def upload(storage: StorageBackend = Depends(get_storage)):
|
|
863
|
+
...
|
|
864
|
+
```
|
|
865
|
+
|
|
866
|
+
### StorageBackend Protocol
|
|
867
|
+
|
|
868
|
+
All backends implement these methods:
|
|
869
|
+
|
|
870
|
+
#### `async put(key, data, content_type, metadata=None)`
|
|
871
|
+
|
|
872
|
+
Store file and return URL.
|
|
873
|
+
|
|
874
|
+
**Parameters**:
|
|
875
|
+
- `key: str` - Storage key (path)
|
|
876
|
+
- `data: bytes` - File content
|
|
877
|
+
- `content_type: str` - MIME type
|
|
878
|
+
- `metadata: Optional[dict]` - Custom metadata
|
|
879
|
+
|
|
880
|
+
**Returns**: `str` - File URL
|
|
881
|
+
|
|
882
|
+
**Raises**: `InvalidKeyError`, `PermissionDeniedError`, `QuotaExceededError`, `StorageError`
|
|
883
|
+
|
|
884
|
+
#### `async get(key)`
|
|
885
|
+
|
|
886
|
+
Retrieve file content.
|
|
887
|
+
|
|
888
|
+
**Parameters**:
|
|
889
|
+
- `key: str` - Storage key
|
|
890
|
+
|
|
891
|
+
**Returns**: `bytes` - File content
|
|
892
|
+
|
|
893
|
+
**Raises**: `FileNotFoundError`, `PermissionDeniedError`, `StorageError`
|
|
894
|
+
|
|
895
|
+
#### `async delete(key)`
|
|
896
|
+
|
|
897
|
+
Remove file.
|
|
898
|
+
|
|
899
|
+
**Parameters**:
|
|
900
|
+
- `key: str` - Storage key
|
|
901
|
+
|
|
902
|
+
**Returns**: `bool` - True if deleted, False if not found
|
|
903
|
+
|
|
904
|
+
**Raises**: `PermissionDeniedError`, `StorageError`
|
|
905
|
+
|
|
906
|
+
#### `async exists(key)`
|
|
907
|
+
|
|
908
|
+
Check if file exists.
|
|
909
|
+
|
|
910
|
+
**Parameters**:
|
|
911
|
+
- `key: str` - Storage key
|
|
912
|
+
|
|
913
|
+
**Returns**: `bool` - True if exists
|
|
914
|
+
|
|
915
|
+
#### `async get_url(key, expires_in=3600, download=False)`
|
|
916
|
+
|
|
917
|
+
Generate signed URL.
|
|
918
|
+
|
|
919
|
+
**Parameters**:
|
|
920
|
+
- `key: str` - Storage key
|
|
921
|
+
- `expires_in: int` - Expiration in seconds (default: 3600)
|
|
922
|
+
- `download: bool` - Force download vs display (default: False)
|
|
923
|
+
|
|
924
|
+
**Returns**: `str` - Signed URL
|
|
925
|
+
|
|
926
|
+
**Raises**: `FileNotFoundError`, `StorageError`
|
|
927
|
+
|
|
928
|
+
#### `async list_keys(prefix="", limit=1000)`
|
|
929
|
+
|
|
930
|
+
List stored files.
|
|
931
|
+
|
|
932
|
+
**Parameters**:
|
|
933
|
+
- `prefix: str` - Key prefix filter (default: "")
|
|
934
|
+
- `limit: int` - Max results (default: 1000)
|
|
935
|
+
|
|
936
|
+
**Returns**: `List[str]` - List of keys
|
|
937
|
+
|
|
938
|
+
#### `async get_metadata(key)`
|
|
939
|
+
|
|
940
|
+
Get file metadata.
|
|
941
|
+
|
|
942
|
+
**Parameters**:
|
|
943
|
+
- `key: str` - Storage key
|
|
944
|
+
|
|
945
|
+
**Returns**: `dict` - Metadata dictionary
|
|
946
|
+
|
|
947
|
+
**Raises**: `FileNotFoundError`, `StorageError`
|
|
948
|
+
|
|
949
|
+
### Exceptions
|
|
950
|
+
|
|
951
|
+
All exceptions inherit from `StorageError`:
|
|
952
|
+
|
|
953
|
+
- `StorageError` - Base exception
|
|
954
|
+
- `FileNotFoundError` - File doesn't exist
|
|
955
|
+
- `PermissionDeniedError` - Access denied
|
|
956
|
+
- `QuotaExceededError` - Storage quota exceeded
|
|
957
|
+
- `InvalidKeyError` - Invalid key format
|
|
958
|
+
|
|
959
|
+
## Health Checks
|
|
960
|
+
|
|
961
|
+
Storage backend health is automatically registered when using `add_storage()`:
|
|
962
|
+
|
|
963
|
+
```python
|
|
964
|
+
# Health check endpoint
|
|
965
|
+
GET /_ops/health
|
|
966
|
+
|
|
967
|
+
# Response
|
|
968
|
+
{
|
|
969
|
+
"status": "healthy",
|
|
970
|
+
"storage": {
|
|
971
|
+
"backend": "S3Backend",
|
|
972
|
+
"status": "connected"
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
```
|
|
976
|
+
|
|
977
|
+
## See Also
|
|
978
|
+
|
|
979
|
+
- [ADR-0012: Generic File Storage System](/src/svc_infra/docs/adr/0012-storage-system.md) - Design decisions
|
|
980
|
+
- [API Integration Guide](/src/svc_infra/docs/api.md) - FastAPI integration patterns
|
|
981
|
+
- [Tenancy Guide](/src/svc_infra/docs/tenancy.md) - Multi-tenant file isolation
|
|
982
|
+
- [Security Guide](/src/svc_infra/docs/security.md) - Security best practices
|