svc-infra 0.1.600__py3-none-any.whl → 0.1.664__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of svc-infra might be problematic. Click here for more details.

Files changed (140) hide show
  1. svc_infra/api/fastapi/admin/__init__.py +3 -0
  2. svc_infra/api/fastapi/admin/add.py +231 -0
  3. svc_infra/api/fastapi/apf_payments/setup.py +0 -2
  4. svc_infra/api/fastapi/auth/add.py +0 -4
  5. svc_infra/api/fastapi/auth/routers/oauth_router.py +19 -4
  6. svc_infra/api/fastapi/billing/router.py +64 -0
  7. svc_infra/api/fastapi/billing/setup.py +19 -0
  8. svc_infra/api/fastapi/cache/add.py +9 -5
  9. svc_infra/api/fastapi/db/nosql/mongo/add.py +33 -27
  10. svc_infra/api/fastapi/db/sql/add.py +40 -18
  11. svc_infra/api/fastapi/db/sql/crud_router.py +176 -14
  12. svc_infra/api/fastapi/db/sql/session.py +16 -0
  13. svc_infra/api/fastapi/dependencies/ratelimit.py +57 -7
  14. svc_infra/api/fastapi/docs/add.py +160 -0
  15. svc_infra/api/fastapi/docs/landing.py +1 -1
  16. svc_infra/api/fastapi/docs/scoped.py +41 -6
  17. svc_infra/api/fastapi/middleware/errors/handlers.py +45 -7
  18. svc_infra/api/fastapi/middleware/graceful_shutdown.py +87 -0
  19. svc_infra/api/fastapi/middleware/ratelimit.py +59 -1
  20. svc_infra/api/fastapi/middleware/ratelimit_store.py +12 -6
  21. svc_infra/api/fastapi/middleware/timeout.py +148 -0
  22. svc_infra/api/fastapi/openapi/mutators.py +114 -0
  23. svc_infra/api/fastapi/ops/add.py +73 -0
  24. svc_infra/api/fastapi/pagination.py +3 -1
  25. svc_infra/api/fastapi/routers/ping.py +1 -0
  26. svc_infra/api/fastapi/setup.py +21 -13
  27. svc_infra/api/fastapi/tenancy/add.py +19 -0
  28. svc_infra/api/fastapi/tenancy/context.py +112 -0
  29. svc_infra/api/fastapi/versioned.py +101 -0
  30. svc_infra/app/README.md +5 -5
  31. svc_infra/billing/__init__.py +23 -0
  32. svc_infra/billing/async_service.py +147 -0
  33. svc_infra/billing/jobs.py +230 -0
  34. svc_infra/billing/models.py +131 -0
  35. svc_infra/billing/quotas.py +101 -0
  36. svc_infra/billing/schemas.py +33 -0
  37. svc_infra/billing/service.py +115 -0
  38. svc_infra/bundled_docs/README.md +5 -0
  39. svc_infra/bundled_docs/__init__.py +1 -0
  40. svc_infra/bundled_docs/getting-started.md +6 -0
  41. svc_infra/cache/__init__.py +4 -0
  42. svc_infra/cache/add.py +158 -0
  43. svc_infra/cache/backend.py +5 -2
  44. svc_infra/cache/decorators.py +19 -1
  45. svc_infra/cache/keys.py +24 -4
  46. svc_infra/cli/__init__.py +28 -8
  47. svc_infra/cli/cmds/__init__.py +8 -0
  48. svc_infra/cli/cmds/db/nosql/mongo/mongo_cmds.py +4 -3
  49. svc_infra/cli/cmds/db/nosql/mongo/mongo_scaffold_cmds.py +4 -4
  50. svc_infra/cli/cmds/db/sql/alembic_cmds.py +80 -11
  51. svc_infra/cli/cmds/db/sql/sql_export_cmds.py +80 -0
  52. svc_infra/cli/cmds/db/sql/sql_scaffold_cmds.py +3 -3
  53. svc_infra/cli/cmds/docs/docs_cmds.py +140 -0
  54. svc_infra/cli/cmds/dx/__init__.py +12 -0
  55. svc_infra/cli/cmds/dx/dx_cmds.py +99 -0
  56. svc_infra/cli/cmds/help.py +4 -0
  57. svc_infra/cli/cmds/obs/obs_cmds.py +4 -3
  58. svc_infra/cli/cmds/sdk/__init__.py +0 -0
  59. svc_infra/cli/cmds/sdk/sdk_cmds.py +102 -0
  60. svc_infra/data/add.py +61 -0
  61. svc_infra/data/backup.py +53 -0
  62. svc_infra/data/erasure.py +45 -0
  63. svc_infra/data/fixtures.py +40 -0
  64. svc_infra/data/retention.py +55 -0
  65. svc_infra/db/nosql/mongo/README.md +13 -13
  66. svc_infra/db/sql/repository.py +51 -11
  67. svc_infra/db/sql/resource.py +5 -0
  68. svc_infra/db/sql/templates/models_schemas/auth/models.py.tmpl +7 -56
  69. svc_infra/db/sql/templates/setup/env_async.py.tmpl +34 -12
  70. svc_infra/db/sql/templates/setup/env_sync.py.tmpl +29 -7
  71. svc_infra/db/sql/tenant.py +79 -0
  72. svc_infra/db/sql/utils.py +18 -4
  73. svc_infra/docs/acceptance-matrix.md +88 -0
  74. svc_infra/docs/acceptance.md +44 -0
  75. svc_infra/docs/admin.md +425 -0
  76. svc_infra/docs/adr/0002-background-jobs-and-scheduling.md +40 -0
  77. svc_infra/docs/adr/0003-webhooks-framework.md +24 -0
  78. svc_infra/docs/adr/0004-tenancy-model.md +42 -0
  79. svc_infra/docs/adr/0005-data-lifecycle.md +86 -0
  80. svc_infra/docs/adr/0006-ops-slos-and-metrics.md +47 -0
  81. svc_infra/docs/adr/0007-docs-and-sdks.md +83 -0
  82. svc_infra/docs/adr/0008-billing-primitives.md +143 -0
  83. svc_infra/docs/adr/0009-acceptance-harness.md +40 -0
  84. svc_infra/docs/adr/0010-timeouts-and-resource-limits.md +54 -0
  85. svc_infra/docs/adr/0011-admin-scope-and-impersonation.md +73 -0
  86. svc_infra/docs/adr/0012-generic-file-storage.md +498 -0
  87. svc_infra/docs/api.md +186 -0
  88. svc_infra/docs/auth.md +11 -0
  89. svc_infra/docs/billing.md +190 -0
  90. svc_infra/docs/cache.md +76 -0
  91. svc_infra/docs/cli.md +74 -0
  92. svc_infra/docs/contributing.md +34 -0
  93. svc_infra/docs/data-lifecycle.md +52 -0
  94. svc_infra/docs/database.md +14 -0
  95. svc_infra/docs/docs-and-sdks.md +62 -0
  96. svc_infra/docs/environment.md +114 -0
  97. svc_infra/docs/getting-started.md +63 -0
  98. svc_infra/docs/idempotency.md +111 -0
  99. svc_infra/docs/jobs.md +67 -0
  100. svc_infra/docs/observability.md +16 -0
  101. svc_infra/docs/ops.md +37 -0
  102. svc_infra/docs/rate-limiting.md +125 -0
  103. svc_infra/docs/repo-review.md +48 -0
  104. svc_infra/docs/security.md +176 -0
  105. svc_infra/docs/storage.md +982 -0
  106. svc_infra/docs/tenancy.md +35 -0
  107. svc_infra/docs/timeouts-and-resource-limits.md +147 -0
  108. svc_infra/docs/versioned-integrations.md +146 -0
  109. svc_infra/docs/webhooks.md +112 -0
  110. svc_infra/dx/add.py +63 -0
  111. svc_infra/dx/changelog.py +74 -0
  112. svc_infra/dx/checks.py +67 -0
  113. svc_infra/http/__init__.py +13 -0
  114. svc_infra/http/client.py +72 -0
  115. svc_infra/jobs/builtins/webhook_delivery.py +14 -2
  116. svc_infra/jobs/queue.py +9 -1
  117. svc_infra/jobs/runner.py +75 -0
  118. svc_infra/jobs/worker.py +17 -1
  119. svc_infra/mcp/svc_infra_mcp.py +85 -28
  120. svc_infra/obs/add.py +54 -7
  121. svc_infra/obs/grafana/dashboards/http-overview.json +45 -0
  122. svc_infra/security/headers.py +15 -2
  123. svc_infra/security/hibp.py +6 -2
  124. svc_infra/security/models.py +27 -7
  125. svc_infra/security/oauth_models.py +59 -0
  126. svc_infra/security/permissions.py +1 -0
  127. svc_infra/storage/__init__.py +93 -0
  128. svc_infra/storage/add.py +250 -0
  129. svc_infra/storage/backends/__init__.py +11 -0
  130. svc_infra/storage/backends/local.py +331 -0
  131. svc_infra/storage/backends/memory.py +214 -0
  132. svc_infra/storage/backends/s3.py +329 -0
  133. svc_infra/storage/base.py +239 -0
  134. svc_infra/storage/easy.py +182 -0
  135. svc_infra/storage/settings.py +192 -0
  136. svc_infra/webhooks/service.py +10 -2
  137. {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/METADATA +45 -14
  138. {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/RECORD +140 -52
  139. {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/WHEEL +0 -0
  140. {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,982 @@
1
+ # Storage System
2
+
3
+ `svc_infra.storage` provides a backend-agnostic file storage abstraction with support for multiple providers (local filesystem, S3-compatible services, Google Cloud Storage, Cloudinary, and in-memory storage). The system enables applications to store and retrieve files without coupling to a specific storage provider, making it easy to switch backends or support multiple environments.
4
+
5
+ ## Overview
6
+
7
+ The storage system provides:
8
+
9
+ - **Backend abstraction**: Write code once, deploy to any storage provider
10
+ - **Multiple backends**: Local filesystem, S3-compatible (AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio), Google Cloud Storage (coming soon), Cloudinary (coming soon), in-memory (testing)
11
+ - **Signed URLs**: Secure, time-limited access to files without exposing raw paths
12
+ - **Metadata support**: Attach custom metadata (user_id, tenant_id, tags) to stored files
13
+ - **Key validation**: Automatic validation of storage keys to prevent path traversal and other attacks
14
+ - **FastAPI integration**: One-line setup with dependency injection
15
+ - **Health checks**: Built-in storage backend health monitoring
16
+ - **Auto-detection**: Automatically detect and configure backend from environment variables
17
+
18
+ ## Architecture
19
+
20
+ All storage backends implement the `StorageBackend` protocol with these core operations:
21
+
22
+ - `put(key, data, content_type, metadata)` → Store file and return URL
23
+ - `get(key)` → Retrieve file content
24
+ - `delete(key)` → Remove file
25
+ - `exists(key)` → Check if file exists
26
+ - `get_url(key, expires_in, download)` → Generate signed/public URL
27
+ - `list_keys(prefix, limit)` → List stored files
28
+ - `get_metadata(key)` → Get file metadata
29
+
30
+ This abstraction enables:
31
+ - Switching storage providers without code changes
32
+ - Testing with in-memory backend
33
+ - Multi-region/multi-provider deployments
34
+ - Provider-specific features (S3 presigned URLs, Cloudinary transformations)
35
+
36
+ ## Quick Start
37
+
38
+ ### Installation
39
+
40
+ Storage dependencies are included in svc-infra. For S3 support, ensure `aioboto3` is installed:
41
+
42
+ ```bash
43
+ poetry add svc-infra
44
+ ```
45
+
46
+ ### One-Line Integration
47
+
48
+ ```python
49
+ from fastapi import FastAPI
50
+ from svc_infra.storage import add_storage
51
+
52
+ app = FastAPI()
53
+
54
+ # Auto-detect backend from environment
55
+ storage = add_storage(app)
56
+ ```
57
+
58
+ ### Using Storage in Routes
59
+
60
+ ```python
61
+ from fastapi import APIRouter, Depends, UploadFile
62
+ from svc_infra.storage import get_storage, StorageBackend
63
+
64
+ router = APIRouter()
65
+
66
+ @router.post("/upload")
67
+ async def upload_file(
68
+ file: UploadFile,
69
+ storage: StorageBackend = Depends(get_storage),
70
+ ):
71
+ """Upload a file and return its URL."""
72
+ content = await file.read()
73
+
74
+ url = await storage.put(
75
+ key=f"uploads/{file.filename}",
76
+ data=content,
77
+ content_type=file.content_type or "application/octet-stream",
78
+ metadata={"uploader": "user_123", "timestamp": "2025-11-18"}
79
+ )
80
+
81
+ return {"url": url, "filename": file.filename}
82
+
83
+ @router.get("/download/{filename}")
84
+ async def download_file(
85
+ filename: str,
86
+ storage: StorageBackend = Depends(get_storage),
87
+ ):
88
+ """Download a file by filename."""
89
+ key = f"uploads/{filename}"
90
+
91
+ try:
92
+ content = await storage.get(key)
93
+ return Response(content=content, media_type="application/octet-stream")
94
+ except FileNotFoundError:
95
+ raise HTTPException(status_code=404, detail="File not found")
96
+
97
+ @router.delete("/files/{filename}")
98
+ async def delete_file(
99
+ filename: str,
100
+ storage: StorageBackend = Depends(get_storage),
101
+ ):
102
+ """Delete a file."""
103
+ key = f"uploads/{filename}"
104
+ await storage.delete(key)
105
+ return {"status": "deleted"}
106
+ ```
107
+
108
+ ## Configuration
109
+
110
+ ### Environment Variables
111
+
112
+ #### Backend Selection
113
+
114
+ - `STORAGE_BACKEND`: Explicit backend type (`local`, `s3`, `gcs`, `cloudinary`, `memory`)
115
+ - If not set, auto-detection is used (see Auto-Detection section)
116
+
117
+ #### Local Backend
118
+
119
+ For Railway persistent volumes, Render disks, or local development:
120
+
121
+ - `STORAGE_BASE_PATH`: Directory for files (default: `/data/uploads`)
122
+ - `STORAGE_BASE_URL`: URL for file serving (default: `http://localhost:8000/files`)
123
+ - `STORAGE_URL_SECRET`: Secret for signing URLs (auto-generated if not set)
124
+ - `STORAGE_URL_EXPIRATION`: Default URL expiration in seconds (default: `3600`)
125
+
126
+ #### S3 Backend
127
+
128
+ For AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio, or any S3-compatible service:
129
+
130
+ - `STORAGE_S3_BUCKET`: Bucket name (required)
131
+ - `STORAGE_S3_REGION`: AWS region (default: `us-east-1`)
132
+ - `STORAGE_S3_ENDPOINT`: Custom endpoint URL for S3-compatible services
133
+ - `STORAGE_S3_ACCESS_KEY`: Access key (falls back to `AWS_ACCESS_KEY_ID`)
134
+ - `STORAGE_S3_SECRET_KEY`: Secret key (falls back to `AWS_SECRET_ACCESS_KEY`)
135
+
136
+ #### GCS Backend (Coming Soon)
137
+
138
+ For Google Cloud Storage:
139
+
140
+ - `STORAGE_GCS_BUCKET`: Bucket name
141
+ - `STORAGE_GCS_PROJECT`: GCP project ID
142
+ - `GOOGLE_APPLICATION_CREDENTIALS`: Path to service account JSON
143
+
144
+ #### Cloudinary Backend (Coming Soon)
145
+
146
+ For image optimization and transformations:
147
+
148
+ - `CLOUDINARY_URL`: Cloudinary connection URL
149
+ - `STORAGE_CLOUDINARY_CLOUD_NAME`: Cloud name
150
+ - `STORAGE_CLOUDINARY_API_KEY`: API key
151
+ - `STORAGE_CLOUDINARY_API_SECRET`: API secret
152
+
153
+ ### Auto-Detection
154
+
155
+ When `STORAGE_BACKEND` is not set, the system auto-detects the backend in this order:
156
+
157
+ 1. **Railway Volume**: If `RAILWAY_VOLUME_MOUNT_PATH` exists → `LocalBackend`
158
+ 2. **S3 Credentials**: If `AWS_ACCESS_KEY_ID` or `STORAGE_S3_BUCKET` exists → `S3Backend`
159
+ 3. **GCS Credentials**: If `GOOGLE_APPLICATION_CREDENTIALS` exists → `GCSBackend` (coming soon)
160
+ 4. **Cloudinary**: If `CLOUDINARY_URL` exists → `CloudinaryBackend` (coming soon)
161
+ 5. **Default**: `MemoryBackend` (with warning about data loss)
162
+
163
+ **Production Recommendation**: Always set `STORAGE_BACKEND` explicitly to avoid unexpected behavior.
164
+
165
+ ## Backend Comparison
166
+
167
+ ### When to Use Each Backend
168
+
169
+ | Backend | Best For | Pros | Cons |
170
+ |---------|----------|------|------|
171
+ | **LocalBackend** | Railway, Render, small deployments, development | Simple, no external dependencies, fast | Not scalable across multiple servers, requires persistent volumes |
172
+ | **S3Backend** | Production deployments, multi-region, CDN integration | Highly scalable, durable, integrates with CloudFront/CDN | Requires AWS account or S3-compatible service, potential egress costs |
173
+ | **GCSBackend** | GCP-native deployments | GCP integration, global CDN | Requires GCP account |
174
+ | **CloudinaryBackend** | Image-heavy applications | Automatic image optimization, transformations, CDN | Additional service cost, image-focused |
175
+ | **MemoryBackend** | Testing, CI/CD | Fast, no setup | Data lost on restart, limited by RAM |
176
+
177
+ ### Provider-Specific Notes
178
+
179
+ #### Railway Persistent Volumes
180
+
181
+ ```bash
182
+ # Railway automatically sets this variable
183
+ RAILWAY_VOLUME_MOUNT_PATH=/data
184
+
185
+ # Storage auto-detects and uses LocalBackend
186
+ STORAGE_BASE_PATH=/data/uploads
187
+ ```
188
+
189
+ #### AWS S3
190
+
191
+ ```bash
192
+ STORAGE_BACKEND=s3
193
+ STORAGE_S3_BUCKET=my-app-uploads
194
+ STORAGE_S3_REGION=us-east-1
195
+ AWS_ACCESS_KEY_ID=AKIA...
196
+ AWS_SECRET_ACCESS_KEY=...
197
+ ```
198
+
199
+ #### DigitalOcean Spaces
200
+
201
+ ```bash
202
+ STORAGE_BACKEND=s3
203
+ STORAGE_S3_BUCKET=my-app-uploads
204
+ STORAGE_S3_REGION=nyc3
205
+ STORAGE_S3_ENDPOINT=https://nyc3.digitaloceanspaces.com
206
+ STORAGE_S3_ACCESS_KEY=...
207
+ STORAGE_S3_SECRET_KEY=...
208
+ ```
209
+
210
+ #### Wasabi
211
+
212
+ ```bash
213
+ STORAGE_BACKEND=s3
214
+ STORAGE_S3_BUCKET=my-app-uploads
215
+ STORAGE_S3_REGION=us-east-1
216
+ STORAGE_S3_ENDPOINT=https://s3.wasabisys.com
217
+ STORAGE_S3_ACCESS_KEY=...
218
+ STORAGE_S3_SECRET_KEY=...
219
+ ```
220
+
221
+ #### Backblaze B2
222
+
223
+ ```bash
224
+ STORAGE_BACKEND=s3
225
+ STORAGE_S3_BUCKET=my-app-uploads
226
+ STORAGE_S3_REGION=us-west-000
227
+ STORAGE_S3_ENDPOINT=https://s3.us-west-000.backblazeb2.com
228
+ STORAGE_S3_ACCESS_KEY=...
229
+ STORAGE_S3_SECRET_KEY=...
230
+ ```
231
+
232
+ #### Minio (Self-Hosted)
233
+
234
+ ```bash
235
+ STORAGE_BACKEND=s3
236
+ STORAGE_S3_BUCKET=my-app-uploads
237
+ STORAGE_S3_REGION=us-east-1
238
+ STORAGE_S3_ENDPOINT=https://minio.example.com
239
+ STORAGE_S3_ACCESS_KEY=minioadmin
240
+ STORAGE_S3_SECRET_KEY=minioadmin
241
+ ```
242
+
243
+ ## Examples
244
+
245
+ ### Profile Picture Upload
246
+
247
+ ```python
248
+ from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
249
+ from svc_infra.storage import get_storage, StorageBackend
250
+ from PIL import Image
251
+ import io
252
+
253
+ router = APIRouter()
254
+
255
+ MAX_SIZE = 2 * 1024 * 1024 # 2MB
256
+ ALLOWED_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
257
+
258
+ @router.post("/users/me/avatar")
259
+ async def upload_avatar(
260
+ file: UploadFile = File(...),
261
+ storage: StorageBackend = Depends(get_storage),
262
+ current_user=Depends(get_current_user), # Your auth dependency
263
+ ):
264
+ """Upload user profile picture."""
265
+ # Validate content type
266
+ if file.content_type not in ALLOWED_TYPES:
267
+ raise HTTPException(
268
+ status_code=415,
269
+ detail=f"Unsupported file type. Allowed: {ALLOWED_TYPES}"
270
+ )
271
+
272
+ # Read and validate size
273
+ content = await file.read()
274
+ if len(content) > MAX_SIZE:
275
+ raise HTTPException(
276
+ status_code=413,
277
+ detail=f"File too large. Max size: {MAX_SIZE} bytes"
278
+ )
279
+
280
+ # Validate image and resize
281
+ try:
282
+ image = Image.open(io.BytesIO(content))
283
+ image.verify() # Verify it's a valid image
284
+
285
+ # Reopen and resize
286
+ image = Image.open(io.BytesIO(content))
287
+ image.thumbnail((200, 200))
288
+
289
+ # Save to bytes
290
+ output = io.BytesIO()
291
+ image.save(output, format=image.format)
292
+ resized_content = output.getvalue()
293
+ except Exception as e:
294
+ raise HTTPException(status_code=400, detail="Invalid image file")
295
+
296
+ # Store with user-specific key
297
+ key = f"avatars/{current_user.id}/profile.{file.filename.split('.')[-1]}"
298
+
299
+ url = await storage.put(
300
+ key=key,
301
+ data=resized_content,
302
+ content_type=file.content_type,
303
+ metadata={
304
+ "user_id": str(current_user.id),
305
+ "original_filename": file.filename,
306
+ "uploaded_at": datetime.utcnow().isoformat(),
307
+ }
308
+ )
309
+
310
+ # Update user record with new avatar URL
311
+ # await update_user_avatar(current_user.id, url)
312
+
313
+ return {"avatar_url": url}
314
+ ```
315
+
316
+ ### Document Storage with Metadata
317
+
318
+ ```python
319
+ from fastapi import APIRouter, Depends, UploadFile, Query
320
+ from svc_infra.storage import get_storage, StorageBackend
321
+ from typing import List, Optional
322
+
323
+ router = APIRouter()
324
+
325
+ @router.post("/documents/upload")
326
+ async def upload_document(
327
+ file: UploadFile,
328
+ tags: List[str] = Query(default=[]),
329
+ category: str = Query(default="general"),
330
+ storage: StorageBackend = Depends(get_storage),
331
+ current_user=Depends(get_current_user),
332
+ ):
333
+ """Upload a document with metadata."""
334
+ content = await file.read()
335
+
336
+ # Generate unique key
337
+ file_id = uuid4()
338
+ key = f"documents/{current_user.id}/{category}/{file_id}/{file.filename}"
339
+
340
+ url = await storage.put(
341
+ key=key,
342
+ data=content,
343
+ content_type=file.content_type or "application/octet-stream",
344
+ metadata={
345
+ "user_id": str(current_user.id),
346
+ "document_id": str(file_id),
347
+ "category": category,
348
+ "tags": ",".join(tags),
349
+ "original_filename": file.filename,
350
+ "size": len(content),
351
+ "uploaded_at": datetime.utcnow().isoformat(),
352
+ }
353
+ )
354
+
355
+ # Store document record in database
356
+ # document = await create_document_record(...)
357
+
358
+ return {
359
+ "document_id": str(file_id),
360
+ "url": url,
361
+ "filename": file.filename,
362
+ "size": len(content),
363
+ "category": category,
364
+ "tags": tags,
365
+ }
366
+
367
+ @router.get("/documents")
368
+ async def list_documents(
369
+ category: Optional[str] = None,
370
+ storage: StorageBackend = Depends(get_storage),
371
+ current_user=Depends(get_current_user),
372
+ ):
373
+ """List user's documents."""
374
+ prefix = f"documents/{current_user.id}/"
375
+ if category:
376
+ prefix += f"{category}/"
377
+
378
+ keys = await storage.list_keys(prefix=prefix, limit=100)
379
+
380
+ # Get metadata for each file
381
+ documents = []
382
+ for key in keys:
383
+ metadata = await storage.get_metadata(key)
384
+ documents.append({
385
+ "key": key,
386
+ "filename": metadata.get("original_filename"),
387
+ "size": metadata.get("size"),
388
+ "category": metadata.get("category"),
389
+ "uploaded_at": metadata.get("uploaded_at"),
390
+ })
391
+
392
+ return {"documents": documents}
393
+ ```
394
+
395
+ ### Tenant-Scoped File Storage
396
+
397
+ ```python
398
+ from fastapi import APIRouter, Depends, UploadFile
399
+ from svc_infra.storage import get_storage, StorageBackend
400
+ from svc_infra.tenancy import require_tenant_id
401
+
402
+ router = APIRouter()
403
+
404
+ @router.post("/tenant-files/upload")
405
+ async def upload_tenant_file(
406
+ file: UploadFile,
407
+ storage: StorageBackend = Depends(get_storage),
408
+ tenant_id: str = Depends(require_tenant_id),
409
+ current_user=Depends(get_current_user),
410
+ ):
411
+ """Upload a file scoped to current tenant."""
412
+ content = await file.read()
413
+
414
+ # Tenant-scoped key ensures isolation
415
+ key = f"tenants/{tenant_id}/files/{uuid4()}/{file.filename}"
416
+
417
+ url = await storage.put(
418
+ key=key,
419
+ data=content,
420
+ content_type=file.content_type or "application/octet-stream",
421
+ metadata={
422
+ "tenant_id": tenant_id,
423
+ "user_id": str(current_user.id),
424
+ "filename": file.filename,
425
+ "uploaded_at": datetime.utcnow().isoformat(),
426
+ }
427
+ )
428
+
429
+ return {"url": url, "filename": file.filename}
430
+
431
+ @router.get("/tenant-files")
432
+ async def list_tenant_files(
433
+ storage: StorageBackend = Depends(get_storage),
434
+ tenant_id: str = Depends(require_tenant_id),
435
+ ):
436
+ """List files for current tenant only."""
437
+ # Prefix ensures tenant isolation
438
+ prefix = f"tenants/{tenant_id}/files/"
439
+
440
+ keys = await storage.list_keys(prefix=prefix, limit=100)
441
+
442
+ return {"files": keys, "count": len(keys)}
443
+ ```
444
+
445
+ ### Signed URL Generation
446
+
447
+ ```python
448
+ from fastapi import APIRouter, Depends, HTTPException
449
+ from svc_infra.storage import get_storage, StorageBackend
450
+
451
+ router = APIRouter()
452
+
453
+ @router.get("/files/{file_id}/download-url")
454
+ async def get_download_url(
455
+ file_id: str,
456
+ expires_in: int = Query(default=3600, ge=60, le=86400), # 1 min to 24 hours
457
+ download: bool = Query(default=True),
458
+ storage: StorageBackend = Depends(get_storage),
459
+ current_user=Depends(get_current_user),
460
+ ):
461
+ """Generate a signed URL for file download."""
462
+ # Verify user owns the file
463
+ # file = await get_file_record(file_id)
464
+ # if file.user_id != current_user.id:
465
+ # raise HTTPException(status_code=403, detail="Access denied")
466
+
467
+ key = f"uploads/{file_id}/document.pdf"
468
+
469
+ # Check file exists
470
+ if not await storage.exists(key):
471
+ raise HTTPException(status_code=404, detail="File not found")
472
+
473
+ # Generate signed URL
474
+ url = await storage.get_url(
475
+ key=key,
476
+ expires_in=expires_in,
477
+ download=download # If True, browser downloads instead of displaying
478
+ )
479
+
480
+ return {
481
+ "url": url,
482
+ "expires_in": expires_in,
483
+ "expires_at": (datetime.utcnow() + timedelta(seconds=expires_in)).isoformat()
484
+ }
485
+ ```
486
+
487
+ ### Large File Uploads with Progress
488
+
489
+ ```python
490
+ from fastapi import APIRouter, Depends, UploadFile, BackgroundTasks
491
+ from svc_infra.storage import get_storage, StorageBackend
492
+
493
+ router = APIRouter()
494
+
495
+ @router.post("/large-files/upload")
496
+ async def upload_large_file(
497
+ file: UploadFile,
498
+ background_tasks: BackgroundTasks,
499
+ storage: StorageBackend = Depends(get_storage),
500
+ current_user=Depends(get_current_user),
501
+ ):
502
+ """Upload large file with background processing."""
503
+ # For very large files, consider chunked uploads
504
+ # This is a simple example that reads entire file
505
+
506
+ file_id = uuid4()
507
+ key = f"large-files/{current_user.id}/{file_id}/{file.filename}"
508
+
509
+ # Read file in chunks
510
+ chunks = []
511
+ while chunk := await file.read(1024 * 1024): # 1MB chunks
512
+ chunks.append(chunk)
513
+
514
+ content = b"".join(chunks)
515
+
516
+ # Store file
517
+ url = await storage.put(
518
+ key=key,
519
+ data=content,
520
+ content_type=file.content_type or "application/octet-stream",
521
+ metadata={
522
+ "user_id": str(current_user.id),
523
+ "file_id": str(file_id),
524
+ "size": len(content),
525
+ "uploaded_at": datetime.utcnow().isoformat(),
526
+ }
527
+ )
528
+
529
+ # Background task for post-processing (virus scan, thumbnail generation, etc.)
530
+ # background_tasks.add_task(process_file, file_id, key)
531
+
532
+ return {
533
+ "file_id": str(file_id),
534
+ "url": url,
535
+ "size": len(content),
536
+ "status": "uploaded"
537
+ }
538
+ ```
539
+
540
+ ## Production Recommendations
541
+
542
+ ### Railway Deployments
543
+
544
+ Railway persistent volumes are ideal for simple deployments:
545
+
546
+ ```bash
547
+ # Railway automatically mounts volume
548
+ RAILWAY_VOLUME_MOUNT_PATH=/data
549
+
550
+ # Storage auto-detects LocalBackend
551
+ # No additional configuration needed
552
+ ```
553
+
554
+ **Pros**:
555
+ - Simple setup, no external services
556
+ - Cost-effective for small/medium apps
557
+ - Fast local access
558
+
559
+ **Cons**:
560
+ - Single server only (not suitable for horizontal scaling)
561
+ - Manual backups required
562
+ - Volume size limits
563
+
564
+ ### AWS Deployments
565
+
566
+ S3 is recommended for production:
567
+
568
+ ```bash
569
+ STORAGE_BACKEND=s3
570
+ STORAGE_S3_BUCKET=myapp-uploads-prod
571
+ STORAGE_S3_REGION=us-east-1
572
+ AWS_ACCESS_KEY_ID=AKIA...
573
+ AWS_SECRET_ACCESS_KEY=...
574
+ ```
575
+
576
+ **Additional recommendations**:
577
+ - Enable versioning for backup/recovery
578
+ - Configure lifecycle policies to archive old files to Glacier
579
+ - Use CloudFront CDN for global distribution
580
+ - Enable server-side encryption (SSE-S3 or SSE-KMS)
581
+ - Set up bucket policies for least-privilege access
582
+
583
+ ### DigitalOcean Deployments
584
+
585
+ DigitalOcean Spaces (S3-compatible) offers simple pricing:
586
+
587
+ ```bash
588
+ STORAGE_BACKEND=s3
589
+ STORAGE_S3_BUCKET=myapp-uploads
590
+ STORAGE_S3_REGION=nyc3
591
+ STORAGE_S3_ENDPOINT=https://nyc3.digitaloceanspaces.com
592
+ STORAGE_S3_ACCESS_KEY=...
593
+ STORAGE_S3_SECRET_KEY=...
594
+ ```
595
+
596
+ **Pros**:
597
+ - Predictable pricing ($5/250GB)
598
+ - Built-in CDN
599
+ - S3-compatible API
600
+
601
+ ### GCP Deployments
602
+
603
+ Google Cloud Storage for GCP-native apps:
604
+
605
+ ```bash
606
+ STORAGE_BACKEND=gcs
607
+ STORAGE_GCS_BUCKET=myapp-uploads
608
+ STORAGE_GCS_PROJECT=my-gcp-project
609
+ GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
610
+ ```
611
+
612
+ (Coming soon)
613
+
614
+ ### Image-Heavy Applications
615
+
616
+ Consider Cloudinary for automatic optimization:
617
+
618
+ ```bash
619
+ STORAGE_BACKEND=cloudinary
620
+ CLOUDINARY_URL=cloudinary://...
621
+ ```
622
+
623
+ **Features**:
624
+ - Automatic image optimization
625
+ - On-the-fly transformations (resize, crop, format)
626
+ - Global CDN
627
+ - Video support
628
+
629
+ (Coming soon)
630
+
631
+ ## Security Considerations
632
+
633
+ ### Never Expose Raw File Paths
634
+
635
+ ❌ **Bad**:
636
+ ```python
637
+ # Don't return raw storage keys or file paths
638
+ return {"path": "/data/uploads/secret-document.pdf"}
639
+ ```
640
+
641
+ ✅ **Good**:
642
+ ```python
643
+ # Return signed URLs with expiration
644
+ url = await storage.get_url(key, expires_in=3600)
645
+ return {"url": url}
646
+ ```
647
+
648
+ ### Always Use Signed URLs with Expiration
649
+
650
+ ```python
651
+ # Short expiration for sensitive documents
652
+ url = await storage.get_url(key, expires_in=300) # 5 minutes
653
+
654
+ # Longer expiration for public assets
655
+ url = await storage.get_url(key, expires_in=86400) # 24 hours
656
+ ```
657
+
658
+ ### Validate File Types and Sizes Before Upload
659
+
660
+ ```python
661
+ MAX_SIZE = 10 * 1024 * 1024 # 10MB
662
+ ALLOWED_TYPES = {"image/jpeg", "image/png", "application/pdf"}
663
+
664
+ if file.content_type not in ALLOWED_TYPES:
665
+ raise HTTPException(status_code=415, detail="Unsupported file type")
666
+
667
+ content = await file.read()
668
+ if len(content) > MAX_SIZE:
669
+ raise HTTPException(status_code=413, detail="File too large")
670
+ ```
671
+
672
+ ### Scan for Viruses
673
+
674
+ Integration with ClamAV or similar (coming in future version):
675
+
676
+ ```python
677
+ # Future API
678
+ from svc_infra.storage.scanners import scan_file
679
+
680
+ result = await scan_file(content)
681
+ if result.is_infected:
682
+ raise HTTPException(status_code=400, detail="File contains malware")
683
+ ```
684
+
685
+ ### Implement Tenant Isolation via Key Prefixes
686
+
687
+ ```python
688
+ # Always scope keys by tenant
689
+ key = f"tenants/{tenant_id}/documents/{file_id}"
690
+
691
+ # Verify access before operations
692
+ if not await verify_tenant_access(current_user, tenant_id):
693
+ raise HTTPException(status_code=403, detail="Access denied")
694
+ ```
695
+
696
+ ### Use IAM Policies for Least-Privilege Access
697
+
698
+ For S3/GCS, create service accounts with minimal permissions:
699
+
700
+ ```json
701
+ {
702
+ "Version": "2012-10-17",
703
+ "Statement": [
704
+ {
705
+ "Effect": "Allow",
706
+ "Action": [
707
+ "s3:PutObject",
708
+ "s3:GetObject",
709
+ "s3:DeleteObject",
710
+ "s3:ListBucket"
711
+ ],
712
+ "Resource": [
713
+ "arn:aws:s3:::myapp-uploads-prod",
714
+ "arn:aws:s3:::myapp-uploads-prod/*"
715
+ ]
716
+ }
717
+ ]
718
+ }
719
+ ```
720
+
721
+ ### Enable Encryption at Rest
722
+
723
+ For S3:
724
+ ```bash
725
+ # Enable default encryption in bucket settings
726
+ aws s3api put-bucket-encryption \
727
+ --bucket myapp-uploads-prod \
728
+ --server-side-encryption-configuration '{
729
+ "Rules": [{
730
+ "ApplyServerSideEncryptionByDefault": {
731
+ "SSEAlgorithm": "AES256"
732
+ }
733
+ }]
734
+ }'
735
+ ```
736
+
737
+ ## Troubleshooting
738
+
739
+ ### Error: "Storage not configured"
740
+
741
+ **Cause**: `add_storage()` was not called or `get_storage()` dependency used without configuration.
742
+
743
+ **Solution**:
744
+ ```python
745
+ from svc_infra.storage import add_storage
746
+
747
+ app = FastAPI()
748
+ storage = add_storage(app) # Add this line
749
+ ```
750
+
751
+ ### Error: "No module named 'aioboto3'"
752
+
753
+ **Cause**: S3Backend requires `aioboto3` dependency.
754
+
755
+ **Solution**:
756
+ ```bash
757
+ poetry add aioboto3
758
+ ```
759
+
760
+ ### Error: "Access Denied" (S3)
761
+
762
+ **Cause**: Invalid credentials or insufficient IAM permissions.
763
+
764
+ **Solution**:
765
+ - Verify `STORAGE_S3_ACCESS_KEY` and `STORAGE_S3_SECRET_KEY`
766
+ - Check IAM policy allows required S3 actions
767
+ - Verify bucket name and region are correct
768
+
769
+ ### Error: "Bucket does not exist"
770
+
771
+ **Cause**: S3 bucket not created or wrong bucket name.
772
+
773
+ **Solution**:
774
+ ```bash
775
+ # Create bucket
776
+ aws s3 mb s3://myapp-uploads-prod --region us-east-1
777
+
778
+ # Or via S3 console
779
+ ```
780
+
781
+ ### Files Not Persisting (LocalBackend)
782
+
783
+ **Cause**: Using in-memory filesystem or container without persistent volume.
784
+
785
+ **Solution**:
786
+ - Railway: Ensure persistent volume is mounted
787
+ - Docker: Mount volume: `docker run -v /data/uploads:/data/uploads ...`
788
+ - Render: Use persistent disks feature
789
+
790
+ ### URLs Expire Too Quickly
791
+
792
+ **Cause**: Default expiration is 1 hour.
793
+
794
+ **Solution**:
795
+ ```python
796
+ # Increase expiration
797
+ url = await storage.get_url(key, expires_in=86400) # 24 hours
798
+
799
+ # Or set default in environment
800
+ STORAGE_URL_EXPIRATION=86400
801
+ ```
802
+
803
+ ### Large File Uploads Fail
804
+
805
+ **Cause**: Request timeout or size limits.
806
+
807
+ **Solution**:
808
+ ```python
809
+ # Increase timeouts in uvicorn
810
+ uvicorn main:app --timeout-keep-alive 300
811
+
812
+ # Or chunk uploads for very large files (>100MB)
813
+ ```
814
+
815
+ ## API Reference
816
+
817
+ ### Core Functions
818
+
819
+ #### `add_storage(app, backend, serve_files, file_route_prefix)`
820
+
821
+ Integrate storage backend with FastAPI application.
822
+
823
+ **Parameters**:
824
+ - `app: FastAPI` - Application instance
825
+ - `backend: Optional[StorageBackend]` - Storage backend (auto-detected if None)
826
+ - `serve_files: bool` - Mount file serving route (LocalBackend only, default: False)
827
+ - `file_route_prefix: str` - URL prefix for files (default: "/files")
828
+
829
+ **Returns**: `StorageBackend` instance
830
+
831
+ **Example**:
832
+ ```python
833
+ storage = add_storage(app)
834
+ ```
835
+
836
+ #### `easy_storage(backend, **kwargs)`
837
+
838
+ Create storage backend with auto-detection.
839
+
840
+ **Parameters**:
841
+ - `backend: Optional[str]` - Backend type ("local", "s3", "memory") or None for auto-detect
842
+ - `**kwargs` - Backend-specific configuration
843
+
844
+ **Returns**: `StorageBackend` instance
845
+
846
+ **Example**:
847
+ ```python
848
+ storage = easy_storage(backend="s3", bucket="uploads", region="us-east-1")
849
+ ```
850
+
851
+ #### `get_storage(request)`
852
+
853
+ FastAPI dependency to inject storage backend.
854
+
855
+ **Parameters**:
856
+ - `request: Request` - FastAPI request
857
+
858
+ **Returns**: `StorageBackend` from app.state.storage
859
+
860
+ **Example**:
861
+ ```python
862
+ async def upload(storage: StorageBackend = Depends(get_storage)):
863
+ ...
864
+ ```
865
+
866
+ ### StorageBackend Protocol
867
+
868
+ All backends implement these methods:
869
+
870
+ #### `async put(key, data, content_type, metadata=None)`
871
+
872
+ Store file and return URL.
873
+
874
+ **Parameters**:
875
+ - `key: str` - Storage key (path)
876
+ - `data: bytes` - File content
877
+ - `content_type: str` - MIME type
878
+ - `metadata: Optional[dict]` - Custom metadata
879
+
880
+ **Returns**: `str` - File URL
881
+
882
+ **Raises**: `InvalidKeyError`, `PermissionDeniedError`, `QuotaExceededError`, `StorageError`
883
+
884
+ #### `async get(key)`
885
+
886
+ Retrieve file content.
887
+
888
+ **Parameters**:
889
+ - `key: str` - Storage key
890
+
891
+ **Returns**: `bytes` - File content
892
+
893
+ **Raises**: `FileNotFoundError`, `PermissionDeniedError`, `StorageError`
894
+
895
+ #### `async delete(key)`
896
+
897
+ Remove file.
898
+
899
+ **Parameters**:
900
+ - `key: str` - Storage key
901
+
902
+ **Returns**: `bool` - True if deleted, False if not found
903
+
904
+ **Raises**: `PermissionDeniedError`, `StorageError`
905
+
906
+ #### `async exists(key)`
907
+
908
+ Check if file exists.
909
+
910
+ **Parameters**:
911
+ - `key: str` - Storage key
912
+
913
+ **Returns**: `bool` - True if exists
914
+
915
+ #### `async get_url(key, expires_in=3600, download=False)`
916
+
917
+ Generate signed URL.
918
+
919
+ **Parameters**:
920
+ - `key: str` - Storage key
921
+ - `expires_in: int` - Expiration in seconds (default: 3600)
922
+ - `download: bool` - Force download vs display (default: False)
923
+
924
+ **Returns**: `str` - Signed URL
925
+
926
+ **Raises**: `FileNotFoundError`, `StorageError`
927
+
928
+ #### `async list_keys(prefix="", limit=1000)`
929
+
930
+ List stored files.
931
+
932
+ **Parameters**:
933
+ - `prefix: str` - Key prefix filter (default: "")
934
+ - `limit: int` - Max results (default: 1000)
935
+
936
+ **Returns**: `List[str]` - List of keys
937
+
938
+ #### `async get_metadata(key)`
939
+
940
+ Get file metadata.
941
+
942
+ **Parameters**:
943
+ - `key: str` - Storage key
944
+
945
+ **Returns**: `dict` - Metadata dictionary
946
+
947
+ **Raises**: `FileNotFoundError`, `StorageError`
948
+
949
+ ### Exceptions
950
+
951
+ All exceptions inherit from `StorageError`:
952
+
953
+ - `StorageError` - Base exception
954
+ - `FileNotFoundError` - File doesn't exist
955
+ - `PermissionDeniedError` - Access denied
956
+ - `QuotaExceededError` - Storage quota exceeded
957
+ - `InvalidKeyError` - Invalid key format
958
+
959
+ ## Health Checks
960
+
961
+ Storage backend health is automatically registered when using `add_storage()`:
962
+
963
+ ```python
964
+ # Health check endpoint
965
+ GET /_ops/health
966
+
967
+ # Response
968
+ {
969
+ "status": "healthy",
970
+ "storage": {
971
+ "backend": "S3Backend",
972
+ "status": "connected"
973
+ }
974
+ }
975
+ ```
976
+
977
+ ## See Also
978
+
979
+ - [ADR-0012: Generic File Storage System](/src/svc_infra/docs/adr/0012-storage-system.md) - Design decisions
980
+ - [API Integration Guide](/src/svc_infra/docs/api.md) - FastAPI integration patterns
981
+ - [Tenancy Guide](/src/svc_infra/docs/tenancy.md) - Multi-tenant file isolation
982
+ - [Security Guide](/src/svc_infra/docs/security.md) - Security best practices