svc-infra 0.1.600__py3-none-any.whl → 0.1.664__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of svc-infra might be problematic. Click here for more details.

Files changed (140) hide show
  1. svc_infra/api/fastapi/admin/__init__.py +3 -0
  2. svc_infra/api/fastapi/admin/add.py +231 -0
  3. svc_infra/api/fastapi/apf_payments/setup.py +0 -2
  4. svc_infra/api/fastapi/auth/add.py +0 -4
  5. svc_infra/api/fastapi/auth/routers/oauth_router.py +19 -4
  6. svc_infra/api/fastapi/billing/router.py +64 -0
  7. svc_infra/api/fastapi/billing/setup.py +19 -0
  8. svc_infra/api/fastapi/cache/add.py +9 -5
  9. svc_infra/api/fastapi/db/nosql/mongo/add.py +33 -27
  10. svc_infra/api/fastapi/db/sql/add.py +40 -18
  11. svc_infra/api/fastapi/db/sql/crud_router.py +176 -14
  12. svc_infra/api/fastapi/db/sql/session.py +16 -0
  13. svc_infra/api/fastapi/dependencies/ratelimit.py +57 -7
  14. svc_infra/api/fastapi/docs/add.py +160 -0
  15. svc_infra/api/fastapi/docs/landing.py +1 -1
  16. svc_infra/api/fastapi/docs/scoped.py +41 -6
  17. svc_infra/api/fastapi/middleware/errors/handlers.py +45 -7
  18. svc_infra/api/fastapi/middleware/graceful_shutdown.py +87 -0
  19. svc_infra/api/fastapi/middleware/ratelimit.py +59 -1
  20. svc_infra/api/fastapi/middleware/ratelimit_store.py +12 -6
  21. svc_infra/api/fastapi/middleware/timeout.py +148 -0
  22. svc_infra/api/fastapi/openapi/mutators.py +114 -0
  23. svc_infra/api/fastapi/ops/add.py +73 -0
  24. svc_infra/api/fastapi/pagination.py +3 -1
  25. svc_infra/api/fastapi/routers/ping.py +1 -0
  26. svc_infra/api/fastapi/setup.py +21 -13
  27. svc_infra/api/fastapi/tenancy/add.py +19 -0
  28. svc_infra/api/fastapi/tenancy/context.py +112 -0
  29. svc_infra/api/fastapi/versioned.py +101 -0
  30. svc_infra/app/README.md +5 -5
  31. svc_infra/billing/__init__.py +23 -0
  32. svc_infra/billing/async_service.py +147 -0
  33. svc_infra/billing/jobs.py +230 -0
  34. svc_infra/billing/models.py +131 -0
  35. svc_infra/billing/quotas.py +101 -0
  36. svc_infra/billing/schemas.py +33 -0
  37. svc_infra/billing/service.py +115 -0
  38. svc_infra/bundled_docs/README.md +5 -0
  39. svc_infra/bundled_docs/__init__.py +1 -0
  40. svc_infra/bundled_docs/getting-started.md +6 -0
  41. svc_infra/cache/__init__.py +4 -0
  42. svc_infra/cache/add.py +158 -0
  43. svc_infra/cache/backend.py +5 -2
  44. svc_infra/cache/decorators.py +19 -1
  45. svc_infra/cache/keys.py +24 -4
  46. svc_infra/cli/__init__.py +28 -8
  47. svc_infra/cli/cmds/__init__.py +8 -0
  48. svc_infra/cli/cmds/db/nosql/mongo/mongo_cmds.py +4 -3
  49. svc_infra/cli/cmds/db/nosql/mongo/mongo_scaffold_cmds.py +4 -4
  50. svc_infra/cli/cmds/db/sql/alembic_cmds.py +80 -11
  51. svc_infra/cli/cmds/db/sql/sql_export_cmds.py +80 -0
  52. svc_infra/cli/cmds/db/sql/sql_scaffold_cmds.py +3 -3
  53. svc_infra/cli/cmds/docs/docs_cmds.py +140 -0
  54. svc_infra/cli/cmds/dx/__init__.py +12 -0
  55. svc_infra/cli/cmds/dx/dx_cmds.py +99 -0
  56. svc_infra/cli/cmds/help.py +4 -0
  57. svc_infra/cli/cmds/obs/obs_cmds.py +4 -3
  58. svc_infra/cli/cmds/sdk/__init__.py +0 -0
  59. svc_infra/cli/cmds/sdk/sdk_cmds.py +102 -0
  60. svc_infra/data/add.py +61 -0
  61. svc_infra/data/backup.py +53 -0
  62. svc_infra/data/erasure.py +45 -0
  63. svc_infra/data/fixtures.py +40 -0
  64. svc_infra/data/retention.py +55 -0
  65. svc_infra/db/nosql/mongo/README.md +13 -13
  66. svc_infra/db/sql/repository.py +51 -11
  67. svc_infra/db/sql/resource.py +5 -0
  68. svc_infra/db/sql/templates/models_schemas/auth/models.py.tmpl +7 -56
  69. svc_infra/db/sql/templates/setup/env_async.py.tmpl +34 -12
  70. svc_infra/db/sql/templates/setup/env_sync.py.tmpl +29 -7
  71. svc_infra/db/sql/tenant.py +79 -0
  72. svc_infra/db/sql/utils.py +18 -4
  73. svc_infra/docs/acceptance-matrix.md +88 -0
  74. svc_infra/docs/acceptance.md +44 -0
  75. svc_infra/docs/admin.md +425 -0
  76. svc_infra/docs/adr/0002-background-jobs-and-scheduling.md +40 -0
  77. svc_infra/docs/adr/0003-webhooks-framework.md +24 -0
  78. svc_infra/docs/adr/0004-tenancy-model.md +42 -0
  79. svc_infra/docs/adr/0005-data-lifecycle.md +86 -0
  80. svc_infra/docs/adr/0006-ops-slos-and-metrics.md +47 -0
  81. svc_infra/docs/adr/0007-docs-and-sdks.md +83 -0
  82. svc_infra/docs/adr/0008-billing-primitives.md +143 -0
  83. svc_infra/docs/adr/0009-acceptance-harness.md +40 -0
  84. svc_infra/docs/adr/0010-timeouts-and-resource-limits.md +54 -0
  85. svc_infra/docs/adr/0011-admin-scope-and-impersonation.md +73 -0
  86. svc_infra/docs/adr/0012-generic-file-storage.md +498 -0
  87. svc_infra/docs/api.md +186 -0
  88. svc_infra/docs/auth.md +11 -0
  89. svc_infra/docs/billing.md +190 -0
  90. svc_infra/docs/cache.md +76 -0
  91. svc_infra/docs/cli.md +74 -0
  92. svc_infra/docs/contributing.md +34 -0
  93. svc_infra/docs/data-lifecycle.md +52 -0
  94. svc_infra/docs/database.md +14 -0
  95. svc_infra/docs/docs-and-sdks.md +62 -0
  96. svc_infra/docs/environment.md +114 -0
  97. svc_infra/docs/getting-started.md +63 -0
  98. svc_infra/docs/idempotency.md +111 -0
  99. svc_infra/docs/jobs.md +67 -0
  100. svc_infra/docs/observability.md +16 -0
  101. svc_infra/docs/ops.md +37 -0
  102. svc_infra/docs/rate-limiting.md +125 -0
  103. svc_infra/docs/repo-review.md +48 -0
  104. svc_infra/docs/security.md +176 -0
  105. svc_infra/docs/storage.md +982 -0
  106. svc_infra/docs/tenancy.md +35 -0
  107. svc_infra/docs/timeouts-and-resource-limits.md +147 -0
  108. svc_infra/docs/versioned-integrations.md +146 -0
  109. svc_infra/docs/webhooks.md +112 -0
  110. svc_infra/dx/add.py +63 -0
  111. svc_infra/dx/changelog.py +74 -0
  112. svc_infra/dx/checks.py +67 -0
  113. svc_infra/http/__init__.py +13 -0
  114. svc_infra/http/client.py +72 -0
  115. svc_infra/jobs/builtins/webhook_delivery.py +14 -2
  116. svc_infra/jobs/queue.py +9 -1
  117. svc_infra/jobs/runner.py +75 -0
  118. svc_infra/jobs/worker.py +17 -1
  119. svc_infra/mcp/svc_infra_mcp.py +85 -28
  120. svc_infra/obs/add.py +54 -7
  121. svc_infra/obs/grafana/dashboards/http-overview.json +45 -0
  122. svc_infra/security/headers.py +15 -2
  123. svc_infra/security/hibp.py +6 -2
  124. svc_infra/security/models.py +27 -7
  125. svc_infra/security/oauth_models.py +59 -0
  126. svc_infra/security/permissions.py +1 -0
  127. svc_infra/storage/__init__.py +93 -0
  128. svc_infra/storage/add.py +250 -0
  129. svc_infra/storage/backends/__init__.py +11 -0
  130. svc_infra/storage/backends/local.py +331 -0
  131. svc_infra/storage/backends/memory.py +214 -0
  132. svc_infra/storage/backends/s3.py +329 -0
  133. svc_infra/storage/base.py +239 -0
  134. svc_infra/storage/easy.py +182 -0
  135. svc_infra/storage/settings.py +192 -0
  136. svc_infra/webhooks/service.py +10 -2
  137. {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/METADATA +45 -14
  138. {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/RECORD +140 -52
  139. {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/WHEEL +0 -0
  140. {svc_infra-0.1.600.dist-info → svc_infra-0.1.664.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,498 @@
1
+ # ADR 0012: Generic File Storage System
2
+
3
+ ## Status
4
+
5
+ Proposed — Design phase (2025-11-17)
6
+
7
+ ## Context
8
+
9
+ svc-infra currently lacks a generic file storage abstraction. Applications built on svc-infra need to store user-uploaded files (profile pictures, documents, media, attachments) with backend flexibility. Current state:
10
+
11
+ - **svc-infra**: No file storage system exists (only commented references to "use svc-infra storage")
12
+ - **fin-infra**: Documents module uses in-memory placeholder dictionaries (`_documents`, `_file_storage`)
13
+ - **Applications**: No standard way to store files, forcing each app to implement custom solutions
14
+
15
+ ### Requirements
16
+
17
+ 1. **Backend Agnostic**: Work with ANY storage provider without code changes
18
+ - Local filesystem (Railway persistent volumes, Render, development)
19
+ - S3-compatible (AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio)
20
+ - Google Cloud Storage
21
+ - Cloudinary (image optimization)
22
+ - In-memory (testing)
23
+
24
+ 2. **Security First**:
25
+ - Signed URLs with expiration by default
26
+ - No exposure of raw file paths
27
+ - Tenant isolation via key prefixes
28
+ - Metadata validation
29
+
30
+ 3. **Production Ready**:
31
+ - Async operations (non-blocking)
32
+ - Connection pooling
33
+ - Retry logic for transient failures
34
+ - Health checks
35
+ - Observability (metrics, logging)
36
+
37
+ 4. **Developer Experience**:
38
+ - One-line integration via `add_storage(app)`
39
+ - Auto-detection from environment variables
40
+ - Type-safe with Pydantic settings
41
+ - Clear error messages
42
+
43
+ 5. **Separation of Concerns**:
44
+ - **svc-infra/storage/**: Generic file storage infrastructure (reusable)
45
+ - **Domain packages** (fin-infra, etc.): Domain-specific features built ON TOP
46
+ - Example: fin-infra documents keep OCR/AI analysis, delegate storage to svc-infra
47
+
48
+ ## Decisions
49
+
50
+ ### 1. Abstract Storage Backend Interface
51
+
52
+ Define a protocol-based interface that all backends must implement:
53
+
54
+ ```python
55
+ from typing import Protocol, Optional
56
+
57
+ class StorageBackend(Protocol):
58
+ """Abstract storage backend interface."""
59
+
60
+ async def put(
61
+ self,
62
+ key: str,
63
+ data: bytes,
64
+ content_type: str,
65
+ metadata: Optional[dict] = None,
66
+ ) -> str:
67
+ """Store file and return public/signed URL."""
68
+ ...
69
+
70
+ async def get(self, key: str) -> bytes:
71
+ """Retrieve file content."""
72
+ ...
73
+
74
+ async def delete(self, key: str) -> bool:
75
+ """Remove file. Returns True if deleted, False if not found."""
76
+ ...
77
+
78
+ async def exists(self, key: str) -> bool:
79
+ """Check if file exists."""
80
+ ...
81
+
82
+ async def get_url(
83
+ self,
84
+ key: str,
85
+ expires_in: int = 3600,
86
+ download: bool = False,
87
+ ) -> str:
88
+ """Generate signed/public URL."""
89
+ ...
90
+
91
+ async def list_keys(
92
+ self,
93
+ prefix: str = "",
94
+ limit: int = 100,
95
+ ) -> list[str]:
96
+ """List stored file keys with optional prefix filter."""
97
+ ...
98
+
99
+ async def get_metadata(self, key: str) -> dict:
100
+ """Get file metadata (size, content_type, custom metadata)."""
101
+ ...
102
+ ```
103
+
104
+ **Rationale**: Protocol-based design (PEP 544) allows structural subtyping without inheritance, making it easy to add new backends without modifying existing code.
105
+
106
+ ### 2. Exception Hierarchy
107
+
108
+ ```python
109
+ class StorageError(Exception):
110
+ """Base exception for storage operations."""
111
+ pass
112
+
113
+ class FileNotFoundError(StorageError):
114
+ """Raised when file does not exist."""
115
+ pass
116
+
117
+ class PermissionDeniedError(StorageError):
118
+ """Raised when lacking permissions for operation."""
119
+ pass
120
+
121
+ class QuotaExceededError(StorageError):
122
+ """Raised when storage quota is exceeded."""
123
+ pass
124
+
125
+ class InvalidKeyError(StorageError):
126
+ """Raised when key format is invalid."""
127
+ pass
128
+ ```
129
+
130
+ **Rationale**: Specific exceptions enable fine-grained error handling and better user feedback.
131
+
132
+ ### 3. Backend Implementations
133
+
134
+ #### LocalBackend (Filesystem)
135
+
136
+ Use cases: Railway persistent volumes, Render disks, local development
137
+
138
+ ```python
139
+ class LocalBackend:
140
+ def __init__(
141
+ self,
142
+ base_path: str = "/data/uploads",
143
+ base_url: str = "http://localhost:8000/files",
144
+ signing_secret: Optional[str] = None,
145
+ ):
146
+ self.base_path = Path(base_path)
147
+ self.base_url = base_url
148
+ self.signing_secret = signing_secret or secrets.token_urlsafe(32)
149
+ ```
150
+
151
+ Features:
152
+ - Async file I/O using `aiofiles`
153
+ - HMAC-based URL signing with expiration
154
+ - Metadata stored as JSON sidecar files (`{key}.meta.json`)
155
+ - Atomic writes using temp files
156
+ - Automatic directory creation
157
+
158
+ **Railway Integration**: Detects `RAILWAY_VOLUME_MOUNT_PATH` and uses it as base_path
159
+
160
+ #### S3Backend (S3-Compatible)
161
+
162
+ Use cases: AWS S3, DigitalOcean Spaces, Wasabi, Backblaze B2, Minio
163
+
164
+ ```python
165
+ class S3Backend:
166
+ def __init__(
167
+ self,
168
+ bucket: str,
169
+ region: str = "us-east-1",
170
+ endpoint: Optional[str] = None,
171
+ access_key: Optional[str] = None,
172
+ secret_key: Optional[str] = None,
173
+ ):
174
+ self.bucket = bucket
175
+ self.region = region
176
+ self.endpoint = endpoint # For DigitalOcean, Wasabi, etc.
177
+ ```
178
+
179
+ Features:
180
+ - Uses `aioboto3` for async operations
181
+ - Custom endpoint support for S3-compatible services
182
+ - Presigned URLs with configurable expiration
183
+ - Multipart upload for large files (>5MB)
184
+ - Connection pooling
185
+ - Metadata stored in S3 object metadata
186
+ - Retry logic with exponential backoff
187
+
188
+ **DigitalOcean Spaces Example**:
189
+ ```python
190
+ S3Backend(
191
+ bucket="my-uploads",
192
+ region="nyc3",
193
+ endpoint="https://nyc3.digitaloceanspaces.com",
194
+ )
195
+ ```
196
+
197
+ #### MemoryBackend (In-Memory)
198
+
199
+ Use cases: Unit tests, development, temporary storage
200
+
201
+ ```python
202
+ class MemoryBackend:
203
+ def __init__(self, max_size: int = 100_000_000): # 100MB default
204
+ self._storage: dict[str, bytes] = {}
205
+ self._metadata: dict[str, dict] = {}
206
+ self.max_size = max_size
207
+ ```
208
+
209
+ Features:
210
+ - Dictionary-based storage
211
+ - Thread-safe operations using `asyncio.Lock`
212
+ - TTL support for automatic expiration
213
+ - Size limits
214
+ - No persistence across restarts
215
+
216
+ #### GCSBackend (Google Cloud Storage) - Optional for v1
217
+
218
+ Use cases: Google Cloud Platform deployments
219
+
220
+ ```python
221
+ class GCSBackend:
222
+ def __init__(
223
+ self,
224
+ bucket: str,
225
+ project: Optional[str] = None,
226
+ credentials_path: Optional[str] = None,
227
+ ):
228
+ self.bucket = bucket
229
+ self.project = project
230
+ ```
231
+
232
+ Features:
233
+ - Uses `google-cloud-storage` SDK
234
+ - Async wrapper around sync SDK
235
+ - Signed URLs with service account
236
+ - Metadata in GCS object metadata
237
+
238
+ **Defer to fast-follow** if time constrained for v1.
239
+
240
+ #### CloudinaryBackend (Image Optimization) - Optional for v1
241
+
242
+ Use cases: Image-heavy applications, automatic optimization
243
+
244
+ ```python
245
+ class CloudinaryBackend:
246
+ def __init__(
247
+ self,
248
+ cloud_name: str,
249
+ api_key: str,
250
+ api_secret: str,
251
+ ):
252
+ self.cloud_name = cloud_name
253
+ ```
254
+
255
+ Features:
256
+ - Uses `cloudinary` SDK
257
+ - Automatic image optimization
258
+ - URL transformations (resize, crop, format)
259
+ - CDN delivery
260
+
261
+ **Defer to fast-follow** if time constrained for v1.
262
+
263
+ ### 4. Configuration and Auto-Detection
264
+
265
+ Use Pydantic settings with environment variables:
266
+
267
+ ```python
268
+ class StorageSettings(BaseSettings):
269
+ # Backend selection
270
+ storage_backend: Optional[str] = None # "local", "s3", "gcs", "cloudinary", "memory"
271
+
272
+ # Local backend
273
+ storage_base_path: str = "/data/uploads"
274
+ storage_base_url: str = "http://localhost:8000/files"
275
+ storage_signing_secret: Optional[str] = None
276
+
277
+ # S3 backend
278
+ storage_s3_bucket: Optional[str] = None
279
+ storage_s3_region: str = "us-east-1"
280
+ storage_s3_endpoint: Optional[str] = None
281
+ storage_s3_access_key: Optional[str] = None
282
+ storage_s3_secret_key: Optional[str] = None
283
+
284
+ # GCS backend
285
+ storage_gcs_bucket: Optional[str] = None
286
+ storage_gcs_project: Optional[str] = None
287
+ storage_gcs_credentials_path: Optional[str] = None
288
+
289
+ # Cloudinary backend
290
+ storage_cloudinary_cloud_name: Optional[str] = None
291
+ storage_cloudinary_api_key: Optional[str] = None
292
+ storage_cloudinary_api_secret: Optional[str] = None
293
+
294
+ class Config:
295
+ env_file = ".env"
296
+ ```
297
+
298
+ **Auto-Detection Logic** (when `storage_backend` not explicitly set):
299
+
300
+ 1. Check for Railway: `RAILWAY_VOLUME_MOUNT_PATH` exists → LocalBackend
301
+ 2. Check for S3: `AWS_ACCESS_KEY_ID` or `storage_s3_bucket` set → S3Backend
302
+ 3. Check for GCS: `GOOGLE_APPLICATION_CREDENTIALS` or `storage_gcs_bucket` set → GCSBackend
303
+ 4. Check for Cloudinary: `CLOUDINARY_URL` set → CloudinaryBackend
304
+ 5. Default: MemoryBackend (with warning log)
305
+
306
+ **Rationale**: Zero-config for common platforms (Railway, AWS) while allowing explicit override.
307
+
308
+ ### 5. FastAPI Integration Pattern
309
+
310
+ ```python
311
+ from svc_infra.storage import add_storage, easy_storage
312
+
313
+ # Option 1: Auto-detect backend
314
+ storage = easy_storage()
315
+ add_storage(app, storage)
316
+
317
+ # Option 2: Explicit backend
318
+ storage = easy_storage(backend="s3", bucket="my-uploads")
319
+ add_storage(app, storage)
320
+
321
+ # Option 3: Custom backend instance
322
+ from svc_infra.storage.backends import S3Backend
323
+ storage = S3Backend(bucket="my-uploads", region="us-west-2")
324
+ add_storage(app, storage)
325
+ ```
326
+
327
+ The `add_storage` helper:
328
+ - Stores backend in `app.state.storage`
329
+ - Registers startup hook for connection testing
330
+ - Registers shutdown hook for cleanup
331
+ - Adds health check endpoint (`/_health/storage`)
332
+ - Optionally mounts file serving route (`/files/{path:path}`)
333
+
334
+ Dependency injection in routes:
335
+
336
+ ```python
337
+ from svc_infra.storage import get_storage
338
+ from fastapi import Depends, UploadFile
339
+
340
+ @router.post("/avatar")
341
+ async def upload_avatar(
342
+ file: UploadFile,
343
+ storage: StorageBackend = Depends(get_storage),
344
+ ):
345
+ content = await file.read()
346
+ url = await storage.put(
347
+ key=f"avatars/{user_id}/{file.filename}",
348
+ data=content,
349
+ content_type=file.content_type,
350
+ metadata={"user_id": user_id},
351
+ )
352
+ return {"url": url}
353
+ ```
354
+
355
+ ### 6. Key Naming Conventions
356
+
357
+ Support tenant-scoped and resource-scoped keys:
358
+
359
+ ```
360
+ {tenant_id}/{resource_type}/{resource_id}/{filename}
361
+ ```
362
+
363
+ Examples:
364
+ - `tenant_123/avatars/user_456/profile.jpg`
365
+ - `tenant_123/documents/doc_789/invoice.pdf`
366
+ - `public/logos/company-logo.png` (no tenant isolation)
367
+
368
+ **Validation**: Keys must be:
369
+ - Relative paths (no leading `/`)
370
+ - No `..` path traversal
371
+ - Max 1024 characters
372
+ - Safe characters: `a-zA-Z0-9._-/`
373
+
374
+ ### 7. Security Considerations
375
+
376
+ 1. **Signed URLs by Default**: All URLs should have expiration (default 1 hour)
377
+ 2. **No Raw Path Exposure**: Never expose filesystem paths or S3 keys directly
378
+ 3. **Content-Type Validation**: Validate MIME types before storage
379
+ 4. **Size Limits**: Enforce max file size (default 10MB, configurable)
380
+ 5. **Virus Scanning**: Hook for integration (ClamAV, VirusTotal) - defer to fast-follow
381
+ 6. **Tenant Isolation**: Enforce key prefixes, prevent cross-tenant access
382
+ 7. **Rate Limiting**: Apply to upload endpoints
383
+
384
+ ### 8. Observability
385
+
386
+ Metrics (Prometheus):
387
+ ```
388
+ storage_operations_total{backend, operation, status}
389
+ storage_operation_duration_seconds{backend, operation}
390
+ storage_file_size_bytes{backend}
391
+ storage_errors_total{backend, error_type}
392
+ ```
393
+
394
+ Logging:
395
+ - Info: File uploaded/deleted with key and size
396
+ - Error: Operation failures with backend and error details
397
+ - Debug: URL generation, metadata operations
398
+
399
+ ### 9. Migration Path
400
+
401
+ For existing applications using placeholder storage (like fin-infra documents):
402
+
403
+ 1. **Add svc-infra storage dependency**
404
+ 2. **Configure backend via environment variables**
405
+ 3. **Replace direct storage calls with svc-infra storage**
406
+ 4. **Keep domain-specific logic** (OCR, analysis, retention policies)
407
+ 5. **Test with MemoryBackend first**, then switch to production backend
408
+ 6. **Migrate existing files** (one-time script to copy from old storage to new)
409
+
410
+ Example refactor for fin-infra documents:
411
+
412
+ ```python
413
+ # Before (in-memory)
414
+ _file_storage: Dict[str, bytes] = {}
415
+
416
+ def download_document(doc_id: str) -> bytes:
417
+ return _file_storage[doc_id]
418
+
419
+ # After (svc-infra storage)
420
+ from svc_infra.storage import get_storage
421
+
422
+ async def download_document(doc_id: str, storage: StorageBackend) -> bytes:
423
+ return await storage.get(f"documents/{doc_id}")
424
+ ```
425
+
426
+ ### 10. Testing Strategy
427
+
428
+ - **Unit Tests**: Mock storage backends, test each implementation separately
429
+ - **Integration Tests**: Real backends (S3, GCS) with test credentials (marked for CI skip)
430
+ - **Acceptance Tests**: End-to-end scenarios (upload → retrieve → delete)
431
+ - **Property Tests**: Key validation, URL signing, metadata preservation
432
+ - **Performance Tests**: Large file uploads, concurrent operations
433
+
434
+ ## Alternatives Considered
435
+
436
+ ### 1. Use Third-Party Library (e.g., fsspec, cloudpathlib)
437
+
438
+ **Rejected**: These libraries are sync-first and don't integrate well with FastAPI's async patterns. Custom implementation gives us full control over async operations, error handling, and FastAPI integration.
439
+
440
+ ### 2. Build Only S3 Backend
441
+
442
+ **Rejected**: Many users deploy on Railway or Render with persistent volumes (no S3). Supporting local filesystem from day one makes svc-infra accessible to all deployment scenarios.
443
+
444
+ ### 3. Store Files in Database (PostgreSQL BYTEA)
445
+
446
+ **Rejected**: Database storage doesn't scale well for large files. BYTEA columns increase backup size, slow down queries, and don't support features like CDN delivery or image transformations.
447
+
448
+ ### 4. Use APF Payments for File Storage
449
+
450
+ **Rejected**: APF Payments is for payment processing, not file storage. Mixing concerns would violate separation of concerns principle.
451
+
452
+ ## Consequences
453
+
454
+ ### Positive
455
+
456
+ 1. **Reusability**: All applications built on svc-infra get file storage for free
457
+ 2. **Flexibility**: Switch backends without code changes (local → S3 → GCS)
458
+ 3. **Railway-Friendly**: Works perfectly with Railway persistent volumes
459
+ 4. **Testing**: MemoryBackend makes unit tests fast and isolated
460
+ 5. **Security**: Signed URLs and validation baked in from day one
461
+ 6. **Observability**: Built-in metrics and logging for production monitoring
462
+
463
+ ### Negative
464
+
465
+ 1. **Maintenance Burden**: Need to maintain multiple backend implementations
466
+ 2. **Dependency Management**: Each backend adds dependencies (boto3, google-cloud-storage, etc.)
467
+ 3. **Testing Complexity**: Need credentials for integration tests
468
+ 4. **Learning Curve**: Users need to understand backend configuration
469
+
470
+ ### Neutral
471
+
472
+ 1. **API Surface**: Adds new public APIs to svc-infra
473
+ 2. **Documentation**: Requires comprehensive docs for each backend
474
+ 3. **Migration**: Existing apps need to migrate from placeholder storage
475
+
476
+ ## Implementation Plan
477
+
478
+ See `.github/PLAN.md` Section 22 for detailed implementation checklist.
479
+
480
+ **Priority**: MUST-HAVE for v1 (foundational infrastructure)
481
+
482
+ **Timeline Estimate**: ~15 days (3 weeks, 1 developer)
483
+
484
+ **Dependencies**: None (can be built in parallel)
485
+
486
+ ## References
487
+
488
+ - [AWS S3 API](https://docs.aws.amazon.com/s3/)
489
+ - [DigitalOcean Spaces](https://docs.digitalocean.com/products/spaces/)
490
+ - [Railway Volumes](https://docs.railway.app/reference/volumes)
491
+ - [Google Cloud Storage](https://cloud.google.com/storage/docs)
492
+ - [Cloudinary API](https://cloudinary.com/documentation)
493
+ - [PEP 544: Protocols](https://peps.python.org/pep-0544/)
494
+ - [FastAPI Dependency Injection](https://fastapi.tiangolo.com/tutorial/dependencies/)
495
+
496
+ ## Revision History
497
+
498
+ - 2025-11-17: Initial draft (Research & Design phase)
svc_infra/docs/api.md ADDED
@@ -0,0 +1,186 @@
1
+ # FastAPI helper guide
2
+
3
+ The `svc_infra.api.fastapi` package provides a one-call bootstrap (`easy_service_app`) that wires request IDs, idempotency, rate limiting, and shared docs defaults for every mounted version. 【F:src/svc_infra/api/fastapi/ease.py†L176-L220】【F:src/svc_infra/api/fastapi/setup.py†L55-L129】
4
+
5
+ ```python
6
+ from svc_infra.api.fastapi.ease import easy_service_app
7
+
8
+ app = easy_service_app(
9
+ name="Payments",
10
+ release="1.0.0",
11
+ versions=[("v1", "myapp.api.v1", None)],
12
+ public_cors_origins=["https://app.example.com"],
13
+ )
14
+ ```
15
+
16
+ ### Environment
17
+
18
+ `easy_service_app` merges explicit flags with `EasyAppOptions.from_env()` so you can flip behavior without code changes:
19
+
20
+ - `ENABLE_LOGGING`, `LOG_LEVEL`, `LOG_FORMAT` – control structured logging defaults. 【F:src/svc_infra/api/fastapi/ease.py†L67-L104】
21
+ - `ENABLE_OBS`, `METRICS_PATH`, `OBS_SKIP_PATHS` – opt into Prometheus/OTEL middleware and tweak metrics exposure. 【F:src/svc_infra/api/fastapi/ease.py†L67-L111】
22
+ - `CORS_ALLOW_ORIGINS` – add allow-listed origins when you don’t pass `public_cors_origins`. 【F:src/svc_infra/api/fastapi/setup.py†L47-L88】
23
+
24
+ ## Quickstart
25
+
26
+ Use `easy_service_app` for a batteries-included FastAPI with sensible defaults:
27
+
28
+ Inputs
29
+ - name: service display name used in docs and logs
30
+ - release: version string (shown in docs and headers)
31
+ - versions: list of tuples of (prefix, import_path, router_name_or_None)
32
+ - public_cors_origins: list of allowed origins for CORS (default deny if omitted)
33
+
34
+ Defaults
35
+ - Logging: enabled with JSON or plain format based on `LOG_FORMAT`; level from `LOG_LEVEL`
36
+ - Observability: Prometheus metrics and OTEL when `ENABLE_OBS=true`; metrics path from `METRICS_PATH` (default `/metrics`)
37
+ - Security headers: strict defaults; CORS disabled unless allowlist provided or `CORS_ALLOW_ORIGINS` set
38
+ - Health: `/ping`, `/healthz`, `/readyz`, `/startupz` are wired
39
+
40
+ Example
41
+ ```python
42
+ from svc_infra.api.fastapi.ease import easy_service_app
43
+
44
+ app = easy_service_app(
45
+ name="Example API",
46
+ release="1.0.0",
47
+ versions=[("v1", "example.api.v1", None)],
48
+ public_cors_origins=["https://app.example.com"],
49
+ )
50
+ ```
51
+
52
+ Override with environment
53
+ ```bash
54
+ export ENABLE_LOGGING=true
55
+ export LOG_LEVEL=INFO
56
+ export ENABLE_OBS=true
57
+ export METRICS_PATH=/metrics
58
+ export CORS_ALLOW_ORIGINS=https://app.example.com,https://admin.example.com
59
+ ```
60
+
61
+ ## Integration Helpers
62
+
63
+ svc-infra provides one-line `add_*` helpers to integrate common functionality into your FastAPI application. Each helper follows the same pattern: wire dependencies, register lifecycle hooks, and expose via `app.state` for dependency injection.
64
+
65
+ ### Storage (`add_storage`)
66
+
67
+ Add file storage backend with auto-detection or explicit configuration.
68
+
69
+ ```python
70
+ from fastapi import FastAPI, Depends, UploadFile
71
+ from svc_infra.storage import add_storage, get_storage, StorageBackend
72
+
73
+ app = FastAPI()
74
+
75
+ # Auto-detect backend from environment (Railway, S3, etc.)
76
+ storage = add_storage(app)
77
+
78
+ # Or explicit backend
79
+ from svc_infra.storage import easy_storage
80
+ backend = easy_storage(backend="s3", bucket="my-uploads")
81
+ storage = add_storage(app, backend)
82
+
83
+ # With file serving for LocalBackend
84
+ backend = easy_storage(backend="local")
85
+ storage = add_storage(app, backend, serve_files=True)
86
+
87
+ # Use in routes via dependency injection
88
+ @app.post("/upload")
89
+ async def upload_file(
90
+ file: UploadFile,
91
+ storage: StorageBackend = Depends(get_storage),
92
+ ):
93
+ content = await file.read()
94
+ url = await storage.put(
95
+ key=f"uploads/{file.filename}",
96
+ data=content,
97
+ content_type=file.content_type or "application/octet-stream",
98
+ metadata={"user": "current_user"}
99
+ )
100
+ return {"url": url}
101
+ ```
102
+
103
+ **Environment variables**:
104
+ - `STORAGE_BACKEND`: Backend type (`local`, `s3`, `memory`) or auto-detect
105
+ - `STORAGE_S3_BUCKET`, `STORAGE_S3_REGION`: S3 configuration
106
+ - `STORAGE_BASE_PATH`: Local backend directory (default: `/data/uploads`)
107
+ - Auto-detects Railway volumes via `RAILWAY_VOLUME_MOUNT_PATH`
108
+
109
+ **See**: [Storage Guide](storage.md) for comprehensive documentation.
110
+
111
+ ### Database (`add_sql_db`)
112
+
113
+ Wire SQLAlchemy connection with health checks and lifecycle management.
114
+
115
+ ```python
116
+ from svc_infra.api.fastapi.db.sql.add import add_sql_db
117
+
118
+ app = FastAPI()
119
+ add_sql_db(app) # Reads SQL_URL or DB_* environment variables
120
+ ```
121
+
122
+ **See**: [Database Guide](database.md)
123
+
124
+ ### Auth (`add_auth_users`)
125
+
126
+ Wire FastAPI Users with sessions, OAuth, MFA, and API keys.
127
+
128
+ ```python
129
+ from svc_infra.api.fastapi.auth.add import add_auth_users
130
+
131
+ app = FastAPI()
132
+ add_auth_users(app, User, UserCreate, UserRead, UserUpdate)
133
+ ```
134
+
135
+ **See**: [Auth Guide](auth.md)
136
+
137
+ ### Observability (`add_observability`)
138
+
139
+ Add Prometheus metrics, request tracking, and health endpoints.
140
+
141
+ ```python
142
+ from svc_infra.obs.add import add_observability
143
+
144
+ app = FastAPI()
145
+ add_observability(app) # Honors ENABLE_OBS, METRICS_PATH environment variables
146
+ ```
147
+
148
+ **See**: [Observability Guide](observability.md)
149
+
150
+ ### Webhooks (`add_webhooks`)
151
+
152
+ Wire webhook producer and verification middleware.
153
+
154
+ ```python
155
+ from svc_infra.webhooks.add import add_webhooks
156
+
157
+ app = FastAPI()
158
+ add_webhooks(app) # Mounts /_webhooks routes and verification middleware
159
+ ```
160
+
161
+ **See**: [Webhooks Guide](webhooks.md)
162
+
163
+ ### Jobs (`easy_jobs`)
164
+
165
+ Initialize job queue and scheduler.
166
+
167
+ ```python
168
+ from svc_infra.jobs.easy import easy_jobs
169
+
170
+ queue, scheduler = easy_jobs() # Reads JOBS_DRIVER, REDIS_URL
171
+ ```
172
+
173
+ **See**: [Jobs Guide](jobs.md)
174
+
175
+ ### Pattern
176
+
177
+ All integration helpers follow this pattern:
178
+
179
+ 1. **Accept app instance**: `add_*(app, ...)`
180
+ 2. **Auto-configure from environment**: Read from env vars with sensible defaults
181
+ 3. **Store in app.state**: Make available via `app.state.storage`, `app.state.db`, etc.
182
+ 4. **Provide dependency**: Export `get_*` function for route injection
183
+ 5. **Register lifecycle hooks**: Handle startup/shutdown (connection pools, cleanup)
184
+ 6. **Add health checks**: Integrate with `/healthz` endpoints
185
+
186
+ This enables one-line integration with zero configuration in most cases, while supporting explicit overrides when needed.
svc_infra/docs/auth.md ADDED
@@ -0,0 +1,11 @@
1
+ # Auth settings
2
+
3
+ `svc_infra.api.fastapi.auth` wraps FastAPI Users with sensible defaults for sessions, OAuth, MFA, and API keys via `add_auth_users`. Configuration comes from `AuthSettings`, which reads environment variables with the `AUTH_` prefix. 【F:src/svc_infra/api/fastapi/auth/add.py†L240-L321】【F:src/svc_infra/api/fastapi/auth/settings.py†L23-L91】
4
+
5
+ ### Key environment variables
6
+
7
+ - `AUTH_JWT__SECRET`, `AUTH_JWT__OLD_SECRETS` – rotate signing keys without downtime. 【F:docs/security.md†L63-L70】
8
+ - `AUTH_SMTP_HOST`, `AUTH_SMTP_USERNAME`, `AUTH_SMTP_PASSWORD`, `AUTH_SMTP_FROM` – enable SMTP delivery; required in production. 【F:src/svc_infra/api/fastapi/auth/settings.py†L44-L60】【F:src/svc_infra/api/fastapi/auth/sender.py†L33-L59】
9
+ - `AUTH_SESSION_COOKIE_SECURE`, `AUTH_SESSION_COOKIE_NAME`, `AUTH_SESSION_COOKIE_SAMESITE` – shape session middleware. 【F:src/svc_infra/api/fastapi/auth/settings.py†L65-L88】【F:src/svc_infra/api/fastapi/auth/add.py†L279-L303】
10
+ - `AUTH_PASSWORD_MIN_LENGTH`, `AUTH_PASSWORD_REQUIRE_SYMBOL`, `AUTH_PASSWORD_BREACH_CHECK` – enforce password policy. 【F:docs/security.md†L24-L35】
11
+ - `AUTH_MFA_DEFAULT_ENABLED_FOR_NEW_USERS`, `AUTH_MFA_ENFORCE_FOR_ALL_USERS` – adjust MFA enforcement. 【F:src/svc_infra/api/fastapi/auth/settings.py†L32-L40】