svc-infra 0.1.600__py3-none-any.whl → 0.1.640__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of svc-infra might be problematic. Click here for more details.
- svc_infra/api/fastapi/admin/__init__.py +3 -0
- svc_infra/api/fastapi/admin/add.py +231 -0
- svc_infra/api/fastapi/billing/router.py +64 -0
- svc_infra/api/fastapi/billing/setup.py +19 -0
- svc_infra/api/fastapi/db/sql/add.py +32 -13
- svc_infra/api/fastapi/db/sql/crud_router.py +178 -16
- svc_infra/api/fastapi/db/sql/session.py +16 -0
- svc_infra/api/fastapi/dependencies/ratelimit.py +57 -7
- svc_infra/api/fastapi/docs/add.py +160 -0
- svc_infra/api/fastapi/docs/landing.py +1 -1
- svc_infra/api/fastapi/middleware/errors/handlers.py +45 -7
- svc_infra/api/fastapi/middleware/graceful_shutdown.py +87 -0
- svc_infra/api/fastapi/middleware/ratelimit.py +59 -1
- svc_infra/api/fastapi/middleware/ratelimit_store.py +12 -6
- svc_infra/api/fastapi/middleware/timeout.py +148 -0
- svc_infra/api/fastapi/openapi/mutators.py +114 -0
- svc_infra/api/fastapi/ops/add.py +73 -0
- svc_infra/api/fastapi/pagination.py +3 -1
- svc_infra/api/fastapi/routers/ping.py +1 -0
- svc_infra/api/fastapi/setup.py +11 -1
- svc_infra/api/fastapi/tenancy/add.py +19 -0
- svc_infra/api/fastapi/tenancy/context.py +112 -0
- svc_infra/app/README.md +5 -5
- svc_infra/billing/__init__.py +23 -0
- svc_infra/billing/async_service.py +147 -0
- svc_infra/billing/jobs.py +230 -0
- svc_infra/billing/models.py +131 -0
- svc_infra/billing/quotas.py +101 -0
- svc_infra/billing/schemas.py +33 -0
- svc_infra/billing/service.py +115 -0
- svc_infra/bundled_docs/README.md +5 -0
- svc_infra/bundled_docs/__init__.py +1 -0
- svc_infra/bundled_docs/getting-started.md +6 -0
- svc_infra/cache/__init__.py +4 -0
- svc_infra/cache/add.py +158 -0
- svc_infra/cache/backend.py +5 -2
- svc_infra/cache/decorators.py +19 -1
- svc_infra/cache/keys.py +24 -4
- svc_infra/cli/__init__.py +28 -8
- svc_infra/cli/cmds/__init__.py +8 -0
- svc_infra/cli/cmds/db/nosql/mongo/mongo_cmds.py +4 -3
- svc_infra/cli/cmds/db/nosql/mongo/mongo_scaffold_cmds.py +4 -4
- svc_infra/cli/cmds/db/sql/alembic_cmds.py +80 -11
- svc_infra/cli/cmds/db/sql/sql_export_cmds.py +80 -0
- svc_infra/cli/cmds/db/sql/sql_scaffold_cmds.py +3 -3
- svc_infra/cli/cmds/docs/docs_cmds.py +140 -0
- svc_infra/cli/cmds/dx/__init__.py +12 -0
- svc_infra/cli/cmds/dx/dx_cmds.py +99 -0
- svc_infra/cli/cmds/help.py +4 -0
- svc_infra/cli/cmds/obs/obs_cmds.py +4 -3
- svc_infra/cli/cmds/sdk/__init__.py +0 -0
- svc_infra/cli/cmds/sdk/sdk_cmds.py +102 -0
- svc_infra/data/add.py +61 -0
- svc_infra/data/backup.py +53 -0
- svc_infra/data/erasure.py +45 -0
- svc_infra/data/fixtures.py +40 -0
- svc_infra/data/retention.py +55 -0
- svc_infra/db/nosql/mongo/README.md +13 -13
- svc_infra/db/sql/repository.py +51 -11
- svc_infra/db/sql/resource.py +5 -0
- svc_infra/db/sql/templates/setup/env_async.py.tmpl +9 -1
- svc_infra/db/sql/templates/setup/env_sync.py.tmpl +9 -2
- svc_infra/db/sql/tenant.py +79 -0
- svc_infra/db/sql/utils.py +18 -4
- svc_infra/docs/acceptance-matrix.md +71 -0
- svc_infra/docs/acceptance.md +44 -0
- svc_infra/docs/admin.md +425 -0
- svc_infra/docs/adr/0002-background-jobs-and-scheduling.md +40 -0
- svc_infra/docs/adr/0003-webhooks-framework.md +24 -0
- svc_infra/docs/adr/0004-tenancy-model.md +42 -0
- svc_infra/docs/adr/0005-data-lifecycle.md +86 -0
- svc_infra/docs/adr/0006-ops-slos-and-metrics.md +47 -0
- svc_infra/docs/adr/0007-docs-and-sdks.md +83 -0
- svc_infra/docs/adr/0008-billing-primitives.md +143 -0
- svc_infra/docs/adr/0009-acceptance-harness.md +40 -0
- svc_infra/docs/adr/0010-timeouts-and-resource-limits.md +54 -0
- svc_infra/docs/adr/0011-admin-scope-and-impersonation.md +73 -0
- svc_infra/docs/api.md +59 -0
- svc_infra/docs/auth.md +11 -0
- svc_infra/docs/billing.md +190 -0
- svc_infra/docs/cache.md +76 -0
- svc_infra/docs/cli.md +74 -0
- svc_infra/docs/contributing.md +34 -0
- svc_infra/docs/data-lifecycle.md +52 -0
- svc_infra/docs/database.md +14 -0
- svc_infra/docs/docs-and-sdks.md +62 -0
- svc_infra/docs/environment.md +114 -0
- svc_infra/docs/getting-started.md +63 -0
- svc_infra/docs/idempotency.md +111 -0
- svc_infra/docs/jobs.md +67 -0
- svc_infra/docs/observability.md +16 -0
- svc_infra/docs/ops.md +37 -0
- svc_infra/docs/rate-limiting.md +125 -0
- svc_infra/docs/repo-review.md +48 -0
- svc_infra/docs/security.md +176 -0
- svc_infra/docs/tenancy.md +35 -0
- svc_infra/docs/timeouts-and-resource-limits.md +147 -0
- svc_infra/docs/webhooks.md +112 -0
- svc_infra/dx/add.py +63 -0
- svc_infra/dx/changelog.py +74 -0
- svc_infra/dx/checks.py +67 -0
- svc_infra/http/__init__.py +13 -0
- svc_infra/http/client.py +72 -0
- svc_infra/jobs/builtins/webhook_delivery.py +14 -2
- svc_infra/jobs/queue.py +9 -1
- svc_infra/jobs/runner.py +75 -0
- svc_infra/jobs/worker.py +17 -1
- svc_infra/mcp/svc_infra_mcp.py +85 -28
- svc_infra/obs/add.py +54 -7
- svc_infra/obs/grafana/dashboards/http-overview.json +45 -0
- svc_infra/security/headers.py +15 -2
- svc_infra/security/hibp.py +6 -2
- svc_infra/security/permissions.py +1 -0
- svc_infra/webhooks/service.py +10 -2
- {svc_infra-0.1.600.dist-info → svc_infra-0.1.640.dist-info}/METADATA +40 -14
- {svc_infra-0.1.600.dist-info → svc_infra-0.1.640.dist-info}/RECORD +118 -44
- {svc_infra-0.1.600.dist-info → svc_infra-0.1.640.dist-info}/WHEEL +0 -0
- {svc_infra-0.1.600.dist-info → svc_infra-0.1.640.dist-info}/entry_points.txt +0 -0
svc_infra/docs/jobs.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Background Jobs & Scheduling
|
|
2
|
+
|
|
3
|
+
This module provides a lightweight JobQueue abstraction with a Redis backend for production and in-memory implementations for local/tests. A simple interval scheduler and CLI runner are included.
|
|
4
|
+
|
|
5
|
+
> ℹ️ Job-related environment variables are documented in [Environment Reference](environment.md).
|
|
6
|
+
|
|
7
|
+
## Quickstart
|
|
8
|
+
|
|
9
|
+
- Initialize in app code:
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
from svc_infra.jobs.easy import easy_jobs
|
|
13
|
+
queue, scheduler = easy_jobs() # uses JOBS_DRIVER=memory|redis
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
- Enqueue a job:
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
job = queue.enqueue("send_email", {"to": "user@example.com"})
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
- Process one job (async):
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from svc_infra.jobs.worker import process_one
|
|
26
|
+
await process_one(queue, handler)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
- Run the CLI runner:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
svc-infra jobs run
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Redis Backend
|
|
36
|
+
|
|
37
|
+
Set environment variables:
|
|
38
|
+
- JOBS_DRIVER=redis
|
|
39
|
+
- REDIS_URL=redis://localhost:6379/0
|
|
40
|
+
|
|
41
|
+
Features:
|
|
42
|
+
- Visibility timeout during processing
|
|
43
|
+
- Exponential backoff on failures (base backoff_seconds * attempts)
|
|
44
|
+
- DLQ after max_attempts
|
|
45
|
+
|
|
46
|
+
## Scheduler
|
|
47
|
+
|
|
48
|
+
An interval-based scheduler runs async callables. You can define tasks via environment JSON:
|
|
49
|
+
|
|
50
|
+
- JOBS_SCHEDULE_JSON:
|
|
51
|
+
|
|
52
|
+
```json
|
|
53
|
+
[
|
|
54
|
+
{"name": "ping", "interval_seconds": 60, "target": "myapp.tasks:ping"}
|
|
55
|
+
]
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The `target` must be an import path of the form `module:function`. Sync functions are wrapped.
|
|
59
|
+
|
|
60
|
+
## Testing
|
|
61
|
+
|
|
62
|
+
- In-memory queue and scheduler enable fast, deterministic tests.
|
|
63
|
+
- Redis tests use `fakeredis` and cover enqueue/reserve/ack/fail and DLQ path.
|
|
64
|
+
|
|
65
|
+
## Next
|
|
66
|
+
|
|
67
|
+
- Add metrics, Redis Lua for atomic multi-ops, SQL-backed queue, distributed scheduler.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Observability guide
|
|
2
|
+
|
|
3
|
+
`svc_infra.obs` adds Prometheus metrics, OpenTelemetry instrumentation, and a CLI to spin up Grafana dashboards.
|
|
4
|
+
|
|
5
|
+
```python
|
|
6
|
+
from svc_infra.api.fastapi.ease import easy_service_app
|
|
7
|
+
|
|
8
|
+
app = easy_service_app(name="Billing", release="1.2.3") # ENABLE_OBS toggles middleware
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
### Environment
|
|
12
|
+
|
|
13
|
+
- `ENABLE_OBS`, `METRICS_PATH`, `OBS_SKIP_PATHS` – FastAPI bootstrap toggles for metrics exposure. 【F:src/svc_infra/api/fastapi/ease.py†L67-L111】
|
|
14
|
+
- `SVC_INFRA_DISABLE_PROMETHEUS` – disable the ASGI middleware entirely (used by tests/sidecars). 【F:src/svc_infra/obs/metrics/asgi.py†L49-L206】
|
|
15
|
+
- `SVC_INFRA_RATE_WINDOW`, `SVC_INFRA_DASHBOARD_REFRESH`, `SVC_INFRA_DASHBOARD_RANGE` – tune the Grafana dashboards generated by the CLI. 【F:src/svc_infra/obs/cloud_dash.py†L85-L108】
|
|
16
|
+
- Grafana Cloud support looks for `GRAFANA_CLOUD_URL`, `GRAFANA_CLOUD_TOKEN`, `GRAFANA_CLOUD_PROM_URL`, `GRAFANA_CLOUD_PROM_USERNAME`, `GRAFANA_CLOUD_RW_TOKEN`. 【F:src/svc_infra/obs/README.md†L37-L119】
|
svc_infra/docs/ops.md
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# SLOs & Ops
|
|
2
|
+
|
|
3
|
+
This guide explains how to use svc-infra’s probes, circuit breaker, and metrics for SLOs.
|
|
4
|
+
|
|
5
|
+
## Probes
|
|
6
|
+
|
|
7
|
+
- `add_probes(app, prefix="/_ops")` exposes:
|
|
8
|
+
- `GET /_ops/live` — liveness
|
|
9
|
+
- `GET /_ops/ready` — readiness
|
|
10
|
+
- `GET /_ops/startup` — startup
|
|
11
|
+
|
|
12
|
+
## Circuit breaker (placeholder)
|
|
13
|
+
|
|
14
|
+
- `circuit_breaker_dependency()` returns a dependency that returns 503 when `CIRCUIT_OPEN` is truthy.
|
|
15
|
+
- Use it per-route: `@app.get("/x", dependencies=[Depends(circuit_breaker_dependency())])`.
|
|
16
|
+
|
|
17
|
+
## Metrics and route classification
|
|
18
|
+
|
|
19
|
+
- `add_observability(app, ...)` enables Prometheus metrics and optional DB pool metrics.
|
|
20
|
+
- You can pass an optional `route_classifier(base_path, method) -> class` callable. When provided, the metrics middleware encodes the resolved route label as `"{base}|{class}"`. Dashboards can split this label to filter public/internal/admin routes.
|
|
21
|
+
|
|
22
|
+
## Dashboards
|
|
23
|
+
|
|
24
|
+
- A minimal Grafana dashboard JSON is provided at `src/svc_infra/obs/grafana/dashboards/http-overview.json` (import into Grafana).
|
|
25
|
+
- It shows:
|
|
26
|
+
- Success rate over 5 minutes
|
|
27
|
+
- p99 latency
|
|
28
|
+
- Top routes by 5xx rate
|
|
29
|
+
|
|
30
|
+
## Defaults & environment
|
|
31
|
+
|
|
32
|
+
- Prometheus middleware is enabled unless `SVC_INFRA_DISABLE_PROMETHEUS=1`.
|
|
33
|
+
- Observability settings: `METRICS_ENABLED`, `METRICS_PATH`, and optional histogram buckets.
|
|
34
|
+
|
|
35
|
+
## See also
|
|
36
|
+
|
|
37
|
+
- Timeouts & Resource Limits: `./timeouts-and-resource-limits.md` — request/body/handler timeouts, outbound client timeouts, DB statement timeouts, jobs/webhooks, and graceful shutdown.
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# Rate Limiting & Abuse Protection
|
|
2
|
+
|
|
3
|
+
This guide shows how to enable and tune the built-in rate limiter and request-size guard, and how to hook simple metrics for abuse detection.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- Global middleware-based rate limiting with standard headers
|
|
8
|
+
- Per-route dependency for fine-grained limits
|
|
9
|
+
- 429 responses include `Retry-After`
|
|
10
|
+
- Pluggable store interface (in-memory provided; Redis store available)
|
|
11
|
+
- Request size limit middleware returning 413
|
|
12
|
+
- Metrics hooks for rate-limiting events and suspect payloads
|
|
13
|
+
|
|
14
|
+
## Global middleware
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from svc_infra.api.fastapi.middleware.ratelimit import SimpleRateLimitMiddleware
|
|
18
|
+
|
|
19
|
+
app.add_middleware(
|
|
20
|
+
SimpleRateLimitMiddleware,
|
|
21
|
+
limit=120, # requests
|
|
22
|
+
window=60, # seconds
|
|
23
|
+
key_fn=lambda r: r.headers.get("X-API-Key") or r.client.host,
|
|
24
|
+
)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Responses include headers:
|
|
28
|
+
- `X-RateLimit-Limit`
|
|
29
|
+
- `X-RateLimit-Remaining`
|
|
30
|
+
- `X-RateLimit-Reset` (epoch seconds)
|
|
31
|
+
|
|
32
|
+
When exceeded, responses are 429 with `Retry-After` and the same headers.
|
|
33
|
+
|
|
34
|
+
Advanced options:
|
|
35
|
+
- `scope_by_tenant=True` to automatically include tenant id in the key if tenancy is configured.
|
|
36
|
+
- `limit_resolver=request -> int | None` to override the limit dynamically per request (e.g., per-plan quotas).
|
|
37
|
+
|
|
38
|
+
## Per-route dependency
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from fastapi import Depends
|
|
42
|
+
from svc_infra.api.fastapi.dependencies.ratelimit import rate_limiter
|
|
43
|
+
|
|
44
|
+
limiter = rate_limiter(limit=10, window=60, key_fn=lambda r: r.client.host)
|
|
45
|
+
|
|
46
|
+
@app.get("/resource", dependencies=[Depends(limiter)])
|
|
47
|
+
def get_resource():
|
|
48
|
+
return {"ok": True}
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The dependency supports the same options as the middleware: `scope_by_tenant`, `limit_resolver`, and a custom `store`.
|
|
52
|
+
|
|
53
|
+
## Store interface
|
|
54
|
+
|
|
55
|
+
The limiter uses a store abstraction so you can inject Redis or other backends.
|
|
56
|
+
|
|
57
|
+
- Default: `InMemoryRateLimitStore` (best-effort, single-process)
|
|
58
|
+
- Interface: `RateLimitStore` with `incr(key, window) -> (count, limit, resetEpoch)`
|
|
59
|
+
|
|
60
|
+
### Redis store
|
|
61
|
+
|
|
62
|
+
Use `RedisRateLimitStore` for multi-instance deployments. It implements a fixed-window counter
|
|
63
|
+
with atomic increments and sets expiry to the end of the window.
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
import redis
|
|
67
|
+
from svc_infra.api.fastapi.middleware.ratelimit_store import RedisRateLimitStore
|
|
68
|
+
from svc_infra.api.fastapi.middleware.ratelimit import SimpleRateLimitMiddleware
|
|
69
|
+
|
|
70
|
+
r = redis.Redis.from_url("redis://localhost:6379/0")
|
|
71
|
+
store = RedisRateLimitStore(r, limit=120, prefix="rl")
|
|
72
|
+
|
|
73
|
+
app.add_middleware(SimpleRateLimitMiddleware, limit=120, window=60, store=store)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Notes:
|
|
77
|
+
- Fixed-window counters are simple and usually sufficient. For smoother limits, consider
|
|
78
|
+
sliding window or token bucket in a future iteration.
|
|
79
|
+
- Use a namespace/prefix per environment/tenant if needed.
|
|
80
|
+
|
|
81
|
+
## Request size guard
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from svc_infra.api.fastapi.middleware.request_size_limit import RequestSizeLimitMiddleware
|
|
85
|
+
|
|
86
|
+
app.add_middleware(RequestSizeLimitMiddleware, max_bytes=1_000_000)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
- Returns 413 with a Problem+JSON-like structure when `Content-Length` exceeds `max_bytes`.
|
|
90
|
+
|
|
91
|
+
## Metrics hooks
|
|
92
|
+
|
|
93
|
+
Hooks live in `svc_infra.obs.metrics` and are no-ops by default. Assign them to log or emit metrics.
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
import logging
|
|
97
|
+
import svc_infra.obs.metrics as metrics
|
|
98
|
+
|
|
99
|
+
logger = logging.getLogger(__name__)
|
|
100
|
+
|
|
101
|
+
metrics.on_rate_limit_exceeded = lambda key, limit, retry: logger.warning(
|
|
102
|
+
"rate_limited", extra={"key": key, "limit": limit, "retry_after": retry}
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
metrics.on_suspect_payload = lambda path, size: logger.warning(
|
|
106
|
+
"suspect_payload", extra={"path": path, "size": size}
|
|
107
|
+
)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Tuning tips
|
|
111
|
+
|
|
112
|
+
- Prefer API key or user ID for `key_fn`; fall back to IP if unauthenticated.
|
|
113
|
+
- Keep windows small (e.g., 60s); layer multiple limits when needed.
|
|
114
|
+
- For distributed deployments, use `RedisRateLimitStore` for atomic increments.
|
|
115
|
+
- Consider separate limits for read vs write routes.
|
|
116
|
+
- Combine with request size limits and auth lockout for layered defense.
|
|
117
|
+
|
|
118
|
+
## Related
|
|
119
|
+
|
|
120
|
+
- Timeouts & Resource Limits: `./timeouts-and-resource-limits.md` — complements rate limits by bounding slow uploads, long handlers, and downstream timeouts.
|
|
121
|
+
|
|
122
|
+
## Testing
|
|
123
|
+
|
|
124
|
+
- Use `-m ratelimit` to select rate-limiting tests.
|
|
125
|
+
- `-m security` also includes these in this repo by default.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# svc-infra Integration Review
|
|
2
|
+
|
|
3
|
+
## Executive Summary
|
|
4
|
+
- `setup_service_api` remains the high-level entry point for API construction. It auto-wires common middleware (request IDs, error handling, idempotency, rate limiting) and OpenAPI customisation so downstream services only supply router packages and metadata. 【F:src/svc_infra/api/fastapi/setup.py†L1-L187】
|
|
5
|
+
- Auth, SQL, jobs, cache, and observability still expose single-call helpers (`add_auth_users`, `add_sql_db`/`setup_sql`, `easy_jobs`, `init_cache`, `add_observability`) that hide wiring details while keeping flags and dependency overrides for advanced scenarios. 【F:src/svc_infra/api/fastapi/auth/add.py†L27-L329】【F:src/svc_infra/api/fastapi/db/sql/add.py†L1-L118】【F:src/svc_infra/jobs/easy.py†L1-L28】【F:src/svc_infra/cache/__init__.py†L1-L29】【F:src/svc_infra/obs/add.py†L1-L68】
|
|
6
|
+
- Security primitives (lockout, session rotation, RBAC/ABAC, audit chain) are packaged as small composable functions and documented patterns, making them easy to reuse outside FastAPI while still integrating with the default auth routers. 【F:src/svc_infra/security/lockout.py†L1-L96】【F:src/svc_infra/security/session.py†L1-L98】【F:src/svc_infra/security/permissions.py†L1-L148】【F:src/svc_infra/security/audit_service.py†L1-L73】【F:docs/security.md†L1-L96】
|
|
7
|
+
- Webhook utilities provide signature verification, a subscription service, and a default router, but they do not yet follow the same add-helper pattern (there is no `add_webhooks(app, ...)` or env-driven persistence wiring). This is the primary deviation from the established DX story. 【F:src/svc_infra/webhooks/fastapi.py†L1-L37】【F:src/svc_infra/webhooks/service.py†L1-L45】【F:src/svc_infra/webhooks/router.py†L1-L55】【F:docs/webhooks.md†L1-L59】
|
|
8
|
+
|
|
9
|
+
## Detailed Findings
|
|
10
|
+
|
|
11
|
+
### API bootstrap and middleware
|
|
12
|
+
- `setup_service_api` builds a parent app and any versioned child apps, runs the OpenAPI mutation pipeline, registers default routers, and attaches middleware without extra code in downstream projects. 【F:src/svc_infra/api/fastapi/setup.py†L1-L187】
|
|
13
|
+
- Scoped docs helpers automatically group endpoints when `add_auth_users` or versioned routers are mounted, so projects get organised API docs with minimal configuration. 【F:src/svc_infra/api/fastapi/auth/add.py†L269-L329】
|
|
14
|
+
- Idempotency and optimistic locking remain opt-in via middleware or dependencies with clear docs; defaults are safe for local use and can be swapped for Redis-backed stores. 【F:docs/idempotency.md†L1-L110】
|
|
15
|
+
|
|
16
|
+
### Authentication and security
|
|
17
|
+
- `add_auth_users` continues to be the main integration surface. It mounts routers based on flags, configures session middleware from env, and exposes API-key plus OAuth support as toggles. Projects can override policy objects or provider models if needed, preserving flexibility. 【F:src/svc_infra/api/fastapi/auth/add.py†L27-L329】
|
|
18
|
+
- Runtime security helpers (principal resolution, `RequireRoles`, `RequireScopes`, `RequirePermission`, ABAC predicates) make it straightforward to guard routes consistently with the dual routers that enforce doc security defaults. 【F:src/svc_infra/api/fastapi/auth/security.py†L1-L146】【F:src/svc_infra/api/fastapi/dual/protected.py†L1-L74】【F:src/svc_infra/security/permissions.py†L1-L148】
|
|
19
|
+
- Low-level primitives (lockout, refresh token rotation, audit chain) remain framework agnostic so other stacks can reuse them. Documentation in `docs/security.md` is aligned with the code and offers wiring recipes. 【F:src/svc_infra/security/lockout.py†L1-L96】【F:src/svc_infra/security/session.py†L1-L98】【F:src/svc_infra/security/audit_service.py†L1-L73】【F:docs/security.md†L1-L96】
|
|
20
|
+
- Recommendation: add a security bundle similar to `setup_sql` that can register `SecurityHeadersMiddleware` and strict CORS when FastAPI is not created via `setup_service_api`. 【F:src/svc_infra/security/headers.py†L1-L35】
|
|
21
|
+
|
|
22
|
+
### Data and jobs
|
|
23
|
+
- SQL helpers offer both granular (`add_sql_db`, `add_sql_resources`) and bundled (`setup_sql`) flows. They guard against duplicate setup and generate CRUD schemas when not provided, keeping the quick-start feel while allowing custom services. 【F:src/svc_infra/api/fastapi/db/sql/add.py†L1-L118】
|
|
24
|
+
- Jobs expose `easy_jobs()` that switches between memory and Redis based on env, matching the pick-style ergonomics used in logging. Documentation explains scheduler tasks and the CLI runner. 【F:src/svc_infra/jobs/easy.py†L1-L28】【F:docs/jobs.md†L1-L65】
|
|
25
|
+
- Cache decorators export both `cache_read` and the friendlier aliases (`cached`, `mutates`) so consumers can adopt them incrementally. 【F:src/svc_infra/cache/__init__.py†L1-L29】
|
|
26
|
+
|
|
27
|
+
### Observability
|
|
28
|
+
- `add_observability` lazily imports instrumentation, respects env flags, and returns a shutdown no-op to keep signature compatibility. It tolerates missing dependencies to avoid burdening projects that do not enable metrics. 【F:src/svc_infra/obs/add.py†L1-L68】
|
|
29
|
+
- Grafana and Prometheus templates are packaged separately, but the code path aligns with the simple toggle approach used elsewhere.
|
|
30
|
+
|
|
31
|
+
### Webhooks and outbox
|
|
32
|
+
- Signature utilities (`require_signature`, `verify_any`) and service abstractions (`WebhookService`, `InMemoryWebhookSubscriptions`) follow the composable style, and docs describe usage alongside the shared job and outbox infrastructure. 【F:src/svc_infra/webhooks/fastapi.py†L1-L37】【F:src/svc_infra/webhooks/service.py†L1-L45】【F:docs/webhooks.md†L1-L59】
|
|
33
|
+
- Gaps relative to the rest of the repo:
|
|
34
|
+
- Router dependencies always create new in-memory stores, so deployments must override dependencies manually. Providing an `add_webhooks(app, store=..., subs=...)` helper that honours env (for example `REDIS_URL`) would mirror `add_sql_db` and `add_observability`. 【F:src/svc_infra/webhooks/router.py†L1-L55】
|
|
35
|
+
- No convenience function registers the router or scheduler tick automatically; consumers must read docs to wire `make_outbox_tick` and queue processors. A top-level helper such as `setup_webhooks(app, outbox=None, inbox=None)` could bundle router inclusion and scheduler guidance.
|
|
36
|
+
|
|
37
|
+
### Documentation and DX consistency
|
|
38
|
+
- Feature guides (security, jobs, webhooks, idempotency) contain quickstart snippets that mirror the API surface, reinforcing the ease-of-setup story. 【F:docs/security.md†L1-L96】【F:docs/jobs.md†L1-L65】【F:docs/webhooks.md†L1-L59】【F:docs/idempotency.md†L1-L110】
|
|
39
|
+
- README remains empty; adding a high-level index pointing to these guides would help teams discover the single-call helpers faster. 【F:README.md†L1-L4】
|
|
40
|
+
- Consider adding a matrix listing which environment variables drive each helper (`SQL_URL`, `REDIS_URL`, `METRICS_PATH`, `AUTH_*`) to highlight parity across modules.
|
|
41
|
+
|
|
42
|
+
## Actionable Suggestions
|
|
43
|
+
1. **Webhook setup helper** — provide `add_webhooks` or `setup_webhooks` to mount the router, accept dependency overrides, and optionally wire scheduler tasks. Default to env-driven outbox or inbox selection to match other modules. 【F:src/svc_infra/webhooks/router.py†L1-L55】
|
|
44
|
+
2. **Security bundle** — offer a helper that installs `SecurityHeadersMiddleware`, strict CORS, and optional signed-cookie settings for apps that bypass `setup_service_api`. This keeps manual FastAPI apps aligned with the default hardening posture. 【F:src/svc_infra/security/headers.py†L1-L35】
|
|
45
|
+
3. **Documentation index** — populate `README.md` with links to the feature guides and highlight key add-style helpers so new projects immediately see the plug-and-play building blocks. 【F:README.md†L1-L4】
|
|
46
|
+
4. **Environment reference** — add a doc or table enumerating env vars consumed by helpers (SQL_URL, REDIS_URL, METRICS_PATH, AUTH_*), emphasising how swapping values maintains flexibility across deployments.
|
|
47
|
+
|
|
48
|
+
Overall, aside from the webhook ergonomics gap and missing README guidance, the codebase maintains the quick-start plus override pattern highlighted in your sample usage while exposing low-level hooks for teams that need deeper control.
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# Security: Configuration & Examples
|
|
2
|
+
|
|
3
|
+
This guide covers the security primitives built into svc-infra and how to wire them:
|
|
4
|
+
|
|
5
|
+
> ℹ️ Environment variables for the auth/security helpers are catalogued in [Environment Reference](environment.md).
|
|
6
|
+
|
|
7
|
+
- Password policy and breach checking
|
|
8
|
+
- Account lockout (exponential backoff)
|
|
9
|
+
- Sessions and refresh tokens (rotation + revocation)
|
|
10
|
+
- JWT key rotation
|
|
11
|
+
- Signed cookies
|
|
12
|
+
- CORS and security headers
|
|
13
|
+
- RBAC and ABAC
|
|
14
|
+
- MFA policy hooks
|
|
15
|
+
|
|
16
|
+
Module map (examples reference these):
|
|
17
|
+
- `svc_infra.security.lockout` (LockoutConfig, compute_lockout, record_attempt, get_lockout_status)
|
|
18
|
+
- `svc_infra.security.signed_cookies` (sign_cookie, verify_cookie)
|
|
19
|
+
- `svc_infra.security.audit` and `security.audit_service` (hash-chain audit logs)
|
|
20
|
+
- `svc_infra.api.fastapi.auth.gaurd` (password login with lockout checks)
|
|
21
|
+
- `svc_infra.api.fastapi.auth.routers.*` (sessions, oauth routes, etc.)
|
|
22
|
+
- `svc_infra.api.fastapi.auth.settings.get_auth_settings` (cookie + token settings)
|
|
23
|
+
- `svc_infra.api.fastapi.middleware.security_headers` and CORS setup (strict defaults)
|
|
24
|
+
|
|
25
|
+
## Password policy and breach checking
|
|
26
|
+
- Enforced by validators with a configurable policy.
|
|
27
|
+
- Breach checking uses the HIBP k-Anonymity range API; can be toggled via settings.
|
|
28
|
+
|
|
29
|
+
Example toggles (pseudo-config):
|
|
30
|
+
- `AUTH_PASSWORD_MIN_LENGTH=12`
|
|
31
|
+
- `AUTH_PASSWORD_REQUIRE_SYMBOL=True`
|
|
32
|
+
- `AUTH_PASSWORD_BREACH_CHECK=True`
|
|
33
|
+
|
|
34
|
+
## Account lockout
|
|
35
|
+
- Exponential backoff with a max cooldown cap to deter credential stuffing.
|
|
36
|
+
- Attempts tracked by user_id and/or IP hash.
|
|
37
|
+
- Login endpoint blocks with 429 + `Retry-After` during cooldown.
|
|
38
|
+
|
|
39
|
+
Key API (from `svc_infra.security.lockout`):
|
|
40
|
+
- `LockoutConfig(threshold=5, window_minutes=15, base_cooldown_seconds=30, max_cooldown_seconds=3600)`
|
|
41
|
+
- `compute_lockout(fail_count, cfg)` → `LockoutStatus(locked, next_allowed_at, failure_count)`
|
|
42
|
+
- `record_attempt(session, user_id, ip_hash, success)`
|
|
43
|
+
- `get_lockout_status(session, user_id, ip_hash, cfg)`
|
|
44
|
+
|
|
45
|
+
Login integration (simplified):
|
|
46
|
+
```python
|
|
47
|
+
from svc_infra.security.lockout import get_lockout_status, record_attempt
|
|
48
|
+
|
|
49
|
+
# Compute ip_hash from request.client.host
|
|
50
|
+
status = await get_lockout_status(session, user_id=None, ip_hash=ip_hash)
|
|
51
|
+
if status.locked:
|
|
52
|
+
raise HTTPException(429, headers={"Retry-After": ..})
|
|
53
|
+
|
|
54
|
+
user = await user_manager.user_db.get_by_email(email)
|
|
55
|
+
if not user:
|
|
56
|
+
await record_attempt(session, user_id=None, ip_hash=ip_hash, success=False)
|
|
57
|
+
raise HTTPException(400, "LOGIN_BAD_CREDENTIALS")
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Sessions and refresh tokens
|
|
61
|
+
- Sessions are enumerable and revocable via the sessions router.
|
|
62
|
+
- Refresh tokens are rotated; old tokens are invalidated via a revocation list.
|
|
63
|
+
|
|
64
|
+
Operational notes:
|
|
65
|
+
- Persist sessions/tokens in a durable DB.
|
|
66
|
+
- Favor short access token TTLs if refresh flow is robust.
|
|
67
|
+
|
|
68
|
+
## JWT key rotation
|
|
69
|
+
- Primary secret plus `old_secrets` allow seamless rotation.
|
|
70
|
+
- Set environment variables:
|
|
71
|
+
- `AUTH_JWT__SECRET="..."`
|
|
72
|
+
- `AUTH_JWT__OLD_SECRETS="old1,old2"`
|
|
73
|
+
|
|
74
|
+
## Signed cookies
|
|
75
|
+
Module: `svc_infra.security.signed_cookies`
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from svc_infra.security.signed_cookies import sign_cookie, verify_cookie
|
|
79
|
+
|
|
80
|
+
sig = sign_cookie({"sub": "user-123"}, secret="k1", exp_seconds=3600)
|
|
81
|
+
payload = verify_cookie(sig, secret="k1", old_secrets=["k0"]) # returns dict
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## CORS and security headers
|
|
85
|
+
- Strict CORS defaults (deny by default). Provide allowlist entries.
|
|
86
|
+
- Security headers middleware sets common protections (X-Frame-Options, X-Content-Type-Options, etc.).
|
|
87
|
+
|
|
88
|
+
Use `svc_infra.security.add.add_security` to install the default middlewares on any
|
|
89
|
+
FastAPI app. By default it adds:
|
|
90
|
+
|
|
91
|
+
- `SecurityHeadersMiddleware` with practical defaults:
|
|
92
|
+
- **Content-Security-Policy**: Allows same-origin resources, inline styles/scripts, data URI images, and HTTPS images. Blocks external scripts and framing.
|
|
93
|
+
- **Strict-Transport-Security**: Forces HTTPS with long max-age and subdomain support
|
|
94
|
+
- **X-Frame-Options**: Blocks framing (DENY)
|
|
95
|
+
- **X-Content-Type-Options**: Prevents MIME sniffing (nosniff)
|
|
96
|
+
- **Referrer-Policy**: Limits referrer leakage
|
|
97
|
+
- **X-XSS-Protection**: Disabled (CSP is the modern protection)
|
|
98
|
+
- A strict `CORSMiddleware` that only enables CORS when origins are provided (via
|
|
99
|
+
parameters or environment variables such as `CORS_ALLOW_ORIGINS`).
|
|
100
|
+
|
|
101
|
+
The default CSP policy is:
|
|
102
|
+
```
|
|
103
|
+
default-src 'self';
|
|
104
|
+
script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net;
|
|
105
|
+
style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net;
|
|
106
|
+
img-src 'self' data: https:;
|
|
107
|
+
connect-src 'self';
|
|
108
|
+
font-src 'self' https://cdn.jsdelivr.net;
|
|
109
|
+
frame-ancestors 'none';
|
|
110
|
+
base-uri 'self';
|
|
111
|
+
form-action 'self'
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
This works out-of-the-box for most web applications, including FastAPI's built-in documentation (Swagger UI, ReDoc), while maintaining strong security.
|
|
115
|
+
|
|
116
|
+
The helper also supports optional toggles so you can match the same cookie and
|
|
117
|
+
header configuration that `setup_service_api` uses.
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from fastapi import FastAPI
|
|
121
|
+
|
|
122
|
+
from svc_infra.security.add import add_security
|
|
123
|
+
|
|
124
|
+
app = FastAPI()
|
|
125
|
+
|
|
126
|
+
add_security(
|
|
127
|
+
app,
|
|
128
|
+
cors_origins=["https://app.example.com"],
|
|
129
|
+
headers_overrides={"Content-Security-Policy": "default-src 'self'; script-src 'self'"}, # Stricter CSP
|
|
130
|
+
install_session_middleware=True, # adds Starlette's SessionMiddleware
|
|
131
|
+
)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Environment variables (applied when parameters are omitted):
|
|
135
|
+
|
|
136
|
+
| Variable | Purpose |
|
|
137
|
+
| --- | --- |
|
|
138
|
+
| `CORS_ALLOW_ORIGINS` | Comma-separated CORS origins (e.g. `https://app.example.com, https://admin.example.com`) |
|
|
139
|
+
| `CORS_ALLOW_METHODS` | Allowed HTTP methods (defaults to `*`) |
|
|
140
|
+
| `CORS_ALLOW_HEADERS` | Allowed headers (defaults to `*`) |
|
|
141
|
+
| `CORS_ALLOW_ORIGIN_REGEX` | Regex used when matching origins (ignored if not set) |
|
|
142
|
+
| `CORS_ALLOW_CREDENTIALS` | Toggle credentials support (`true` / `false`) |
|
|
143
|
+
| `SESSION_COOKIE_NAME` | Session cookie name (defaults to `svc_session`) |
|
|
144
|
+
| `SESSION_COOKIE_MAX_AGE_SECONDS` | Max age for the session cookie (defaults to `14400`) |
|
|
145
|
+
| `SESSION_COOKIE_SAMESITE` | SameSite policy (`lax` by default) |
|
|
146
|
+
| `SESSION_COOKIE_SECURE` | Force the session cookie to be HTTPS-only |
|
|
147
|
+
| `SESSION_SECRET` | Secret key for Starlette's SessionMiddleware |
|
|
148
|
+
|
|
149
|
+
When your service already uses `setup_service_api`, call `add_security` after
|
|
150
|
+
building the parent app if you need additional overrides while keeping the
|
|
151
|
+
defaults intact:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
from svc_infra.api.fastapi.setup import setup_service_api
|
|
155
|
+
from svc_infra.security.add import add_security
|
|
156
|
+
|
|
157
|
+
app = setup_service_api(...)
|
|
158
|
+
|
|
159
|
+
add_security(
|
|
160
|
+
app,
|
|
161
|
+
headers_overrides={"Strict-Transport-Security": "max-age=63072000; includeSubDomains"},
|
|
162
|
+
enable_hsts_preload=False,
|
|
163
|
+
)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## RBAC and ABAC
|
|
167
|
+
- RBAC decorators guard endpoints by role/permission.
|
|
168
|
+
- ABAC evaluates resource ownership and attributes (e.g., `owns_resource`).
|
|
169
|
+
|
|
170
|
+
## MFA policy hooks
|
|
171
|
+
- Policy decides when MFA is required; login returns 401 with `MFA_REQUIRED` and a pre-token when applicable.
|
|
172
|
+
|
|
173
|
+
## Troubleshooting
|
|
174
|
+
- 429 on login: lockout active. Check `Retry-After` and `FailedAuthAttempt` rows.
|
|
175
|
+
- Token invalid post-refresh: confirm rotation + revocation writes.
|
|
176
|
+
- Cookie verification errors: check signing keys/exp.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Tenancy model and integration
|
|
2
|
+
|
|
3
|
+
This framework uses a soft-tenant isolation model by default: tenant_id is a column on tenant-scoped tables, and all queries are filtered by this value. Consumers can later adopt schema-per-tenant or DB-per-tenant strategies; the API surfaces remain compatible.
|
|
4
|
+
|
|
5
|
+
## How tenant is resolved
|
|
6
|
+
- `resolve_tenant_id(request)` looks up tenant id in this order:
|
|
7
|
+
1) Global override hook (set via `add_tenancy(app, resolver=...)`)
|
|
8
|
+
2) Auth identity (user.tenant_id or api_key.tenant_id) when auth is enabled
|
|
9
|
+
3) `X-Tenant-Id` request header
|
|
10
|
+
4) `request.state.tenant_id`
|
|
11
|
+
|
|
12
|
+
Use `TenantId` dependency to require it in routes, and `OptionalTenantId` to access it if present.
|
|
13
|
+
|
|
14
|
+
## Enforcement in data layer
|
|
15
|
+
- Wrap services with `TenantSqlService` to automatically:
|
|
16
|
+
- Apply `WHERE model.tenant_id == <tenant>` on list/get/update/delete/search/count.
|
|
17
|
+
- Inject `tenant_id` upon create when the model has the tenant field.
|
|
18
|
+
|
|
19
|
+
## Tenant-aware CRUD router
|
|
20
|
+
- When defining a `SqlResource`, set `tenant_field="tenant_id"` to mount a tenant-aware CRUD router. All endpoints will require `TenantId` and enforce scoping.
|
|
21
|
+
|
|
22
|
+
## Per-tenant rate limits / quotas
|
|
23
|
+
- Global middleware and per-route dependency support tenant-aware policies:
|
|
24
|
+
- `scope_by_tenant=True` puts requests in independent buckets per tenant.
|
|
25
|
+
- `limit_resolver(request, tenant_id)` lets you return dynamic limits (e.g., plan-based quotas).
|
|
26
|
+
|
|
27
|
+
## Export a tenant’s data (SQL)
|
|
28
|
+
- CLI command: `sql export-tenant`
|
|
29
|
+
- Example:
|
|
30
|
+
- `python -m svc_infra.cli sql export-tenant items --tenant-id t1 --output out.json`
|
|
31
|
+
- Flags:
|
|
32
|
+
- `--tenant-id` (required), `--tenant-field` (default `tenant_id`), `--limit` (optional), `--database-url` (or set `SQL_URL`).
|
|
33
|
+
|
|
34
|
+
## Migration to other isolation strategies
|
|
35
|
+
- Schema-per-tenant or DB-per-tenant can be layered by adapting the session factory or repository to select the schema/DB based on `tenant_id`. Your application code that relies on `TenantId` and tenant-aware services/routers remains the same.
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# Timeouts & Resource Limits
|
|
2
|
+
|
|
3
|
+
This guide covers request/handler timeouts, outbound HTTP client timeouts, database statement timeouts, job/webhook delivery timeouts, and graceful shutdown. It explains defaults, configuration, wiring, and recommended tuning by environment.
|
|
4
|
+
|
|
5
|
+
## Why timeouts?
|
|
6
|
+
|
|
7
|
+
- Protects your service from slowloris uploads and hanging requests
|
|
8
|
+
- Limits blast radius of slow downstreams (HTTP, DB, webhooks)
|
|
9
|
+
- Enables predictable backpressure and faster recovery during incidents
|
|
10
|
+
|
|
11
|
+
## Configuration overview
|
|
12
|
+
|
|
13
|
+
The library exposes simple environment variables with sensible defaults. Use floats for second values unless noted.
|
|
14
|
+
|
|
15
|
+
- REQUEST_BODY_TIMEOUT_SECONDS (int)
|
|
16
|
+
- Default: prod=15, nonprod=30
|
|
17
|
+
- Purpose: Abort slow request body reads (slowloris defense)
|
|
18
|
+
- REQUEST_TIMEOUT_SECONDS (int)
|
|
19
|
+
- Default: prod=30, nonprod=15
|
|
20
|
+
- Purpose: Cap overall handler execution time
|
|
21
|
+
- HTTP_CLIENT_TIMEOUT_SECONDS (float)
|
|
22
|
+
- Default: 10.0
|
|
23
|
+
- Purpose: Default timeout for outbound httpx clients created via helpers
|
|
24
|
+
- DB_STATEMENT_TIMEOUT_MS (int)
|
|
25
|
+
- Default: unset (disabled)
|
|
26
|
+
- Purpose: Per-transaction statement timeout (Postgres via SET LOCAL)
|
|
27
|
+
- JOB_DEFAULT_TIMEOUT_SECONDS (float)
|
|
28
|
+
- Default: unset (disabled)
|
|
29
|
+
- Purpose: Caps per-job handler runtime in the in-process jobs runner
|
|
30
|
+
- WEBHOOK_DELIVERY_TIMEOUT_SECONDS (float)
|
|
31
|
+
- Default: falls back to HTTP client default (10.0)
|
|
32
|
+
- Purpose: Timeout for webhook delivery HTTP calls
|
|
33
|
+
- SHUTDOWN_GRACE_PERIOD_SECONDS (float)
|
|
34
|
+
- Default: prod=20.0, nonprod=5.0
|
|
35
|
+
- Purpose: Wait time for in-flight requests to drain on shutdown
|
|
36
|
+
|
|
37
|
+
See ADR-0010 for design rationale: `src/svc_infra/docs/adr/0010-timeouts-and-resource-limits.md`.
|
|
38
|
+
|
|
39
|
+
## Request/handler timeouts (FastAPI)
|
|
40
|
+
|
|
41
|
+
Two middlewares enforce timeouts inside your ASGI app:
|
|
42
|
+
|
|
43
|
+
- BodyReadTimeoutMiddleware
|
|
44
|
+
- Enforces a per-chunk timeout while reading the incoming request body.
|
|
45
|
+
- If reads stall beyond the timeout, responds with 408 application/problem+json.
|
|
46
|
+
- Module: `svc_infra.api.fastapi.middleware.timeout.BodyReadTimeoutMiddleware`
|
|
47
|
+
- HandlerTimeoutMiddleware
|
|
48
|
+
- Caps overall request handler execution time using asyncio.wait_for.
|
|
49
|
+
- If exceeded, responds with 504 application/problem+json.
|
|
50
|
+
- Module: `svc_infra.api.fastapi.middleware.timeout.HandlerTimeoutMiddleware`
|
|
51
|
+
|
|
52
|
+
Example wiring:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from fastapi import FastAPI
|
|
56
|
+
from svc_infra.api.fastapi.middleware.timeout import (
|
|
57
|
+
BodyReadTimeoutMiddleware,
|
|
58
|
+
HandlerTimeoutMiddleware,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
app = FastAPI()
|
|
62
|
+
|
|
63
|
+
# Abort slow uploads (slowloris) after 15s in prod / 30s nonprod by default
|
|
64
|
+
app.add_middleware(BodyReadTimeoutMiddleware) # or timeout_seconds=20
|
|
65
|
+
|
|
66
|
+
# Cap total handler time (e.g., 30s in prod by default)
|
|
67
|
+
app.add_middleware(HandlerTimeoutMiddleware) # or timeout_seconds=25
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
HTTP semantics:
|
|
71
|
+
|
|
72
|
+
- Body timeout → 408 Request Timeout (Problem+JSON) with fields: type, title, status, detail, instance, trace_id
|
|
73
|
+
- Handler timeout → 504 Gateway Timeout (Problem+JSON) with fields: type, title, status, detail, instance, trace_id
|
|
74
|
+
|
|
75
|
+
## Outbound HTTP client timeouts (httpx)
|
|
76
|
+
|
|
77
|
+
Use the provided helpers to create httpx clients with the default timeout (driven by HTTP_CLIENT_TIMEOUT_SECONDS).
|
|
78
|
+
|
|
79
|
+
- Module: `svc_infra.http.client`
|
|
80
|
+
- `get_default_timeout_seconds()` → float
|
|
81
|
+
- `make_timeout(seconds=None) -> httpx.Timeout`
|
|
82
|
+
- `new_httpx_client(timeout_seconds=None, ...) -> httpx.Client`
|
|
83
|
+
- `new_async_httpx_client(timeout_seconds=None, ...) -> httpx.AsyncClient`
|
|
84
|
+
|
|
85
|
+
Error mapping:
|
|
86
|
+
|
|
87
|
+
- `httpx.TimeoutException` is mapped to 504 Gateway Timeout with Problem+JSON by default when `register_error_handlers(app)` is used.
|
|
88
|
+
- Module: `svc_infra.api.fastapi.middleware.errors.handlers.register_error_handlers`
|
|
89
|
+
|
|
90
|
+
## Database statement timeouts (SQLAlchemy / Postgres)
|
|
91
|
+
|
|
92
|
+
If `DB_STATEMENT_TIMEOUT_MS` is set and Postgres is used, a per-transaction `SET LOCAL statement_timeout = :ms` is executed for sessions yielded by the built-in dependency.
|
|
93
|
+
|
|
94
|
+
- Module: `svc_infra.api.fastapi.db.sql.session.get_session`
|
|
95
|
+
- Non-Postgres dialects (e.g., SQLite) ignore this gracefully.
|
|
96
|
+
|
|
97
|
+
## Jobs and webhooks
|
|
98
|
+
|
|
99
|
+
- Jobs runner
|
|
100
|
+
- Env: `JOB_DEFAULT_TIMEOUT_SECONDS`
|
|
101
|
+
- Module: `svc_infra.jobs.worker.process_one` — wraps job handler with `asyncio.wait_for()` when configured.
|
|
102
|
+
- Webhook delivery
|
|
103
|
+
- Env: `WEBHOOK_DELIVERY_TIMEOUT_SECONDS` (falls back to HTTP client default when unset)
|
|
104
|
+
- Module: `svc_infra.jobs.builtins.webhook_delivery.make_webhook_handler` — uses `new_async_httpx_client` with derived timeout.
|
|
105
|
+
|
|
106
|
+
## Graceful shutdown
|
|
107
|
+
|
|
108
|
+
Install graceful shutdown to wait for in-flight requests (up to a grace period) during application shutdown.
|
|
109
|
+
|
|
110
|
+
- Module: `svc_infra.api.fastapi.middleware.graceful_shutdown.install_graceful_shutdown`
|
|
111
|
+
- Env: `SHUTDOWN_GRACE_PERIOD_SECONDS` (prod=20.0, nonprod=5.0 by default)
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from svc_infra.api.fastapi.middleware.graceful_shutdown import install_graceful_shutdown
|
|
115
|
+
|
|
116
|
+
install_graceful_shutdown(app) # or grace_seconds=30.0
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Tuning recommendations
|
|
120
|
+
|
|
121
|
+
- Production
|
|
122
|
+
- REQUEST_BODY_TIMEOUT_SECONDS: 10–20s (shorter for public APIs)
|
|
123
|
+
- REQUEST_TIMEOUT_SECONDS: 20–30s (align with upstream proxy/gateway timeouts)
|
|
124
|
+
- HTTP_CLIENT_TIMEOUT_SECONDS: 3–10s (favor quick failover with retries)
|
|
125
|
+
- DB_STATEMENT_TIMEOUT_MS: set per-route/transaction if queries are constrained
|
|
126
|
+
- SHUTDOWN_GRACE_PERIOD_SECONDS: 20–60s depending on peak latencies
|
|
127
|
+
- Staging/Dev
|
|
128
|
+
- Relax timeouts slightly to reduce test flakiness (defaults already reflect this)
|
|
129
|
+
- Gateways/Proxies
|
|
130
|
+
- Ensure upstream (e.g., NGINX, ALB) timeouts exceed app’s body timeout and are aligned with handler timeout to avoid double timeouts.
|
|
131
|
+
|
|
132
|
+
## Testing and acceptance
|
|
133
|
+
|
|
134
|
+
- Unit tests cover body read timeout, handler timeout, outbound timeout mapping, and a smoke check for DB statement timeout.
|
|
135
|
+
- Acceptance tests:
|
|
136
|
+
- A2-04: slow handler → 504 Problem
|
|
137
|
+
- A2-05: slow body → 408 Problem or 413 (size) as applicable
|
|
138
|
+
- A2-06: outbound httpx timeout → 504 Problem
|
|
139
|
+
|
|
140
|
+
## Troubleshooting
|
|
141
|
+
|
|
142
|
+
- Seeing 200 instead of 408 for slow uploads under some servers?
|
|
143
|
+
- Some servers buffer the entire body before invoking the app. The BodyReadTimeoutMiddleware greedily drains with per-chunk timeouts and replays to reliably detect slowloris. Ensure HTTP/1.1 parsing with a streaming-capable server implementation (e.g., uvicorn+httptools) in acceptance tests.
|
|
144
|
+
- Outbound timeouts not mapped to Problem?
|
|
145
|
+
- Ensure `register_error_handlers(app)` is installed so `httpx.TimeoutException` returns a 504 Problem.
|
|
146
|
+
- Statement timeout ignored on SQLite?
|
|
147
|
+
- Expected. Non-Postgres dialects skip `SET LOCAL` safely.
|