tigzig-concurrency 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tigzig_concurrency-0.1.0/PKG-INFO +270 -0
- tigzig_concurrency-0.1.0/README.md +244 -0
- tigzig_concurrency-0.1.0/pyproject.toml +45 -0
- tigzig_concurrency-0.1.0/src/tigzig_concurrency/__init__.py +15 -0
- tigzig_concurrency-0.1.0/src/tigzig_concurrency/client_ip.py +56 -0
- tigzig_concurrency-0.1.0/src/tigzig_concurrency/headers.py +43 -0
- tigzig_concurrency-0.1.0/src/tigzig_concurrency/queue.py +212 -0
- tigzig_concurrency-0.1.0/src/tigzig_concurrency/responses.py +42 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tigzig-concurrency
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Bounded wait queue + client-IP forwarding for FastAPI backends
|
|
5
|
+
Project-URL: Homepage, https://github.com/amararun/tigzig-concurrency
|
|
6
|
+
Project-URL: Repository, https://github.com/amararun/tigzig-concurrency
|
|
7
|
+
Project-URL: Issues, https://github.com/amararun/tigzig-concurrency/issues
|
|
8
|
+
Author-email: Amar Harolikar <amar@harolikar.com>
|
|
9
|
+
License: MIT
|
|
10
|
+
Keywords: backpressure,concurrency,fastapi,queue,rate-limit
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Framework :: FastAPI
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
|
|
23
|
+
Requires-Python: >=3.8
|
|
24
|
+
Requires-Dist: starlette>=0.27.0
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# tigzig-concurrency
|
|
28
|
+
|
|
29
|
+
Bounded wait queue + client-IP forwarding for FastAPI / Starlette backends.
|
|
30
|
+
|
|
31
|
+
A standing-rule pattern for the tigzig fleet. Replaces ad-hoc per-app
|
|
32
|
+
concurrency-check and IP-extraction code so all backends behave the same
|
|
33
|
+
way under load.
|
|
34
|
+
|
|
35
|
+
## Why this exists
|
|
36
|
+
|
|
37
|
+
Two recurring problems across the fleet:
|
|
38
|
+
|
|
39
|
+
1. **Fast-fail concurrency caps reject legitimate users.** A user
|
|
40
|
+
double-clicks submit, two of their parallel sub-requests collide, and
|
|
41
|
+
one gets a 429 even though the server is mostly idle. This package
|
|
42
|
+
replaces the fast-fail check with a small bounded queue: the second
|
|
43
|
+
request waits up to N seconds for a slot, then either runs or 429s.
|
|
44
|
+
The server is still protected from sustained overload (queue is
|
|
45
|
+
bounded), but micro-bursts succeed.
|
|
46
|
+
|
|
47
|
+
2. **Backend-to-backend calls trip downstream per-IP limits.** When
|
|
48
|
+
backend A (e.g. TA) calls backend B (e.g. yfin) on behalf of an end
|
|
49
|
+
user, B sees the request from A's container IP - so two unrelated
|
|
50
|
+
end users routed via A look like ONE user at B and trip B's per-IP
|
|
51
|
+
cap. This package provides ``forward_client_headers()`` so A passes
|
|
52
|
+
the real user IP, and ``get_real_client_ip()`` (which B already uses)
|
|
53
|
+
reads it.
|
|
54
|
+
|
|
55
|
+
## Installation
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install tigzig-concurrency
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Add to your ``requirements.txt``:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
tigzig-concurrency>=0.1.0
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Quick start
|
|
68
|
+
|
|
69
|
+
### 1. Wrap your endpoint with the queue
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from fastapi import FastAPI, Request
|
|
73
|
+
from tigzig_concurrency import BoundedQueue, get_real_client_ip
|
|
74
|
+
|
|
75
|
+
app = FastAPI()
|
|
76
|
+
queue = BoundedQueue.from_env() # reads MAX_INFLIGHT_*, MAX_QUEUE_* from env
|
|
77
|
+
|
|
78
|
+
@app.post("/api/work")
|
|
79
|
+
async def do_work(request: Request):
|
|
80
|
+
client_ip = get_real_client_ip(request)
|
|
81
|
+
await queue.acquire(client_ip)
|
|
82
|
+
try:
|
|
83
|
+
# ... your endpoint work ...
|
|
84
|
+
return {"ok": True}
|
|
85
|
+
finally:
|
|
86
|
+
queue.release(client_ip)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### 2. Forward the real user IP on backend-to-backend calls
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
import httpx
|
|
93
|
+
from tigzig_concurrency import forward_client_headers, get_real_client_ip
|
|
94
|
+
|
|
95
|
+
@app.post("/api/work")
|
|
96
|
+
async def do_work(request: Request):
|
|
97
|
+
client_ip = get_real_client_ip(request)
|
|
98
|
+
await queue.acquire(client_ip)
|
|
99
|
+
try:
|
|
100
|
+
async with httpx.AsyncClient() as http:
|
|
101
|
+
response = await http.get(
|
|
102
|
+
"https://other-backend.tigzig.com/data",
|
|
103
|
+
headers=forward_client_headers(client_ip),
|
|
104
|
+
)
|
|
105
|
+
return response.json()
|
|
106
|
+
finally:
|
|
107
|
+
queue.release(client_ip)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
The downstream backend's ``get_real_client_ip()`` will read the
|
|
111
|
+
``X-Original-Client-IP`` header and rate-limit per real end user.
|
|
112
|
+
|
|
113
|
+
## Configuration cheatsheet
|
|
114
|
+
|
|
115
|
+
Five environment variables. **You will forget what these mean in three
|
|
116
|
+
weeks - this is the cheatsheet.** Future-you, hello.
|
|
117
|
+
|
|
118
|
+
### The five knobs
|
|
119
|
+
|
|
120
|
+
| Variable | What it does | When to raise / lower |
|
|
121
|
+
|---|---|---|
|
|
122
|
+
| ``MAX_INFLIGHT_PER_IP`` | Active requests per IP at one time. | One real user can have this many running at once. Higher = more permissive for power-users; lower = stricter fairness. |
|
|
123
|
+
| ``MAX_INFLIGHT_GLOBAL`` | Active requests across all IPs combined. | Cap on total CPU/memory the server commits at once. Higher = more parallelism if you have headroom; lower if memory is tight. |
|
|
124
|
+
| ``MAX_QUEUE_PER_IP`` | Extra requests per IP that may wait in line. | Bigger queue = more forgiveness for "user clicked submit 4 times in 5 seconds". 0 = no queueing per-IP (fast-fail behaviour). |
|
|
125
|
+
| ``MAX_QUEUE_GLOBAL`` | Extra requests across all IPs that may wait. | Total system buffer for bursts. Beyond this -> instant 503. |
|
|
126
|
+
| ``MAX_QUEUE_WAIT_SECONDS`` | How long any one request waits in queue before 429. | Match to typical request length; ``wait + work + slack`` should stay under your edge timeout (Cloudflare free is 100s). |
|
|
127
|
+
|
|
128
|
+
### Behaviour
|
|
129
|
+
|
|
130
|
+
| Situation | Response |
|
|
131
|
+
|---|---|
|
|
132
|
+
| Slot available | Run immediately (no queue overhead). |
|
|
133
|
+
| Slot taken, queue not full | Wait up to ``MAX_QUEUE_WAIT_SECONDS`` for a slot, then run, OR 429 if the wait expired. |
|
|
134
|
+
| Per-IP queue full at arrival | 503 immediately (this user is hammering). |
|
|
135
|
+
| Global queue full at arrival | 503 immediately (server is overloaded). |
|
|
136
|
+
|
|
137
|
+
Both 429 and 503 include a ``Retry-After: 10`` header so the frontend
|
|
138
|
+
can show a friendly "server busy, retry in 10s" spinner instead of a
|
|
139
|
+
hard error.
|
|
140
|
+
|
|
141
|
+
### Tuning profiles for the tigzig fleet
|
|
142
|
+
|
|
143
|
+
Three workload types cover most apps. Pick the closest profile, set the
|
|
144
|
+
env vars accordingly, and tune from there if needed.
|
|
145
|
+
|
|
146
|
+
#### Profile A: fast IO-bound (1-5s per request)
|
|
147
|
+
For yfin, mdtopdf, llama-parse, markitdown, neon-proxy, supabase-proxy,
|
|
148
|
+
and any wrapper that mostly does network I/O + light pandas.
|
|
149
|
+
|
|
150
|
+
```
|
|
151
|
+
MAX_INFLIGHT_PER_IP=6
|
|
152
|
+
MAX_INFLIGHT_GLOBAL=15
|
|
153
|
+
MAX_QUEUE_PER_IP=4
|
|
154
|
+
MAX_QUEUE_GLOBAL=10
|
|
155
|
+
MAX_QUEUE_WAIT_SECONDS=5
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Why: requests are quick, so being aggressive with concurrency is safe
|
|
159
|
+
(no memory pressure). Short wait window because end users expect fast
|
|
160
|
+
responses; queue absorbs occasional bursts.
|
|
161
|
+
|
|
162
|
+
#### Profile B: slow LLM / compute-bound (10-45s per request)
|
|
163
|
+
For TA, FFN/SPR, QuantStats, codeinter (Python sandbox), codeinter-gemini.
|
|
164
|
+
|
|
165
|
+
```
|
|
166
|
+
MAX_INFLIGHT_PER_IP=2
|
|
167
|
+
MAX_INFLIGHT_GLOBAL=6
|
|
168
|
+
MAX_QUEUE_PER_IP=3
|
|
169
|
+
MAX_QUEUE_GLOBAL=8
|
|
170
|
+
MAX_QUEUE_WAIT_SECONDS=30
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Why: each request hogs a CPU or holds an LLM connection for a long time.
|
|
174
|
+
Tighter in-flight cap to avoid memory blowup. Longer wait because users
|
|
175
|
+
already expect slow; an extra 15-30s of queue is invisible vs the 30s of
|
|
176
|
+
actual work. Total ``wait + work`` stays under Cloudflare 100s.
|
|
177
|
+
|
|
178
|
+
#### Profile C: heavy memory / database (DuckDB queries, Flowise)
|
|
179
|
+
For duckdb-backend, duckdb-dashboards-backend, flowise-docker-custom.
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
MAX_INFLIGHT_PER_IP=2
|
|
183
|
+
MAX_INFLIGHT_GLOBAL=4
|
|
184
|
+
MAX_QUEUE_PER_IP=2
|
|
185
|
+
MAX_QUEUE_GLOBAL=4
|
|
186
|
+
MAX_QUEUE_WAIT_SECONDS=15
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Why: a single heavy DuckDB query can pin a CPU AND eat a few hundred MB
|
|
190
|
+
of memory. Tight global cap matters more than per-IP fairness. Bump the
|
|
191
|
+
container memory limit if you raise the global cap.
|
|
192
|
+
|
|
193
|
+
### Container-limit reminder
|
|
194
|
+
|
|
195
|
+
These knobs only protect the server if the container itself has enough
|
|
196
|
+
CPU and memory. For the tigzig fleet's standard:
|
|
197
|
+
|
|
198
|
+
```
|
|
199
|
+
limits_cpus=2.0
|
|
200
|
+
limits_memory=1024m (or 2048m for memory-heavy or many-concurrent profiles)
|
|
201
|
+
limits_memory_swap=2048m
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
A queue with ``MAX_INFLIGHT_GLOBAL=15`` on a container capped at
|
|
205
|
+
``cpus=0.25`` will just queue everything and run them slowly. The CPU
|
|
206
|
+
bump is the bigger lever.
|
|
207
|
+
|
|
208
|
+
## Multiple queues in one app
|
|
209
|
+
|
|
210
|
+
If one endpoint is much heavier than others, you can run two queues
|
|
211
|
+
with different settings via the ``prefix`` argument:
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
light_queue = BoundedQueue.from_env(prefix="LIGHT_", name="light")
|
|
215
|
+
heavy_queue = BoundedQueue.from_env(prefix="HEAVY_", name="heavy")
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Then set ``LIGHT_MAX_INFLIGHT_PER_IP``, ``HEAVY_MAX_INFLIGHT_PER_IP`` etc.
|
|
219
|
+
in your environment.
|
|
220
|
+
|
|
221
|
+
## Health / introspection
|
|
222
|
+
|
|
223
|
+
The queue exposes its current state via a ``stats`` property. Useful
|
|
224
|
+
for ``/healthz`` endpoints:
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
@app.get("/healthz/queue")
|
|
228
|
+
def queue_health():
|
|
229
|
+
return queue.stats
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Returns:
|
|
233
|
+
|
|
234
|
+
```json
|
|
235
|
+
{
|
|
236
|
+
"name": "queue",
|
|
237
|
+
"limits": {"inflight_per_ip": 2, "inflight_global": 6, ...},
|
|
238
|
+
"current": {"inflight_global": 3, "queued_global": 1, "queued_per_ip": {"1.2.3.4": 1}}
|
|
239
|
+
}
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
## Reference: header chain that ``get_real_client_ip`` reads
|
|
243
|
+
|
|
244
|
+
In order:
|
|
245
|
+
|
|
246
|
+
1. ``X-Original-Client-IP`` (canonical for trusted upstream backends)
|
|
247
|
+
2. ``CF-Connecting-IP`` (Cloudflare)
|
|
248
|
+
3. ``X-Forwarded-For`` (first IP if comma-separated)
|
|
249
|
+
4. ``X-Real-IP``
|
|
250
|
+
5. ``request.client.host`` (socket peer; usually docker proxy)
|
|
251
|
+
|
|
252
|
+
Returns ``"unknown"`` if no IP can be resolved.
|
|
253
|
+
|
|
254
|
+
## Why ``X-Original-Client-IP`` over ``X-Forwarded-For``
|
|
255
|
+
|
|
256
|
+
XFF gets rewritten by every hop (Cloudflare, Caddy, nginx). For trusted
|
|
257
|
+
internal forwarding it's safer to use a custom header that only the
|
|
258
|
+
trusted upstream sets. ``X-Original-Client-IP`` is the convention across
|
|
259
|
+
the tigzig fleet; the helper sets both to maximize compatibility but
|
|
260
|
+
the canonical reader prefers the custom header.
|
|
261
|
+
|
|
262
|
+
## Versioning
|
|
263
|
+
|
|
264
|
+
Semantic versioning. Breaking changes to function signatures or env-var
|
|
265
|
+
names bump the major version. Tuning-profile recommendations may evolve
|
|
266
|
+
across minor versions; check the README of the version you're using.
|
|
267
|
+
|
|
268
|
+
## License
|
|
269
|
+
|
|
270
|
+
MIT.
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
# tigzig-concurrency
|
|
2
|
+
|
|
3
|
+
Bounded wait queue + client-IP forwarding for FastAPI / Starlette backends.
|
|
4
|
+
|
|
5
|
+
A standing-rule pattern for the tigzig fleet. Replaces ad-hoc per-app
|
|
6
|
+
concurrency-check and IP-extraction code so all backends behave the same
|
|
7
|
+
way under load.
|
|
8
|
+
|
|
9
|
+
## Why this exists
|
|
10
|
+
|
|
11
|
+
Two recurring problems across the fleet:
|
|
12
|
+
|
|
13
|
+
1. **Fast-fail concurrency caps reject legitimate users.** A user
|
|
14
|
+
double-clicks submit, two of their parallel sub-requests collide, and
|
|
15
|
+
one gets a 429 even though the server is mostly idle. This package
|
|
16
|
+
replaces the fast-fail check with a small bounded queue: the second
|
|
17
|
+
request waits up to N seconds for a slot, then either runs or 429s.
|
|
18
|
+
The server is still protected from sustained overload (queue is
|
|
19
|
+
bounded), but micro-bursts succeed.
|
|
20
|
+
|
|
21
|
+
2. **Backend-to-backend calls trip downstream per-IP limits.** When
|
|
22
|
+
backend A (e.g. TA) calls backend B (e.g. yfin) on behalf of an end
|
|
23
|
+
user, B sees the request from A's container IP - so two unrelated
|
|
24
|
+
end users routed via A look like ONE user at B and trip B's per-IP
|
|
25
|
+
cap. This package provides ``forward_client_headers()`` so A passes
|
|
26
|
+
the real user IP, and ``get_real_client_ip()`` (which B already uses)
|
|
27
|
+
reads it.
|
|
28
|
+
|
|
29
|
+
## Installation
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install tigzig-concurrency
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Add to your ``requirements.txt``:
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
tigzig-concurrency>=0.1.0
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Quick start
|
|
42
|
+
|
|
43
|
+
### 1. Wrap your endpoint with the queue
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from fastapi import FastAPI, Request
|
|
47
|
+
from tigzig_concurrency import BoundedQueue, get_real_client_ip
|
|
48
|
+
|
|
49
|
+
app = FastAPI()
|
|
50
|
+
queue = BoundedQueue.from_env() # reads MAX_INFLIGHT_*, MAX_QUEUE_* from env
|
|
51
|
+
|
|
52
|
+
@app.post("/api/work")
|
|
53
|
+
async def do_work(request: Request):
|
|
54
|
+
client_ip = get_real_client_ip(request)
|
|
55
|
+
await queue.acquire(client_ip)
|
|
56
|
+
try:
|
|
57
|
+
# ... your endpoint work ...
|
|
58
|
+
return {"ok": True}
|
|
59
|
+
finally:
|
|
60
|
+
queue.release(client_ip)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### 2. Forward the real user IP on backend-to-backend calls
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
import httpx
|
|
67
|
+
from tigzig_concurrency import forward_client_headers, get_real_client_ip
|
|
68
|
+
|
|
69
|
+
@app.post("/api/work")
|
|
70
|
+
async def do_work(request: Request):
|
|
71
|
+
client_ip = get_real_client_ip(request)
|
|
72
|
+
await queue.acquire(client_ip)
|
|
73
|
+
try:
|
|
74
|
+
async with httpx.AsyncClient() as http:
|
|
75
|
+
response = await http.get(
|
|
76
|
+
"https://other-backend.tigzig.com/data",
|
|
77
|
+
headers=forward_client_headers(client_ip),
|
|
78
|
+
)
|
|
79
|
+
return response.json()
|
|
80
|
+
finally:
|
|
81
|
+
queue.release(client_ip)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The downstream backend's ``get_real_client_ip()`` will read the
|
|
85
|
+
``X-Original-Client-IP`` header and rate-limit per real end user.
|
|
86
|
+
|
|
87
|
+
## Configuration cheatsheet
|
|
88
|
+
|
|
89
|
+
Five environment variables. **You will forget what these mean in three
|
|
90
|
+
weeks - this is the cheatsheet.** Future-you, hello.
|
|
91
|
+
|
|
92
|
+
### The five knobs
|
|
93
|
+
|
|
94
|
+
| Variable | What it does | When to raise / lower |
|
|
95
|
+
|---|---|---|
|
|
96
|
+
| ``MAX_INFLIGHT_PER_IP`` | Active requests per IP at one time. | One real user can have this many running at once. Higher = more permissive for power-users; lower = stricter fairness. |
|
|
97
|
+
| ``MAX_INFLIGHT_GLOBAL`` | Active requests across all IPs combined. | Cap on total CPU/memory the server commits at once. Higher = more parallelism if you have headroom; lower if memory is tight. |
|
|
98
|
+
| ``MAX_QUEUE_PER_IP`` | Extra requests per IP that may wait in line. | Bigger queue = more forgiveness for "user clicked submit 4 times in 5 seconds". 0 = no queueing per-IP (fast-fail behaviour). |
|
|
99
|
+
| ``MAX_QUEUE_GLOBAL`` | Extra requests across all IPs that may wait. | Total system buffer for bursts. Beyond this -> instant 503. |
|
|
100
|
+
| ``MAX_QUEUE_WAIT_SECONDS`` | How long any one request waits in queue before 429. | Match to typical request length; ``wait + work + slack`` should stay under your edge timeout (Cloudflare free is 100s). |
|
|
101
|
+
|
|
102
|
+
### Behaviour
|
|
103
|
+
|
|
104
|
+
| Situation | Response |
|
|
105
|
+
|---|---|
|
|
106
|
+
| Slot available | Run immediately (no queue overhead). |
|
|
107
|
+
| Slot taken, queue not full | Wait up to ``MAX_QUEUE_WAIT_SECONDS`` for a slot, then run, OR 429 if the wait expired. |
|
|
108
|
+
| Per-IP queue full at arrival | 503 immediately (this user is hammering). |
|
|
109
|
+
| Global queue full at arrival | 503 immediately (server is overloaded). |
|
|
110
|
+
|
|
111
|
+
Both 429 and 503 include a ``Retry-After: 10`` header so the frontend
|
|
112
|
+
can show a friendly "server busy, retry in 10s" spinner instead of a
|
|
113
|
+
hard error.
|
|
114
|
+
|
|
115
|
+
### Tuning profiles for the tigzig fleet
|
|
116
|
+
|
|
117
|
+
Three workload types cover most apps. Pick the closest profile, set the
|
|
118
|
+
env vars accordingly, and tune from there if needed.
|
|
119
|
+
|
|
120
|
+
#### Profile A: fast IO-bound (1-5s per request)
|
|
121
|
+
For yfin, mdtopdf, llama-parse, markitdown, neon-proxy, supabase-proxy,
|
|
122
|
+
and any wrapper that mostly does network I/O + light pandas.
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
MAX_INFLIGHT_PER_IP=6
|
|
126
|
+
MAX_INFLIGHT_GLOBAL=15
|
|
127
|
+
MAX_QUEUE_PER_IP=4
|
|
128
|
+
MAX_QUEUE_GLOBAL=10
|
|
129
|
+
MAX_QUEUE_WAIT_SECONDS=5
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Why: requests are quick, so being aggressive with concurrency is safe
|
|
133
|
+
(no memory pressure). Short wait window because end users expect fast
|
|
134
|
+
responses; queue absorbs occasional bursts.
|
|
135
|
+
|
|
136
|
+
#### Profile B: slow LLM / compute-bound (10-45s per request)
|
|
137
|
+
For TA, FFN/SPR, QuantStats, codeinter (Python sandbox), codeinter-gemini.
|
|
138
|
+
|
|
139
|
+
```
|
|
140
|
+
MAX_INFLIGHT_PER_IP=2
|
|
141
|
+
MAX_INFLIGHT_GLOBAL=6
|
|
142
|
+
MAX_QUEUE_PER_IP=3
|
|
143
|
+
MAX_QUEUE_GLOBAL=8
|
|
144
|
+
MAX_QUEUE_WAIT_SECONDS=30
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Why: each request hogs a CPU or holds an LLM connection for a long time.
|
|
148
|
+
Tighter in-flight cap to avoid memory blowup. Longer wait because users
|
|
149
|
+
already expect slow; an extra 15-30s of queue is invisible vs the 30s of
|
|
150
|
+
actual work. Total ``wait + work`` stays under Cloudflare 100s.
|
|
151
|
+
|
|
152
|
+
#### Profile C: heavy memory / database (DuckDB queries, Flowise)
|
|
153
|
+
For duckdb-backend, duckdb-dashboards-backend, flowise-docker-custom.
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
MAX_INFLIGHT_PER_IP=2
|
|
157
|
+
MAX_INFLIGHT_GLOBAL=4
|
|
158
|
+
MAX_QUEUE_PER_IP=2
|
|
159
|
+
MAX_QUEUE_GLOBAL=4
|
|
160
|
+
MAX_QUEUE_WAIT_SECONDS=15
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Why: a single heavy DuckDB query can pin a CPU AND eat a few hundred MB
|
|
164
|
+
of memory. Tight global cap matters more than per-IP fairness. Bump the
|
|
165
|
+
container memory limit if you raise the global cap.
|
|
166
|
+
|
|
167
|
+
### Container-limit reminder
|
|
168
|
+
|
|
169
|
+
These knobs only protect the server if the container itself has enough
|
|
170
|
+
CPU and memory. For the tigzig fleet's standard:
|
|
171
|
+
|
|
172
|
+
```
|
|
173
|
+
limits_cpus=2.0
|
|
174
|
+
limits_memory=1024m (or 2048m for memory-heavy or many-concurrent profiles)
|
|
175
|
+
limits_memory_swap=2048m
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
A queue with ``MAX_INFLIGHT_GLOBAL=15`` on a container capped at
|
|
179
|
+
``cpus=0.25`` will just queue everything and run them slowly. The CPU
|
|
180
|
+
bump is the bigger lever.
|
|
181
|
+
|
|
182
|
+
## Multiple queues in one app
|
|
183
|
+
|
|
184
|
+
If one endpoint is much heavier than others, you can run two queues
|
|
185
|
+
with different settings via the ``prefix`` argument:
|
|
186
|
+
|
|
187
|
+
```python
|
|
188
|
+
light_queue = BoundedQueue.from_env(prefix="LIGHT_", name="light")
|
|
189
|
+
heavy_queue = BoundedQueue.from_env(prefix="HEAVY_", name="heavy")
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Then set ``LIGHT_MAX_INFLIGHT_PER_IP``, ``HEAVY_MAX_INFLIGHT_PER_IP`` etc.
|
|
193
|
+
in your environment.
|
|
194
|
+
|
|
195
|
+
## Health / introspection
|
|
196
|
+
|
|
197
|
+
The queue exposes its current state via a ``stats`` property. Useful
|
|
198
|
+
for ``/healthz`` endpoints:
|
|
199
|
+
|
|
200
|
+
```python
|
|
201
|
+
@app.get("/healthz/queue")
|
|
202
|
+
def queue_health():
|
|
203
|
+
return queue.stats
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
|
|
208
|
+
```json
|
|
209
|
+
{
|
|
210
|
+
"name": "queue",
|
|
211
|
+
"limits": {"inflight_per_ip": 2, "inflight_global": 6, ...},
|
|
212
|
+
"current": {"inflight_global": 3, "queued_global": 1, "queued_per_ip": {"1.2.3.4": 1}}
|
|
213
|
+
}
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
## Reference: header chain that ``get_real_client_ip`` reads
|
|
217
|
+
|
|
218
|
+
In order:
|
|
219
|
+
|
|
220
|
+
1. ``X-Original-Client-IP`` (canonical for trusted upstream backends)
|
|
221
|
+
2. ``CF-Connecting-IP`` (Cloudflare)
|
|
222
|
+
3. ``X-Forwarded-For`` (first IP if comma-separated)
|
|
223
|
+
4. ``X-Real-IP``
|
|
224
|
+
5. ``request.client.host`` (socket peer; usually docker proxy)
|
|
225
|
+
|
|
226
|
+
Returns ``"unknown"`` if no IP can be resolved.
|
|
227
|
+
|
|
228
|
+
## Why ``X-Original-Client-IP`` over ``X-Forwarded-For``
|
|
229
|
+
|
|
230
|
+
XFF gets rewritten by every hop (Cloudflare, Caddy, nginx). For trusted
|
|
231
|
+
internal forwarding it's safer to use a custom header that only the
|
|
232
|
+
trusted upstream sets. ``X-Original-Client-IP`` is the convention across
|
|
233
|
+
the tigzig fleet; the helper sets both to maximize compatibility but
|
|
234
|
+
the canonical reader prefers the custom header.
|
|
235
|
+
|
|
236
|
+
## Versioning
|
|
237
|
+
|
|
238
|
+
Semantic versioning. Breaking changes to function signatures or env-var
|
|
239
|
+
names bump the major version. Tuning-profile recommendations may evolve
|
|
240
|
+
across minor versions; check the README of the version you're using.
|
|
241
|
+
|
|
242
|
+
## License
|
|
243
|
+
|
|
244
|
+
MIT.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tigzig-concurrency"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Bounded wait queue + client-IP forwarding for FastAPI backends"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
requires-python = ">=3.8"
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Amar Harolikar", email = "amar@harolikar.com"}
|
|
14
|
+
]
|
|
15
|
+
keywords = ["fastapi", "concurrency", "rate-limit", "queue", "backpressure"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Framework :: FastAPI",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"License :: OSI Approved :: MIT License",
|
|
21
|
+
"Operating System :: OS Independent",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.8",
|
|
24
|
+
"Programming Language :: Python :: 3.9",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Topic :: Internet :: WWW/HTTP :: HTTP Servers",
|
|
29
|
+
]
|
|
30
|
+
dependencies = [
|
|
31
|
+
"starlette>=0.27.0",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/amararun/tigzig-concurrency"
|
|
36
|
+
Repository = "https://github.com/amararun/tigzig-concurrency"
|
|
37
|
+
Issues = "https://github.com/amararun/tigzig-concurrency/issues"
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build.targets.sdist]
|
|
40
|
+
include = [
|
|
41
|
+
"/src",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.targets.wheel]
|
|
45
|
+
packages = ["src/tigzig_concurrency"]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""tigzig-concurrency: bounded wait queue + client-IP forwarding for FastAPI."""
|
|
2
|
+
from .client_ip import get_real_client_ip
|
|
3
|
+
from .headers import forward_client_headers
|
|
4
|
+
from .queue import BoundedQueue
|
|
5
|
+
from .responses import queue_busy_429, queue_full_503
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"BoundedQueue",
|
|
11
|
+
"forward_client_headers",
|
|
12
|
+
"get_real_client_ip",
|
|
13
|
+
"queue_busy_429",
|
|
14
|
+
"queue_full_503",
|
|
15
|
+
]
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Real client-IP extraction for FastAPI / Starlette apps behind proxies.
|
|
2
|
+
|
|
3
|
+
Network topology in the tigzig fleet:
|
|
4
|
+
Browser -> Cloudflare -> Caddy -> Container (typical, orange-cloud)
|
|
5
|
+
Browser -> Caddy -> Container (no Cloudflare, grey-cloud)
|
|
6
|
+
Browser -> CF -> Vercel -> CF -> Hetzner (multi-hop via Vercel proxy)
|
|
7
|
+
Browser -> Backend A -> Backend B (server-to-server, internal)
|
|
8
|
+
|
|
9
|
+
Behind Caddy, ``request.client.host`` is always the docker proxy IP
|
|
10
|
+
(172.18.0.10). The real client must be read from headers. Header preference,
|
|
11
|
+
in order, matches what every tigzig backend expects:
|
|
12
|
+
|
|
13
|
+
1. ``X-Original-Client-IP`` - canonical for trusted upstream backends
|
|
14
|
+
(Vercel proxy, internal service-to-service). This is the one a
|
|
15
|
+
sibling backend should set when calling another backend on behalf
|
|
16
|
+
of an end user, so the downstream rate-limits per real user instead
|
|
17
|
+
of per upstream container IP.
|
|
18
|
+
2. ``CF-Connecting-IP`` - set by Cloudflare on every proxied request.
|
|
19
|
+
3. ``X-Forwarded-For`` - first IP in the comma-separated list. Used
|
|
20
|
+
when CF is grey-clouded so Caddy sees the real client directly.
|
|
21
|
+
4. ``X-Real-IP`` - some proxies set this instead of XFF.
|
|
22
|
+
5. ``request.client.host`` - socket peer; usually the docker proxy.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from typing import TYPE_CHECKING
|
|
27
|
+
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
from starlette.requests import Request
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
_HEADER_ORDER = (
|
|
33
|
+
"x-original-client-ip",
|
|
34
|
+
"cf-connecting-ip",
|
|
35
|
+
"x-forwarded-for",
|
|
36
|
+
"x-real-ip",
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_real_client_ip(request: "Request") -> str:
|
|
41
|
+
"""Return the best-known real client IP for ``request``.
|
|
42
|
+
|
|
43
|
+
Checks the canonical header chain (see module docstring) and falls back
|
|
44
|
+
to the socket peer. Strips whitespace and takes the first IP if the
|
|
45
|
+
header carries a comma-separated list (as XFF can).
|
|
46
|
+
|
|
47
|
+
Returns ``"unknown"`` if no IP can be determined.
|
|
48
|
+
"""
|
|
49
|
+
for header_name in _HEADER_ORDER:
|
|
50
|
+
value = request.headers.get(header_name, "").strip()
|
|
51
|
+
if value:
|
|
52
|
+
# XFF may be "client, proxy1, proxy2"; take the leftmost entry.
|
|
53
|
+
return value.split(",", 1)[0].strip()
|
|
54
|
+
if request.client and request.client.host:
|
|
55
|
+
return request.client.host
|
|
56
|
+
return "unknown"
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Header forwarding for backend-to-backend HTTP calls.
|
|
2
|
+
|
|
3
|
+
When backend A calls backend B on behalf of an end user, B should see the
|
|
4
|
+
end user's IP, not A's container IP. Otherwise B's per-IP rate limits
|
|
5
|
+
treat all traffic from A as a single "user" and trip on legitimate
|
|
6
|
+
multi-user load.
|
|
7
|
+
|
|
8
|
+
The pattern: A extracts the real client IP via :func:`get_real_client_ip`,
|
|
9
|
+
then includes :func:`forward_client_headers` on every outbound httpx call
|
|
10
|
+
to B. B reads the header chain (which prefers ``X-Original-Client-IP``)
|
|
11
|
+
and rate-limits per real end user.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import Dict
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def forward_client_headers(real_client_ip: str) -> Dict[str, str]:
|
|
19
|
+
"""Build the headers backend A should include when calling backend B.
|
|
20
|
+
|
|
21
|
+
Returns an empty dict if ``real_client_ip`` is empty or ``"unknown"``,
|
|
22
|
+
so callers can pass the result unconditionally without poisoning the
|
|
23
|
+
downstream rate limiter with a bogus IP.
|
|
24
|
+
|
|
25
|
+
Example::
|
|
26
|
+
|
|
27
|
+
from tigzig_concurrency import forward_client_headers, get_real_client_ip
|
|
28
|
+
|
|
29
|
+
client_ip = get_real_client_ip(request)
|
|
30
|
+
async with httpx.AsyncClient() as http:
|
|
31
|
+
response = await http.get(
|
|
32
|
+
"https://yfin-h.tigzig.com/get-all-prices/",
|
|
33
|
+
headers=forward_client_headers(client_ip),
|
|
34
|
+
)
|
|
35
|
+
"""
|
|
36
|
+
if not real_client_ip or real_client_ip == "unknown":
|
|
37
|
+
return {}
|
|
38
|
+
return {
|
|
39
|
+
# Canonical: every tigzig backend's get_real_client_ip checks this first.
|
|
40
|
+
"X-Original-Client-IP": real_client_ip,
|
|
41
|
+
# Fallback for downstreams that only know XFF.
|
|
42
|
+
"X-Forwarded-For": real_client_ip,
|
|
43
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Bounded wait queue for FastAPI / asyncio backends.
|
|
2
|
+
|
|
3
|
+
Replaces the classic fast-fail concurrency check with a small queue that
|
|
4
|
+
absorbs micro-bursts (a user double-clicking submit, an upstream backend
|
|
5
|
+
fanning out a few parallel calls, etc.) without immediately 429-ing.
|
|
6
|
+
The queue is bounded in both depth and wait time, so sustained overload
|
|
7
|
+
still backpressure-rejects quickly.
|
|
8
|
+
|
|
9
|
+
Five knobs (typically read from environment via :meth:`BoundedQueue.from_env`):
|
|
10
|
+
|
|
11
|
+
* ``MAX_INFLIGHT_PER_IP`` - active requests per IP at one time.
|
|
12
|
+
* ``MAX_INFLIGHT_GLOBAL`` - active requests across all IPs.
|
|
13
|
+
* ``MAX_QUEUE_PER_IP`` - extra requests per IP that may wait in line.
|
|
14
|
+
* ``MAX_QUEUE_GLOBAL`` - extra requests across all IPs that may wait.
|
|
15
|
+
* ``MAX_QUEUE_WAIT_SECONDS`` - max time any one request waits before 429.
|
|
16
|
+
|
|
17
|
+
Behaviour matrix (one acquire call):
|
|
18
|
+
|
|
19
|
+
============================== =========================================
|
|
20
|
+
Slot available Run immediately.
|
|
21
|
+
Slot taken, queue not full Wait up to ``MAX_QUEUE_WAIT_SECONDS``,
|
|
22
|
+
then either acquire or 429.
|
|
23
|
+
Per-IP queue full at arrival 503 immediately.
|
|
24
|
+
Global queue full at arrival 503 immediately.
|
|
25
|
+
============================== =========================================
|
|
26
|
+
|
|
27
|
+
Both 429 and 503 responses include a ``Retry-After: 10`` header so
|
|
28
|
+
frontends can show a friendly busy message + retry instead of a hard error.
|
|
29
|
+
|
|
30
|
+
Tuning rule of thumb: pick ``MAX_QUEUE_WAIT_SECONDS`` so that
|
|
31
|
+
``wait + work + slack`` stays well under the edge timeout (Cloudflare
|
|
32
|
+
free is 100s). For a 30-45s LLM-bound endpoint, 30s wait is fine. For a
|
|
33
|
+
1-5s API endpoint, 5s wait is plenty.
|
|
34
|
+
"""
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import asyncio
|
|
38
|
+
import logging
|
|
39
|
+
import os
|
|
40
|
+
from collections import defaultdict
|
|
41
|
+
from typing import Dict, Optional
|
|
42
|
+
|
|
43
|
+
from .responses import queue_busy_429, queue_full_503
|
|
44
|
+
|
|
45
|
+
logger = logging.getLogger("tigzig_concurrency")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class BoundedQueue:
|
|
49
|
+
"""An async-safe bounded wait queue with per-IP and global limits.
|
|
50
|
+
|
|
51
|
+
Use :meth:`from_env` to construct from environment variables, or pass
|
|
52
|
+
the five integer/float settings directly to the constructor.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
max_inflight_per_ip: int,
|
|
58
|
+
max_inflight_global: int,
|
|
59
|
+
max_queue_per_ip: int,
|
|
60
|
+
max_queue_global: int,
|
|
61
|
+
max_queue_wait_seconds: float,
|
|
62
|
+
name: str = "queue",
|
|
63
|
+
) -> None:
|
|
64
|
+
if max_inflight_per_ip < 1:
|
|
65
|
+
raise ValueError("max_inflight_per_ip must be >= 1")
|
|
66
|
+
if max_inflight_global < 1:
|
|
67
|
+
raise ValueError("max_inflight_global must be >= 1")
|
|
68
|
+
if max_queue_per_ip < 0:
|
|
69
|
+
raise ValueError("max_queue_per_ip must be >= 0")
|
|
70
|
+
if max_queue_global < 0:
|
|
71
|
+
raise ValueError("max_queue_global must be >= 0")
|
|
72
|
+
if max_queue_wait_seconds < 0:
|
|
73
|
+
raise ValueError("max_queue_wait_seconds must be >= 0")
|
|
74
|
+
|
|
75
|
+
self.max_inflight_per_ip = max_inflight_per_ip
|
|
76
|
+
self.max_inflight_global = max_inflight_global
|
|
77
|
+
self.max_queue_per_ip = max_queue_per_ip
|
|
78
|
+
self.max_queue_global = max_queue_global
|
|
79
|
+
self.max_queue_wait_seconds = max_queue_wait_seconds
|
|
80
|
+
self.name = name
|
|
81
|
+
|
|
82
|
+
self._global_sem = asyncio.Semaphore(max_inflight_global)
|
|
83
|
+
self._ip_sems: Dict[str, asyncio.Semaphore] = {}
|
|
84
|
+
self._ip_sems_lock = asyncio.Lock()
|
|
85
|
+
self._global_queued: int = 0
|
|
86
|
+
self._ip_queued: Dict[str, int] = defaultdict(int)
|
|
87
|
+
self._queue_state_lock = asyncio.Lock()
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def from_env(cls, prefix: str = "", name: Optional[str] = None) -> "BoundedQueue":
|
|
91
|
+
"""Construct from environment variables.
|
|
92
|
+
|
|
93
|
+
Reads ``{prefix}MAX_INFLIGHT_PER_IP``, ``{prefix}MAX_INFLIGHT_GLOBAL``,
|
|
94
|
+
``{prefix}MAX_QUEUE_PER_IP``, ``{prefix}MAX_QUEUE_GLOBAL``,
|
|
95
|
+
``{prefix}MAX_QUEUE_WAIT_SECONDS``. Use a non-empty ``prefix`` to
|
|
96
|
+
run multiple queues in one app with different settings (e.g.
|
|
97
|
+
``prefix="HEAVY_"`` reads ``HEAVY_MAX_INFLIGHT_PER_IP`` etc.).
|
|
98
|
+
|
|
99
|
+
Defaults are conservative ("slow LLM-bound" profile from the
|
|
100
|
+
tigzig fleet); override per-app via environment.
|
|
101
|
+
"""
|
|
102
|
+
return cls(
|
|
103
|
+
max_inflight_per_ip=int(os.getenv(f"{prefix}MAX_INFLIGHT_PER_IP", "2")),
|
|
104
|
+
max_inflight_global=int(os.getenv(f"{prefix}MAX_INFLIGHT_GLOBAL", "6")),
|
|
105
|
+
max_queue_per_ip=int(os.getenv(f"{prefix}MAX_QUEUE_PER_IP", "3")),
|
|
106
|
+
max_queue_global=int(os.getenv(f"{prefix}MAX_QUEUE_GLOBAL", "8")),
|
|
107
|
+
max_queue_wait_seconds=float(os.getenv(f"{prefix}MAX_QUEUE_WAIT_SECONDS", "15")),
|
|
108
|
+
name=name or (prefix.rstrip("_").lower() if prefix else "queue"),
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
async def _get_ip_sem(self, client_ip: str) -> asyncio.Semaphore:
|
|
112
|
+
async with self._ip_sems_lock:
|
|
113
|
+
sem = self._ip_sems.get(client_ip)
|
|
114
|
+
if sem is None:
|
|
115
|
+
sem = asyncio.Semaphore(self.max_inflight_per_ip)
|
|
116
|
+
self._ip_sems[client_ip] = sem
|
|
117
|
+
return sem
|
|
118
|
+
|
|
119
|
+
async def acquire(self, client_ip: str) -> None:
|
|
120
|
+
"""Wait for an in-flight slot. Raises :class:`HTTPException` on
|
|
121
|
+
429 (queue wait timed out) or 503 (queue full at arrival)."""
|
|
122
|
+
deadline = asyncio.get_event_loop().time() + self.max_queue_wait_seconds
|
|
123
|
+
|
|
124
|
+
ip_sem = await self._get_ip_sem(client_ip)
|
|
125
|
+
|
|
126
|
+
# Reserve a queue slot, or fail fast if the queue is already full.
|
|
127
|
+
async with self._queue_state_lock:
|
|
128
|
+
if self._global_queued >= self.max_queue_global:
|
|
129
|
+
logger.warning(
|
|
130
|
+
"[%s] global queue full (%d), rejecting from %s",
|
|
131
|
+
self.name, self._global_queued, client_ip,
|
|
132
|
+
)
|
|
133
|
+
raise queue_full_503()
|
|
134
|
+
ip_queued_now = self._ip_queued[client_ip]
|
|
135
|
+
if ip_queued_now >= self.max_queue_per_ip:
|
|
136
|
+
logger.warning(
|
|
137
|
+
"[%s] per-IP queue full for %s (%d), rejecting",
|
|
138
|
+
self.name, client_ip, ip_queued_now,
|
|
139
|
+
)
|
|
140
|
+
raise queue_full_503(
|
|
141
|
+
message="Too many requests in flight from your address. Please wait and try again.",
|
|
142
|
+
)
|
|
143
|
+
self._global_queued += 1
|
|
144
|
+
self._ip_queued[client_ip] = ip_queued_now + 1
|
|
145
|
+
|
|
146
|
+
acquired_ip_sem = False
|
|
147
|
+
try:
|
|
148
|
+
# Step 1: per-IP semaphore.
|
|
149
|
+
remaining = max(0.0, deadline - asyncio.get_event_loop().time())
|
|
150
|
+
try:
|
|
151
|
+
await asyncio.wait_for(ip_sem.acquire(), timeout=remaining)
|
|
152
|
+
acquired_ip_sem = True
|
|
153
|
+
except asyncio.TimeoutError:
|
|
154
|
+
logger.warning(
|
|
155
|
+
"[%s] per-IP slot wait timed out for %s after %.1fs",
|
|
156
|
+
self.name, client_ip, self.max_queue_wait_seconds,
|
|
157
|
+
)
|
|
158
|
+
raise queue_busy_429()
|
|
159
|
+
|
|
160
|
+
# Step 2: global semaphore.
|
|
161
|
+
remaining = max(0.0, deadline - asyncio.get_event_loop().time())
|
|
162
|
+
try:
|
|
163
|
+
await asyncio.wait_for(self._global_sem.acquire(), timeout=remaining)
|
|
164
|
+
except asyncio.TimeoutError:
|
|
165
|
+
ip_sem.release()
|
|
166
|
+
logger.warning(
|
|
167
|
+
"[%s] global slot wait timed out for %s",
|
|
168
|
+
self.name, client_ip,
|
|
169
|
+
)
|
|
170
|
+
raise queue_busy_429()
|
|
171
|
+
finally:
|
|
172
|
+
async with self._queue_state_lock:
|
|
173
|
+
self._global_queued = max(0, self._global_queued - 1)
|
|
174
|
+
ip_q = self._ip_queued[client_ip] - 1
|
|
175
|
+
if ip_q <= 0:
|
|
176
|
+
self._ip_queued.pop(client_ip, None)
|
|
177
|
+
else:
|
|
178
|
+
self._ip_queued[client_ip] = ip_q
|
|
179
|
+
|
|
180
|
+
def release(self, client_ip: str) -> None:
|
|
181
|
+
"""Release an in-flight slot. Safe to call even if :meth:`acquire`
|
|
182
|
+
was not completed; releasing a never-acquired semaphore is a no-op
|
|
183
|
+
beyond bumping the counter (asyncio.Semaphore allows this)."""
|
|
184
|
+
self._global_sem.release()
|
|
185
|
+
sem = self._ip_sems.get(client_ip)
|
|
186
|
+
if sem is not None:
|
|
187
|
+
sem.release()
|
|
188
|
+
|
|
189
|
+
@property
|
|
190
|
+
def stats(self) -> dict:
|
|
191
|
+
"""Snapshot of current state. Useful for /healthz endpoints.
|
|
192
|
+
|
|
193
|
+
Note: counts are read without the lock, so values are eventually
|
|
194
|
+
consistent. Good enough for monitoring; do not use for
|
|
195
|
+
decision-making logic."""
|
|
196
|
+
# _value is the implementation detail; semaphore = (max - inflight).
|
|
197
|
+
global_inflight = self.max_inflight_global - self._global_sem._value
|
|
198
|
+
return {
|
|
199
|
+
"name": self.name,
|
|
200
|
+
"limits": {
|
|
201
|
+
"inflight_per_ip": self.max_inflight_per_ip,
|
|
202
|
+
"inflight_global": self.max_inflight_global,
|
|
203
|
+
"queue_per_ip": self.max_queue_per_ip,
|
|
204
|
+
"queue_global": self.max_queue_global,
|
|
205
|
+
"wait_seconds": self.max_queue_wait_seconds,
|
|
206
|
+
},
|
|
207
|
+
"current": {
|
|
208
|
+
"inflight_global": global_inflight,
|
|
209
|
+
"queued_global": self._global_queued,
|
|
210
|
+
"queued_per_ip": dict(self._ip_queued),
|
|
211
|
+
},
|
|
212
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Standard HTTP error responses for queue-saturation cases.
|
|
2
|
+
|
|
3
|
+
Three failure modes, two response codes:
|
|
4
|
+
|
|
5
|
+
* ``429 Too Many Requests`` - the request entered the queue and waited but
|
|
6
|
+
no slot freed within ``MAX_QUEUE_WAIT_SECONDS``. The frontend should
|
|
7
|
+
show a "server busy, retrying in N seconds" spinner and retry after
|
|
8
|
+
``Retry-After``.
|
|
9
|
+
* ``503 Service Unavailable`` - the queue itself was full at arrival
|
|
10
|
+
(either per-IP or global). The system is genuinely overloaded; the
|
|
11
|
+
frontend should back off longer.
|
|
12
|
+
|
|
13
|
+
Both responses include a ``Retry-After`` header (seconds) so a polite
|
|
14
|
+
client can wait before retrying.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from starlette.exceptions import HTTPException
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def queue_busy_429(
|
|
22
|
+
message: str = "Server is busy with other requests. Please retry in a few seconds.",
|
|
23
|
+
retry_after: int = 10,
|
|
24
|
+
) -> HTTPException:
|
|
25
|
+
"""Build a 429 for "queued but the wait timed out before a slot freed"."""
|
|
26
|
+
return HTTPException(
|
|
27
|
+
status_code=429,
|
|
28
|
+
detail=message,
|
|
29
|
+
headers={"Retry-After": str(retry_after)},
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def queue_full_503(
|
|
34
|
+
message: str = "Server is at capacity right now. Please wait a moment and try again.",
|
|
35
|
+
retry_after: int = 10,
|
|
36
|
+
) -> HTTPException:
|
|
37
|
+
"""Build a 503 for "queue cap exceeded at arrival, didn't even queue"."""
|
|
38
|
+
return HTTPException(
|
|
39
|
+
status_code=503,
|
|
40
|
+
detail=message,
|
|
41
|
+
headers={"Retry-After": str(retry_after)},
|
|
42
|
+
)
|