tigzig-concurrency 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,15 @@
1
+ """tigzig-concurrency: bounded wait queue + client-IP forwarding for FastAPI."""
2
+ from .client_ip import get_real_client_ip
3
+ from .headers import forward_client_headers
4
+ from .queue import BoundedQueue
5
+ from .responses import queue_busy_429, queue_full_503
6
+
7
+ __version__ = "0.1.0"
8
+
9
+ __all__ = [
10
+ "BoundedQueue",
11
+ "forward_client_headers",
12
+ "get_real_client_ip",
13
+ "queue_busy_429",
14
+ "queue_full_503",
15
+ ]
@@ -0,0 +1,56 @@
1
+ """Real client-IP extraction for FastAPI / Starlette apps behind proxies.
2
+
3
+ Network topology in the tigzig fleet:
4
+ Browser -> Cloudflare -> Caddy -> Container (typical, orange-cloud)
5
+ Browser -> Caddy -> Container (no Cloudflare, grey-cloud)
6
+ Browser -> CF -> Vercel -> CF -> Hetzner (multi-hop via Vercel proxy)
7
+ Browser -> Backend A -> Backend B (server-to-server, internal)
8
+
9
+ Behind Caddy, ``request.client.host`` is always the docker proxy IP
10
+ (172.18.0.10). The real client must be read from headers. Header preference,
11
+ in order, matches what every tigzig backend expects:
12
+
13
+ 1. ``X-Original-Client-IP`` - canonical for trusted upstream backends
14
+ (Vercel proxy, internal service-to-service). This is the one a
15
+ sibling backend should set when calling another backend on behalf
16
+ of an end user, so the downstream rate-limits per real user instead
17
+ of per upstream container IP.
18
+ 2. ``CF-Connecting-IP`` - set by Cloudflare on every proxied request.
19
+ 3. ``X-Forwarded-For`` - first IP in the comma-separated list. Used
20
+ when CF is grey-clouded so Caddy sees the real client directly.
21
+ 4. ``X-Real-IP`` - some proxies set this instead of XFF.
22
+ 5. ``request.client.host`` - socket peer; usually the docker proxy.
23
+ """
24
+ from __future__ import annotations
25
+
26
+ from typing import TYPE_CHECKING
27
+
28
+ if TYPE_CHECKING:
29
+ from starlette.requests import Request
30
+
31
+
32
+ _HEADER_ORDER = (
33
+ "x-original-client-ip",
34
+ "cf-connecting-ip",
35
+ "x-forwarded-for",
36
+ "x-real-ip",
37
+ )
38
+
39
+
40
+ def get_real_client_ip(request: "Request") -> str:
41
+ """Return the best-known real client IP for ``request``.
42
+
43
+ Checks the canonical header chain (see module docstring) and falls back
44
+ to the socket peer. Strips whitespace and takes the first IP if the
45
+ header carries a comma-separated list (as XFF can).
46
+
47
+ Returns ``"unknown"`` if no IP can be determined.
48
+ """
49
+ for header_name in _HEADER_ORDER:
50
+ value = request.headers.get(header_name, "").strip()
51
+ if value:
52
+ # XFF may be "client, proxy1, proxy2"; take the leftmost entry.
53
+ return value.split(",", 1)[0].strip()
54
+ if request.client and request.client.host:
55
+ return request.client.host
56
+ return "unknown"
@@ -0,0 +1,43 @@
1
+ """Header forwarding for backend-to-backend HTTP calls.
2
+
3
+ When backend A calls backend B on behalf of an end user, B should see the
4
+ end user's IP, not A's container IP. Otherwise B's per-IP rate limits
5
+ treat all traffic from A as a single "user" and trip on legitimate
6
+ multi-user load.
7
+
8
+ The pattern: A extracts the real client IP via :func:`get_real_client_ip`,
9
+ then includes :func:`forward_client_headers` on every outbound httpx call
10
+ to B. B reads the header chain (which prefers ``X-Original-Client-IP``)
11
+ and rate-limits per real end user.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ from typing import Dict
16
+
17
+
18
+ def forward_client_headers(real_client_ip: str) -> Dict[str, str]:
19
+ """Build the headers backend A should include when calling backend B.
20
+
21
+ Returns an empty dict if ``real_client_ip`` is empty or ``"unknown"``,
22
+ so callers can pass the result unconditionally without poisoning the
23
+ downstream rate limiter with a bogus IP.
24
+
25
+ Example::
26
+
27
+ from tigzig_concurrency import forward_client_headers, get_real_client_ip
28
+
29
+ client_ip = get_real_client_ip(request)
30
+ async with httpx.AsyncClient() as http:
31
+ response = await http.get(
32
+ "https://yfin-h.tigzig.com/get-all-prices/",
33
+ headers=forward_client_headers(client_ip),
34
+ )
35
+ """
36
+ if not real_client_ip or real_client_ip == "unknown":
37
+ return {}
38
+ return {
39
+ # Canonical: every tigzig backend's get_real_client_ip checks this first.
40
+ "X-Original-Client-IP": real_client_ip,
41
+ # Fallback for downstreams that only know XFF.
42
+ "X-Forwarded-For": real_client_ip,
43
+ }
@@ -0,0 +1,212 @@
1
+ """Bounded wait queue for FastAPI / asyncio backends.
2
+
3
+ Replaces the classic fast-fail concurrency check with a small queue that
4
+ absorbs micro-bursts (a user double-clicking submit, an upstream backend
5
+ fanning out a few parallel calls, etc.) without immediately 429-ing.
6
+ The queue is bounded in both depth and wait time, so sustained overload
7
+ still backpressure-rejects quickly.
8
+
9
+ Five knobs (typically read from environment via :meth:`BoundedQueue.from_env`):
10
+
11
+ * ``MAX_INFLIGHT_PER_IP`` - active requests per IP at one time.
12
+ * ``MAX_INFLIGHT_GLOBAL`` - active requests across all IPs.
13
+ * ``MAX_QUEUE_PER_IP`` - extra requests per IP that may wait in line.
14
+ * ``MAX_QUEUE_GLOBAL`` - extra requests across all IPs that may wait.
15
+ * ``MAX_QUEUE_WAIT_SECONDS`` - max time any one request waits before 429.
16
+
17
+ Behaviour matrix (one acquire call):
18
+
19
+ ============================== =========================================
20
+ Slot available Run immediately.
21
+ Slot taken, queue not full Wait up to ``MAX_QUEUE_WAIT_SECONDS``,
22
+ then either acquire or 429.
23
+ Per-IP queue full at arrival 503 immediately.
24
+ Global queue full at arrival 503 immediately.
25
+ ============================== =========================================
26
+
27
+ Both 429 and 503 responses include a ``Retry-After: 10`` header so
28
+ frontends can show a friendly busy message + retry instead of a hard error.
29
+
30
+ Tuning rule of thumb: pick ``MAX_QUEUE_WAIT_SECONDS`` so that
31
+ ``wait + work + slack`` stays well under the edge timeout (Cloudflare
32
+ free is 100s). For a 30-45s LLM-bound endpoint, 30s wait is fine. For a
33
+ 1-5s API endpoint, 5s wait is plenty.
34
+ """
35
+ from __future__ import annotations
36
+
37
+ import asyncio
38
+ import logging
39
+ import os
40
+ from collections import defaultdict
41
+ from typing import Dict, Optional
42
+
43
+ from .responses import queue_busy_429, queue_full_503
44
+
45
+ logger = logging.getLogger("tigzig_concurrency")
46
+
47
+
48
+ class BoundedQueue:
49
+ """An async-safe bounded wait queue with per-IP and global limits.
50
+
51
+ Use :meth:`from_env` to construct from environment variables, or pass
52
+ the five integer/float settings directly to the constructor.
53
+ """
54
+
55
+ def __init__(
56
+ self,
57
+ max_inflight_per_ip: int,
58
+ max_inflight_global: int,
59
+ max_queue_per_ip: int,
60
+ max_queue_global: int,
61
+ max_queue_wait_seconds: float,
62
+ name: str = "queue",
63
+ ) -> None:
64
+ if max_inflight_per_ip < 1:
65
+ raise ValueError("max_inflight_per_ip must be >= 1")
66
+ if max_inflight_global < 1:
67
+ raise ValueError("max_inflight_global must be >= 1")
68
+ if max_queue_per_ip < 0:
69
+ raise ValueError("max_queue_per_ip must be >= 0")
70
+ if max_queue_global < 0:
71
+ raise ValueError("max_queue_global must be >= 0")
72
+ if max_queue_wait_seconds < 0:
73
+ raise ValueError("max_queue_wait_seconds must be >= 0")
74
+
75
+ self.max_inflight_per_ip = max_inflight_per_ip
76
+ self.max_inflight_global = max_inflight_global
77
+ self.max_queue_per_ip = max_queue_per_ip
78
+ self.max_queue_global = max_queue_global
79
+ self.max_queue_wait_seconds = max_queue_wait_seconds
80
+ self.name = name
81
+
82
+ self._global_sem = asyncio.Semaphore(max_inflight_global)
83
+ self._ip_sems: Dict[str, asyncio.Semaphore] = {}
84
+ self._ip_sems_lock = asyncio.Lock()
85
+ self._global_queued: int = 0
86
+ self._ip_queued: Dict[str, int] = defaultdict(int)
87
+ self._queue_state_lock = asyncio.Lock()
88
+
89
+ @classmethod
90
+ def from_env(cls, prefix: str = "", name: Optional[str] = None) -> "BoundedQueue":
91
+ """Construct from environment variables.
92
+
93
+ Reads ``{prefix}MAX_INFLIGHT_PER_IP``, ``{prefix}MAX_INFLIGHT_GLOBAL``,
94
+ ``{prefix}MAX_QUEUE_PER_IP``, ``{prefix}MAX_QUEUE_GLOBAL``,
95
+ ``{prefix}MAX_QUEUE_WAIT_SECONDS``. Use a non-empty ``prefix`` to
96
+ run multiple queues in one app with different settings (e.g.
97
+ ``prefix="HEAVY_"`` reads ``HEAVY_MAX_INFLIGHT_PER_IP`` etc.).
98
+
99
+ Defaults are conservative ("slow LLM-bound" profile from the
100
+ tigzig fleet); override per-app via environment.
101
+ """
102
+ return cls(
103
+ max_inflight_per_ip=int(os.getenv(f"{prefix}MAX_INFLIGHT_PER_IP", "2")),
104
+ max_inflight_global=int(os.getenv(f"{prefix}MAX_INFLIGHT_GLOBAL", "6")),
105
+ max_queue_per_ip=int(os.getenv(f"{prefix}MAX_QUEUE_PER_IP", "3")),
106
+ max_queue_global=int(os.getenv(f"{prefix}MAX_QUEUE_GLOBAL", "8")),
107
+ max_queue_wait_seconds=float(os.getenv(f"{prefix}MAX_QUEUE_WAIT_SECONDS", "15")),
108
+ name=name or (prefix.rstrip("_").lower() if prefix else "queue"),
109
+ )
110
+
111
+ async def _get_ip_sem(self, client_ip: str) -> asyncio.Semaphore:
112
+ async with self._ip_sems_lock:
113
+ sem = self._ip_sems.get(client_ip)
114
+ if sem is None:
115
+ sem = asyncio.Semaphore(self.max_inflight_per_ip)
116
+ self._ip_sems[client_ip] = sem
117
+ return sem
118
+
119
+ async def acquire(self, client_ip: str) -> None:
120
+ """Wait for an in-flight slot. Raises :class:`HTTPException` on
121
+ 429 (queue wait timed out) or 503 (queue full at arrival)."""
122
+ deadline = asyncio.get_event_loop().time() + self.max_queue_wait_seconds
123
+
124
+ ip_sem = await self._get_ip_sem(client_ip)
125
+
126
+ # Reserve a queue slot, or fail fast if the queue is already full.
127
+ async with self._queue_state_lock:
128
+ if self._global_queued >= self.max_queue_global:
129
+ logger.warning(
130
+ "[%s] global queue full (%d), rejecting from %s",
131
+ self.name, self._global_queued, client_ip,
132
+ )
133
+ raise queue_full_503()
134
+ ip_queued_now = self._ip_queued[client_ip]
135
+ if ip_queued_now >= self.max_queue_per_ip:
136
+ logger.warning(
137
+ "[%s] per-IP queue full for %s (%d), rejecting",
138
+ self.name, client_ip, ip_queued_now,
139
+ )
140
+ raise queue_full_503(
141
+ message="Too many requests in flight from your address. Please wait and try again.",
142
+ )
143
+ self._global_queued += 1
144
+ self._ip_queued[client_ip] = ip_queued_now + 1
145
+
146
+ acquired_ip_sem = False
147
+ try:
148
+ # Step 1: per-IP semaphore.
149
+ remaining = max(0.0, deadline - asyncio.get_event_loop().time())
150
+ try:
151
+ await asyncio.wait_for(ip_sem.acquire(), timeout=remaining)
152
+ acquired_ip_sem = True
153
+ except asyncio.TimeoutError:
154
+ logger.warning(
155
+ "[%s] per-IP slot wait timed out for %s after %.1fs",
156
+ self.name, client_ip, self.max_queue_wait_seconds,
157
+ )
158
+ raise queue_busy_429()
159
+
160
+ # Step 2: global semaphore.
161
+ remaining = max(0.0, deadline - asyncio.get_event_loop().time())
162
+ try:
163
+ await asyncio.wait_for(self._global_sem.acquire(), timeout=remaining)
164
+ except asyncio.TimeoutError:
165
+ ip_sem.release()
166
+ logger.warning(
167
+ "[%s] global slot wait timed out for %s",
168
+ self.name, client_ip,
169
+ )
170
+ raise queue_busy_429()
171
+ finally:
172
+ async with self._queue_state_lock:
173
+ self._global_queued = max(0, self._global_queued - 1)
174
+ ip_q = self._ip_queued[client_ip] - 1
175
+ if ip_q <= 0:
176
+ self._ip_queued.pop(client_ip, None)
177
+ else:
178
+ self._ip_queued[client_ip] = ip_q
179
+
180
+ def release(self, client_ip: str) -> None:
181
+ """Release an in-flight slot. Safe to call even if :meth:`acquire`
182
+ was not completed; releasing a never-acquired semaphore is a no-op
183
+ beyond bumping the counter (asyncio.Semaphore allows this)."""
184
+ self._global_sem.release()
185
+ sem = self._ip_sems.get(client_ip)
186
+ if sem is not None:
187
+ sem.release()
188
+
189
+ @property
190
+ def stats(self) -> dict:
191
+ """Snapshot of current state. Useful for /healthz endpoints.
192
+
193
+ Note: counts are read without the lock, so values are eventually
194
+ consistent. Good enough for monitoring; do not use for
195
+ decision-making logic."""
196
+ # _value is the implementation detail; semaphore = (max - inflight).
197
+ global_inflight = self.max_inflight_global - self._global_sem._value
198
+ return {
199
+ "name": self.name,
200
+ "limits": {
201
+ "inflight_per_ip": self.max_inflight_per_ip,
202
+ "inflight_global": self.max_inflight_global,
203
+ "queue_per_ip": self.max_queue_per_ip,
204
+ "queue_global": self.max_queue_global,
205
+ "wait_seconds": self.max_queue_wait_seconds,
206
+ },
207
+ "current": {
208
+ "inflight_global": global_inflight,
209
+ "queued_global": self._global_queued,
210
+ "queued_per_ip": dict(self._ip_queued),
211
+ },
212
+ }
@@ -0,0 +1,42 @@
1
+ """Standard HTTP error responses for queue-saturation cases.
2
+
3
+ Three failure modes, two response codes:
4
+
5
+ * ``429 Too Many Requests`` - the request entered the queue and waited but
6
+ no slot freed within ``MAX_QUEUE_WAIT_SECONDS``. The frontend should
7
+ show a "server busy, retrying in N seconds" spinner and retry after
8
+ ``Retry-After``.
9
+ * ``503 Service Unavailable`` - the queue itself was full at arrival
10
+ (either per-IP or global). The system is genuinely overloaded; the
11
+ frontend should back off longer.
12
+
13
+ Both responses include a ``Retry-After`` header (seconds) so a polite
14
+ client can wait before retrying.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ from starlette.exceptions import HTTPException
19
+
20
+
21
+ def queue_busy_429(
22
+ message: str = "Server is busy with other requests. Please retry in a few seconds.",
23
+ retry_after: int = 10,
24
+ ) -> HTTPException:
25
+ """Build a 429 for "queued but the wait timed out before a slot freed"."""
26
+ return HTTPException(
27
+ status_code=429,
28
+ detail=message,
29
+ headers={"Retry-After": str(retry_after)},
30
+ )
31
+
32
+
33
+ def queue_full_503(
34
+ message: str = "Server is at capacity right now. Please wait a moment and try again.",
35
+ retry_after: int = 10,
36
+ ) -> HTTPException:
37
+ """Build a 503 for "queue cap exceeded at arrival, didn't even queue"."""
38
+ return HTTPException(
39
+ status_code=503,
40
+ detail=message,
41
+ headers={"Retry-After": str(retry_after)},
42
+ )
@@ -0,0 +1,270 @@
1
+ Metadata-Version: 2.4
2
+ Name: tigzig-concurrency
3
+ Version: 0.1.0
4
+ Summary: Bounded wait queue + client-IP forwarding for FastAPI backends
5
+ Project-URL: Homepage, https://github.com/amararun/tigzig-concurrency
6
+ Project-URL: Repository, https://github.com/amararun/tigzig-concurrency
7
+ Project-URL: Issues, https://github.com/amararun/tigzig-concurrency/issues
8
+ Author-email: Amar Harolikar <amar@harolikar.com>
9
+ License: MIT
10
+ Keywords: backpressure,concurrency,fastapi,queue,rate-limit
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Framework :: FastAPI
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.8
18
+ Classifier: Programming Language :: Python :: 3.9
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
23
+ Requires-Python: >=3.8
24
+ Requires-Dist: starlette>=0.27.0
25
+ Description-Content-Type: text/markdown
26
+
27
+ # tigzig-concurrency
28
+
29
+ Bounded wait queue + client-IP forwarding for FastAPI / Starlette backends.
30
+
31
+ A standing-rule pattern for the tigzig fleet. Replaces ad-hoc per-app
32
+ concurrency-check and IP-extraction code so all backends behave the same
33
+ way under load.
34
+
35
+ ## Why this exists
36
+
37
+ Two recurring problems across the fleet:
38
+
39
+ 1. **Fast-fail concurrency caps reject legitimate users.** A user
40
+ double-clicks submit, two of their parallel sub-requests collide, and
41
+ one gets a 429 even though the server is mostly idle. This package
42
+ replaces the fast-fail check with a small bounded queue: the second
43
+ request waits up to N seconds for a slot, then either runs or 429s.
44
+ The server is still protected from sustained overload (queue is
45
+ bounded), but micro-bursts succeed.
46
+
47
+ 2. **Backend-to-backend calls trip downstream per-IP limits.** When
48
+ backend A (e.g. TA) calls backend B (e.g. yfin) on behalf of an end
49
+ user, B sees the request from A's container IP - so two unrelated
50
+ end users routed via A look like ONE user at B and trip B's per-IP
51
+ cap. This package provides ``forward_client_headers()`` so A passes
52
+ the real user IP, and ``get_real_client_ip()`` (which B already uses)
53
+ reads it.
54
+
55
+ ## Installation
56
+
57
+ ```bash
58
+ pip install tigzig-concurrency
59
+ ```
60
+
61
+ Add to your ``requirements.txt``:
62
+
63
+ ```
64
+ tigzig-concurrency>=0.1.0
65
+ ```
66
+
67
+ ## Quick start
68
+
69
+ ### 1. Wrap your endpoint with the queue
70
+
71
+ ```python
72
+ from fastapi import FastAPI, Request
73
+ from tigzig_concurrency import BoundedQueue, get_real_client_ip
74
+
75
+ app = FastAPI()
76
+ queue = BoundedQueue.from_env() # reads MAX_INFLIGHT_*, MAX_QUEUE_* from env
77
+
78
+ @app.post("/api/work")
79
+ async def do_work(request: Request):
80
+ client_ip = get_real_client_ip(request)
81
+ await queue.acquire(client_ip)
82
+ try:
83
+ # ... your endpoint work ...
84
+ return {"ok": True}
85
+ finally:
86
+ queue.release(client_ip)
87
+ ```
88
+
89
+ ### 2. Forward the real user IP on backend-to-backend calls
90
+
91
+ ```python
92
+ import httpx
93
+ from tigzig_concurrency import forward_client_headers, get_real_client_ip
94
+
95
+ @app.post("/api/work")
96
+ async def do_work(request: Request):
97
+ client_ip = get_real_client_ip(request)
98
+ await queue.acquire(client_ip)
99
+ try:
100
+ async with httpx.AsyncClient() as http:
101
+ response = await http.get(
102
+ "https://other-backend.tigzig.com/data",
103
+ headers=forward_client_headers(client_ip),
104
+ )
105
+ return response.json()
106
+ finally:
107
+ queue.release(client_ip)
108
+ ```
109
+
110
+ The downstream backend's ``get_real_client_ip()`` will read the
111
+ ``X-Original-Client-IP`` header and rate-limit per real end user.
112
+
113
+ ## Configuration cheatsheet
114
+
115
+ Five environment variables. **You will forget what these mean in three
116
+ weeks - this is the cheatsheet.** Future-you, hello.
117
+
118
+ ### The five knobs
119
+
120
+ | Variable | What it does | When to raise / lower |
121
+ |---|---|---|
122
+ | ``MAX_INFLIGHT_PER_IP`` | Active requests per IP at one time. | One real user can have this many running at once. Higher = more permissive for power-users; lower = stricter fairness. |
123
+ | ``MAX_INFLIGHT_GLOBAL`` | Active requests across all IPs combined. | Cap on total CPU/memory the server commits at once. Higher = more parallelism if you have headroom; lower if memory is tight. |
124
+ | ``MAX_QUEUE_PER_IP`` | Extra requests per IP that may wait in line. | Bigger queue = more forgiveness for "user clicked submit 4 times in 5 seconds". 0 = no queueing per-IP (fast-fail behaviour). |
125
+ | ``MAX_QUEUE_GLOBAL`` | Extra requests across all IPs that may wait. | Total system buffer for bursts. Beyond this -> instant 503. |
126
+ | ``MAX_QUEUE_WAIT_SECONDS`` | How long any one request waits in queue before 429. | Match to typical request length; ``wait + work + slack`` should stay under your edge timeout (Cloudflare free is 100s). |
127
+
128
+ ### Behaviour
129
+
130
+ | Situation | Response |
131
+ |---|---|
132
+ | Slot available | Run immediately (no queue overhead). |
133
+ | Slot taken, queue not full | Wait up to ``MAX_QUEUE_WAIT_SECONDS`` for a slot, then run, OR 429 if the wait expired. |
134
+ | Per-IP queue full at arrival | 503 immediately (this user is hammering). |
135
+ | Global queue full at arrival | 503 immediately (server is overloaded). |
136
+
137
+ Both 429 and 503 include a ``Retry-After: 10`` header so the frontend
138
+ can show a friendly "server busy, retry in 10s" spinner instead of a
139
+ hard error.
140
+
141
+ ### Tuning profiles for the tigzig fleet
142
+
143
+ Three workload types cover most apps. Pick the closest profile, set the
144
+ env vars accordingly, and tune from there if needed.
145
+
146
+ #### Profile A: fast IO-bound (1-5s per request)
147
+ For yfin, mdtopdf, llama-parse, markitdown, neon-proxy, supabase-proxy,
148
+ and any wrapper that mostly does network I/O + light pandas.
149
+
150
+ ```
151
+ MAX_INFLIGHT_PER_IP=6
152
+ MAX_INFLIGHT_GLOBAL=15
153
+ MAX_QUEUE_PER_IP=4
154
+ MAX_QUEUE_GLOBAL=10
155
+ MAX_QUEUE_WAIT_SECONDS=5
156
+ ```
157
+
158
+ Why: requests are quick, so being aggressive with concurrency is safe
159
+ (no memory pressure). Short wait window because end users expect fast
160
+ responses; queue absorbs occasional bursts.
161
+
162
+ #### Profile B: slow LLM / compute-bound (10-45s per request)
163
+ For TA, FFN/SPR, QuantStats, codeinter (Python sandbox), codeinter-gemini.
164
+
165
+ ```
166
+ MAX_INFLIGHT_PER_IP=2
167
+ MAX_INFLIGHT_GLOBAL=6
168
+ MAX_QUEUE_PER_IP=3
169
+ MAX_QUEUE_GLOBAL=8
170
+ MAX_QUEUE_WAIT_SECONDS=30
171
+ ```
172
+
173
+ Why: each request hogs a CPU or holds an LLM connection for a long time.
174
+ Tighter in-flight cap to avoid memory blowup. Longer wait because users
175
+ already expect slow; an extra 15-30s of queue is invisible vs the 30s of
176
+ actual work. Total ``wait + work`` stays under Cloudflare 100s.
177
+
178
+ #### Profile C: heavy memory / database (DuckDB queries, Flowise)
179
+ For duckdb-backend, duckdb-dashboards-backend, flowise-docker-custom.
180
+
181
+ ```
182
+ MAX_INFLIGHT_PER_IP=2
183
+ MAX_INFLIGHT_GLOBAL=4
184
+ MAX_QUEUE_PER_IP=2
185
+ MAX_QUEUE_GLOBAL=4
186
+ MAX_QUEUE_WAIT_SECONDS=15
187
+ ```
188
+
189
+ Why: a single heavy DuckDB query can pin a CPU AND eat a few hundred MB
190
+ of memory. Tight global cap matters more than per-IP fairness. Bump the
191
+ container memory limit if you raise the global cap.
192
+
193
+ ### Container-limit reminder
194
+
195
+ These knobs only protect the server if the container itself has enough
196
+ CPU and memory. For the tigzig fleet's standard:
197
+
198
+ ```
199
+ limits_cpus=2.0
200
+ limits_memory=1024m (or 2048m for memory-heavy or many-concurrent profiles)
201
+ limits_memory_swap=2048m
202
+ ```
203
+
204
+ A queue with ``MAX_INFLIGHT_GLOBAL=15`` on a container capped at
205
+ ``cpus=0.25`` will just queue everything and run them slowly. The CPU
206
+ bump is the bigger lever.
207
+
208
+ ## Multiple queues in one app
209
+
210
+ If one endpoint is much heavier than others, you can run two queues
211
+ with different settings via the ``prefix`` argument:
212
+
213
+ ```python
214
+ light_queue = BoundedQueue.from_env(prefix="LIGHT_", name="light")
215
+ heavy_queue = BoundedQueue.from_env(prefix="HEAVY_", name="heavy")
216
+ ```
217
+
218
+ Then set ``LIGHT_MAX_INFLIGHT_PER_IP``, ``HEAVY_MAX_INFLIGHT_PER_IP`` etc.
219
+ in your environment.
220
+
221
+ ## Health / introspection
222
+
223
+ The queue exposes its current state via a ``stats`` property. Useful
224
+ for ``/healthz`` endpoints:
225
+
226
+ ```python
227
+ @app.get("/healthz/queue")
228
+ def queue_health():
229
+ return queue.stats
230
+ ```
231
+
232
+ Returns:
233
+
234
+ ```json
235
+ {
236
+ "name": "queue",
237
+ "limits": {"inflight_per_ip": 2, "inflight_global": 6, ...},
238
+ "current": {"inflight_global": 3, "queued_global": 1, "queued_per_ip": {"1.2.3.4": 1}}
239
+ }
240
+ ```
241
+
242
+ ## Reference: header chain that ``get_real_client_ip`` reads
243
+
244
+ In order:
245
+
246
+ 1. ``X-Original-Client-IP`` (canonical for trusted upstream backends)
247
+ 2. ``CF-Connecting-IP`` (Cloudflare)
248
+ 3. ``X-Forwarded-For`` (first IP if comma-separated)
249
+ 4. ``X-Real-IP``
250
+ 5. ``request.client.host`` (socket peer; usually docker proxy)
251
+
252
+ Returns ``"unknown"`` if no IP can be resolved.
253
+
254
+ ## Why ``X-Original-Client-IP`` over ``X-Forwarded-For``
255
+
256
+ XFF gets rewritten by every hop (Cloudflare, Caddy, nginx). For trusted
257
+ internal forwarding it's safer to use a custom header that only the
258
+ trusted upstream sets. ``X-Original-Client-IP`` is the convention across
259
+ the tigzig fleet; the helper sets both to maximize compatibility but
260
+ the canonical reader prefers the custom header.
261
+
262
+ ## Versioning
263
+
264
+ Semantic versioning. Breaking changes to function signatures or env-var
265
+ names bump the major version. Tuning-profile recommendations may evolve
266
+ across minor versions; check the README of the version you're using.
267
+
268
+ ## License
269
+
270
+ MIT.
@@ -0,0 +1,8 @@
1
+ tigzig_concurrency/__init__.py,sha256=YhK4XLsKzB_wNxy2wfcOjtrF0EW3RUfgSaWtIeBXZeo,411
2
+ tigzig_concurrency/client_ip.py,sha256=6VWfLc1GlNpDDwO0raNGcKBbgCTM5c50FHeWqaih3kw,2321
3
+ tigzig_concurrency/headers.py,sha256=TBhKsgVbB9ubYHd3NGpllf3l4ATMxuKG62EeL4i-5Ts,1642
4
+ tigzig_concurrency/queue.py,sha256=w6DLeXJm-GQxGXVGrgynbIHIW3mG1TFiEH48eTb-r2w,9130
5
+ tigzig_concurrency/responses.py,sha256=VvUYqCWOsKh8GMh9Sz-KqBxN__Q15ucNuogSvfLLVx0,1442
6
+ tigzig_concurrency-0.1.0.dist-info/METADATA,sha256=rtalz7FhQB_0oqC3Pc437aZlgptiaHCR0D7nTDXzFKY,9351
7
+ tigzig_concurrency-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
8
+ tigzig_concurrency-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.29.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any