fastapi-stream-lease 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastapi_stream_lease-0.1.0/.github/workflows/ci.yml +64 -0
- fastapi_stream_lease-0.1.0/.github/workflows/publish.yml +30 -0
- fastapi_stream_lease-0.1.0/.gitignore +30 -0
- fastapi_stream_lease-0.1.0/LICENSE +21 -0
- fastapi_stream_lease-0.1.0/PKG-INFO +213 -0
- fastapi_stream_lease-0.1.0/README.md +176 -0
- fastapi_stream_lease-0.1.0/pyproject.toml +83 -0
- fastapi_stream_lease-0.1.0/src/fastapi_stream_lease/__init__.py +17 -0
- fastapi_stream_lease-0.1.0/src/fastapi_stream_lease/config.py +47 -0
- fastapi_stream_lease-0.1.0/src/fastapi_stream_lease/exceptions.py +58 -0
- fastapi_stream_lease-0.1.0/src/fastapi_stream_lease/lease.py +95 -0
- fastapi_stream_lease-0.1.0/src/fastapi_stream_lease/lua.py +95 -0
- fastapi_stream_lease-0.1.0/src/fastapi_stream_lease/manager.py +147 -0
- fastapi_stream_lease-0.1.0/src/fastapi_stream_lease/py.typed +1 -0
- fastapi_stream_lease-0.1.0/tests/conftest.py +32 -0
- fastapi_stream_lease-0.1.0/tests/test_fastapi_integration.py +109 -0
- fastapi_stream_lease-0.1.0/tests/test_lease_lifecycle.py +159 -0
- fastapi_stream_lease-0.1.0/tests/test_lua_scripts.py +170 -0
- fastapi_stream_lease-0.1.0/uv.lock +1028 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
lint:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
|
|
15
|
+
- name: Install uv
|
|
16
|
+
uses: astral-sh/setup-uv@v5
|
|
17
|
+
with:
|
|
18
|
+
enable-cache: true
|
|
19
|
+
|
|
20
|
+
- name: Set up Python
|
|
21
|
+
run: uv python install 3.10
|
|
22
|
+
|
|
23
|
+
- name: Install dependencies
|
|
24
|
+
run: uv sync --extra dev
|
|
25
|
+
|
|
26
|
+
- name: Run Ruff Lint
|
|
27
|
+
run: uv run ruff check .
|
|
28
|
+
|
|
29
|
+
- name: Run Ruff Format Check
|
|
30
|
+
run: uv run ruff format --check .
|
|
31
|
+
|
|
32
|
+
- name: Run Mypy Type Checking
|
|
33
|
+
run: uv run mypy src
|
|
34
|
+
|
|
35
|
+
test:
|
|
36
|
+
runs-on: ubuntu-latest
|
|
37
|
+
strategy:
|
|
38
|
+
fail-fast: false
|
|
39
|
+
matrix:
|
|
40
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
41
|
+
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/checkout@v4
|
|
44
|
+
|
|
45
|
+
- name: Install uv
|
|
46
|
+
uses: astral-sh/setup-uv@v5
|
|
47
|
+
with:
|
|
48
|
+
enable-cache: true
|
|
49
|
+
|
|
50
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
51
|
+
run: uv python install ${{ matrix.python-version }}
|
|
52
|
+
|
|
53
|
+
- name: Install dependencies
|
|
54
|
+
run: uv sync --extra dev
|
|
55
|
+
|
|
56
|
+
- name: Run Pytest with Coverage
|
|
57
|
+
run: uv run pytest --cov=fastapi_stream_lease --cov-report=xml --cov-report=term-missing
|
|
58
|
+
|
|
59
|
+
- name: Upload coverage artifact
|
|
60
|
+
uses: actions/upload-artifact@v4
|
|
61
|
+
if: matrix.python-version == '3.13'
|
|
62
|
+
with:
|
|
63
|
+
name: coverage-report
|
|
64
|
+
path: coverage.xml
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
id-token: write
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
pypi-publish:
|
|
13
|
+
name: Build and publish Python 🐍 distributions 📦 to PyPI
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
environment:
|
|
16
|
+
name: pypi
|
|
17
|
+
url: https://pypi.org/p/fastapi-stream-lease
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- name: Install uv
|
|
22
|
+
uses: astral-sh/setup-uv@v5
|
|
23
|
+
with:
|
|
24
|
+
enable-cache: true
|
|
25
|
+
|
|
26
|
+
- name: Build sdist and wheel
|
|
27
|
+
run: uv build
|
|
28
|
+
|
|
29
|
+
- name: Publish distribution 📦 to PyPI
|
|
30
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
dist/
|
|
8
|
+
build/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
.eggs/
|
|
11
|
+
|
|
12
|
+
# Environments
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
ENV/
|
|
16
|
+
|
|
17
|
+
# Testing & Coverage
|
|
18
|
+
.pytest_cache/
|
|
19
|
+
.coverage
|
|
20
|
+
coverage.xml
|
|
21
|
+
htmlcov/
|
|
22
|
+
|
|
23
|
+
# Type checking & linting
|
|
24
|
+
.mypy_cache/
|
|
25
|
+
.ruff_cache/
|
|
26
|
+
|
|
27
|
+
# IDE & OS
|
|
28
|
+
.vscode/
|
|
29
|
+
.idea/
|
|
30
|
+
.DS_Store
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Agustin Saiz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fastapi-stream-lease
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Distributed stream and SSE concurrency lease manager for FastAPI and Starlette backed by atomic Redis Lua scripts.
|
|
5
|
+
Project-URL: Homepage, https://github.com/agustin18/fastapi-stream-lease
|
|
6
|
+
Project-URL: Repository, https://github.com/agustin18/fastapi-stream-lease
|
|
7
|
+
Project-URL: Issues, https://github.com/agustin18/fastapi-stream-lease/issues
|
|
8
|
+
Author-email: Agustin Saiz <agustinsaiz02@gmail.com>
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: concurrency,fastapi,llm-streaming,rate-limiting,redis,server-sent-events,sse,starlette,streaming
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Framework :: FastAPI
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: redis>=5.0.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: fakeredis[lua]>=2.20.0; extra == 'dev'
|
|
27
|
+
Requires-Dist: fastapi>=0.100.0; extra == 'dev'
|
|
28
|
+
Requires-Dist: httpx>=0.25.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: mypy>=1.10.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
34
|
+
Provides-Extra: fastapi
|
|
35
|
+
Requires-Dist: fastapi>=0.100.0; extra == 'fastapi'
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# fastapi-stream-lease
|
|
39
|
+
|
|
40
|
+
[](https://github.com/agustin18/fastapi-stream-lease/actions)
|
|
41
|
+
[](https://pypi.org/project/fastapi-stream-lease/)
|
|
42
|
+
[](https://www.python.org/downloads/)
|
|
43
|
+
[](https://opensource.org/licenses/MIT)
|
|
44
|
+
[](https://github.com/astral-sh/ruff)
|
|
45
|
+
|
|
46
|
+
**Distributed stream and SSE concurrency lease manager for FastAPI and Starlette, backed by atomic Redis Lua scripts.**
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## ⚡ The Problem: Why Traditional Rate Limiters Fail for Streams & LLMs
|
|
51
|
+
|
|
52
|
+
Standard rate limiters (such as `fastapi-limiter` or `slowapi`) count **requests per unit of time** (e.g. *5 requests per minute*).
|
|
53
|
+
|
|
54
|
+
While this works for standard REST APIs, it completely breaks down for **long-lived streaming connections** (Server-Sent Events, WebSockets, or streaming LLM tokens from OpenAI / Claude / Ollama):
|
|
55
|
+
|
|
56
|
+
1. **Duration Blindness:** A user can make a single request that stays open for 15 minutes, consuming a server socket the entire time. A rate limiter considers this "1 request" and allows the user to open 50 more tabs.
|
|
57
|
+
2. **Zombie Connection Leaks:** When mobile users switch networks or close tabs abruptly without clean TCP closure, worker connections remain blocked until timeout, causing connection pool exhaustion and denial of service.
|
|
58
|
+
3. **Multi-Worker Desynchronization:** In-memory concurrency limiters (like `asyncio.Semaphore`) fail across multi-process deployments (Gunicorn / Docker containers) because workers cannot share state.
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
Traditional Rate Limiter: fastapi-stream-lease:
|
|
62
|
+
┌───────────────────────┐ ┌──────────────────────────────────────────────┐
|
|
63
|
+
│ Request 1 -> ALLOWED │ │ Stream 1 (Active) -> LEASE ACQUIRED (1/2) │
|
|
64
|
+
│ Request 2 -> ALLOWED │ │ Stream 2 (Active) -> LEASE ACQUIRED (2/2) │
|
|
65
|
+
│ (Both streams active │ │ Stream 3 (Attempt) -> REJECTED: HTTP 429 │
|
|
66
|
+
│ for 10 minutes, │ │ Retry-After: 5 │
|
|
67
|
+
│ server sockets exhausted!) │ Stream 1 disconnects -> LEASE RELEASED │
|
|
68
|
+
└───────────────────────┘ │ Stream 3 re-attempt -> LEASE ACQUIRED (2/2) │
|
|
69
|
+
└──────────────────────────────────────────────┘
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
`fastapi-stream-lease` solves this with **Sliding Distributed Leases** inside **atomic Redis Lua scripts**:
|
|
73
|
+
- Enforces strict concurrency limits **per user** (`max_per_user`) and **globally** (`max_global`).
|
|
74
|
+
- Leases automatically self-expire if the client or worker dies without clean closure (zero zombies).
|
|
75
|
+
- A background renewal task keeps long-running streams alive even during slow Time-To-First-Token (TTFT) pauses.
|
|
76
|
+
- Released immediately when the stream finishes or client disconnects.
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## 🚀 Installation
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
pip install fastapi-stream-lease
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Or using `uv`:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
uv add fastapi-stream-lease
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
*(Requires Redis 5.0+ and Python 3.10+)*
|
|
93
|
+
|
|
94
|
+
---
|
|
95
|
+
|
|
96
|
+
## 💡 Quickstart
|
|
97
|
+
|
|
98
|
+
Protect an SSE or LLM streaming endpoint in just a few lines:
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from fastapi import FastAPI, Depends, Request
|
|
102
|
+
from fastapi.responses import StreamingResponse
|
|
103
|
+
import redis.asyncio as redis
|
|
104
|
+
|
|
105
|
+
from fastapi_stream_lease import (
|
|
106
|
+
StreamLeaseManager,
|
|
107
|
+
LeaseConfig,
|
|
108
|
+
StreamLeaseRejected,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
app = FastAPI()
|
|
112
|
+
redis_client = redis.from_url("redis://localhost:6379")
|
|
113
|
+
|
|
114
|
+
# Configure lease boundaries:
|
|
115
|
+
# Each user can hold at most 2 concurrent streams; cluster max is 500.
|
|
116
|
+
lease_manager = StreamLeaseManager(
|
|
117
|
+
redis=redis_client,
|
|
118
|
+
config=LeaseConfig(
|
|
119
|
+
max_per_user=2,
|
|
120
|
+
max_global=500,
|
|
121
|
+
lease_seconds=30.0,
|
|
122
|
+
),
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# Convert lease rejections into clean HTTP 429 Too Many Requests responses:
|
|
127
|
+
@app.exception_handler(StreamLeaseRejected)
|
|
128
|
+
async def lease_rejected_handler(request: Request, exc: StreamLeaseRejected):
|
|
129
|
+
return exc.as_response()
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@app.get("/api/chat/stream")
|
|
133
|
+
async def chat_stream(user_id: str = "user_123"):
|
|
134
|
+
# 1. Acquire lease (raises StreamLeaseRejected if limit reached)
|
|
135
|
+
lease = await lease_manager.acquire(user_id)
|
|
136
|
+
|
|
137
|
+
async def token_generator():
|
|
138
|
+
# Example: streaming tokens from an LLM
|
|
139
|
+
for word in ["Hello", "world", "this", "is", "streamed!"]:
|
|
140
|
+
yield f"data: {word}\n\n"
|
|
141
|
+
|
|
142
|
+
# 2. Wrap generator: guarantees background auto-renewal and release on disconnect
|
|
143
|
+
return StreamingResponse(
|
|
144
|
+
lease.wrap(token_generator()),
|
|
145
|
+
media_type="text/event-stream",
|
|
146
|
+
)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
### Protecting WebSockets
|
|
150
|
+
|
|
151
|
+
For WebSockets and scoped async routines, use the `lease_manager.lease(...)` context manager:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
@app.websocket("/ws/chat/{user_id}")
|
|
155
|
+
async def websocket_chat(websocket: WebSocket, user_id: str):
|
|
156
|
+
await websocket.accept()
|
|
157
|
+
# Acquires lease on enter, automatically releases when socket closes or disconnects
|
|
158
|
+
async with lease_manager.lease(user_id):
|
|
159
|
+
while True:
|
|
160
|
+
msg = await websocket.receive_text()
|
|
161
|
+
await websocket.send_text(f"Echo: {msg}")
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## 🛠️ How It Works (Algorithmic Math)
|
|
167
|
+
|
|
168
|
+
All concurrency validations, expirations, and insertions run inside **atomic Lua scripts** on Redis:
|
|
169
|
+
|
|
170
|
+
1. **Sorted Sets (`ZSET`):** Active streams are stored in Redis `ZSET`s where the value is a unique `lease_id` and the score is the epoch expiration timestamp (`now + lease_seconds`).
|
|
171
|
+
2. **Atomic Eviction:** Before checking capacity, `ZREMRANGEBYSCORE` purges all expired entries in $O(\log N + M)$.
|
|
172
|
+
3. **Capacity Check:** `ZCARD` verifies current stream count in $O(1)$ against `max_per_user` and `max_global`. Setting either to `0` disables that limit.
|
|
173
|
+
4. **Redis Cluster Slot Affinity:** Keys automatically use `{prefix}` hash tags (e.g. `{stream_lease}:user:123` and `{stream_lease}:global`), guaranteeing zero `CROSSSLOT` errors across distributed Redis clusters.
|
|
174
|
+
5. **Acquisition:** If capacity permits, `ZADD` registers the lease in $O(\log N)$ and updates the key TTL.
|
|
175
|
+
6. **Auto-Renewal:** While the stream is active, `lease.wrap()` spawns a lightweight background worker that calls `ZADD` to advance the expiration score every `lease_seconds / 2`.
|
|
176
|
+
7. **Guaranteed Release:** When the stream completes or the client disconnects, `ZREM` removes the lease immediately in the `finally:` block.
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## ⚙️ Configuration Options
|
|
181
|
+
|
|
182
|
+
Customize `LeaseConfig`:
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
from fastapi_stream_lease import LeaseConfig
|
|
186
|
+
|
|
187
|
+
config = LeaseConfig(
|
|
188
|
+
lease_seconds=30.0, # Lease expiration window (seconds)
|
|
189
|
+
max_per_user=3, # Max active streams per user (set 0 to disable)
|
|
190
|
+
max_global=1000, # Max active streams cluster-wide (set 0 to disable)
|
|
191
|
+
key_prefix="my_app:sse", # Custom Redis key prefix (hash-tag safe)
|
|
192
|
+
)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## 🧪 Testing & Observability
|
|
198
|
+
|
|
199
|
+
You can inspect the live count of active streams at any time:
|
|
200
|
+
|
|
201
|
+
```python
|
|
202
|
+
# Active streams for a specific user:
|
|
203
|
+
active_user_streams = await lease_manager.get_active_count("user_123")
|
|
204
|
+
|
|
205
|
+
# Active streams across the entire cluster:
|
|
206
|
+
active_global_streams = await lease_manager.get_active_count()
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## 📄 License
|
|
212
|
+
|
|
213
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# fastapi-stream-lease
|
|
2
|
+
|
|
3
|
+
[](https://github.com/agustin18/fastapi-stream-lease/actions)
|
|
4
|
+
[](https://pypi.org/project/fastapi-stream-lease/)
|
|
5
|
+
[](https://www.python.org/downloads/)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
[](https://github.com/astral-sh/ruff)
|
|
8
|
+
|
|
9
|
+
**Distributed stream and SSE concurrency lease manager for FastAPI and Starlette, backed by atomic Redis Lua scripts.**
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## ⚡ The Problem: Why Traditional Rate Limiters Fail for Streams & LLMs
|
|
14
|
+
|
|
15
|
+
Standard rate limiters (such as `fastapi-limiter` or `slowapi`) count **requests per unit of time** (e.g. *5 requests per minute*).
|
|
16
|
+
|
|
17
|
+
While this works for standard REST APIs, it completely breaks down for **long-lived streaming connections** (Server-Sent Events, WebSockets, or streaming LLM tokens from OpenAI / Claude / Ollama):
|
|
18
|
+
|
|
19
|
+
1. **Duration Blindness:** A user can make a single request that stays open for 15 minutes, consuming a server socket the entire time. A rate limiter considers this "1 request" and allows the user to open 50 more tabs.
|
|
20
|
+
2. **Zombie Connection Leaks:** When mobile users switch networks or close tabs abruptly without clean TCP closure, worker connections remain blocked until timeout, causing connection pool exhaustion and denial of service.
|
|
21
|
+
3. **Multi-Worker Desynchronization:** In-memory concurrency limiters (like `asyncio.Semaphore`) fail across multi-process deployments (Gunicorn / Docker containers) because workers cannot share state.
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
Traditional Rate Limiter: fastapi-stream-lease:
|
|
25
|
+
┌───────────────────────┐ ┌──────────────────────────────────────────────┐
|
|
26
|
+
│ Request 1 -> ALLOWED │ │ Stream 1 (Active) -> LEASE ACQUIRED (1/2) │
|
|
27
|
+
│ Request 2 -> ALLOWED │ │ Stream 2 (Active) -> LEASE ACQUIRED (2/2) │
|
|
28
|
+
│ (Both streams active │ │ Stream 3 (Attempt) -> REJECTED: HTTP 429 │
|
|
29
|
+
│ for 10 minutes, │ │ Retry-After: 5 │
|
|
30
|
+
│ server sockets exhausted!) │ Stream 1 disconnects -> LEASE RELEASED │
|
|
31
|
+
└───────────────────────┘ │ Stream 3 re-attempt -> LEASE ACQUIRED (2/2) │
|
|
32
|
+
└──────────────────────────────────────────────┘
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
`fastapi-stream-lease` solves this with **Sliding Distributed Leases** inside **atomic Redis Lua scripts**:
|
|
36
|
+
- Enforces strict concurrency limits **per user** (`max_per_user`) and **globally** (`max_global`).
|
|
37
|
+
- Leases automatically self-expire if the client or worker dies without clean closure (zero zombies).
|
|
38
|
+
- A background renewal task keeps long-running streams alive even during slow Time-To-First-Token (TTFT) pauses.
|
|
39
|
+
- Released immediately when the stream finishes or client disconnects.
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## 🚀 Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install fastapi-stream-lease
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Or using `uv`:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
uv add fastapi-stream-lease
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
*(Requires Redis 5.0+ and Python 3.10+)*
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## 💡 Quickstart
|
|
60
|
+
|
|
61
|
+
Protect an SSE or LLM streaming endpoint in just a few lines:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from fastapi import FastAPI, Depends, Request
|
|
65
|
+
from fastapi.responses import StreamingResponse
|
|
66
|
+
import redis.asyncio as redis
|
|
67
|
+
|
|
68
|
+
from fastapi_stream_lease import (
|
|
69
|
+
StreamLeaseManager,
|
|
70
|
+
LeaseConfig,
|
|
71
|
+
StreamLeaseRejected,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
app = FastAPI()
|
|
75
|
+
redis_client = redis.from_url("redis://localhost:6379")
|
|
76
|
+
|
|
77
|
+
# Configure lease boundaries:
|
|
78
|
+
# Each user can hold at most 2 concurrent streams; cluster max is 500.
|
|
79
|
+
lease_manager = StreamLeaseManager(
|
|
80
|
+
redis=redis_client,
|
|
81
|
+
config=LeaseConfig(
|
|
82
|
+
max_per_user=2,
|
|
83
|
+
max_global=500,
|
|
84
|
+
lease_seconds=30.0,
|
|
85
|
+
),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# Convert lease rejections into clean HTTP 429 Too Many Requests responses:
|
|
90
|
+
@app.exception_handler(StreamLeaseRejected)
|
|
91
|
+
async def lease_rejected_handler(request: Request, exc: StreamLeaseRejected):
|
|
92
|
+
return exc.as_response()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@app.get("/api/chat/stream")
|
|
96
|
+
async def chat_stream(user_id: str = "user_123"):
|
|
97
|
+
# 1. Acquire lease (raises StreamLeaseRejected if limit reached)
|
|
98
|
+
lease = await lease_manager.acquire(user_id)
|
|
99
|
+
|
|
100
|
+
async def token_generator():
|
|
101
|
+
# Example: streaming tokens from an LLM
|
|
102
|
+
for word in ["Hello", "world", "this", "is", "streamed!"]:
|
|
103
|
+
yield f"data: {word}\n\n"
|
|
104
|
+
|
|
105
|
+
# 2. Wrap generator: guarantees background auto-renewal and release on disconnect
|
|
106
|
+
return StreamingResponse(
|
|
107
|
+
lease.wrap(token_generator()),
|
|
108
|
+
media_type="text/event-stream",
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Protecting WebSockets
|
|
113
|
+
|
|
114
|
+
For WebSockets and scoped async routines, use the `lease_manager.lease(...)` context manager:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
@app.websocket("/ws/chat/{user_id}")
|
|
118
|
+
async def websocket_chat(websocket: WebSocket, user_id: str):
|
|
119
|
+
await websocket.accept()
|
|
120
|
+
# Acquires lease on enter, automatically releases when socket closes or disconnects
|
|
121
|
+
async with lease_manager.lease(user_id):
|
|
122
|
+
while True:
|
|
123
|
+
msg = await websocket.receive_text()
|
|
124
|
+
await websocket.send_text(f"Echo: {msg}")
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## 🛠️ How It Works (Algorithmic Math)
|
|
130
|
+
|
|
131
|
+
All concurrency validations, expirations, and insertions run inside **atomic Lua scripts** on Redis:
|
|
132
|
+
|
|
133
|
+
1. **Sorted Sets (`ZSET`):** Active streams are stored in Redis `ZSET`s where the value is a unique `lease_id` and the score is the epoch expiration timestamp (`now + lease_seconds`).
|
|
134
|
+
2. **Atomic Eviction:** Before checking capacity, `ZREMRANGEBYSCORE` purges all expired entries in $O(\log N + M)$.
|
|
135
|
+
3. **Capacity Check:** `ZCARD` verifies current stream count in $O(1)$ against `max_per_user` and `max_global`. Setting either to `0` disables that limit.
|
|
136
|
+
4. **Redis Cluster Slot Affinity:** Keys automatically use `{prefix}` hash tags (e.g. `{stream_lease}:user:123` and `{stream_lease}:global`), guaranteeing zero `CROSSSLOT` errors across distributed Redis clusters.
|
|
137
|
+
5. **Acquisition:** If capacity permits, `ZADD` registers the lease in $O(\log N)$ and updates the key TTL.
|
|
138
|
+
6. **Auto-Renewal:** While the stream is active, `lease.wrap()` spawns a lightweight background worker that calls `ZADD` to advance the expiration score every `lease_seconds / 2`.
|
|
139
|
+
7. **Guaranteed Release:** When the stream completes or the client disconnects, `ZREM` removes the lease immediately in the `finally:` block.
|
|
140
|
+
|
|
141
|
+
---
|
|
142
|
+
|
|
143
|
+
## ⚙️ Configuration Options
|
|
144
|
+
|
|
145
|
+
Customize `LeaseConfig`:
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from fastapi_stream_lease import LeaseConfig
|
|
149
|
+
|
|
150
|
+
config = LeaseConfig(
|
|
151
|
+
lease_seconds=30.0, # Lease expiration window (seconds)
|
|
152
|
+
max_per_user=3, # Max active streams per user (set 0 to disable)
|
|
153
|
+
max_global=1000, # Max active streams cluster-wide (set 0 to disable)
|
|
154
|
+
key_prefix="my_app:sse", # Custom Redis key prefix (hash-tag safe)
|
|
155
|
+
)
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## 🧪 Testing & Observability
|
|
161
|
+
|
|
162
|
+
You can inspect the live count of active streams at any time:
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
# Active streams for a specific user:
|
|
166
|
+
active_user_streams = await lease_manager.get_active_count("user_123")
|
|
167
|
+
|
|
168
|
+
# Active streams across the entire cluster:
|
|
169
|
+
active_global_streams = await lease_manager.get_active_count()
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## 📄 License
|
|
175
|
+
|
|
176
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "fastapi-stream-lease"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Distributed stream and SSE concurrency lease manager for FastAPI and Starlette backed by atomic Redis Lua scripts."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{ name = "Agustin Saiz", email = "agustinsaiz02@gmail.com" }
|
|
12
|
+
]
|
|
13
|
+
license = { text = "MIT" }
|
|
14
|
+
keywords = [
|
|
15
|
+
"fastapi",
|
|
16
|
+
"starlette",
|
|
17
|
+
"sse",
|
|
18
|
+
"server-sent-events",
|
|
19
|
+
"streaming",
|
|
20
|
+
"concurrency",
|
|
21
|
+
"rate-limiting",
|
|
22
|
+
"redis",
|
|
23
|
+
"llm-streaming",
|
|
24
|
+
]
|
|
25
|
+
classifiers = [
|
|
26
|
+
"Development Status :: 4 - Beta",
|
|
27
|
+
"Intended Audience :: Developers",
|
|
28
|
+
"License :: OSI Approved :: MIT License",
|
|
29
|
+
"Programming Language :: Python :: 3",
|
|
30
|
+
"Programming Language :: Python :: 3.10",
|
|
31
|
+
"Programming Language :: Python :: 3.11",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Framework :: FastAPI",
|
|
35
|
+
"Topic :: Internet :: WWW/HTTP :: HTTP Servers",
|
|
36
|
+
"Typing :: Typed",
|
|
37
|
+
]
|
|
38
|
+
requires-python = ">=3.10"
|
|
39
|
+
dependencies = [
|
|
40
|
+
"redis>=5.0.0",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.optional-dependencies]
|
|
44
|
+
fastapi = [
|
|
45
|
+
"fastapi>=0.100.0",
|
|
46
|
+
]
|
|
47
|
+
dev = [
|
|
48
|
+
"fastapi>=0.100.0",
|
|
49
|
+
"httpx>=0.25.0",
|
|
50
|
+
"pytest>=8.0.0",
|
|
51
|
+
"pytest-asyncio>=0.23.0",
|
|
52
|
+
"pytest-cov>=5.0.0",
|
|
53
|
+
"fakeredis[lua]>=2.20.0",
|
|
54
|
+
"ruff>=0.4.0",
|
|
55
|
+
"mypy>=1.10.0",
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
[project.urls]
|
|
59
|
+
Homepage = "https://github.com/agustin18/fastapi-stream-lease"
|
|
60
|
+
Repository = "https://github.com/agustin18/fastapi-stream-lease"
|
|
61
|
+
Issues = "https://github.com/agustin18/fastapi-stream-lease/issues"
|
|
62
|
+
|
|
63
|
+
[tool.hatch.build.targets.wheel]
|
|
64
|
+
packages = ["src/fastapi_stream_lease"]
|
|
65
|
+
|
|
66
|
+
[tool.pytest.ini_options]
|
|
67
|
+
asyncio_mode = "auto"
|
|
68
|
+
testpaths = ["tests"]
|
|
69
|
+
addopts = "--cov=fastapi_stream_lease --cov-report=term-missing --cov-report=xml"
|
|
70
|
+
|
|
71
|
+
[tool.ruff]
|
|
72
|
+
line-length = 100
|
|
73
|
+
target-version = "py310"
|
|
74
|
+
|
|
75
|
+
[tool.ruff.lint]
|
|
76
|
+
select = ["E", "F", "I", "W", "UP", "B"]
|
|
77
|
+
ignore = []
|
|
78
|
+
|
|
79
|
+
[tool.mypy]
|
|
80
|
+
python_version = "3.10"
|
|
81
|
+
strict = true
|
|
82
|
+
warn_return_any = true
|
|
83
|
+
warn_unused_configs = true
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from fastapi_stream_lease.config import LeaseConfig
|
|
4
|
+
from fastapi_stream_lease.exceptions import StreamLeaseError, StreamLeaseRejected
|
|
5
|
+
from fastapi_stream_lease.lease import StreamLease
|
|
6
|
+
from fastapi_stream_lease.manager import StreamLeaseManager
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"LeaseConfig",
|
|
12
|
+
"StreamLease",
|
|
13
|
+
"StreamLeaseError",
|
|
14
|
+
"StreamLeaseManager",
|
|
15
|
+
"StreamLeaseRejected",
|
|
16
|
+
"__version__",
|
|
17
|
+
]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(frozen=True)
|
|
7
|
+
class LeaseConfig:
|
|
8
|
+
"""Configuration for stream lease management."""
|
|
9
|
+
|
|
10
|
+
lease_seconds: float = 30.0
|
|
11
|
+
"""Duration (in seconds) of each lease before expiring in Redis if not renewed."""
|
|
12
|
+
|
|
13
|
+
max_per_user: int = 3
|
|
14
|
+
"""Maximum concurrent streams allowed for a single user/principal key."""
|
|
15
|
+
|
|
16
|
+
max_global: int = 500
|
|
17
|
+
"""Maximum concurrent streams allowed across the entire cluster."""
|
|
18
|
+
|
|
19
|
+
key_prefix: str = "stream_lease"
|
|
20
|
+
"""Prefix for Redis keys (e.g. stream_lease:user:{id}, stream_lease:global)."""
|
|
21
|
+
|
|
22
|
+
def __post_init__(self) -> None:
|
|
23
|
+
if self.lease_seconds <= 0:
|
|
24
|
+
raise ValueError("lease_seconds must be greater than 0")
|
|
25
|
+
if self.max_per_user < 0:
|
|
26
|
+
raise ValueError("max_per_user cannot be negative")
|
|
27
|
+
if self.max_global < 0:
|
|
28
|
+
raise ValueError("max_global cannot be negative")
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def _cluster_prefix(self) -> str:
|
|
32
|
+
"""Ensure prefix uses Redis hash tags {...} for slot affinity in Redis Cluster."""
|
|
33
|
+
if "{" in self.key_prefix and "}" in self.key_prefix:
|
|
34
|
+
return self.key_prefix
|
|
35
|
+
return f"{{{self.key_prefix}}}"
|
|
36
|
+
|
|
37
|
+
def user_key(self, user_id: str | int) -> str:
|
|
38
|
+
return f"{self._cluster_prefix}:user:{user_id}"
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def global_key(self) -> str:
|
|
42
|
+
return f"{self._cluster_prefix}:global"
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def redis_ttl(self) -> int:
|
|
46
|
+
"""TTL set on Redis keys to ensure dead keys self-clean (twice lease duration)."""
|
|
47
|
+
return max(60, int(self.lease_seconds * 2))
|