wred-lifespan 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wred_lifespan-0.1.0/.gitignore +8 -0
- wred_lifespan-0.1.0/LICENSE +29 -0
- wred_lifespan-0.1.0/PKG-INFO +173 -0
- wred_lifespan-0.1.0/README.md +156 -0
- wred_lifespan-0.1.0/benchmarks/request_overhead.py +57 -0
- wred_lifespan-0.1.0/examples/wrapped.py +23 -0
- wred_lifespan-0.1.0/pyproject.toml +29 -0
- wred_lifespan-0.1.0/tests/test_lifespan.py +320 -0
- wred_lifespan-0.1.0/tests/test_workers.py +141 -0
- wred_lifespan-0.1.0/tests/worker_app.py +34 -0
- wred_lifespan-0.1.0/wred_lifespan/__init__.py +145 -0
- wred_lifespan-0.1.0/wred_lifespan/py.typed +0 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Harrison Erd
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
10
|
+
list of conditions and the following disclaimer.
|
|
11
|
+
|
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
and/or other materials provided with the distribution.
|
|
15
|
+
|
|
16
|
+
3. Neither the name of the copyright holder nor the names of its contributors
|
|
17
|
+
may be used to endorse or promote products derived from this software
|
|
18
|
+
without specific prior written permission.
|
|
19
|
+
|
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wred-lifespan
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: ASGI startup and shutdown hooks for Wred
|
|
5
|
+
Author-email: Harrison Erd <harrisonerd@gmail.com>
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-Expression: BSD-3-Clause
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: wred>=0.14,<0.15
|
|
11
|
+
Requires-Dist: pytest>=8 ; extra == "test"
|
|
12
|
+
Requires-Dist: pytest-asyncio>=0.24 ; extra == "test"
|
|
13
|
+
Requires-Dist: httpx>=0.27 ; extra == "test"
|
|
14
|
+
Requires-Dist: uvicorn>=0.30,<1 ; extra == "test"
|
|
15
|
+
Provides-Extra: test
|
|
16
|
+
|
|
17
|
+
# wred-lifespan
|
|
18
|
+
|
|
19
|
+
Async startup and shutdown hooks for Wred, through one function:
|
|
20
|
+
`lifespan_application`. Requires Python 3.11+, Wred 0.14, and an asyncio ASGI
|
|
21
|
+
server such as Uvicorn.
|
|
22
|
+
|
|
23
|
+
Install from PyPI:
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
python -m pip install wred-lifespan
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from wred import App
|
|
31
|
+
from wred_lifespan import lifespan_application
|
|
32
|
+
|
|
33
|
+
api = App()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
async def startup():
|
|
37
|
+
print("Starting up")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def shutdown():
|
|
41
|
+
print("Shutting down")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@api.route("/")
|
|
45
|
+
async def hello(request):
|
|
46
|
+
return {"hello": "world"}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
app = lifespan_application(api, on_start=startup, on_shutdown=shutdown)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Save the example as `app.py`. Register routes on `api` and serve `app`:
|
|
53
|
+
|
|
54
|
+
```sh
|
|
55
|
+
python -m pip install uvicorn
|
|
56
|
+
uvicorn app:app --lifespan on --workers 2
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Configuration
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
app = lifespan_application(
|
|
63
|
+
api,
|
|
64
|
+
on_start=[connect_database, warm_cache],
|
|
65
|
+
on_shutdown=[close_cache, close_database],
|
|
66
|
+
hook_timeout=10.0,
|
|
67
|
+
)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Both hook arguments are optional and accept one async callable or an iterable
|
|
71
|
+
of async callables. Each takes no arguments. Hooks run sequentially in the
|
|
72
|
+
supplied order; place dependent resource cleanup before its dependencies.
|
|
73
|
+
Iterables are captured when wrapping the app and can be reused across lifespans.
|
|
74
|
+
Use `functools.partial` on an async function to bind arguments.
|
|
75
|
+
|
|
76
|
+
`hook_timeout` defaults to **30 seconds per hook** and must be positive and
|
|
77
|
+
finite. Each cleanup hook gets a fresh timeout, so a hung hook cannot consume
|
|
78
|
+
the next hook's budget. For N shutdown hooks, allow at least N × `hook_timeout`
|
|
79
|
+
for cleanup in your process supervisor's termination grace period, in addition
|
|
80
|
+
to time for draining requests. Startup failure can take the startup hooks' time
|
|
81
|
+
plus the entire cleanup budget before it is reported.
|
|
82
|
+
|
|
83
|
+
Synchronous hooks are rejected during configuration. Hooks must use
|
|
84
|
+
nonblocking async operations and cooperate with cancellation. Python's
|
|
85
|
+
[`asyncio.timeout`](https://docs.python.org/3.11/library/asyncio-task.html#asyncio.timeout)
|
|
86
|
+
cannot forcibly stop blocking code or a coroutine that indefinitely suppresses
|
|
87
|
+
cancellation. Set timeouts in external clients too. Moving blocking work to a
|
|
88
|
+
thread does not make that thread cancellable.
|
|
89
|
+
|
|
90
|
+
## Cleanup and failure behavior
|
|
91
|
+
|
|
92
|
+
- Startup completion is sent only after every startup hook finishes.
|
|
93
|
+
- Startup exception or timeout stops subsequent startup hooks and runs **all
|
|
94
|
+
shutdown hooks** as rollback before sending `lifespan.startup.failed`.
|
|
95
|
+
- Shutdown attempts every hook even if an earlier hook fails or times out.
|
|
96
|
+
It sends `lifespan.shutdown.failed` if any cleanup failed, including all
|
|
97
|
+
ordinary errors in the diagnostic; otherwise it sends completion.
|
|
98
|
+
- Cancellation, a broken lifespan connection, or an unexpected message after
|
|
99
|
+
startup begins also triggers cleanup. The original exception propagates;
|
|
100
|
+
secondary cleanup errors are logged through the `wred_lifespan` logger.
|
|
101
|
+
- Cancellation during cleanup interrupts that hook; remaining hooks are still
|
|
102
|
+
attempted before cancellation propagates. Interrupted hooks are not retried.
|
|
103
|
+
Process-exit exceptions and forced process termination can prevent cleanup.
|
|
104
|
+
|
|
105
|
+
Startup and shutdown callbacks are independent, so the wrapper cannot infer
|
|
106
|
+
which acquisitions correspond to which cleanup. Cleanup hooks must tolerate
|
|
107
|
+
missing or partially initialized resources. For example, with HTTPX installed:
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
import httpx
|
|
111
|
+
|
|
112
|
+
client = None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
async def startup():
|
|
116
|
+
global client
|
|
117
|
+
client = httpx.AsyncClient(timeout=5.0)
|
|
118
|
+
# If this fails, shutdown still closes the client.
|
|
119
|
+
response = await client.get("https://example.com")
|
|
120
|
+
response.raise_for_status()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
async def shutdown():
|
|
124
|
+
global client
|
|
125
|
+
if client is not None:
|
|
126
|
+
await client.aclose()
|
|
127
|
+
client = None
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Assign acquired resources before performing later operations that can fail.
|
|
131
|
+
An acquisition that fails before returning a resource must clean up its own
|
|
132
|
+
partial work. Prefer separate shutdown hooks for independent resources so a
|
|
133
|
+
failure closing one cannot skip closing another.
|
|
134
|
+
|
|
135
|
+
## Deployment
|
|
136
|
+
|
|
137
|
+
Hooks run once per lifespan connection, usually once per worker, in the event
|
|
138
|
+
loop serving that worker. They do not run at import or per request. Hook-owned
|
|
139
|
+
resources must belong to the correct worker/event loop; the wrapper does not
|
|
140
|
+
provide shared application state or cluster-wide coordination. Registering a
|
|
141
|
+
scheduled job in startup therefore starts one job per worker.
|
|
142
|
+
|
|
143
|
+
Wrap the application once. The wrapper owns the ASGI lifespan protocol and does
|
|
144
|
+
not compose another application's existing lifespan handler. HTTP and other
|
|
145
|
+
scopes are forwarded unchanged. The earlier `on_start` and `on_shutdown`
|
|
146
|
+
decorator exports have been removed; those names now appear only as keyword
|
|
147
|
+
arguments to `lifespan_application`.
|
|
148
|
+
|
|
149
|
+
The Linux integration test runs two real Uvicorn workers, concurrent HTTP
|
|
150
|
+
clients, worker replacement, SIGHUP restarts, and SIGTERM shutdown. It checks
|
|
151
|
+
that requests only see initialized resources and every gracefully stopped
|
|
152
|
+
worker executes cleanup. Uvicorn may close accepted idle connections during
|
|
153
|
+
restart; the test records these transport disconnects separately and permits
|
|
154
|
+
them only during restart. For deployments requiring uninterrupted requests,
|
|
155
|
+
drain traffic at the load balancer before terminating instances. Lifespan hooks
|
|
156
|
+
alone cannot guarantee zero downtime or cleanup after SIGKILL.
|
|
157
|
+
|
|
158
|
+
## Development
|
|
159
|
+
|
|
160
|
+
From a source checkout:
|
|
161
|
+
|
|
162
|
+
```sh
|
|
163
|
+
python -m pip install -e '.[test]'
|
|
164
|
+
python -m pytest -s
|
|
165
|
+
python -m pytest -m 'not integration' # fast protocol tests only
|
|
166
|
+
python -m build
|
|
167
|
+
python benchmarks/request_overhead.py
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
The benchmark compares raw Wred with the wrapper using in-process HTTP ASGI
|
|
171
|
+
calls. It measures wrapper overhead, not network throughput or production
|
|
172
|
+
capacity. The worker test is a finite deployment check, not a long-running soak.
|
|
173
|
+
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# wred-lifespan
|
|
2
|
+
|
|
3
|
+
Async startup and shutdown hooks for Wred, through one function:
|
|
4
|
+
`lifespan_application`. Requires Python 3.11+, Wred 0.14, and an asyncio ASGI
|
|
5
|
+
server such as Uvicorn.
|
|
6
|
+
|
|
7
|
+
Install from PyPI:
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
python -m pip install wred-lifespan
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from wred import App
|
|
15
|
+
from wred_lifespan import lifespan_application
|
|
16
|
+
|
|
17
|
+
api = App()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
async def startup():
|
|
21
|
+
print("Starting up")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
async def shutdown():
|
|
25
|
+
print("Shutting down")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@api.route("/")
|
|
29
|
+
async def hello(request):
|
|
30
|
+
return {"hello": "world"}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
app = lifespan_application(api, on_start=startup, on_shutdown=shutdown)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Save the example as `app.py`. Register routes on `api` and serve `app`:
|
|
37
|
+
|
|
38
|
+
```sh
|
|
39
|
+
python -m pip install uvicorn
|
|
40
|
+
uvicorn app:app --lifespan on --workers 2
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Configuration
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
app = lifespan_application(
|
|
47
|
+
api,
|
|
48
|
+
on_start=[connect_database, warm_cache],
|
|
49
|
+
on_shutdown=[close_cache, close_database],
|
|
50
|
+
hook_timeout=10.0,
|
|
51
|
+
)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Both hook arguments are optional and accept one async callable or an iterable
|
|
55
|
+
of async callables. Each takes no arguments. Hooks run sequentially in the
|
|
56
|
+
supplied order; place dependent resource cleanup before its dependencies.
|
|
57
|
+
Iterables are captured when wrapping the app and can be reused across lifespans.
|
|
58
|
+
Use `functools.partial` on an async function to bind arguments.
|
|
59
|
+
|
|
60
|
+
`hook_timeout` defaults to **30 seconds per hook** and must be positive and
|
|
61
|
+
finite. Each cleanup hook gets a fresh timeout, so a hung hook cannot consume
|
|
62
|
+
the next hook's budget. For N shutdown hooks, allow at least N × `hook_timeout`
|
|
63
|
+
for cleanup in your process supervisor's termination grace period, in addition
|
|
64
|
+
to time for draining requests. Startup failure can take the startup hooks' time
|
|
65
|
+
plus the entire cleanup budget before it is reported.
|
|
66
|
+
|
|
67
|
+
Synchronous hooks are rejected during configuration. Hooks must use
|
|
68
|
+
nonblocking async operations and cooperate with cancellation. Python's
|
|
69
|
+
[`asyncio.timeout`](https://docs.python.org/3.11/library/asyncio-task.html#asyncio.timeout)
|
|
70
|
+
cannot forcibly stop blocking code or a coroutine that indefinitely suppresses
|
|
71
|
+
cancellation. Set timeouts in external clients too. Moving blocking work to a
|
|
72
|
+
thread does not make that thread cancellable.
|
|
73
|
+
|
|
74
|
+
## Cleanup and failure behavior
|
|
75
|
+
|
|
76
|
+
- Startup completion is sent only after every startup hook finishes.
|
|
77
|
+
- Startup exception or timeout stops subsequent startup hooks and runs **all
|
|
78
|
+
shutdown hooks** as rollback before sending `lifespan.startup.failed`.
|
|
79
|
+
- Shutdown attempts every hook even if an earlier hook fails or times out.
|
|
80
|
+
It sends `lifespan.shutdown.failed` if any cleanup failed, including all
|
|
81
|
+
ordinary errors in the diagnostic; otherwise it sends completion.
|
|
82
|
+
- Cancellation, a broken lifespan connection, or an unexpected message after
|
|
83
|
+
startup begins also triggers cleanup. The original exception propagates;
|
|
84
|
+
secondary cleanup errors are logged through the `wred_lifespan` logger.
|
|
85
|
+
- Cancellation during cleanup interrupts that hook; remaining hooks are still
|
|
86
|
+
attempted before cancellation propagates. Interrupted hooks are not retried.
|
|
87
|
+
Process-exit exceptions and forced process termination can prevent cleanup.
|
|
88
|
+
|
|
89
|
+
Startup and shutdown callbacks are independent, so the wrapper cannot infer
|
|
90
|
+
which acquisitions correspond to which cleanup. Cleanup hooks must tolerate
|
|
91
|
+
missing or partially initialized resources. For example, with HTTPX installed:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
import httpx
|
|
95
|
+
|
|
96
|
+
client = None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
async def startup():
|
|
100
|
+
global client
|
|
101
|
+
client = httpx.AsyncClient(timeout=5.0)
|
|
102
|
+
# If this fails, shutdown still closes the client.
|
|
103
|
+
response = await client.get("https://example.com")
|
|
104
|
+
response.raise_for_status()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
async def shutdown():
|
|
108
|
+
global client
|
|
109
|
+
if client is not None:
|
|
110
|
+
await client.aclose()
|
|
111
|
+
client = None
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Assign acquired resources before performing later operations that can fail.
|
|
115
|
+
An acquisition that fails before returning a resource must clean up its own
|
|
116
|
+
partial work. Prefer separate shutdown hooks for independent resources so a
|
|
117
|
+
failure closing one cannot skip closing another.
|
|
118
|
+
|
|
119
|
+
## Deployment
|
|
120
|
+
|
|
121
|
+
Hooks run once per lifespan connection, usually once per worker, in the event
|
|
122
|
+
loop serving that worker. They do not run at import or per request. Hook-owned
|
|
123
|
+
resources must belong to the correct worker/event loop; the wrapper does not
|
|
124
|
+
provide shared application state or cluster-wide coordination. Registering a
|
|
125
|
+
scheduled job in startup therefore starts one job per worker.
|
|
126
|
+
|
|
127
|
+
Wrap the application once. The wrapper owns the ASGI lifespan protocol and does
|
|
128
|
+
not compose another application's existing lifespan handler. HTTP and other
|
|
129
|
+
scopes are forwarded unchanged. The earlier `on_start` and `on_shutdown`
|
|
130
|
+
decorator exports have been removed; those names now appear only as keyword
|
|
131
|
+
arguments to `lifespan_application`.
|
|
132
|
+
|
|
133
|
+
The Linux integration test runs two real Uvicorn workers, concurrent HTTP
|
|
134
|
+
clients, worker replacement, SIGHUP restarts, and SIGTERM shutdown. It checks
|
|
135
|
+
that requests only see initialized resources and every gracefully stopped
|
|
136
|
+
worker executes cleanup. Uvicorn may close accepted idle connections during
|
|
137
|
+
restart; the test records these transport disconnects separately and permits
|
|
138
|
+
them only during restart. For deployments requiring uninterrupted requests,
|
|
139
|
+
drain traffic at the load balancer before terminating instances. Lifespan hooks
|
|
140
|
+
alone cannot guarantee zero downtime or cleanup after SIGKILL.
|
|
141
|
+
|
|
142
|
+
## Development
|
|
143
|
+
|
|
144
|
+
From a source checkout:
|
|
145
|
+
|
|
146
|
+
```sh
|
|
147
|
+
python -m pip install -e '.[test]'
|
|
148
|
+
python -m pytest -s
|
|
149
|
+
python -m pytest -m 'not integration' # fast protocol tests only
|
|
150
|
+
python -m build
|
|
151
|
+
python benchmarks/request_overhead.py
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
The benchmark compares raw Wred with the wrapper using in-process HTTP ASGI
|
|
155
|
+
calls. It measures wrapper overhead, not network throughput or production
|
|
156
|
+
capacity. The worker test is a finite deployment check, not a long-running soak.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""In-process request overhead; excludes network, server, and concurrency costs."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import asyncio
|
|
5
|
+
import statistics
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
from wred import App
|
|
9
|
+
|
|
10
|
+
from wred_lifespan import lifespan_application
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
async def main(requests, rounds):
|
|
14
|
+
api = App()
|
|
15
|
+
|
|
16
|
+
@api.route("/")
|
|
17
|
+
async def hello(request):
|
|
18
|
+
return {"hello": "world"}
|
|
19
|
+
|
|
20
|
+
scope = {"type": "http", "method": "GET", "path": "/"}
|
|
21
|
+
|
|
22
|
+
async def receive():
|
|
23
|
+
return {"type": "http.request", "body": b""}
|
|
24
|
+
|
|
25
|
+
async def send(message):
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
wrapped = lifespan_application(api)
|
|
29
|
+
samples = {"wred": [], "wrapped": []}
|
|
30
|
+
for turn in range(rounds + 1):
|
|
31
|
+
order = [("wred", api), ("wrapped", wrapped)]
|
|
32
|
+
if turn % 2:
|
|
33
|
+
order.reverse()
|
|
34
|
+
for name, app in order:
|
|
35
|
+
start = time.perf_counter()
|
|
36
|
+
for _ in range(requests):
|
|
37
|
+
await app(scope, receive, send)
|
|
38
|
+
elapsed = time.perf_counter() - start
|
|
39
|
+
if turn: # Discard warmup; alternate order to reduce timing bias.
|
|
40
|
+
samples[name].append(elapsed / requests * 1_000_000)
|
|
41
|
+
raw, wrapped_time = (statistics.median(samples[name]) for name in samples)
|
|
42
|
+
print(
|
|
43
|
+
f"{requests:,} requests/round; {rounds} measured rounds; median microseconds/request"
|
|
44
|
+
)
|
|
45
|
+
print(
|
|
46
|
+
f"Wred: {raw:.3f}; wrapped: {wrapped_time:.3f}; added: {wrapped_time - raw:.3f}"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
if __name__ == "__main__":
|
|
51
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
52
|
+
parser.add_argument("--requests", type=int, default=100_000)
|
|
53
|
+
parser.add_argument("--rounds", type=int, default=7)
|
|
54
|
+
args = parser.parse_args()
|
|
55
|
+
if args.requests < 1 or args.rounds < 1:
|
|
56
|
+
parser.error("requests and rounds must be positive")
|
|
57
|
+
asyncio.run(main(args.requests, args.rounds))
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Run with: uvicorn examples.wrapped:app --lifespan on"""
|
|
2
|
+
|
|
3
|
+
from wred import App
|
|
4
|
+
|
|
5
|
+
from wred_lifespan import lifespan_application
|
|
6
|
+
|
|
7
|
+
api = App()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
async def startup():
|
|
11
|
+
print("Starting up")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
async def shutdown():
|
|
15
|
+
print("Shutting down")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@api.route("/")
|
|
19
|
+
async def hello(request):
|
|
20
|
+
return {"hello": "world"}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
app = lifespan_application(api, on_start=startup, on_shutdown=shutdown)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["flit_core>=3.12,<4"]
|
|
3
|
+
build-backend = "flit_core.buildapi"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "wred-lifespan"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "ASGI startup and shutdown hooks for Wred"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "BSD-3-Clause"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Harrison Erd", email = "harrisonerd@gmail.com" }]
|
|
14
|
+
dependencies = ["wred>=0.14,<0.15"]
|
|
15
|
+
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
test = ["pytest>=8", "pytest-asyncio>=0.24", "httpx>=0.27", "uvicorn>=0.30,<1"]
|
|
18
|
+
|
|
19
|
+
[tool.flit.sdist]
|
|
20
|
+
include = ["tests/", "examples/", "benchmarks/"]
|
|
21
|
+
|
|
22
|
+
[tool.pytest.ini_options]
|
|
23
|
+
asyncio_mode = "auto"
|
|
24
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
25
|
+
testpaths = ["tests"]
|
|
26
|
+
markers = ["integration: real worker processes and network sockets"]
|
|
27
|
+
|
|
28
|
+
[tool.ruff]
|
|
29
|
+
target-version = "py311"
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
|
|
3
|
+
import httpx
|
|
4
|
+
import pytest
|
|
5
|
+
from wred import App
|
|
6
|
+
|
|
7
|
+
from wred_lifespan import lifespan_application
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
async def exchange(app, events=("lifespan.startup", "lifespan.shutdown")):
|
|
11
|
+
"""Drive the ASGI protocol and return messages sent by the application."""
|
|
12
|
+
incoming = asyncio.Queue()
|
|
13
|
+
outgoing = []
|
|
14
|
+
for event in events:
|
|
15
|
+
incoming.put_nowait({"type": event})
|
|
16
|
+
|
|
17
|
+
async def send(message):
|
|
18
|
+
outgoing.append(message)
|
|
19
|
+
|
|
20
|
+
async with asyncio.timeout(2):
|
|
21
|
+
await app({"type": "lifespan"}, incoming.get, send)
|
|
22
|
+
return outgoing
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
async def test_wred_serves_requests_between_completed_hooks():
|
|
26
|
+
api = App()
|
|
27
|
+
events = []
|
|
28
|
+
|
|
29
|
+
async def startup():
|
|
30
|
+
await asyncio.sleep(0)
|
|
31
|
+
events.append("ready")
|
|
32
|
+
|
|
33
|
+
async def shutdown():
|
|
34
|
+
await asyncio.sleep(0)
|
|
35
|
+
events.append("closed")
|
|
36
|
+
|
|
37
|
+
@api.route("/")
|
|
38
|
+
async def hello(request):
|
|
39
|
+
return {"hello": "world", "ready": events == ["ready"]}
|
|
40
|
+
|
|
41
|
+
app = lifespan_application(api, on_start=startup, on_shutdown=shutdown)
|
|
42
|
+
|
|
43
|
+
assert events == []
|
|
44
|
+
incoming, outgoing = asyncio.Queue(), asyncio.Queue()
|
|
45
|
+
async with asyncio.timeout(2), asyncio.TaskGroup() as tasks:
|
|
46
|
+
tasks.create_task(app({"type": "lifespan"}, incoming.get, outgoing.put))
|
|
47
|
+
await incoming.put({"type": "lifespan.startup"})
|
|
48
|
+
assert await outgoing.get() == {"type": "lifespan.startup.complete"}
|
|
49
|
+
assert events == ["ready"]
|
|
50
|
+
async with httpx.AsyncClient(
|
|
51
|
+
transport=httpx.ASGITransport(app), base_url="http://test"
|
|
52
|
+
) as client:
|
|
53
|
+
for _ in range(2):
|
|
54
|
+
response = await client.get("/")
|
|
55
|
+
assert response.status_code == 200
|
|
56
|
+
assert response.json() == {"hello": "world", "ready": True}
|
|
57
|
+
assert events == ["ready"]
|
|
58
|
+
await incoming.put({"type": "lifespan.shutdown"})
|
|
59
|
+
assert await outgoing.get() == {"type": "lifespan.shutdown.complete"}
|
|
60
|
+
assert events == ["ready", "closed"]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def test_other_scopes_and_downstream_errors_pass_through():
|
|
64
|
+
scope = {"type": "websocket", "state": {"token": "hello"}}
|
|
65
|
+
incoming = asyncio.Queue()
|
|
66
|
+
incoming.put_nowait({"type": "websocket.connect"})
|
|
67
|
+
outgoing = []
|
|
68
|
+
failure = RuntimeError("downstream error")
|
|
69
|
+
|
|
70
|
+
async def downstream(scope, receive, send):
|
|
71
|
+
assert await receive() == {"type": "websocket.connect"}
|
|
72
|
+
await send({"type": "websocket.send", "text": scope["state"]["token"]})
|
|
73
|
+
scope["state"]["visited"] = True
|
|
74
|
+
raise failure
|
|
75
|
+
|
|
76
|
+
async def unexpected():
|
|
77
|
+
pytest.fail("Lifespan callbacks must not run for WebSocket scopes")
|
|
78
|
+
|
|
79
|
+
async def send(message):
|
|
80
|
+
outgoing.append(message)
|
|
81
|
+
|
|
82
|
+
app = lifespan_application(downstream, on_start=unexpected, on_shutdown=unexpected)
|
|
83
|
+
async with asyncio.timeout(2):
|
|
84
|
+
with pytest.raises(RuntimeError) as caught:
|
|
85
|
+
await app(scope, incoming.get, send)
|
|
86
|
+
assert caught.value is failure
|
|
87
|
+
assert scope["state"]["visited"] is True
|
|
88
|
+
assert outgoing == [{"type": "websocket.send", "text": "hello"}]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@pytest.mark.parametrize("transport", ["receive", "send"])
|
|
92
|
+
async def test_transport_errors_propagate(transport):
|
|
93
|
+
calls = []
|
|
94
|
+
failure = OSError("connection lost")
|
|
95
|
+
|
|
96
|
+
async def receive():
|
|
97
|
+
if transport == "receive":
|
|
98
|
+
raise failure
|
|
99
|
+
return {"type": "lifespan.startup"}
|
|
100
|
+
|
|
101
|
+
async def send(message):
|
|
102
|
+
calls.append(message)
|
|
103
|
+
raise failure
|
|
104
|
+
|
|
105
|
+
with pytest.raises(OSError) as caught:
|
|
106
|
+
await lifespan_application(App())({"type": "lifespan"}, receive, send)
|
|
107
|
+
assert caught.value is failure
|
|
108
|
+
assert calls == (
|
|
109
|
+
[{"type": "lifespan.startup.complete"}] if transport == "send" else []
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@pytest.mark.parametrize(
|
|
114
|
+
"events",
|
|
115
|
+
[
|
|
116
|
+
["lifespan.shutdown"],
|
|
117
|
+
["lifespan.startup", "lifespan.startup"],
|
|
118
|
+
["unknown"],
|
|
119
|
+
],
|
|
120
|
+
)
|
|
121
|
+
async def test_out_of_order_messages_are_rejected(events):
|
|
122
|
+
with pytest.raises(RuntimeError, match="Expected lifespan\\."):
|
|
123
|
+
await exchange(lifespan_application(App()), events)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@pytest.mark.parametrize("option", ["on_start", "on_shutdown"])
|
|
127
|
+
@pytest.mark.parametrize("invalid", [lambda: None, [None], [lambda: None]])
|
|
128
|
+
def test_non_async_hooks_are_rejected_at_configuration(option, invalid):
|
|
129
|
+
with pytest.raises(TypeError, match="hooks must be async"):
|
|
130
|
+
lifespan_application(App(), **{option: invalid})
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@pytest.mark.parametrize("timeout", [0, -1, float("nan"), float("inf"), None, True])
|
|
134
|
+
def test_timeouts_must_be_positive_and_finite(timeout):
|
|
135
|
+
with pytest.raises(ValueError, match="positive finite"):
|
|
136
|
+
lifespan_application(App(), hook_timeout=timeout)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@pytest.mark.parametrize("option", [None, "on_start", "on_shutdown"])
|
|
140
|
+
async def test_optional_hooks_and_iterators_are_reusable_per_connection(option):
|
|
141
|
+
events = []
|
|
142
|
+
|
|
143
|
+
class Hook:
|
|
144
|
+
async def __call__(self):
|
|
145
|
+
events.append("called")
|
|
146
|
+
|
|
147
|
+
options = {} if option is None else {option: iter([Hook()])}
|
|
148
|
+
app = lifespan_application(App(), **options)
|
|
149
|
+
for _ in range(2):
|
|
150
|
+
assert await exchange(app) == [
|
|
151
|
+
{"type": "lifespan.startup.complete"},
|
|
152
|
+
{"type": "lifespan.shutdown.complete"},
|
|
153
|
+
]
|
|
154
|
+
assert events == ([] if option is None else ["called", "called"])
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@pytest.mark.parametrize("phase", ["startup", "shutdown"])
|
|
158
|
+
@pytest.mark.parametrize("suppress", [False, True])
|
|
159
|
+
async def test_timeouts_cancel_hooks_and_allow_remaining_cleanup(phase, suppress):
|
|
160
|
+
events = []
|
|
161
|
+
|
|
162
|
+
async def hang():
|
|
163
|
+
try:
|
|
164
|
+
await asyncio.Event().wait()
|
|
165
|
+
except asyncio.CancelledError:
|
|
166
|
+
if not suppress:
|
|
167
|
+
raise
|
|
168
|
+
finally:
|
|
169
|
+
events.append("cancelled")
|
|
170
|
+
|
|
171
|
+
async def cleanup():
|
|
172
|
+
events.append("cleaned")
|
|
173
|
+
|
|
174
|
+
async def skipped():
|
|
175
|
+
events.append("unexpected")
|
|
176
|
+
|
|
177
|
+
app = lifespan_application(
|
|
178
|
+
App(),
|
|
179
|
+
on_start=[hang, skipped] if phase == "startup" else None,
|
|
180
|
+
on_shutdown=cleanup if phase == "startup" else [hang, cleanup],
|
|
181
|
+
hook_timeout=0.01,
|
|
182
|
+
)
|
|
183
|
+
messages = await exchange(app)
|
|
184
|
+
assert events == ["cancelled", "cleaned"]
|
|
185
|
+
assert messages[-1]["type"] == f"lifespan.{phase}.failed"
|
|
186
|
+
assert "TimeoutError" in messages[-1]["message"]
|
|
187
|
+
assert len(messages) == (1 if phase == "startup" else 2)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@pytest.mark.parametrize("phase", ["startup", "running", "shutdown"])
|
|
191
|
+
async def test_external_cancellation_cleans_up_then_propagates(phase):
|
|
192
|
+
incoming, outgoing = asyncio.Queue(), asyncio.Queue()
|
|
193
|
+
entered = asyncio.Event()
|
|
194
|
+
events = []
|
|
195
|
+
incoming.put_nowait({"type": "lifespan.startup"})
|
|
196
|
+
if phase == "shutdown":
|
|
197
|
+
incoming.put_nowait({"type": "lifespan.shutdown"})
|
|
198
|
+
|
|
199
|
+
async def startup():
|
|
200
|
+
if phase == "startup":
|
|
201
|
+
entered.set()
|
|
202
|
+
await asyncio.Event().wait()
|
|
203
|
+
|
|
204
|
+
async def first_cleanup():
|
|
205
|
+
events.append("first")
|
|
206
|
+
if phase == "shutdown":
|
|
207
|
+
entered.set()
|
|
208
|
+
await asyncio.Event().wait()
|
|
209
|
+
|
|
210
|
+
async def last_cleanup():
|
|
211
|
+
events.append("last")
|
|
212
|
+
|
|
213
|
+
app = lifespan_application(
|
|
214
|
+
App(), on_start=startup, on_shutdown=[first_cleanup, last_cleanup]
|
|
215
|
+
)
|
|
216
|
+
async with asyncio.timeout(2), asyncio.TaskGroup() as tasks:
|
|
217
|
+
task = tasks.create_task(app({"type": "lifespan"}, incoming.get, outgoing.put))
|
|
218
|
+
if phase == "running":
|
|
219
|
+
assert await outgoing.get() == {"type": "lifespan.startup.complete"}
|
|
220
|
+
else:
|
|
221
|
+
await entered.wait()
|
|
222
|
+
task.cancel()
|
|
223
|
+
with pytest.raises(asyncio.CancelledError):
|
|
224
|
+
await task
|
|
225
|
+
assert events == ["first", "last"]
|
|
226
|
+
remaining = []
|
|
227
|
+
while not outgoing.empty():
|
|
228
|
+
remaining.append(outgoing.get_nowait())
|
|
229
|
+
assert remaining == (
|
|
230
|
+
[{"type": "lifespan.startup.complete"}] if phase == "shutdown" else []
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@pytest.mark.parametrize(
|
|
235
|
+
"broken_at", ["startup_ack", "shutdown_receive", "shutdown_ack"]
|
|
236
|
+
)
|
|
237
|
+
async def test_broken_connection_cleans_up_exactly_once(broken_at):
|
|
238
|
+
events = []
|
|
239
|
+
received = 0
|
|
240
|
+
failure = OSError("connection lost")
|
|
241
|
+
|
|
242
|
+
async def cleanup():
|
|
243
|
+
events.append("closed")
|
|
244
|
+
|
|
245
|
+
async def receive():
|
|
246
|
+
nonlocal received
|
|
247
|
+
received += 1
|
|
248
|
+
if received == 1:
|
|
249
|
+
return {"type": "lifespan.startup"}
|
|
250
|
+
if broken_at == "shutdown_receive":
|
|
251
|
+
raise failure
|
|
252
|
+
return {"type": "lifespan.shutdown"}
|
|
253
|
+
|
|
254
|
+
async def send(message):
|
|
255
|
+
if (
|
|
256
|
+
broken_at == "startup_ack"
|
|
257
|
+
and message["type"] == "lifespan.startup.complete"
|
|
258
|
+
) or (
|
|
259
|
+
broken_at == "shutdown_ack"
|
|
260
|
+
and message["type"] == "lifespan.shutdown.complete"
|
|
261
|
+
):
|
|
262
|
+
raise failure
|
|
263
|
+
|
|
264
|
+
with pytest.raises(OSError) as caught:
|
|
265
|
+
await lifespan_application(App(), on_shutdown=cleanup)(
|
|
266
|
+
{"type": "lifespan"}, receive, send
|
|
267
|
+
)
|
|
268
|
+
assert caught.value is failure
|
|
269
|
+
assert events == ["closed"]
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
async def test_shutdown_attempts_remaining_hooks_and_reports_all_errors():
|
|
273
|
+
events = []
|
|
274
|
+
|
|
275
|
+
async def first():
|
|
276
|
+
events.append("first")
|
|
277
|
+
raise ValueError("first cleanup failed")
|
|
278
|
+
|
|
279
|
+
async def second():
|
|
280
|
+
events.append("second")
|
|
281
|
+
raise RuntimeError("second cleanup failed")
|
|
282
|
+
|
|
283
|
+
async def last():
|
|
284
|
+
events.append("last")
|
|
285
|
+
|
|
286
|
+
messages = await exchange(
|
|
287
|
+
lifespan_application(App(), on_shutdown=[first, second, last])
|
|
288
|
+
)
|
|
289
|
+
assert events == ["first", "second", "last"]
|
|
290
|
+
assert messages[-1]["type"] == "lifespan.shutdown.failed"
|
|
291
|
+
assert "first cleanup failed" in messages[-1]["message"]
|
|
292
|
+
assert "second cleanup failed" in messages[-1]["message"]
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
async def test_partial_startup_rolls_back_acquired_resources_and_reports_both_errors():
|
|
296
|
+
resources = []
|
|
297
|
+
|
|
298
|
+
async def acquire():
|
|
299
|
+
resources.append("connection")
|
|
300
|
+
|
|
301
|
+
async def fail():
|
|
302
|
+
raise RuntimeError("startup failed")
|
|
303
|
+
|
|
304
|
+
async def must_not_run():
|
|
305
|
+
resources.append("unexpected")
|
|
306
|
+
|
|
307
|
+
async def close():
|
|
308
|
+
resources.clear()
|
|
309
|
+
raise ValueError("rollback failed after closing")
|
|
310
|
+
|
|
311
|
+
messages = await exchange(
|
|
312
|
+
lifespan_application(
|
|
313
|
+
App(), on_start=[acquire, fail, must_not_run], on_shutdown=close
|
|
314
|
+
)
|
|
315
|
+
)
|
|
316
|
+
assert resources == []
|
|
317
|
+
assert len(messages) == 1
|
|
318
|
+
assert messages[0]["type"] == "lifespan.startup.failed"
|
|
319
|
+
assert "startup failed" in messages[0]["message"]
|
|
320
|
+
assert "rollback failed after closing" in messages[0]["message"]
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Actual sockets, process signals, and worker replacement (POSIX only)."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import os
|
|
5
|
+
import signal
|
|
6
|
+
import socket
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
import pytest
|
|
12
|
+
|
|
13
|
+
pytestmark = [
|
|
14
|
+
pytest.mark.integration,
|
|
15
|
+
pytest.mark.skipif(sys.platform != "linux", reason="Linux process signals"),
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
async def test_workers_serve_during_replacement_and_rolling_restart(tmp_path):
|
|
20
|
+
event_dir = tmp_path / "events"
|
|
21
|
+
event_dir.mkdir()
|
|
22
|
+
environment = os.environ.copy()
|
|
23
|
+
# Carry the exact Wred installation/source path used by this test process.
|
|
24
|
+
environment["PYTHONPATH"] = os.pathsep.join(
|
|
25
|
+
[str(Path(__file__).parent), *[str(Path(p).resolve()) for p in sys.path if p]]
|
|
26
|
+
)
|
|
27
|
+
environment["WRED_LIFESPAN_EVENTS"] = str(event_dir)
|
|
28
|
+
|
|
29
|
+
def pids(event):
|
|
30
|
+
return {int(path.stem) for path in event_dir.glob(f"*.{event}")}
|
|
31
|
+
|
|
32
|
+
async def wait_for(predicate):
|
|
33
|
+
async with asyncio.timeout(15):
|
|
34
|
+
while not predicate():
|
|
35
|
+
assert process.returncode is None, log_path.read_text()
|
|
36
|
+
await asyncio.sleep(0.025)
|
|
37
|
+
|
|
38
|
+
log_path = tmp_path / "server.log"
|
|
39
|
+
counts = [0] * 8
|
|
40
|
+
seen = set()
|
|
41
|
+
transition = False
|
|
42
|
+
disconnects = []
|
|
43
|
+
stop = asyncio.Event()
|
|
44
|
+
with socket.socket() as listener, log_path.open("w") as log:
|
|
45
|
+
listener.bind(("127.0.0.1", 0))
|
|
46
|
+
port = listener.getsockname()[1]
|
|
47
|
+
listener.listen(128)
|
|
48
|
+
process = await asyncio.create_subprocess_exec(
|
|
49
|
+
*[
|
|
50
|
+
sys.executable,
|
|
51
|
+
"-m",
|
|
52
|
+
"uvicorn",
|
|
53
|
+
"worker_app:app",
|
|
54
|
+
"--fd",
|
|
55
|
+
str(listener.fileno()),
|
|
56
|
+
"--workers",
|
|
57
|
+
"2",
|
|
58
|
+
"--lifespan",
|
|
59
|
+
"on",
|
|
60
|
+
"--no-access-log",
|
|
61
|
+
"--timeout-graceful-shutdown",
|
|
62
|
+
"5",
|
|
63
|
+
],
|
|
64
|
+
env=environment,
|
|
65
|
+
pass_fds=(listener.fileno(),),
|
|
66
|
+
stdout=log,
|
|
67
|
+
stderr=log,
|
|
68
|
+
start_new_session=True,
|
|
69
|
+
)
|
|
70
|
+
try:
|
|
71
|
+
await wait_for(lambda: len(pids("started")) == 2)
|
|
72
|
+
original = pids("started")
|
|
73
|
+
async with httpx.AsyncClient(
|
|
74
|
+
base_url=f"http://127.0.0.1:{port}",
|
|
75
|
+
timeout=5,
|
|
76
|
+
limits=httpx.Limits(max_connections=8, max_keepalive_connections=0),
|
|
77
|
+
) as client:
|
|
78
|
+
|
|
79
|
+
async def traffic(index):
|
|
80
|
+
while not stop.is_set():
|
|
81
|
+
during_restart = transition
|
|
82
|
+
try:
|
|
83
|
+
response = await client.get("/")
|
|
84
|
+
except (
|
|
85
|
+
httpx.RemoteProtocolError,
|
|
86
|
+
httpx.ReadError,
|
|
87
|
+
httpx.ConnectError,
|
|
88
|
+
):
|
|
89
|
+
# Uvicorn closes idle sockets during shutdown. Track
|
|
90
|
+
# transport losses only in the explicit restart window.
|
|
91
|
+
if not during_restart:
|
|
92
|
+
raise
|
|
93
|
+
disconnects.append(index)
|
|
94
|
+
continue
|
|
95
|
+
assert response.status_code == 200
|
|
96
|
+
result = response.json()
|
|
97
|
+
assert result["ready"] is True
|
|
98
|
+
seen.add(result["pid"])
|
|
99
|
+
counts[index] += 1
|
|
100
|
+
|
|
101
|
+
async with asyncio.timeout(45), asyncio.TaskGroup() as tasks:
|
|
102
|
+
for index in range(len(counts)):
|
|
103
|
+
tasks.create_task(traffic(index))
|
|
104
|
+
await wait_for(lambda: sum(counts) >= 100 and original <= seen)
|
|
105
|
+
# Gracefully stop one worker; the supervisor must replace it.
|
|
106
|
+
transition = True
|
|
107
|
+
victim = min(original)
|
|
108
|
+
os.kill(victim, signal.SIGTERM)
|
|
109
|
+
await wait_for(lambda: len(pids("started")) == 3)
|
|
110
|
+
await wait_for(lambda: victim in pids("stopped"))
|
|
111
|
+
replacement = pids("started") - original
|
|
112
|
+
await wait_for(lambda: replacement <= seen)
|
|
113
|
+
before_restart = pids("started")
|
|
114
|
+
process.send_signal(signal.SIGHUP)
|
|
115
|
+
await wait_for(lambda: len(pids("started")) == 5)
|
|
116
|
+
await wait_for(lambda: before_restart <= pids("stopped"))
|
|
117
|
+
await wait_for(lambda: pids("started") - before_restart <= seen)
|
|
118
|
+
transition = False
|
|
119
|
+
stable_target = sum(counts) + 1000
|
|
120
|
+
await wait_for(lambda: sum(counts) >= stable_target)
|
|
121
|
+
stop.set()
|
|
122
|
+
process.send_signal(signal.SIGTERM)
|
|
123
|
+
code = await asyncio.wait_for(process.wait(), timeout=10)
|
|
124
|
+
assert code == 0, log_path.read_text()
|
|
125
|
+
assert pids("started") == pids("stopped")
|
|
126
|
+
assert all(count > 0 for count in counts)
|
|
127
|
+
print(
|
|
128
|
+
f"{sum(counts)} successful requests; {len(seen)} worker PIDs; "
|
|
129
|
+
f"{len(disconnects)} restart transport disconnects; all cleaned up"
|
|
130
|
+
)
|
|
131
|
+
finally:
|
|
132
|
+
# Reap the process tree even when an assertion or deadline fails.
|
|
133
|
+
try:
|
|
134
|
+
os.killpg(process.pid, signal.SIGTERM)
|
|
135
|
+
except ProcessLookupError:
|
|
136
|
+
pass
|
|
137
|
+
try:
|
|
138
|
+
await asyncio.wait_for(process.wait(), timeout=5)
|
|
139
|
+
except TimeoutError:
|
|
140
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
141
|
+
await asyncio.wait_for(process.wait(), timeout=5)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Instrumented application for real process lifecycle tests."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from wred import App
|
|
8
|
+
|
|
9
|
+
from wred_lifespan import lifespan_application
|
|
10
|
+
|
|
11
|
+
api = App()
|
|
12
|
+
resource = None
|
|
13
|
+
events = Path(os.environ["WRED_LIFESPAN_EVENTS"])
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
async def startup():
|
|
17
|
+
global resource
|
|
18
|
+
resource = (events / f"{os.getpid()}.resource").open("w")
|
|
19
|
+
(events / f"{os.getpid()}.started").touch()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def shutdown():
|
|
23
|
+
if resource is not None:
|
|
24
|
+
resource.close()
|
|
25
|
+
(events / f"{os.getpid()}.stopped").touch()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@api.route("/")
|
|
29
|
+
async def hello(request):
|
|
30
|
+
await asyncio.sleep(0.005)
|
|
31
|
+
return {"pid": os.getpid(), "ready": resource is not None and not resource.closed}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
app = lifespan_application(api, on_start=startup, on_shutdown=shutdown, hook_timeout=1)
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Bounded startup and best-effort cleanup for asyncio ASGI applications."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Awaitable, Callable, Iterable
|
|
7
|
+
from inspect import iscoroutinefunction
|
|
8
|
+
from traceback import format_exception
|
|
9
|
+
from typing import Any, TypeAlias
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
12
|
+
__all__ = ["lifespan_application"]
|
|
13
|
+
|
|
14
|
+
Scope: TypeAlias = dict[str, Any]
|
|
15
|
+
Message: TypeAlias = dict[str, Any]
|
|
16
|
+
Receive: TypeAlias = Callable[[], Awaitable[Message]]
|
|
17
|
+
Send: TypeAlias = Callable[[Message], Awaitable[None]]
|
|
18
|
+
ASGIApplication: TypeAlias = Callable[[Scope, Receive, Send], Awaitable[None]]
|
|
19
|
+
LifespanHandler: TypeAlias = Callable[[], Awaitable[None]]
|
|
20
|
+
Handlers: TypeAlias = LifespanHandler | Iterable[LifespanHandler] | None
|
|
21
|
+
_logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _handlers(value: Handlers) -> tuple[LifespanHandler, ...]:
|
|
25
|
+
if value is None:
|
|
26
|
+
return ()
|
|
27
|
+
handlers = (value,) if callable(value) else tuple(value)
|
|
28
|
+
for handler in handlers:
|
|
29
|
+
if not (
|
|
30
|
+
iscoroutinefunction(handler)
|
|
31
|
+
or iscoroutinefunction(getattr(handler, "__call__", None)) # noqa: B004 - async callable objects.
|
|
32
|
+
):
|
|
33
|
+
raise TypeError(
|
|
34
|
+
"Lifespan hooks must be async functions taking no arguments"
|
|
35
|
+
)
|
|
36
|
+
return handlers
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
async def _run(handler: LifespanHandler, seconds: float) -> None:
|
|
40
|
+
async with asyncio.timeout(seconds) as deadline:
|
|
41
|
+
await handler()
|
|
42
|
+
# A hook that catches cancellation and returns still missed its deadline.
|
|
43
|
+
if deadline.expired():
|
|
44
|
+
raise TimeoutError("Lifespan hook exceeded its timeout")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def _cleanup(
|
|
48
|
+
handlers: tuple[LifespanHandler, ...], seconds: float
|
|
49
|
+
) -> list[BaseException]:
|
|
50
|
+
errors: list[BaseException] = []
|
|
51
|
+
for handler in handlers:
|
|
52
|
+
try:
|
|
53
|
+
await _run(handler, seconds)
|
|
54
|
+
except (Exception, asyncio.CancelledError) as exc: # noqa: BLE001 - attempt every cleanup.
|
|
55
|
+
# Each cleanup gets its own deadline; one failure cannot skip the rest.
|
|
56
|
+
errors.append(exc)
|
|
57
|
+
return errors
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _log_errors(errors: list[BaseException]) -> None:
|
|
61
|
+
for error in errors:
|
|
62
|
+
if not isinstance(error, asyncio.CancelledError):
|
|
63
|
+
_logger.error(
|
|
64
|
+
"Lifespan cleanup failed",
|
|
65
|
+
exc_info=(type(error), error, error.__traceback__),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _result(phase: str, errors: list[BaseException]) -> Message:
|
|
70
|
+
for error in errors:
|
|
71
|
+
if isinstance(error, asyncio.CancelledError):
|
|
72
|
+
_log_errors(errors)
|
|
73
|
+
raise error
|
|
74
|
+
if errors:
|
|
75
|
+
failure = BaseExceptionGroup(f"Lifespan {phase} failed", errors)
|
|
76
|
+
return {
|
|
77
|
+
"type": f"lifespan.{phase}.failed",
|
|
78
|
+
"message": "".join(format_exception(failure)),
|
|
79
|
+
}
|
|
80
|
+
return {"type": f"lifespan.{phase}.complete"}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
async def _receive(receive: Receive, phase: str) -> None:
|
|
84
|
+
message = await receive()
|
|
85
|
+
if message["type"] != f"lifespan.{phase}":
|
|
86
|
+
raise RuntimeError(f"Expected lifespan.{phase}, received {message['type']}")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def lifespan_application(
|
|
90
|
+
app: ASGIApplication,
|
|
91
|
+
*,
|
|
92
|
+
on_start: Handlers = None,
|
|
93
|
+
on_shutdown: Handlers = None,
|
|
94
|
+
hook_timeout: float = 30.0,
|
|
95
|
+
) -> ASGIApplication:
|
|
96
|
+
"""Wrap an ASGI app with async startup and shutdown hooks.
|
|
97
|
+
|
|
98
|
+
Hooks run in supplied order, each with its own positive, finite timeout in
|
|
99
|
+
seconds. Shutdown hooks also run after partial startup failure, cancellation,
|
|
100
|
+
or a broken lifespan connection; they must tolerate uninitialized resources.
|
|
101
|
+
All cleanup hooks are attempted even when another fails or times out.
|
|
102
|
+
|
|
103
|
+
Timeouts require cooperative asyncio code. Blocking code or suppressed
|
|
104
|
+
cancellation cannot be forcibly stopped. This wrapper owns lifespan scopes;
|
|
105
|
+
all other scopes are forwarded unchanged to app.
|
|
106
|
+
"""
|
|
107
|
+
if not callable(app):
|
|
108
|
+
raise TypeError("app must be an ASGI callable")
|
|
109
|
+
if (
|
|
110
|
+
isinstance(hook_timeout, bool)
|
|
111
|
+
or not isinstance(hook_timeout, (int, float))
|
|
112
|
+
or not math.isfinite(hook_timeout)
|
|
113
|
+
or hook_timeout <= 0
|
|
114
|
+
):
|
|
115
|
+
raise ValueError("hook_timeout must be a positive finite number of seconds")
|
|
116
|
+
startup, shutdown = _handlers(on_start), _handlers(on_shutdown)
|
|
117
|
+
|
|
118
|
+
async def application(scope: Scope, receive: Receive, send: Send) -> None:
|
|
119
|
+
if scope["type"] != "lifespan":
|
|
120
|
+
await app(scope, receive, send)
|
|
121
|
+
return
|
|
122
|
+
|
|
123
|
+
await _receive(receive, "startup")
|
|
124
|
+
cleanup_started = False
|
|
125
|
+
try:
|
|
126
|
+
try:
|
|
127
|
+
for handler in startup:
|
|
128
|
+
await _run(handler, hook_timeout)
|
|
129
|
+
except Exception as exc: # noqa: BLE001 - report failures through ASGI.
|
|
130
|
+
cleanup_started = True
|
|
131
|
+
errors = [exc, *await _cleanup(shutdown, hook_timeout)]
|
|
132
|
+
await send(_result("startup", errors))
|
|
133
|
+
return
|
|
134
|
+
await send({"type": "lifespan.startup.complete"})
|
|
135
|
+
await _receive(receive, "shutdown")
|
|
136
|
+
cleanup_started = True
|
|
137
|
+
errors = await _cleanup(shutdown, hook_timeout)
|
|
138
|
+
await send(_result("shutdown", errors))
|
|
139
|
+
finally:
|
|
140
|
+
# Transport errors, protocol errors, and cancellation still release
|
|
141
|
+
# resources. Preserve the original exception and log cleanup errors.
|
|
142
|
+
if not cleanup_started:
|
|
143
|
+
_log_errors(await _cleanup(shutdown, hook_timeout))
|
|
144
|
+
|
|
145
|
+
return application
|
|
File without changes
|