retrpy 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retrpy-0.1.0/.gitignore +10 -0
- retrpy-0.1.0/LICENSE +21 -0
- retrpy-0.1.0/PKG-INFO +420 -0
- retrpy-0.1.0/README.md +397 -0
- retrpy-0.1.0/pyproject.toml +51 -0
- retrpy-0.1.0/src/retrpy/__init__.py +5 -0
- retrpy-0.1.0/src/retrpy/_plan.py +149 -0
- retrpy-0.1.0/src/retrpy/py.typed +0 -0
- retrpy-0.1.0/tests/__init__.py +0 -0
- retrpy-0.1.0/tests/conftest.py +8 -0
- retrpy-0.1.0/tests/fakes.py +29 -0
- retrpy-0.1.0/tests/test_extras.py +94 -0
- retrpy-0.1.0/tests/test_operators.py +64 -0
- retrpy-0.1.0/tests/test_plan.py +29 -0
- retrpy-0.1.0/tests/test_run.py +80 -0
retrpy-0.1.0/.gitignore
ADDED
retrpy-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 aaltatan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
retrpy-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: retrpy
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Retry and fallback as plans of attempts, composed with * and +.
|
|
5
|
+
Project-URL: Homepage, https://github.com/aaltatan/retrpy
|
|
6
|
+
Project-URL: Repository, https://github.com/aaltatan/retrpy
|
|
7
|
+
Project-URL: Issues, https://github.com/aaltatan/retrpy/issues
|
|
8
|
+
Author-email: aaltatan <a.altatan@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: backoff,fallback,functional,jitter,resilience,retry
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# retrpy
|
|
25
|
+
|
|
26
|
+
Retry and fallback as **plans of attempts**, composed with ordinary Python operators.
|
|
27
|
+
Zero dependencies, fully typed.
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from retrpy import attempt
|
|
31
|
+
|
|
32
|
+
plan = (
|
|
33
|
+
attempt(fetch_rate, client, primary_url) * 3 # try the primary 3 times
|
|
34
|
+
+ attempt(fetch_rate, client, backup_url) * 2 # then the backup twice
|
|
35
|
+
+ attempt(lambda: None) # then give up with None
|
|
36
|
+
)
|
|
37
|
+
rate = plan(delay=0.05, backoff=2.0, catch=(NetworkError, RetryableHTTPError))
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
A plan is a value: build it once, run it as often as you like, pass it around, `len()` it.
|
|
41
|
+
The first attempt that succeeds wins.
|
|
42
|
+
|
|
43
|
+
## The problem it solves
|
|
44
|
+
|
|
45
|
+
A nightly job turns the day's orders into invoices, and each order total has to be converted
|
|
46
|
+
from USD to EUR. The exchange-rate provider is usually fine, but now and then it times out,
|
|
47
|
+
answers `503`, or rate-limits you with `429`. When that happens at 2 a.m., the job dies and
|
|
48
|
+
nobody gets invoiced.
|
|
49
|
+
|
|
50
|
+
You want: retry the main provider a few times with a growing wait, then switch to a second
|
|
51
|
+
provider, and as a last resort use yesterday's rate from your own database. Log every failure.
|
|
52
|
+
Never retry a real bug. And keep it testable without waiting for real.
|
|
53
|
+
|
|
54
|
+
Written by hand, that usually ends up like this:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
def get_rate() -> float:
|
|
58
|
+
last_error = None
|
|
59
|
+
wait = 0.5
|
|
60
|
+
for url in (PRIMARY, SECONDARY):
|
|
61
|
+
for _ in range(3):
|
|
62
|
+
try:
|
|
63
|
+
return fetch_rate(url, "USD", "EUR")
|
|
64
|
+
except (TimeoutError, ConnectionError, RetryableHTTPError) as error:
|
|
65
|
+
log.warning("rate fetch failed: %r", error)
|
|
66
|
+
last_error = error
|
|
67
|
+
time.sleep(wait) # sleeps after the last try too
|
|
68
|
+
wait *= 2 # the secondary inherits the primary's grown wait
|
|
69
|
+
try:
|
|
70
|
+
return load_cached_rate("USD", "EUR")
|
|
71
|
+
except LookupError:
|
|
72
|
+
raise last_error # earlier failures are lost
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
It works, but it's hard to read and has quiet bugs. The secondary provider starts at a 4-second
|
|
76
|
+
wait instead of 0.5. It sleeps for nothing before switching source. Only the last error
|
|
77
|
+
survives. And `time.sleep` is hard-wired, so the tests really sleep.
|
|
78
|
+
|
|
79
|
+
With retrpy, the plan says exactly what you meant, and the bugs go away:
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
import logging
|
|
83
|
+
|
|
84
|
+
from retrpy import attempt
|
|
85
|
+
|
|
86
|
+
log = logging.getLogger("invoicing")
|
|
87
|
+
|
|
88
|
+
rate_plan = (
|
|
89
|
+
attempt(fetch_rate, PRIMARY, "USD", "EUR") * 3 # main provider, 3 tries
|
|
90
|
+
+ attempt(fetch_rate, SECONDARY, "USD", "EUR") * 3 # backup provider, fresh backoff
|
|
91
|
+
+ attempt(load_cached_rate, "USD", "EUR") # yesterday's rate from the database
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def get_rate(sleep=time.sleep) -> float:
|
|
96
|
+
return rate_plan(
|
|
97
|
+
delay=0.5,
|
|
98
|
+
backoff=2.0,
|
|
99
|
+
max_delay=10.0,
|
|
100
|
+
catch=(TimeoutError, ConnectionError, RetryableHTTPError, LookupError),
|
|
101
|
+
on_failure=lambda error, n: log.warning("rate attempt %d failed: %r", n, error),
|
|
102
|
+
sleep=sleep,
|
|
103
|
+
)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
What you get:
|
|
107
|
+
|
|
108
|
+
- **Each provider gets its own backoff**: 0.5 s and 1 s between the primary's tries, then the
|
|
109
|
+
same again for the secondary. There's no wait when switching source.
|
|
110
|
+
- **Bugs fail fast.** A `KeyError` from a changed JSON shape isn't in `catch`, so it surfaces on
|
|
111
|
+
the first try, unwrapped, with its real traceback.
|
|
112
|
+
- **Nothing is lost.** If even the cache is empty, the job raises one `ExceptionGroup` with all
|
|
113
|
+
seven failures, so the morning report shows exactly what went wrong and in what order.
|
|
114
|
+
- **Tests are instant.** Pass `sleep=waits.append` and assert on the waits.
|
|
115
|
+
- **The policy is a value.** `rate_plan` can be imported, reused by another job, or extended
|
|
116
|
+
with `+`.
|
|
117
|
+
|
|
118
|
+
## Install
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
uv add retrpy
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install retrpy
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## Building plans
|
|
129
|
+
|
|
130
|
+
| Expression | Meaning |
|
|
131
|
+
| --- | --- |
|
|
132
|
+
| `attempt(fn, *args, **kwargs)` | one attempt of `fn(*args, **kwargs)` |
|
|
133
|
+
| `p * n` or `n * p` | retry: repeat `n` times (`n >= 1`) |
|
|
134
|
+
| `p + q` | fallback: run `q`'s attempts after `p`'s |
|
|
135
|
+
| `+ attempt(lambda: default)` | a default value is just one more attempt |
|
|
136
|
+
|
|
137
|
+
The operators behave like they do on tuples: `(a * 2) * 3` is six attempts of `a`, and
|
|
138
|
+
`(a * 2 + b) * 3` repeats the whole sequence three times.
|
|
139
|
+
|
|
140
|
+
## Running
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
plan(
|
|
144
|
+
delay=0.0, # first wait between retries, in seconds
|
|
145
|
+
backoff=1.0, # each next wait is multiplied by this
|
|
146
|
+
catch=(Exception,), # only these are retried; anything else propagates at once
|
|
147
|
+
max_delay=None, # cap on a single wait
|
|
148
|
+
jitter=None, # e.g. lambda d: d * random.uniform(0.5, 1.0)
|
|
149
|
+
on_failure=None, # e.g. lambda error, number: log.warning("attempt %d: %r", number, error)
|
|
150
|
+
sleep=time.sleep, # inject a fake in tests
|
|
151
|
+
)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
- Backoff **restarts for each segment**. In `primary * 3 + backup * 2`, the backup starts again
|
|
155
|
+
from `delay` instead of inheriting the primary's grown wait. There's no wait when switching
|
|
156
|
+
to the next source.
|
|
157
|
+
- The wait is `delay * backoff**k`, then `jitter`, then capped by `max_delay`. A wait of zero
|
|
158
|
+
or less doesn't sleep.
|
|
159
|
+
- Long retry budgets are safe with a cap. If `delay * backoff**k` grows too big for a float
|
|
160
|
+
(around 1,000 doublings), the wait becomes infinite instead of raising `OverflowError`, and
|
|
161
|
+
`max_delay` caps it as usual. Without `max_delay`, set a cap or keep the count reasonable:
|
|
162
|
+
an infinite wait makes `time.sleep` itself raise.
|
|
163
|
+
- Errors outside `catch` propagate immediately and unwrapped. `KeyboardInterrupt` and other
|
|
164
|
+
`BaseException`s are never caught.
|
|
165
|
+
- To retry on a bad *result*, make the attempt raise: for example, turn HTTP 429/5xx into a
|
|
166
|
+
`RetryableHTTPError`.
|
|
167
|
+
|
|
168
|
+
## When everything fails
|
|
169
|
+
|
|
170
|
+
All failures are raised together, in order, as an `ExceptionGroup`:
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
try:
|
|
174
|
+
rate = plan(catch=(NetworkError,))
|
|
175
|
+
except* NetworkError as group:
|
|
176
|
+
for error in group.exceptions:
|
|
177
|
+
log.error("failed: %r", error)
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
## Examples
|
|
181
|
+
|
|
182
|
+
### HTTP with a mirror and a default
|
|
183
|
+
|
|
184
|
+
Retry on a bad *response* by making the attempt raise. Here, 429 and 5xx become
|
|
185
|
+
`RetryableHTTPError`; a 404 stays an `HTTPError` and fails fast, because it's not in `catch`.
|
|
186
|
+
|
|
187
|
+
```python
|
|
188
|
+
import json
|
|
189
|
+
import urllib.error
|
|
190
|
+
import urllib.request
|
|
191
|
+
|
|
192
|
+
from retrpy import attempt
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class RetryableHTTPError(Exception):
|
|
196
|
+
pass
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def fetch_rate(url: str, currency: str) -> float:
|
|
200
|
+
try:
|
|
201
|
+
with urllib.request.urlopen(f"{url}/rates/{currency}", timeout=5) as response:
|
|
202
|
+
return float(json.load(response)["rate"])
|
|
203
|
+
except urllib.error.HTTPError as error:
|
|
204
|
+
if error.code == 429 or error.code >= 500:
|
|
205
|
+
raise RetryableHTTPError(error.code) from error
|
|
206
|
+
raise
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
plan = (
|
|
210
|
+
attempt(fetch_rate, "https://api.example.com", "EUR") * 3
|
|
211
|
+
+ attempt(fetch_rate, "https://mirror.example.com", "EUR") * 2
|
|
212
|
+
+ attempt(lambda: None)
|
|
213
|
+
) # Plan[float | None]
|
|
214
|
+
|
|
215
|
+
rate = plan(
|
|
216
|
+
delay=0.2,
|
|
217
|
+
backoff=2.0,
|
|
218
|
+
catch=(urllib.error.URLError, TimeoutError, RetryableHTTPError),
|
|
219
|
+
)
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
The waits are `0.2, 0.4` on the primary, then no wait, then `0.2` on the mirror: the
|
|
223
|
+
mirror's backoff starts fresh.
|
|
224
|
+
|
|
225
|
+
### Exponential backoff with full jitter and a cap
|
|
226
|
+
|
|
227
|
+
Jitter spreads out clients that failed at the same moment, so they don't all retry in sync.
|
|
228
|
+
`jitter` gets the computed wait and returns the one to use; `max_delay` caps the result.
|
|
229
|
+
|
|
230
|
+
```python
|
|
231
|
+
import random
|
|
232
|
+
|
|
233
|
+
result = (attempt(call_service) * 8)(
|
|
234
|
+
delay=0.1,
|
|
235
|
+
backoff=2.0,
|
|
236
|
+
jitter=lambda wait: random.uniform(0, wait), # "full jitter"
|
|
237
|
+
max_delay=5.0,
|
|
238
|
+
catch=(ConnectionError, TimeoutError),
|
|
239
|
+
)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Other common shapes are `lambda w: w / 2 + random.uniform(0, w / 2)` ("equal jitter") and
|
|
243
|
+
`lambda w: w * random.uniform(0.8, 1.2)` (±20%).
|
|
244
|
+
|
|
245
|
+
### Logging every failure
|
|
246
|
+
|
|
247
|
+
`on_failure` sees each caught error and its number across the whole plan, including the last.
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
import logging
|
|
251
|
+
|
|
252
|
+
log = logging.getLogger(__name__)
|
|
253
|
+
|
|
254
|
+
plan = attempt(fetch_rate, primary, "EUR") * 3 + attempt(fetch_rate, mirror, "EUR") * 2
|
|
255
|
+
|
|
256
|
+
rate = plan(
|
|
257
|
+
delay=0.2,
|
|
258
|
+
catch=(OSError, RetryableHTTPError),
|
|
259
|
+
on_failure=lambda error, n: log.warning("attempt %d/%d failed: %r", n, len(plan), error),
|
|
260
|
+
)
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
### Building plans from data
|
|
264
|
+
|
|
265
|
+
Plans are values, so ordinary Python builds them. Fall back through a list of mirrors:
|
|
266
|
+
|
|
267
|
+
```python
|
|
268
|
+
from functools import reduce
|
|
269
|
+
from operator import add
|
|
270
|
+
|
|
271
|
+
MIRRORS = ["https://a.example.com", "https://b.example.com", "https://c.example.com"]
|
|
272
|
+
|
|
273
|
+
plan = reduce(add, (attempt(fetch_rate, url, "EUR") * 2 for url in MIRRORS))
|
|
274
|
+
|
|
275
|
+
len(plan) # 6
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Or rotate through sources instead of exhausting each one first. Multiplying a multi-segment
|
|
279
|
+
plan repeats the whole sequence, like a tuple:
|
|
280
|
+
|
|
281
|
+
```python
|
|
282
|
+
rotation = (attempt(fetch_rate, MIRRORS[0], "EUR") + attempt(fetch_rate, MIRRORS[1], "EUR")) * 3
|
|
283
|
+
# a, b, a, b, a, b
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
### Reusable fragments
|
|
287
|
+
|
|
288
|
+
Name the parts once and combine them where they're used:
|
|
289
|
+
|
|
290
|
+
```python
|
|
291
|
+
from_api = attempt(load_user_from_api, user_id) * 3
|
|
292
|
+
from_cache = attempt(load_user_from_cache, user_id)
|
|
293
|
+
anonymous = attempt(lambda: ANONYMOUS_USER)
|
|
294
|
+
|
|
295
|
+
strict = from_api # must come from the API, or raise
|
|
296
|
+
relaxed = from_api + from_cache + anonymous # always returns a user
|
|
297
|
+
|
|
298
|
+
user = relaxed(delay=0.1, catch=(OSError, CacheMiss))
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
### Waiting for a service to come up
|
|
302
|
+
|
|
303
|
+
Any callable works, including standard library functions with arguments:
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
import socket
|
|
307
|
+
|
|
308
|
+
connection = (attempt(socket.create_connection, ("db.internal", 5432), timeout=2) * 10)(
|
|
309
|
+
delay=0.5,
|
|
310
|
+
backoff=1.5,
|
|
311
|
+
max_delay=5.0,
|
|
312
|
+
catch=(OSError,),
|
|
313
|
+
)
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
### Failing fast on bugs
|
|
317
|
+
|
|
318
|
+
Only the errors in `catch` are retried. Everything else, like a bug in your parsing code,
|
|
319
|
+
propagates on the first try, unwrapped, with its original traceback:
|
|
320
|
+
|
|
321
|
+
```python
|
|
322
|
+
def fetch_and_parse() -> Order:
|
|
323
|
+
payload = fetch_order() # may raise ConnectionError: retried
|
|
324
|
+
return Order.from_json(payload) # may raise KeyError: a bug, raised at once
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
order = (attempt(fetch_and_parse) * 5)(catch=(ConnectionError,))
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
`catch=()` retries nothing, and every error propagates at once. `catch` never accepts
|
|
331
|
+
`BaseException`, so Ctrl-C always stops the run.
|
|
332
|
+
|
|
333
|
+
### Handling the failures by type
|
|
334
|
+
|
|
335
|
+
When every attempt fails, `except*` lets you handle each kind of error separately:
|
|
336
|
+
|
|
337
|
+
```python
|
|
338
|
+
try:
|
|
339
|
+
rate = plan(catch=(urllib.error.URLError, RetryableHTTPError))
|
|
340
|
+
except* RetryableHTTPError as group:
|
|
341
|
+
codes = [error.args[0] for error in group.exceptions]
|
|
342
|
+
log.error("server kept failing: %s", codes)
|
|
343
|
+
rate = None
|
|
344
|
+
except* urllib.error.URLError:
|
|
345
|
+
log.error("network unreachable")
|
|
346
|
+
rate = None
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
Or catch the whole group and inspect it:
|
|
350
|
+
|
|
351
|
+
```python
|
|
352
|
+
try:
|
|
353
|
+
rate = plan(catch=(OSError,))
|
|
354
|
+
except ExceptionGroup as group:
|
|
355
|
+
last = group.exceptions[-1]
|
|
356
|
+
timeouts = group.subgroup(TimeoutError)
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
## Testing
|
|
360
|
+
|
|
361
|
+
Pass a fake `sleep` so tests run instantly and you can assert on the waits:
|
|
362
|
+
|
|
363
|
+
```python
|
|
364
|
+
waits = []
|
|
365
|
+
plan(delay=0.05, backoff=2.0, sleep=waits.append)
|
|
366
|
+
assert waits == [0.05, 0.1]
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
A scripted fake makes each attempt's outcome explicit:
|
|
370
|
+
|
|
371
|
+
```python
|
|
372
|
+
import pytest
|
|
373
|
+
|
|
374
|
+
from retrpy import attempt
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
class Scripted:
|
|
378
|
+
"""Returns or raises each outcome in turn; the last one repeats."""
|
|
379
|
+
|
|
380
|
+
def __init__(self, *outcomes):
|
|
381
|
+
self.outcomes, self.calls = outcomes, 0
|
|
382
|
+
|
|
383
|
+
def __call__(self):
|
|
384
|
+
outcome = self.outcomes[min(self.calls, len(self.outcomes) - 1)]
|
|
385
|
+
self.calls += 1
|
|
386
|
+
if isinstance(outcome, BaseException):
|
|
387
|
+
raise outcome
|
|
388
|
+
return outcome
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def test_falls_back_to_mirror():
|
|
392
|
+
primary = Scripted(ConnectionError("down"))
|
|
393
|
+
mirror = Scripted(ConnectionError("busy"), 1.08)
|
|
394
|
+
waits = []
|
|
395
|
+
|
|
396
|
+
plan = attempt(primary) * 3 + attempt(mirror) * 2
|
|
397
|
+
rate = plan(delay=0.1, backoff=2.0, catch=(ConnectionError,), sleep=waits.append)
|
|
398
|
+
|
|
399
|
+
assert rate == 1.08
|
|
400
|
+
assert (primary.calls, mirror.calls) == (3, 2)
|
|
401
|
+
assert waits == [0.1, 0.2, 0.1]
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def test_gives_up_with_every_error():
|
|
405
|
+
plan = attempt(Scripted(ConnectionError("down"))) * 2
|
|
406
|
+
|
|
407
|
+
with pytest.raises(ExceptionGroup, match="all 2 attempts failed") as excinfo:
|
|
408
|
+
plan(catch=(ConnectionError,), sleep=lambda _: None)
|
|
409
|
+
|
|
410
|
+
assert len(excinfo.value.exceptions) == 2
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
## With pyflowstep
|
|
414
|
+
|
|
415
|
+
A pyflowstep `Flow` is a callable, so it can be an attempt as is:
|
|
416
|
+
|
|
417
|
+
```python
|
|
418
|
+
plan = attempt(login_flow, page, user=user) * 3 + attempt(sso_login_flow, page)
|
|
419
|
+
page = plan(delay=0.5, catch=(TimeoutError,))
|
|
420
|
+
```
|
retrpy-0.1.0/README.md
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
# retrpy
|
|
2
|
+
|
|
3
|
+
Retry and fallback as **plans of attempts**, composed with ordinary Python operators.
|
|
4
|
+
Zero dependencies, fully typed.
|
|
5
|
+
|
|
6
|
+
```python
|
|
7
|
+
from retrpy import attempt
|
|
8
|
+
|
|
9
|
+
plan = (
|
|
10
|
+
attempt(fetch_rate, client, primary_url) * 3 # try the primary 3 times
|
|
11
|
+
+ attempt(fetch_rate, client, backup_url) * 2 # then the backup twice
|
|
12
|
+
+ attempt(lambda: None) # then give up with None
|
|
13
|
+
)
|
|
14
|
+
rate = plan(delay=0.05, backoff=2.0, catch=(NetworkError, RetryableHTTPError))
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
A plan is a value: build it once, run it as often as you like, pass it around, `len()` it.
|
|
18
|
+
The first attempt that succeeds wins.
|
|
19
|
+
|
|
20
|
+
## The problem it solves
|
|
21
|
+
|
|
22
|
+
A nightly job turns the day's orders into invoices, and each order total has to be converted
|
|
23
|
+
from USD to EUR. The exchange-rate provider is usually fine, but now and then it times out,
|
|
24
|
+
answers `503`, or rate-limits you with `429`. When that happens at 2 a.m., the job dies and
|
|
25
|
+
nobody gets invoiced.
|
|
26
|
+
|
|
27
|
+
You want: retry the main provider a few times with a growing wait, then switch to a second
|
|
28
|
+
provider, and as a last resort use yesterday's rate from your own database. Log every failure.
|
|
29
|
+
Never retry a real bug. And keep it testable without waiting for real.
|
|
30
|
+
|
|
31
|
+
Written by hand, that usually ends up like this:
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
def get_rate() -> float:
|
|
35
|
+
last_error = None
|
|
36
|
+
wait = 0.5
|
|
37
|
+
for url in (PRIMARY, SECONDARY):
|
|
38
|
+
for _ in range(3):
|
|
39
|
+
try:
|
|
40
|
+
return fetch_rate(url, "USD", "EUR")
|
|
41
|
+
except (TimeoutError, ConnectionError, RetryableHTTPError) as error:
|
|
42
|
+
log.warning("rate fetch failed: %r", error)
|
|
43
|
+
last_error = error
|
|
44
|
+
time.sleep(wait) # sleeps after the last try too
|
|
45
|
+
wait *= 2 # the secondary inherits the primary's grown wait
|
|
46
|
+
try:
|
|
47
|
+
return load_cached_rate("USD", "EUR")
|
|
48
|
+
except LookupError:
|
|
49
|
+
raise last_error # earlier failures are lost
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
It works, but it's hard to read and has quiet bugs. The secondary provider starts at a 4-second
|
|
53
|
+
wait instead of 0.5. It sleeps for nothing before switching source. Only the last error
|
|
54
|
+
survives. And `time.sleep` is hard-wired, so the tests really sleep.
|
|
55
|
+
|
|
56
|
+
With retrpy, the plan says exactly what you meant, and the bugs go away:
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
import logging
|
|
60
|
+
|
|
61
|
+
from retrpy import attempt
|
|
62
|
+
|
|
63
|
+
log = logging.getLogger("invoicing")
|
|
64
|
+
|
|
65
|
+
rate_plan = (
|
|
66
|
+
attempt(fetch_rate, PRIMARY, "USD", "EUR") * 3 # main provider, 3 tries
|
|
67
|
+
+ attempt(fetch_rate, SECONDARY, "USD", "EUR") * 3 # backup provider, fresh backoff
|
|
68
|
+
+ attempt(load_cached_rate, "USD", "EUR") # yesterday's rate from the database
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def get_rate(sleep=time.sleep) -> float:
|
|
73
|
+
return rate_plan(
|
|
74
|
+
delay=0.5,
|
|
75
|
+
backoff=2.0,
|
|
76
|
+
max_delay=10.0,
|
|
77
|
+
catch=(TimeoutError, ConnectionError, RetryableHTTPError, LookupError),
|
|
78
|
+
on_failure=lambda error, n: log.warning("rate attempt %d failed: %r", n, error),
|
|
79
|
+
sleep=sleep,
|
|
80
|
+
)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
What you get:
|
|
84
|
+
|
|
85
|
+
- **Each provider gets its own backoff**: 0.5 s and 1 s between the primary's tries, then the
|
|
86
|
+
same again for the secondary. There's no wait when switching source.
|
|
87
|
+
- **Bugs fail fast.** A `KeyError` from a changed JSON shape isn't in `catch`, so it surfaces on
|
|
88
|
+
the first try, unwrapped, with its real traceback.
|
|
89
|
+
- **Nothing is lost.** If even the cache is empty, the job raises one `ExceptionGroup` with all
|
|
90
|
+
seven failures, so the morning report shows exactly what went wrong and in what order.
|
|
91
|
+
- **Tests are instant.** Pass `sleep=waits.append` and assert on the waits.
|
|
92
|
+
- **The policy is a value.** `rate_plan` can be imported, reused by another job, or extended
|
|
93
|
+
with `+`.
|
|
94
|
+
|
|
95
|
+
## Install
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
uv add retrpy
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
pip install retrpy
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Building plans
|
|
106
|
+
|
|
107
|
+
| Expression | Meaning |
|
|
108
|
+
| --- | --- |
|
|
109
|
+
| `attempt(fn, *args, **kwargs)` | one attempt of `fn(*args, **kwargs)` |
|
|
110
|
+
| `p * n` or `n * p` | retry: repeat `n` times (`n >= 1`) |
|
|
111
|
+
| `p + q` | fallback: run `q`'s attempts after `p`'s |
|
|
112
|
+
| `+ attempt(lambda: default)` | a default value is just one more attempt |
|
|
113
|
+
|
|
114
|
+
The operators behave like they do on tuples: `(a * 2) * 3` is six attempts of `a`, and
|
|
115
|
+
`(a * 2 + b) * 3` repeats the whole sequence three times.
|
|
116
|
+
|
|
117
|
+
## Running
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
plan(
|
|
121
|
+
delay=0.0, # first wait between retries, in seconds
|
|
122
|
+
backoff=1.0, # each next wait is multiplied by this
|
|
123
|
+
catch=(Exception,), # only these are retried; anything else propagates at once
|
|
124
|
+
max_delay=None, # cap on a single wait
|
|
125
|
+
jitter=None, # e.g. lambda d: d * random.uniform(0.5, 1.0)
|
|
126
|
+
on_failure=None, # e.g. lambda error, number: log.warning("attempt %d: %r", number, error)
|
|
127
|
+
sleep=time.sleep, # inject a fake in tests
|
|
128
|
+
)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
- Backoff **restarts for each segment**. In `primary * 3 + backup * 2`, the backup starts again
|
|
132
|
+
from `delay` instead of inheriting the primary's grown wait. There's no wait when switching
|
|
133
|
+
to the next source.
|
|
134
|
+
- The wait is `delay * backoff**k`, then `jitter`, then capped by `max_delay`. A wait of zero
|
|
135
|
+
or less doesn't sleep.
|
|
136
|
+
- Long retry budgets are safe with a cap. If `delay * backoff**k` grows too big for a float
|
|
137
|
+
(around 1,000 doublings), the wait becomes infinite instead of raising `OverflowError`, and
|
|
138
|
+
`max_delay` caps it as usual. Without `max_delay`, set a cap or keep the count reasonable:
|
|
139
|
+
an infinite wait makes `time.sleep` itself raise.
|
|
140
|
+
- Errors outside `catch` propagate immediately and unwrapped. `KeyboardInterrupt` and other
|
|
141
|
+
`BaseException`s are never caught.
|
|
142
|
+
- To retry on a bad *result*, make the attempt raise: for example, turn HTTP 429/5xx into a
|
|
143
|
+
`RetryableHTTPError`.
|
|
144
|
+
|
|
145
|
+
## When everything fails
|
|
146
|
+
|
|
147
|
+
All failures are raised together, in order, as an `ExceptionGroup`:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
try:
|
|
151
|
+
rate = plan(catch=(NetworkError,))
|
|
152
|
+
except* NetworkError as group:
|
|
153
|
+
for error in group.exceptions:
|
|
154
|
+
log.error("failed: %r", error)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Examples
|
|
158
|
+
|
|
159
|
+
### HTTP with a mirror and a default
|
|
160
|
+
|
|
161
|
+
Retry on a bad *response* by making the attempt raise. Here, 429 and 5xx become
|
|
162
|
+
`RetryableHTTPError`; a 404 stays an `HTTPError` and fails fast, because it's not in `catch`.
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
import json
|
|
166
|
+
import urllib.error
|
|
167
|
+
import urllib.request
|
|
168
|
+
|
|
169
|
+
from retrpy import attempt
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
class RetryableHTTPError(Exception):
|
|
173
|
+
pass
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def fetch_rate(url: str, currency: str) -> float:
|
|
177
|
+
try:
|
|
178
|
+
with urllib.request.urlopen(f"{url}/rates/{currency}", timeout=5) as response:
|
|
179
|
+
return float(json.load(response)["rate"])
|
|
180
|
+
except urllib.error.HTTPError as error:
|
|
181
|
+
if error.code == 429 or error.code >= 500:
|
|
182
|
+
raise RetryableHTTPError(error.code) from error
|
|
183
|
+
raise
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
plan = (
|
|
187
|
+
attempt(fetch_rate, "https://api.example.com", "EUR") * 3
|
|
188
|
+
+ attempt(fetch_rate, "https://mirror.example.com", "EUR") * 2
|
|
189
|
+
+ attempt(lambda: None)
|
|
190
|
+
) # Plan[float | None]
|
|
191
|
+
|
|
192
|
+
rate = plan(
|
|
193
|
+
delay=0.2,
|
|
194
|
+
backoff=2.0,
|
|
195
|
+
catch=(urllib.error.URLError, TimeoutError, RetryableHTTPError),
|
|
196
|
+
)
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
The waits are `0.2, 0.4` on the primary, then no wait, then `0.2` on the mirror: the
|
|
200
|
+
mirror's backoff starts fresh.
|
|
201
|
+
|
|
202
|
+
### Exponential backoff with full jitter and a cap
|
|
203
|
+
|
|
204
|
+
Jitter spreads out clients that failed at the same moment, so they don't all retry in sync.
|
|
205
|
+
`jitter` gets the computed wait and returns the one to use; `max_delay` caps the result.
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
import random
|
|
209
|
+
|
|
210
|
+
result = (attempt(call_service) * 8)(
|
|
211
|
+
delay=0.1,
|
|
212
|
+
backoff=2.0,
|
|
213
|
+
jitter=lambda wait: random.uniform(0, wait), # "full jitter"
|
|
214
|
+
max_delay=5.0,
|
|
215
|
+
catch=(ConnectionError, TimeoutError),
|
|
216
|
+
)
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Other common shapes are `lambda w: w / 2 + random.uniform(0, w / 2)` ("equal jitter") and
|
|
220
|
+
`lambda w: w * random.uniform(0.8, 1.2)` (±20%).
|
|
221
|
+
|
|
222
|
+
### Logging every failure
|
|
223
|
+
|
|
224
|
+
`on_failure` sees each caught error and its number across the whole plan, including the last.
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
import logging
|
|
228
|
+
|
|
229
|
+
log = logging.getLogger(__name__)
|
|
230
|
+
|
|
231
|
+
plan = attempt(fetch_rate, primary, "EUR") * 3 + attempt(fetch_rate, mirror, "EUR") * 2
|
|
232
|
+
|
|
233
|
+
rate = plan(
|
|
234
|
+
delay=0.2,
|
|
235
|
+
catch=(OSError, RetryableHTTPError),
|
|
236
|
+
on_failure=lambda error, n: log.warning("attempt %d/%d failed: %r", n, len(plan), error),
|
|
237
|
+
)
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
### Building plans from data
|
|
241
|
+
|
|
242
|
+
Plans are values, so ordinary Python builds them. Fall back through a list of mirrors:
|
|
243
|
+
|
|
244
|
+
```python
|
|
245
|
+
from functools import reduce
|
|
246
|
+
from operator import add
|
|
247
|
+
|
|
248
|
+
MIRRORS = ["https://a.example.com", "https://b.example.com", "https://c.example.com"]
|
|
249
|
+
|
|
250
|
+
plan = reduce(add, (attempt(fetch_rate, url, "EUR") * 2 for url in MIRRORS))
|
|
251
|
+
|
|
252
|
+
len(plan) # 6
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
Or rotate through sources instead of exhausting each one first. Multiplying a multi-segment
|
|
256
|
+
plan repeats the whole sequence, like a tuple:
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
rotation = (attempt(fetch_rate, MIRRORS[0], "EUR") + attempt(fetch_rate, MIRRORS[1], "EUR")) * 3
|
|
260
|
+
# a, b, a, b, a, b
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
### Reusable fragments
|
|
264
|
+
|
|
265
|
+
Name the parts once and combine them where they're used:
|
|
266
|
+
|
|
267
|
+
```python
|
|
268
|
+
from_api = attempt(load_user_from_api, user_id) * 3
|
|
269
|
+
from_cache = attempt(load_user_from_cache, user_id)
|
|
270
|
+
anonymous = attempt(lambda: ANONYMOUS_USER)
|
|
271
|
+
|
|
272
|
+
strict = from_api # must come from the API, or raise
|
|
273
|
+
relaxed = from_api + from_cache + anonymous # always returns a user
|
|
274
|
+
|
|
275
|
+
user = relaxed(delay=0.1, catch=(OSError, CacheMiss))
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
### Waiting for a service to come up
|
|
279
|
+
|
|
280
|
+
Any callable works, including standard library functions with arguments:
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
import socket
|
|
284
|
+
|
|
285
|
+
connection = (attempt(socket.create_connection, ("db.internal", 5432), timeout=2) * 10)(
|
|
286
|
+
delay=0.5,
|
|
287
|
+
backoff=1.5,
|
|
288
|
+
max_delay=5.0,
|
|
289
|
+
catch=(OSError,),
|
|
290
|
+
)
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
### Failing fast on bugs
|
|
294
|
+
|
|
295
|
+
Only the errors in `catch` are retried. Everything else, like a bug in your parsing code,
|
|
296
|
+
propagates on the first try, unwrapped, with its original traceback:
|
|
297
|
+
|
|
298
|
+
```python
|
|
299
|
+
def fetch_and_parse() -> Order:
|
|
300
|
+
payload = fetch_order() # may raise ConnectionError: retried
|
|
301
|
+
return Order.from_json(payload) # may raise KeyError: a bug, raised at once
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
order = (attempt(fetch_and_parse) * 5)(catch=(ConnectionError,))
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
`catch=()` retries nothing, and every error propagates at once. `catch` never accepts
|
|
308
|
+
`BaseException`, so Ctrl-C always stops the run.
|
|
309
|
+
|
|
310
|
+
### Handling the failures by type
|
|
311
|
+
|
|
312
|
+
When every attempt fails, `except*` lets you handle each kind of error separately:
|
|
313
|
+
|
|
314
|
+
```python
|
|
315
|
+
try:
|
|
316
|
+
rate = plan(catch=(urllib.error.URLError, RetryableHTTPError))
|
|
317
|
+
except* RetryableHTTPError as group:
|
|
318
|
+
codes = [error.args[0] for error in group.exceptions]
|
|
319
|
+
log.error("server kept failing: %s", codes)
|
|
320
|
+
rate = None
|
|
321
|
+
except* urllib.error.URLError:
|
|
322
|
+
log.error("network unreachable")
|
|
323
|
+
rate = None
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
Or catch the whole group and inspect it:
|
|
327
|
+
|
|
328
|
+
```python
|
|
329
|
+
try:
|
|
330
|
+
rate = plan(catch=(OSError,))
|
|
331
|
+
except ExceptionGroup as group:
|
|
332
|
+
last = group.exceptions[-1]
|
|
333
|
+
timeouts = group.subgroup(TimeoutError)
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
## Testing
|
|
337
|
+
|
|
338
|
+
Pass a fake `sleep` so tests run instantly and you can assert on the waits:
|
|
339
|
+
|
|
340
|
+
```python
|
|
341
|
+
waits = []
|
|
342
|
+
plan(delay=0.05, backoff=2.0, sleep=waits.append)
|
|
343
|
+
assert waits == [0.05, 0.1]
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
A scripted fake makes each attempt's outcome explicit:
|
|
347
|
+
|
|
348
|
+
```python
|
|
349
|
+
import pytest
|
|
350
|
+
|
|
351
|
+
from retrpy import attempt
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
class Scripted:
|
|
355
|
+
"""Returns or raises each outcome in turn; the last one repeats."""
|
|
356
|
+
|
|
357
|
+
def __init__(self, *outcomes):
|
|
358
|
+
self.outcomes, self.calls = outcomes, 0
|
|
359
|
+
|
|
360
|
+
def __call__(self):
|
|
361
|
+
outcome = self.outcomes[min(self.calls, len(self.outcomes) - 1)]
|
|
362
|
+
self.calls += 1
|
|
363
|
+
if isinstance(outcome, BaseException):
|
|
364
|
+
raise outcome
|
|
365
|
+
return outcome
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def test_falls_back_to_mirror():
|
|
369
|
+
primary = Scripted(ConnectionError("down"))
|
|
370
|
+
mirror = Scripted(ConnectionError("busy"), 1.08)
|
|
371
|
+
waits = []
|
|
372
|
+
|
|
373
|
+
plan = attempt(primary) * 3 + attempt(mirror) * 2
|
|
374
|
+
rate = plan(delay=0.1, backoff=2.0, catch=(ConnectionError,), sleep=waits.append)
|
|
375
|
+
|
|
376
|
+
assert rate == 1.08
|
|
377
|
+
assert (primary.calls, mirror.calls) == (3, 2)
|
|
378
|
+
assert waits == [0.1, 0.2, 0.1]
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def test_gives_up_with_every_error():
|
|
382
|
+
plan = attempt(Scripted(ConnectionError("down"))) * 2
|
|
383
|
+
|
|
384
|
+
with pytest.raises(ExceptionGroup, match="all 2 attempts failed") as excinfo:
|
|
385
|
+
plan(catch=(ConnectionError,), sleep=lambda _: None)
|
|
386
|
+
|
|
387
|
+
assert len(excinfo.value.exceptions) == 2
|
|
388
|
+
```
|
|
389
|
+
|
|
390
|
+
## With pyflowstep
|
|
391
|
+
|
|
392
|
+
A pyflowstep `Flow` is a callable, so it can be an attempt as is:
|
|
393
|
+
|
|
394
|
+
```python
|
|
395
|
+
plan = attempt(login_flow, page, user=user) * 3 + attempt(sso_login_flow, page)
|
|
396
|
+
page = plan(delay=0.5, catch=(TimeoutError,))
|
|
397
|
+
```
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "retrpy"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Retry and fallback as plans of attempts, composed with * and +."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "aaltatan", email = "a.altatan@gmail.com" }
|
|
8
|
+
]
|
|
9
|
+
license = "MIT"
|
|
10
|
+
license-files = ["LICENSE"]
|
|
11
|
+
requires-python = ">=3.12"
|
|
12
|
+
dependencies = []
|
|
13
|
+
keywords = ["retry", "fallback", "backoff", "resilience", "jitter", "functional"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
23
|
+
"Typing :: Typed",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/aaltatan/retrpy"
|
|
28
|
+
Repository = "https://github.com/aaltatan/retrpy"
|
|
29
|
+
Issues = "https://github.com/aaltatan/retrpy/issues"
|
|
30
|
+
|
|
31
|
+
[build-system]
|
|
32
|
+
requires = ["hatchling"]
|
|
33
|
+
build-backend = "hatchling.build"
|
|
34
|
+
|
|
35
|
+
[tool.hatch.build.targets.sdist]
|
|
36
|
+
include = ["src", "tests", "README.md", "LICENSE"]
|
|
37
|
+
|
|
38
|
+
[dependency-groups]
|
|
39
|
+
dev = [
|
|
40
|
+
"mypy",
|
|
41
|
+
"pytest",
|
|
42
|
+
"ruff",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[tool.ruff]
|
|
46
|
+
line-length = 100
|
|
47
|
+
target-version = "py312"
|
|
48
|
+
extend-exclude = ["docs"]
|
|
49
|
+
|
|
50
|
+
[tool.mypy]
|
|
51
|
+
strict = true
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Plans of attempts: retry is repetition, fallback is concatenation."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import time
|
|
5
|
+
from collections.abc import Callable, Iterator
|
|
6
|
+
from dataclasses import dataclass, replace
|
|
7
|
+
from functools import partial
|
|
8
|
+
from typing import Self, cast
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True, slots=True)
|
|
12
|
+
class Segment[T]:
|
|
13
|
+
"""One zero-argument callable repeated ``count`` times; backoff restarts per segment."""
|
|
14
|
+
|
|
15
|
+
fn: Callable[[], T]
|
|
16
|
+
count: int
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True, slots=True)
|
|
20
|
+
class Plan[T]:
|
|
21
|
+
"""An immutable, ordered sequence of attempts; the first one that succeeds wins.
|
|
22
|
+
|
|
23
|
+
The plan keeps its segments rather than a flat list of calls, because the wait between
|
|
24
|
+
attempts grows within a segment and restarts when the plan falls back to the next one.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
segments: tuple[Segment[T], ...]
|
|
28
|
+
|
|
29
|
+
def __len__(self) -> int:
|
|
30
|
+
return sum(segment.count for segment in self.segments)
|
|
31
|
+
|
|
32
|
+
def __iter__(self) -> Iterator[Callable[[], T]]:
|
|
33
|
+
for segment in self.segments:
|
|
34
|
+
yield from [segment.fn] * segment.count
|
|
35
|
+
|
|
36
|
+
def __call__(
|
|
37
|
+
self,
|
|
38
|
+
*,
|
|
39
|
+
delay: float = 0.0,
|
|
40
|
+
backoff: float = 1.0,
|
|
41
|
+
catch: tuple[type[Exception], ...] = (Exception,),
|
|
42
|
+
max_delay: float | None = None,
|
|
43
|
+
jitter: Callable[[float], float] | None = None,
|
|
44
|
+
on_failure: Callable[[Exception, int], None] | None = None,
|
|
45
|
+
sleep: Callable[[float], object] = time.sleep,
|
|
46
|
+
) -> T:
|
|
47
|
+
"""Run the attempts in order and return the first success.
|
|
48
|
+
|
|
49
|
+
Exceptions outside ``catch`` propagate at once, unwrapped; ``catch`` only takes
|
|
50
|
+
``Exception`` subclasses, so ``KeyboardInterrupt`` and cancellation always get through.
|
|
51
|
+
Within a segment the wait
|
|
52
|
+
before the next try is ``delay * backoff**k``, with ``k`` restarting at each segment,
|
|
53
|
+
so a backup source never inherits the primary's grown delay. Switching segments does
|
|
54
|
+
not wait. ``jitter`` maps each wait to a new one (it owns its RNG, so tests can pass a
|
|
55
|
+
deterministic one), and ``max_delay`` then caps it; a wait of zero or less skips
|
|
56
|
+
``sleep``. ``on_failure(error, number)`` sees every caught failure, numbered from 1
|
|
57
|
+
across the whole plan, including the last. When every attempt fails, an
|
|
58
|
+
``ExceptionGroup`` holds all the failures in order, so callers can use ``except*``.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
if not self.segments:
|
|
62
|
+
msg = "Cannot run an empty plan"
|
|
63
|
+
raise ValueError(msg)
|
|
64
|
+
|
|
65
|
+
# The annotation can't stop untyped callers; catching BaseException would swallow Ctrl-C.
|
|
66
|
+
if not all(issubclass(kind, Exception) for kind in catch):
|
|
67
|
+
msg = f"catch only takes Exception subclasses, never BaseException; got {catch!r}"
|
|
68
|
+
raise TypeError(msg)
|
|
69
|
+
|
|
70
|
+
errors: list[Exception] = []
|
|
71
|
+
|
|
72
|
+
for segment in self.segments:
|
|
73
|
+
for k in range(segment.count):
|
|
74
|
+
try:
|
|
75
|
+
return segment.fn()
|
|
76
|
+
|
|
77
|
+
except catch as error:
|
|
78
|
+
errors.append(error)
|
|
79
|
+
|
|
80
|
+
if on_failure is not None:
|
|
81
|
+
on_failure(error, len(errors))
|
|
82
|
+
|
|
83
|
+
if (
|
|
84
|
+
k < segment.count - 1
|
|
85
|
+
and (wait := _wait(delay, backoff, k, jitter, max_delay)) > 0
|
|
86
|
+
):
|
|
87
|
+
sleep(wait)
|
|
88
|
+
|
|
89
|
+
msg = f"all {len(errors)} attempts failed"
|
|
90
|
+
|
|
91
|
+
raise ExceptionGroup(msg, errors)
|
|
92
|
+
|
|
93
|
+
def __mul__(self, n: int) -> Self:
|
|
94
|
+
if not isinstance(n, int):
|
|
95
|
+
return NotImplemented
|
|
96
|
+
|
|
97
|
+
if n < 1:
|
|
98
|
+
msg = f"Plan can only be repeated at least 1 time, got {n}"
|
|
99
|
+
raise ValueError(msg)
|
|
100
|
+
|
|
101
|
+
# One segment grows its count so its backoff keeps growing; anything else repeats.
|
|
102
|
+
if len(self.segments) == 1:
|
|
103
|
+
(segment,) = self.segments
|
|
104
|
+
|
|
105
|
+
return replace(self, segments=(Segment(segment.fn, segment.count * n),))
|
|
106
|
+
|
|
107
|
+
return replace(self, segments=self.segments * n)
|
|
108
|
+
|
|
109
|
+
__rmul__ = __mul__
|
|
110
|
+
|
|
111
|
+
def __add__[U](self, other: "Plan[U]") -> "Plan[T | U]":
|
|
112
|
+
if not isinstance(other, Plan):
|
|
113
|
+
return NotImplemented
|
|
114
|
+
|
|
115
|
+
segments: tuple[Segment[T | U], ...] = (*self.segments, *other.segments)
|
|
116
|
+
|
|
117
|
+
# Widen self's type so replace() keeps the subclass and accepts the wider segments.
|
|
118
|
+
return replace(cast("Plan[T | U]", self), segments=segments)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def attempt[T, **P](fn: Callable[P, T], /, *args: P.args, **kwargs: P.kwargs) -> Plan[T]:
|
|
122
|
+
"""Bind ``fn`` to its arguments as a one-attempt plan, ready for ``*`` and ``+``."""
|
|
123
|
+
return Plan((Segment(partial(fn, *args, **kwargs), 1),))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# --- private helpers
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _wait(
|
|
130
|
+
delay: float,
|
|
131
|
+
backoff: float,
|
|
132
|
+
k: int,
|
|
133
|
+
jitter: Callable[[float], float] | None,
|
|
134
|
+
max_delay: float | None,
|
|
135
|
+
) -> float:
|
|
136
|
+
try:
|
|
137
|
+
wait = delay * backoff**k
|
|
138
|
+
|
|
139
|
+
except OverflowError:
|
|
140
|
+
# Long retry budgets outgrow floats; max_delay still caps the infinite wait.
|
|
141
|
+
wait = math.inf
|
|
142
|
+
|
|
143
|
+
if jitter is not None:
|
|
144
|
+
wait = jitter(wait)
|
|
145
|
+
|
|
146
|
+
if max_delay is not None:
|
|
147
|
+
wait = min(wait, max_delay)
|
|
148
|
+
|
|
149
|
+
return wait
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
class FakeSleep:
|
|
2
|
+
"""Records requested waits instead of sleeping."""
|
|
3
|
+
|
|
4
|
+
def __init__(self) -> None:
|
|
5
|
+
self.calls: list[float] = []
|
|
6
|
+
|
|
7
|
+
def __call__(self, seconds: float) -> None:
|
|
8
|
+
self.calls.append(seconds)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Scripted:
|
|
12
|
+
"""Returns or raises each outcome in turn; the last outcome repeats."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, *outcomes: object) -> None:
|
|
15
|
+
self.outcomes = outcomes
|
|
16
|
+
self.calls = 0
|
|
17
|
+
|
|
18
|
+
def __call__(self) -> object:
|
|
19
|
+
outcome = self.outcomes[min(self.calls, len(self.outcomes) - 1)]
|
|
20
|
+
self.calls += 1
|
|
21
|
+
|
|
22
|
+
if isinstance(outcome, BaseException):
|
|
23
|
+
raise outcome
|
|
24
|
+
|
|
25
|
+
return outcome
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class NetworkError(Exception):
|
|
29
|
+
pass
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from retrpy import attempt
|
|
4
|
+
from tests.fakes import FakeSleep, NetworkError, Scripted
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TestWait:
|
|
8
|
+
def test_jitter_applied(self, sleep: FakeSleep) -> None:
|
|
9
|
+
plan = attempt(Scripted(NetworkError(), 1)) * 2
|
|
10
|
+
|
|
11
|
+
plan(delay=0.1, jitter=lambda d: d / 2, sleep=sleep)
|
|
12
|
+
|
|
13
|
+
assert sleep.calls == [0.05]
|
|
14
|
+
|
|
15
|
+
def test_max_delay_caps_after_jitter(self, sleep: FakeSleep) -> None:
|
|
16
|
+
plan = attempt(Scripted(NetworkError(), NetworkError(), 1)) * 3
|
|
17
|
+
|
|
18
|
+
plan(delay=1, backoff=10, jitter=lambda d: d + 1, max_delay=5, sleep=sleep)
|
|
19
|
+
|
|
20
|
+
assert sleep.calls == [2, 5]
|
|
21
|
+
|
|
22
|
+
def test_nonpositive_jitter_skips_sleep(self, sleep: FakeSleep) -> None:
|
|
23
|
+
plan = attempt(Scripted(NetworkError(), 1)) * 2
|
|
24
|
+
|
|
25
|
+
plan(delay=1, jitter=lambda d: -1, sleep=sleep)
|
|
26
|
+
|
|
27
|
+
assert sleep.calls == []
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TestOnFailure:
|
|
31
|
+
def test_on_failure_every_caught_failure_including_last(self, sleep: FakeSleep) -> None:
|
|
32
|
+
errors = [NetworkError(1), NetworkError(2), NetworkError(3)]
|
|
33
|
+
seen: list[tuple[Exception, int]] = []
|
|
34
|
+
|
|
35
|
+
with pytest.raises(ExceptionGroup):
|
|
36
|
+
(attempt(Scripted(*errors)) * 3)(
|
|
37
|
+
on_failure=lambda e, n: seen.append((e, n)), sleep=sleep
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
assert seen == [(errors[0], 1), (errors[1], 2), (errors[2], 3)]
|
|
41
|
+
|
|
42
|
+
def test_on_failure_numbers_across_segments(self, sleep: FakeSleep) -> None:
|
|
43
|
+
numbers: list[int] = []
|
|
44
|
+
plan = attempt(Scripted(NetworkError())) * 2 + attempt(lambda: 1)
|
|
45
|
+
|
|
46
|
+
assert plan(on_failure=lambda e, n: numbers.append(n), sleep=sleep) == 1
|
|
47
|
+
|
|
48
|
+
assert numbers == [1, 2]
|
|
49
|
+
|
|
50
|
+
def test_on_failure_numbers_continue_into_fallback(self, sleep: FakeSleep) -> None:
|
|
51
|
+
numbers: list[int] = []
|
|
52
|
+
plan = attempt(Scripted(NetworkError())) * 2 + attempt(Scripted(NetworkError()))
|
|
53
|
+
|
|
54
|
+
with pytest.raises(ExceptionGroup):
|
|
55
|
+
plan(on_failure=lambda e, n: numbers.append(n), sleep=sleep)
|
|
56
|
+
|
|
57
|
+
assert numbers == [1, 2, 3]
|
|
58
|
+
|
|
59
|
+
def test_on_failure_not_called_for_fail_fast(self, sleep: FakeSleep) -> None:
|
|
60
|
+
seen: list[int] = []
|
|
61
|
+
|
|
62
|
+
with pytest.raises(ValueError, match="bad"):
|
|
63
|
+
(attempt(Scripted(ValueError("bad"))) * 2)(
|
|
64
|
+
catch=(NetworkError,), on_failure=lambda e, n: seen.append(n), sleep=sleep
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
assert seen == []
|
|
68
|
+
|
|
69
|
+
def test_on_failure_error_propagates(self, sleep: FakeSleep) -> None:
|
|
70
|
+
fn = Scripted(NetworkError(), 1)
|
|
71
|
+
|
|
72
|
+
def hook(error: Exception, number: int) -> None:
|
|
73
|
+
raise RuntimeError("hook broke")
|
|
74
|
+
|
|
75
|
+
with pytest.raises(RuntimeError, match="hook broke"):
|
|
76
|
+
(attempt(fn) * 2)(on_failure=hook, sleep=sleep)
|
|
77
|
+
|
|
78
|
+
assert fn.calls == 1
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class TestReviewFixes:
|
|
82
|
+
def test_long_capped_backoff_does_not_overflow(self, sleep: FakeSleep) -> None:
|
|
83
|
+
plan = attempt(Scripted(NetworkError())) * 1100 + attempt(lambda: 1)
|
|
84
|
+
|
|
85
|
+
assert plan(delay=0.01, backoff=2.0, max_delay=1, sleep=sleep) == 1
|
|
86
|
+
assert sleep.calls[-1] == 1
|
|
87
|
+
|
|
88
|
+
def test_catch_rejects_base_exception(self, sleep: FakeSleep) -> None:
|
|
89
|
+
fn = Scripted(1)
|
|
90
|
+
|
|
91
|
+
with pytest.raises(TypeError, match="BaseException"):
|
|
92
|
+
attempt(fn)(catch=(BaseException,), sleep=sleep) # type: ignore[arg-type]
|
|
93
|
+
|
|
94
|
+
assert fn.calls == 0
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from retrpy import Plan, attempt
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def shape(plan: Plan[Any]) -> list[tuple[object, int]]:
|
|
9
|
+
return [(segment.fn, segment.count) for segment in plan.segments]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@pytest.fixture
|
|
13
|
+
def a() -> Plan[int]:
|
|
14
|
+
return attempt(int)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture
|
|
18
|
+
def b() -> Plan[str]:
|
|
19
|
+
return attempt(str)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TestMul:
|
|
23
|
+
def test_mul_single_segment_multiplies_count(self, a: Plan[int]) -> None:
|
|
24
|
+
fa = a.segments[0].fn
|
|
25
|
+
|
|
26
|
+
assert shape((a * 2) * 3) == [(fa, 6)]
|
|
27
|
+
|
|
28
|
+
def test_rmul_matches_mul(self, a: Plan[int]) -> None:
|
|
29
|
+
assert 3 * a == a * 3
|
|
30
|
+
|
|
31
|
+
def test_mul_multi_segment_repeats_sequence(self, a: Plan[int], b: Plan[str]) -> None:
|
|
32
|
+
fa, fb = a.segments[0].fn, b.segments[0].fn
|
|
33
|
+
|
|
34
|
+
assert shape((a * 2 + b) * 3) == [(fa, 2), (fb, 1)] * 3
|
|
35
|
+
|
|
36
|
+
@pytest.mark.parametrize("n", [0, -1])
|
|
37
|
+
def test_mul_rejects_zero_and_negative(self, a: Plan[int], n: int) -> None:
|
|
38
|
+
with pytest.raises(ValueError, match="at least 1"):
|
|
39
|
+
a * n
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class TestAdd:
|
|
43
|
+
def test_add_never_merges(self, a: Plan[int]) -> None:
|
|
44
|
+
fa = a.segments[0].fn
|
|
45
|
+
|
|
46
|
+
assert shape(a * 2 + a * 3) == [(fa, 2), (fa, 3)]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class TestOperands:
|
|
50
|
+
def test_wrong_operand_types_raise_type_error(self, a: Plan[int]) -> None:
|
|
51
|
+
with pytest.raises(TypeError):
|
|
52
|
+
a + 3 # type: ignore[operator]
|
|
53
|
+
|
|
54
|
+
with pytest.raises(TypeError):
|
|
55
|
+
a * 2.0 # type: ignore[operator]
|
|
56
|
+
|
|
57
|
+
def test_subclass_preserved(self, a: Plan[int], b: Plan[str]) -> None:
|
|
58
|
+
class P[T](Plan[T]):
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
p = P(a.segments)
|
|
62
|
+
|
|
63
|
+
assert type(p * 2) is P
|
|
64
|
+
assert type(p + b) is P
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import dataclasses
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
import retrpy
|
|
6
|
+
from retrpy import Plan, attempt
|
|
7
|
+
from retrpy._plan import Segment
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TestAttempt:
|
|
11
|
+
def test_binds_arguments(self) -> None:
|
|
12
|
+
plan = attempt(pow, 2, exp=3)
|
|
13
|
+
|
|
14
|
+
assert len(plan.segments) == 1
|
|
15
|
+
assert plan.segments[0].count == 1
|
|
16
|
+
assert plan.segments[0].fn() == 8
|
|
17
|
+
|
|
18
|
+
def test_len_and_iter_expand_counts(self) -> None:
|
|
19
|
+
plan = Plan((Segment(lambda: 1, 2), Segment(lambda: 2, 1)))
|
|
20
|
+
|
|
21
|
+
assert len(plan) == 3
|
|
22
|
+
assert [f() for f in plan] == [1, 1, 2]
|
|
23
|
+
|
|
24
|
+
def test_is_immutable(self) -> None:
|
|
25
|
+
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
26
|
+
attempt(int).segments = () # type: ignore[misc]
|
|
27
|
+
|
|
28
|
+
def test_public_api(self) -> None:
|
|
29
|
+
assert retrpy.__all__ == ["Plan", "attempt"]
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from retrpy import Plan, attempt
|
|
4
|
+
from tests.fakes import FakeSleep, NetworkError, Scripted
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TestSuccess:
|
|
8
|
+
def test_success_first_try(self, sleep: FakeSleep) -> None:
|
|
9
|
+
assert attempt(lambda: 5)(sleep=sleep) == 5
|
|
10
|
+
assert sleep.calls == []
|
|
11
|
+
|
|
12
|
+
def test_retry_then_success(self, sleep: FakeSleep) -> None:
|
|
13
|
+
fn = Scripted(NetworkError(), NetworkError(), 7)
|
|
14
|
+
|
|
15
|
+
assert (attempt(fn) * 3)(delay=0.05, backoff=2.0, sleep=sleep) == 7
|
|
16
|
+
assert sleep.calls == [0.05, 0.1]
|
|
17
|
+
|
|
18
|
+
def test_zero_delay_never_sleeps(self, sleep: FakeSleep) -> None:
|
|
19
|
+
fn = Scripted(NetworkError(), 1)
|
|
20
|
+
|
|
21
|
+
assert (attempt(fn) * 2)(sleep=sleep) == 1
|
|
22
|
+
assert sleep.calls == []
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class TestFallback:
|
|
26
|
+
def test_fallback_backoff_restarts_per_segment(self, sleep: FakeSleep) -> None:
|
|
27
|
+
primary = Scripted(NetworkError())
|
|
28
|
+
backup = Scripted(NetworkError(), 9)
|
|
29
|
+
plan = attempt(primary) * 3 + attempt(backup) * 2
|
|
30
|
+
|
|
31
|
+
assert plan(delay=0.05, backoff=2.0, sleep=sleep) == 9
|
|
32
|
+
assert sleep.calls == [0.05, 0.1, 0.05]
|
|
33
|
+
|
|
34
|
+
def test_default_value_step(self, sleep: FakeSleep) -> None:
|
|
35
|
+
plan = attempt(Scripted(NetworkError())) * 2 + attempt(lambda: None)
|
|
36
|
+
|
|
37
|
+
assert plan(sleep=sleep) is None
|
|
38
|
+
|
|
39
|
+
def test_runs_are_independent(self, sleep: FakeSleep) -> None:
|
|
40
|
+
plan = attempt(Scripted(NetworkError())) * 3 + attempt(lambda: 0)
|
|
41
|
+
|
|
42
|
+
plan(delay=1, backoff=2, sleep=sleep)
|
|
43
|
+
plan(delay=1, backoff=2, sleep=sleep)
|
|
44
|
+
|
|
45
|
+
assert sleep.calls == [1, 2, 1, 2]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class TestFailFast:
|
|
49
|
+
def test_fail_fast_outside_catch(self, sleep: FakeSleep) -> None:
|
|
50
|
+
err = ValueError("bad")
|
|
51
|
+
fn = Scripted(err, 1)
|
|
52
|
+
|
|
53
|
+
with pytest.raises(ValueError, match="bad") as excinfo:
|
|
54
|
+
(attempt(fn) * 3)(catch=(KeyError,), sleep=sleep)
|
|
55
|
+
|
|
56
|
+
assert excinfo.value is err
|
|
57
|
+
assert fn.calls == 1
|
|
58
|
+
|
|
59
|
+
def test_keyboard_interrupt_propagates(self, sleep: FakeSleep) -> None:
|
|
60
|
+
fn = Scripted(KeyboardInterrupt(), 1)
|
|
61
|
+
|
|
62
|
+
with pytest.raises(KeyboardInterrupt):
|
|
63
|
+
(attempt(fn) * 3)(sleep=sleep)
|
|
64
|
+
|
|
65
|
+
assert fn.calls == 1
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class TestExhausted:
|
|
69
|
+
def test_exhausted_raises_exception_group(self, sleep: FakeSleep) -> None:
|
|
70
|
+
errors = [NetworkError(1), NetworkError(2), NetworkError(3)]
|
|
71
|
+
plan = attempt(Scripted(*errors)) * 3
|
|
72
|
+
|
|
73
|
+
with pytest.raises(ExceptionGroup, match="all 3 attempts failed") as excinfo:
|
|
74
|
+
plan(sleep=sleep)
|
|
75
|
+
|
|
76
|
+
assert list(excinfo.value.exceptions) == errors
|
|
77
|
+
|
|
78
|
+
def test_empty_plan_raises(self, sleep: FakeSleep) -> None:
|
|
79
|
+
with pytest.raises(ValueError, match="empty"):
|
|
80
|
+
Plan(())(sleep=sleep)
|