httpx-limiter 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/.gitignore +3 -0
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/PKG-INFO +141 -38
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/README.md +131 -30
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/pyproject.toml +51 -25
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/src/httpx_limiter/__init__.py +9 -7
- httpx_limiter-0.6.0/src/httpx_limiter/abstract_rate_limiter_repository.py → httpx_limiter-0.7.0/src/httpx_limiter/_rate_limiter_repository.py +20 -17
- httpx_limiter-0.7.0/src/httpx_limiter/_types.py +30 -0
- httpx_limiter-0.7.0/src/httpx_limiter/_version.py +24 -0
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/src/httpx_limiter/aiolimiter/__init__.py +1 -1
- httpx_limiter-0.7.0/src/httpx_limiter/httpx/__init__.py +46 -0
- httpx_limiter-0.7.0/src/httpx_limiter/httpx/abstract_rate_limiter_repository.py +39 -0
- {httpx_limiter-0.6.0/src/httpx_limiter → httpx_limiter-0.7.0/src/httpx_limiter/httpx}/async_multi_rate_limited_transport.py +40 -8
- {httpx_limiter-0.6.0/src/httpx_limiter → httpx_limiter-0.7.0/src/httpx_limiter/httpx}/async_rate_limited_transport.py +42 -9
- {httpx_limiter-0.6.0/src/httpx_limiter → httpx_limiter-0.7.0/src/httpx_limiter/httpx}/types.py +2 -12
- httpx_limiter-0.7.0/src/httpx_limiter/httpx2/__init__.py +46 -0
- httpx_limiter-0.7.0/src/httpx_limiter/httpx2/abstract_rate_limiter_repository.py +39 -0
- httpx_limiter-0.7.0/src/httpx_limiter/httpx2/async_multi_rate_limited_transport.py +124 -0
- httpx_limiter-0.7.0/src/httpx_limiter/httpx2/async_rate_limited_transport.py +124 -0
- httpx_limiter-0.7.0/src/httpx_limiter/httpx2/types.py +43 -0
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/src/httpx_limiter/pyrate/__init__.py +4 -1
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/src/httpx_limiter/pyrate/pyrate_async_limiter.py +1 -8
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/src/httpx_limiter/rate.py +1 -0
- httpx_limiter-0.6.0/src/httpx_limiter/_version.py +0 -34
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/LICENSE +0 -0
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/src/httpx_limiter/abstract_async_limiter.py +0 -0
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/src/httpx_limiter/aiolimiter/aiolimiter_async_limiter.py +0 -0
- {httpx_limiter-0.6.0 → httpx_limiter-0.7.0}/src/httpx_limiter/py.typed +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: httpx-limiter
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: A lightweight package that provides rate-limited httpx transports.
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Summary: A lightweight package that provides rate-limited httpx and httpx2 transports.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Midnighter/httpx-limiter
|
|
6
6
|
Project-URL: Documentation, https://midnighter.github.io/httpx-limiter/
|
|
7
7
|
Project-URL: Source Code, https://github.com/Midnighter/httpx-limiter
|
|
@@ -10,7 +10,7 @@ Project-URL: Download, https://pypi.org/project/httpx-limiter/#files
|
|
|
10
10
|
Author-email: "Moritz E. Beber" <midnighter@posteo.net>
|
|
11
11
|
License-Expression: Apache-2.0
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Keywords: httpx,leaky bucket,limiter,rate-limit
|
|
13
|
+
Keywords: httpx,httpx2,leaky bucket,limiter,rate-limit
|
|
14
14
|
Classifier: Development Status :: 2 - Pre-Alpha
|
|
15
15
|
Classifier: Environment :: Web Environment
|
|
16
16
|
Classifier: Intended Audience :: Developers
|
|
@@ -18,18 +18,20 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
18
18
|
Classifier: Natural Language :: English
|
|
19
19
|
Classifier: Operating System :: OS Independent
|
|
20
20
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
22
21
|
Classifier: Programming Language :: Python :: 3.11
|
|
23
22
|
Classifier: Programming Language :: Python :: 3.12
|
|
24
23
|
Classifier: Programming Language :: Python :: 3.13
|
|
25
24
|
Classifier: Programming Language :: Python :: 3.14
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
26
26
|
Classifier: Topic :: Internet :: WWW/HTTP
|
|
27
27
|
Classifier: Typing :: Typed
|
|
28
|
-
Requires-Python: >=3.
|
|
29
|
-
Requires-Dist: httpx~=0.25
|
|
30
|
-
Requires-Dist: typing-extensions~=4.6; python_version < '3.11'
|
|
28
|
+
Requires-Python: >=3.11
|
|
31
29
|
Provides-Extra: aiolimiter
|
|
32
30
|
Requires-Dist: aiolimiter~=1.2; extra == 'aiolimiter'
|
|
31
|
+
Provides-Extra: httpx
|
|
32
|
+
Requires-Dist: httpx~=0.25; extra == 'httpx'
|
|
33
|
+
Provides-Extra: httpx2
|
|
34
|
+
Requires-Dist: httpx2~=2.10; extra == 'httpx2'
|
|
33
35
|
Provides-Extra: pyrate
|
|
34
36
|
Requires-Dist: pyrate-limiter~=4.0; extra == 'pyrate'
|
|
35
37
|
Description-Content-Type: text/markdown
|
|
@@ -37,28 +39,51 @@ Description-Content-Type: text/markdown
|
|
|
37
39
|
# HTTPX Limiter
|
|
38
40
|
|
|
39
41
|
| | |
|
|
40
|
-
|
|
42
|
+
|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
41
43
|
| Package | [](https://pypi.org/project/httpx-limiter/) [](https://pypi.org/project/httpx-limiter/) |
|
|
42
44
|
| Meta | [](LICENSE) [](.github/CODE_OF_CONDUCT.md) [](https://mypy-lang.org/) [](https://github.com/ambv/black) [](https://github.com/astral-sh/ruff) |
|
|
43
|
-
| Automation | [](https://github.com/Midnighter/httpx-limiter/actions/workflows/main.yml)
|
|
45
|
+
| Automation | [](https://github.com/Midnighter/httpx-limiter/actions/workflows/main.yml) [](https://codecov.io/github/Midnighter/httpx-limiter) |
|
|
44
46
|
|
|
45
|
-
_A lightweight package that provides rate-limited httpx transports._
|
|
47
|
+
_A lightweight package that provides rate-limited httpx and httpx2 transports._
|
|
46
48
|
|
|
47
49
|
## Installation
|
|
48
50
|
|
|
49
|
-
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/).
|
|
50
|
-
|
|
51
|
+
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). Plain
|
|
52
|
+
`pip install httpx-limiter` installs only the shared core, which contains no HTTP
|
|
53
|
+
client. Select the HTTP client you use with an extra, `httpx` or `httpx2`, and normally
|
|
54
|
+
also a rate limiter backend, `aiolimiter` or `pyrate`.
|
|
51
55
|
|
|
52
56
|
```sh
|
|
53
|
-
pip install 'httpx-limiter[aiolimiter]'
|
|
57
|
+
pip install 'httpx-limiter[httpx,aiolimiter]'
|
|
58
|
+
pip install 'httpx-limiter[httpx2,pyrate]'
|
|
54
59
|
```
|
|
55
60
|
|
|
56
|
-
|
|
61
|
+
To use both HTTP clients in the same process, install both extras.
|
|
57
62
|
|
|
58
63
|
```sh
|
|
59
|
-
pip install 'httpx-limiter[
|
|
64
|
+
pip install 'httpx-limiter[httpx,httpx2,aiolimiter]'
|
|
60
65
|
```
|
|
61
66
|
|
|
67
|
+
There is no automatic fallback: you choose the integration explicitly by importing from
|
|
68
|
+
`httpx_limiter.httpx` or from `httpx_limiter.httpx2`.
|
|
69
|
+
|
|
70
|
+
### Migrating from <0.7
|
|
71
|
+
|
|
72
|
+
The HTTP-specific classes moved out of the package root and `httpx` is no longer a
|
|
73
|
+
required dependency. Install `httpx-limiter[httpx]` instead of `httpx-limiter` and
|
|
74
|
+
change your imports. `Rate`, `AbstractAsyncLimiter`, and the limiter implementations
|
|
75
|
+
keep their import paths.
|
|
76
|
+
|
|
77
|
+
```diff
|
|
78
|
+
-from httpx_limiter import AsyncRateLimitedTransport, Rate
|
|
79
|
+
+from httpx_limiter import Rate
|
|
80
|
+
+from httpx_limiter.httpx import AsyncRateLimitedTransport
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The same applies to `AsyncMultiRateLimitedTransport`, `AbstractRateLimiterRepository`,
|
|
84
|
+
and `HTTPXAsyncHTTPTransportKeywordArguments`. For HTTPX2, import them from
|
|
85
|
+
`httpx_limiter.httpx2` instead.
|
|
86
|
+
|
|
62
87
|
## Rate Limiter Backends
|
|
63
88
|
|
|
64
89
|
This package provides two different asynchronous rate limiter implementations to choose
|
|
@@ -113,9 +138,17 @@ async def main():
|
|
|
113
138
|
|
|
114
139
|
## Tutorial
|
|
115
140
|
|
|
116
|
-
You can limit the number of requests made by an HTTPX client using the
|
|
117
|
-
provided in this package. That is useful in situations when you need to make
|
|
118
|
-
number of asynchronous requests against endpoints that implement a rate limit.
|
|
141
|
+
You can limit the number of requests made by an HTTPX or HTTPX2 client using the
|
|
142
|
+
transports provided in this package. That is useful in situations when you need to make
|
|
143
|
+
a large number of asynchronous requests against endpoints that implement a rate limit.
|
|
144
|
+
|
|
145
|
+
The examples below use HTTPX. For HTTPX2, import `httpx2` instead of `httpx`, and import
|
|
146
|
+
the transports and repositories from `httpx_limiter.httpx2` instead of
|
|
147
|
+
`httpx_limiter.httpx`; everything else stays the same. The
|
|
148
|
+
[documentation](https://midnighter.github.io/httpx-limiter/) shows every example for both
|
|
149
|
+
clients. Transports, repositories, requests, and responses must all come from the same
|
|
150
|
+
client: `httpx_limiter.httpx` works with `httpx` and `httpx_limiter.httpx2` works with
|
|
151
|
+
`httpx2`.
|
|
119
152
|
|
|
120
153
|
### Single Rate Limit
|
|
121
154
|
|
|
@@ -124,7 +157,8 @@ able to make twenty requests per second, for example, use the following code:
|
|
|
124
157
|
|
|
125
158
|
```python
|
|
126
159
|
import httpx
|
|
127
|
-
from httpx_limiter import
|
|
160
|
+
from httpx_limiter import Rate
|
|
161
|
+
from httpx_limiter.httpx import AsyncRateLimitedTransport
|
|
128
162
|
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
129
163
|
|
|
130
164
|
async def main():
|
|
@@ -138,32 +172,37 @@ async def main():
|
|
|
138
172
|
|
|
139
173
|
> [!IMPORTANT]
|
|
140
174
|
> Due to limitations in the design of the underlying [leaky
|
|
141
|
-
> bucket](https://en.wikipedia.org/wiki/Leaky_bucket) implementation, which is
|
|
142
|
-
>
|
|
143
|
-
>
|
|
144
|
-
>
|
|
145
|
-
>
|
|
146
|
-
>
|
|
147
|
-
>
|
|
175
|
+
> bucket](https://en.wikipedia.org/wiki/Leaky_bucket) implementation, which is used to
|
|
176
|
+
> implement the rate limiting, the magnitude of **the rate is also the maximum
|
|
177
|
+
> capacity of the bucket**. That means, if you **set a rate that is larger than one, a
|
|
178
|
+
> burst of requests equal to that capacity will be allowed**. If you do not want to
|
|
179
|
+
> allow any bursts, set the magnitude to one, but the duration to the inverse of your
|
|
180
|
+
> desired rate. If you want to allow twenty requests per second, for example, set the
|
|
181
|
+
> magnitude to 1 and the duration to 0.05 seconds.
|
|
148
182
|
>
|
|
149
183
|
> ```python
|
|
184
|
+
> from httpx_limiter import Rate
|
|
185
|
+
>
|
|
150
186
|
> Rate.create(magnitude=1, duration=1/20)
|
|
151
187
|
> ```
|
|
152
188
|
|
|
153
189
|
### Multiple Rate Limits
|
|
154
190
|
|
|
155
191
|
For more advanced use cases, you can apply different rate limits based on a concrete
|
|
156
|
-
implementation of the `AbstractRateLimiterRepository
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
192
|
+
implementation of the `AbstractRateLimiterRepository` of your HTTP client, either
|
|
193
|
+
[for HTTPX](https://midnighter.github.io/httpx-limiter/api/#httpx_limiter.httpx.AbstractRateLimiterRepository)
|
|
194
|
+
or
|
|
195
|
+
[for HTTPX2](https://midnighter.github.io/httpx-limiter/api/#httpx_limiter.httpx2.AbstractRateLimiterRepository).
|
|
196
|
+
There are two relevant methods that both get passed the current request. One method
|
|
197
|
+
needs to identify which rate limit to apply, and the other method sets the rate limit
|
|
198
|
+
itself. See the following example:
|
|
160
199
|
|
|
161
200
|
```python
|
|
162
201
|
import httpx
|
|
163
|
-
from httpx_limiter import
|
|
202
|
+
from httpx_limiter import Rate
|
|
203
|
+
from httpx_limiter.httpx import (
|
|
164
204
|
AbstractRateLimiterRepository,
|
|
165
205
|
AsyncMultiRateLimitedTransport,
|
|
166
|
-
Rate
|
|
167
206
|
)
|
|
168
207
|
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
169
208
|
|
|
@@ -186,19 +225,22 @@ async def main():
|
|
|
186
225
|
repository=DomainBasedRateLimiterRepository(),
|
|
187
226
|
),
|
|
188
227
|
) as client:
|
|
189
|
-
|
|
190
|
-
pass
|
|
228
|
+
response = await client.get("https://httpbin.org")
|
|
191
229
|
```
|
|
192
230
|
|
|
193
231
|
> [!TIP]
|
|
194
|
-
> You are free to ignore the request parameter and use global information like
|
|
195
|
-
>
|
|
232
|
+
> You are free to ignore the request parameter and use global information like the
|
|
233
|
+
> time of day to determine the rate limit.
|
|
196
234
|
|
|
197
235
|
```python
|
|
198
236
|
from datetime import datetime, timezone
|
|
199
237
|
|
|
200
238
|
import httpx
|
|
201
|
-
from httpx_limiter import
|
|
239
|
+
from httpx_limiter import Rate
|
|
240
|
+
from httpx_limiter.httpx import (
|
|
241
|
+
AbstractRateLimiterRepository,
|
|
242
|
+
AsyncMultiRateLimitedTransport,
|
|
243
|
+
)
|
|
202
244
|
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
203
245
|
|
|
204
246
|
|
|
@@ -218,6 +260,67 @@ class DayNightRateLimiterRepository(AbstractRateLimiterRepository):
|
|
|
218
260
|
return PyrateAsyncLimiter.create(Rate.create(magnitude=10))
|
|
219
261
|
|
|
220
262
|
return PyrateAsyncLimiter.create(Rate.create(magnitude=100))
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
async def main():
|
|
266
|
+
async with httpx.AsyncClient(
|
|
267
|
+
transport=AsyncMultiRateLimitedTransport.create(
|
|
268
|
+
repository=DayNightRateLimiterRepository(),
|
|
269
|
+
),
|
|
270
|
+
) as client:
|
|
271
|
+
response = await client.get("https://httpbin.org")
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
### Transport Options
|
|
275
|
+
|
|
276
|
+
Both `AsyncRateLimitedTransport.create` and `AsyncMultiRateLimitedTransport.create`
|
|
277
|
+
forward additional keyword arguments to the `AsyncHTTPTransport` of the selected client.
|
|
278
|
+
The accepted keywords are described by the `HTTPXAsyncHTTPTransportKeywordArguments`
|
|
279
|
+
type of `httpx_limiter.httpx` and of `httpx_limiter.httpx2`. Types such as `Limits` and
|
|
280
|
+
`Proxy` must come from the selected client.
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
import httpx
|
|
284
|
+
from httpx_limiter import Rate
|
|
285
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
286
|
+
from httpx_limiter.httpx import AsyncRateLimitedTransport
|
|
287
|
+
|
|
288
|
+
async def main():
|
|
289
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
290
|
+
transport = AsyncRateLimitedTransport.create(
|
|
291
|
+
limiter=limiter,
|
|
292
|
+
limits=httpx.Limits(max_connections=10),
|
|
293
|
+
retries=2,
|
|
294
|
+
)
|
|
295
|
+
async with httpx.AsyncClient(transport=transport) as client:
|
|
296
|
+
response = await client.get("https://httpbin.org")
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
### Using Both Clients Together
|
|
300
|
+
|
|
301
|
+
You can install both extras, `httpx-limiter[httpx,httpx2]`, and use both integrations in
|
|
302
|
+
the same process. The limiters are independent of the HTTP client, so you may even share
|
|
303
|
+
one limiter between them. The HTTP objects, however, are **not interchangeable**: an
|
|
304
|
+
`httpx_limiter.httpx` transport cannot be used with an `httpx2.AsyncClient`, and a
|
|
305
|
+
repository for `httpx.Request` cannot serve `httpx2.Request` objects.
|
|
306
|
+
|
|
307
|
+
```python
|
|
308
|
+
import httpx
|
|
309
|
+
import httpx2
|
|
310
|
+
from httpx_limiter import Rate
|
|
311
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
312
|
+
from httpx_limiter.httpx import AsyncRateLimitedTransport as HTTPXTransport
|
|
313
|
+
from httpx_limiter.httpx2 import AsyncRateLimitedTransport as HTTPX2Transport
|
|
314
|
+
|
|
315
|
+
async def main():
|
|
316
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
317
|
+
|
|
318
|
+
async with (
|
|
319
|
+
httpx.AsyncClient(transport=HTTPXTransport.create(limiter=limiter)) as old,
|
|
320
|
+
httpx2.AsyncClient(transport=HTTPX2Transport.create(limiter=limiter)) as new,
|
|
321
|
+
):
|
|
322
|
+
first = await old.get("https://httpbin.org") # an httpx.Response
|
|
323
|
+
second = await new.get("https://httpbin.org") # an httpx2.Response
|
|
221
324
|
```
|
|
222
325
|
|
|
223
326
|
## Copyright
|
|
@@ -1,28 +1,51 @@
|
|
|
1
1
|
# HTTPX Limiter
|
|
2
2
|
|
|
3
3
|
| | |
|
|
4
|
-
|
|
4
|
+
|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
5
5
|
| Package | [](https://pypi.org/project/httpx-limiter/) [](https://pypi.org/project/httpx-limiter/) |
|
|
6
6
|
| Meta | [](LICENSE) [](.github/CODE_OF_CONDUCT.md) [](https://mypy-lang.org/) [](https://github.com/ambv/black) [](https://github.com/astral-sh/ruff) |
|
|
7
|
-
| Automation | [](https://github.com/Midnighter/httpx-limiter/actions/workflows/main.yml)
|
|
7
|
+
| Automation | [](https://github.com/Midnighter/httpx-limiter/actions/workflows/main.yml) [](https://codecov.io/github/Midnighter/httpx-limiter) |
|
|
8
8
|
|
|
9
|
-
_A lightweight package that provides rate-limited httpx transports._
|
|
9
|
+
_A lightweight package that provides rate-limited httpx and httpx2 transports._
|
|
10
10
|
|
|
11
11
|
## Installation
|
|
12
12
|
|
|
13
|
-
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/).
|
|
14
|
-
|
|
13
|
+
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). Plain
|
|
14
|
+
`pip install httpx-limiter` installs only the shared core, which contains no HTTP
|
|
15
|
+
client. Select the HTTP client you use with an extra, `httpx` or `httpx2`, and normally
|
|
16
|
+
also a rate limiter backend, `aiolimiter` or `pyrate`.
|
|
15
17
|
|
|
16
18
|
```sh
|
|
17
|
-
pip install 'httpx-limiter[aiolimiter]'
|
|
19
|
+
pip install 'httpx-limiter[httpx,aiolimiter]'
|
|
20
|
+
pip install 'httpx-limiter[httpx2,pyrate]'
|
|
18
21
|
```
|
|
19
22
|
|
|
20
|
-
|
|
23
|
+
To use both HTTP clients in the same process, install both extras.
|
|
21
24
|
|
|
22
25
|
```sh
|
|
23
|
-
pip install 'httpx-limiter[
|
|
26
|
+
pip install 'httpx-limiter[httpx,httpx2,aiolimiter]'
|
|
24
27
|
```
|
|
25
28
|
|
|
29
|
+
There is no automatic fallback: you choose the integration explicitly by importing from
|
|
30
|
+
`httpx_limiter.httpx` or from `httpx_limiter.httpx2`.
|
|
31
|
+
|
|
32
|
+
### Migrating from <0.7
|
|
33
|
+
|
|
34
|
+
The HTTP-specific classes moved out of the package root and `httpx` is no longer a
|
|
35
|
+
required dependency. Install `httpx-limiter[httpx]` instead of `httpx-limiter` and
|
|
36
|
+
change your imports. `Rate`, `AbstractAsyncLimiter`, and the limiter implementations
|
|
37
|
+
keep their import paths.
|
|
38
|
+
|
|
39
|
+
```diff
|
|
40
|
+
-from httpx_limiter import AsyncRateLimitedTransport, Rate
|
|
41
|
+
+from httpx_limiter import Rate
|
|
42
|
+
+from httpx_limiter.httpx import AsyncRateLimitedTransport
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The same applies to `AsyncMultiRateLimitedTransport`, `AbstractRateLimiterRepository`,
|
|
46
|
+
and `HTTPXAsyncHTTPTransportKeywordArguments`. For HTTPX2, import them from
|
|
47
|
+
`httpx_limiter.httpx2` instead.
|
|
48
|
+
|
|
26
49
|
## Rate Limiter Backends
|
|
27
50
|
|
|
28
51
|
This package provides two different asynchronous rate limiter implementations to choose
|
|
@@ -77,9 +100,17 @@ async def main():
|
|
|
77
100
|
|
|
78
101
|
## Tutorial
|
|
79
102
|
|
|
80
|
-
You can limit the number of requests made by an HTTPX client using the
|
|
81
|
-
provided in this package. That is useful in situations when you need to make
|
|
82
|
-
number of asynchronous requests against endpoints that implement a rate limit.
|
|
103
|
+
You can limit the number of requests made by an HTTPX or HTTPX2 client using the
|
|
104
|
+
transports provided in this package. That is useful in situations when you need to make
|
|
105
|
+
a large number of asynchronous requests against endpoints that implement a rate limit.
|
|
106
|
+
|
|
107
|
+
The examples below use HTTPX. For HTTPX2, import `httpx2` instead of `httpx`, and import
|
|
108
|
+
the transports and repositories from `httpx_limiter.httpx2` instead of
|
|
109
|
+
`httpx_limiter.httpx`; everything else stays the same. The
|
|
110
|
+
[documentation](https://midnighter.github.io/httpx-limiter/) shows every example for both
|
|
111
|
+
clients. Transports, repositories, requests, and responses must all come from the same
|
|
112
|
+
client: `httpx_limiter.httpx` works with `httpx` and `httpx_limiter.httpx2` works with
|
|
113
|
+
`httpx2`.
|
|
83
114
|
|
|
84
115
|
### Single Rate Limit
|
|
85
116
|
|
|
@@ -88,7 +119,8 @@ able to make twenty requests per second, for example, use the following code:
|
|
|
88
119
|
|
|
89
120
|
```python
|
|
90
121
|
import httpx
|
|
91
|
-
from httpx_limiter import
|
|
122
|
+
from httpx_limiter import Rate
|
|
123
|
+
from httpx_limiter.httpx import AsyncRateLimitedTransport
|
|
92
124
|
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
93
125
|
|
|
94
126
|
async def main():
|
|
@@ -102,32 +134,37 @@ async def main():
|
|
|
102
134
|
|
|
103
135
|
> [!IMPORTANT]
|
|
104
136
|
> Due to limitations in the design of the underlying [leaky
|
|
105
|
-
> bucket](https://en.wikipedia.org/wiki/Leaky_bucket) implementation, which is
|
|
106
|
-
>
|
|
107
|
-
>
|
|
108
|
-
>
|
|
109
|
-
>
|
|
110
|
-
>
|
|
111
|
-
>
|
|
137
|
+
> bucket](https://en.wikipedia.org/wiki/Leaky_bucket) implementation, which is used to
|
|
138
|
+
> implement the rate limiting, the magnitude of **the rate is also the maximum
|
|
139
|
+
> capacity of the bucket**. That means, if you **set a rate that is larger than one, a
|
|
140
|
+
> burst of requests equal to that capacity will be allowed**. If you do not want to
|
|
141
|
+
> allow any bursts, set the magnitude to one, but the duration to the inverse of your
|
|
142
|
+
> desired rate. If you want to allow twenty requests per second, for example, set the
|
|
143
|
+
> magnitude to 1 and the duration to 0.05 seconds.
|
|
112
144
|
>
|
|
113
145
|
> ```python
|
|
146
|
+
> from httpx_limiter import Rate
|
|
147
|
+
>
|
|
114
148
|
> Rate.create(magnitude=1, duration=1/20)
|
|
115
149
|
> ```
|
|
116
150
|
|
|
117
151
|
### Multiple Rate Limits
|
|
118
152
|
|
|
119
153
|
For more advanced use cases, you can apply different rate limits based on a concrete
|
|
120
|
-
implementation of the `AbstractRateLimiterRepository
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
154
|
+
implementation of the `AbstractRateLimiterRepository` of your HTTP client, either
|
|
155
|
+
[for HTTPX](https://midnighter.github.io/httpx-limiter/api/#httpx_limiter.httpx.AbstractRateLimiterRepository)
|
|
156
|
+
or
|
|
157
|
+
[for HTTPX2](https://midnighter.github.io/httpx-limiter/api/#httpx_limiter.httpx2.AbstractRateLimiterRepository).
|
|
158
|
+
There are two relevant methods that both get passed the current request. One method
|
|
159
|
+
needs to identify which rate limit to apply, and the other method sets the rate limit
|
|
160
|
+
itself. See the following example:
|
|
124
161
|
|
|
125
162
|
```python
|
|
126
163
|
import httpx
|
|
127
|
-
from httpx_limiter import
|
|
164
|
+
from httpx_limiter import Rate
|
|
165
|
+
from httpx_limiter.httpx import (
|
|
128
166
|
AbstractRateLimiterRepository,
|
|
129
167
|
AsyncMultiRateLimitedTransport,
|
|
130
|
-
Rate
|
|
131
168
|
)
|
|
132
169
|
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
133
170
|
|
|
@@ -150,19 +187,22 @@ async def main():
|
|
|
150
187
|
repository=DomainBasedRateLimiterRepository(),
|
|
151
188
|
),
|
|
152
189
|
) as client:
|
|
153
|
-
|
|
154
|
-
pass
|
|
190
|
+
response = await client.get("https://httpbin.org")
|
|
155
191
|
```
|
|
156
192
|
|
|
157
193
|
> [!TIP]
|
|
158
|
-
> You are free to ignore the request parameter and use global information like
|
|
159
|
-
>
|
|
194
|
+
> You are free to ignore the request parameter and use global information like the
|
|
195
|
+
> time of day to determine the rate limit.
|
|
160
196
|
|
|
161
197
|
```python
|
|
162
198
|
from datetime import datetime, timezone
|
|
163
199
|
|
|
164
200
|
import httpx
|
|
165
|
-
from httpx_limiter import
|
|
201
|
+
from httpx_limiter import Rate
|
|
202
|
+
from httpx_limiter.httpx import (
|
|
203
|
+
AbstractRateLimiterRepository,
|
|
204
|
+
AsyncMultiRateLimitedTransport,
|
|
205
|
+
)
|
|
166
206
|
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
167
207
|
|
|
168
208
|
|
|
@@ -182,6 +222,67 @@ class DayNightRateLimiterRepository(AbstractRateLimiterRepository):
|
|
|
182
222
|
return PyrateAsyncLimiter.create(Rate.create(magnitude=10))
|
|
183
223
|
|
|
184
224
|
return PyrateAsyncLimiter.create(Rate.create(magnitude=100))
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
async def main():
|
|
228
|
+
async with httpx.AsyncClient(
|
|
229
|
+
transport=AsyncMultiRateLimitedTransport.create(
|
|
230
|
+
repository=DayNightRateLimiterRepository(),
|
|
231
|
+
),
|
|
232
|
+
) as client:
|
|
233
|
+
response = await client.get("https://httpbin.org")
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
### Transport Options
|
|
237
|
+
|
|
238
|
+
Both `AsyncRateLimitedTransport.create` and `AsyncMultiRateLimitedTransport.create`
|
|
239
|
+
forward additional keyword arguments to the `AsyncHTTPTransport` of the selected client.
|
|
240
|
+
The accepted keywords are described by the `HTTPXAsyncHTTPTransportKeywordArguments`
|
|
241
|
+
type of `httpx_limiter.httpx` and of `httpx_limiter.httpx2`. Types such as `Limits` and
|
|
242
|
+
`Proxy` must come from the selected client.
|
|
243
|
+
|
|
244
|
+
```python
|
|
245
|
+
import httpx
|
|
246
|
+
from httpx_limiter import Rate
|
|
247
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
248
|
+
from httpx_limiter.httpx import AsyncRateLimitedTransport
|
|
249
|
+
|
|
250
|
+
async def main():
|
|
251
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
252
|
+
transport = AsyncRateLimitedTransport.create(
|
|
253
|
+
limiter=limiter,
|
|
254
|
+
limits=httpx.Limits(max_connections=10),
|
|
255
|
+
retries=2,
|
|
256
|
+
)
|
|
257
|
+
async with httpx.AsyncClient(transport=transport) as client:
|
|
258
|
+
response = await client.get("https://httpbin.org")
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
### Using Both Clients Together
|
|
262
|
+
|
|
263
|
+
You can install both extras, `httpx-limiter[httpx,httpx2]`, and use both integrations in
|
|
264
|
+
the same process. The limiters are independent of the HTTP client, so you may even share
|
|
265
|
+
one limiter between them. The HTTP objects, however, are **not interchangeable**: an
|
|
266
|
+
`httpx_limiter.httpx` transport cannot be used with an `httpx2.AsyncClient`, and a
|
|
267
|
+
repository for `httpx.Request` cannot serve `httpx2.Request` objects.
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
import httpx
|
|
271
|
+
import httpx2
|
|
272
|
+
from httpx_limiter import Rate
|
|
273
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
274
|
+
from httpx_limiter.httpx import AsyncRateLimitedTransport as HTTPXTransport
|
|
275
|
+
from httpx_limiter.httpx2 import AsyncRateLimitedTransport as HTTPX2Transport
|
|
276
|
+
|
|
277
|
+
async def main():
|
|
278
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
279
|
+
|
|
280
|
+
async with (
|
|
281
|
+
httpx.AsyncClient(transport=HTTPXTransport.create(limiter=limiter)) as old,
|
|
282
|
+
httpx2.AsyncClient(transport=HTTPX2Transport.create(limiter=limiter)) as new,
|
|
283
|
+
):
|
|
284
|
+
first = await old.get("https://httpbin.org") # an httpx.Response
|
|
285
|
+
second = await new.get("https://httpbin.org") # an httpx2.Response
|
|
185
286
|
```
|
|
186
287
|
|
|
187
288
|
## Copyright
|