httpx-limiter 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/PKG-INFO +104 -35
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/README.md +97 -31
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/pyproject.toml +70 -56
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/__init__.py +1 -1
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/_version.py +2 -2
- httpx_limiter-0.6.0/src/httpx_limiter/abstract_async_limiter.py +47 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/abstract_rate_limiter_repository.py +6 -18
- httpx_limiter-0.6.0/src/httpx_limiter/aiolimiter/__init__.py +18 -0
- httpx_limiter-0.6.0/src/httpx_limiter/aiolimiter/aiolimiter_async_limiter.py +76 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/async_multi_rate_limited_transport.py +1 -1
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/async_rate_limited_transport.py +6 -8
- httpx_limiter-0.6.0/src/httpx_limiter/pyrate/__init__.py +18 -0
- httpx_limiter-0.4.0/src/httpx_limiter/async_limiter.py → httpx_limiter-0.6.0/src/httpx_limiter/pyrate/pyrate_async_limiter.py +9 -31
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/rate.py +4 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/types.py +21 -6
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/.gitignore +0 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/LICENSE +0 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/py.typed +0 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: httpx-limiter
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: A lightweight package that provides rate-limited httpx transports.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Midnighter/httpx-limiter
|
|
6
|
-
Project-URL: Documentation, https://httpx-limiter
|
|
6
|
+
Project-URL: Documentation, https://midnighter.github.io/httpx-limiter/
|
|
7
7
|
Project-URL: Source Code, https://github.com/Midnighter/httpx-limiter
|
|
8
8
|
Project-URL: Bug Tracker, https://github.com/Midnighter/httpx-limiter/issues
|
|
9
9
|
Project-URL: Download, https://pypi.org/project/httpx-limiter/#files
|
|
@@ -13,7 +13,6 @@ License-File: LICENSE
|
|
|
13
13
|
Keywords: httpx,leaky bucket,limiter,rate-limit
|
|
14
14
|
Classifier: Development Status :: 2 - Pre-Alpha
|
|
15
15
|
Classifier: Environment :: Web Environment
|
|
16
|
-
Classifier: Framework :: AsyncIO
|
|
17
16
|
Classifier: Intended Audience :: Developers
|
|
18
17
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
19
18
|
Classifier: Natural Language :: English
|
|
@@ -23,12 +22,16 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
23
22
|
Classifier: Programming Language :: Python :: 3.11
|
|
24
23
|
Classifier: Programming Language :: Python :: 3.12
|
|
25
24
|
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
26
26
|
Classifier: Topic :: Internet :: WWW/HTTP
|
|
27
27
|
Classifier: Typing :: Typed
|
|
28
28
|
Requires-Python: >=3.10
|
|
29
29
|
Requires-Dist: httpx~=0.25
|
|
30
|
-
Requires-Dist: pyrate-limiter~=3.9
|
|
31
30
|
Requires-Dist: typing-extensions~=4.6; python_version < '3.11'
|
|
31
|
+
Provides-Extra: aiolimiter
|
|
32
|
+
Requires-Dist: aiolimiter~=1.2; extra == 'aiolimiter'
|
|
33
|
+
Provides-Extra: pyrate
|
|
34
|
+
Requires-Dist: pyrate-limiter~=4.0; extra == 'pyrate'
|
|
32
35
|
Description-Content-Type: text/markdown
|
|
33
36
|
|
|
34
37
|
# HTTPX Limiter
|
|
@@ -43,33 +46,92 @@ _A lightweight package that provides rate-limited httpx transports._
|
|
|
43
46
|
|
|
44
47
|
## Installation
|
|
45
48
|
|
|
46
|
-
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/).
|
|
47
|
-
|
|
49
|
+
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). Install it,
|
|
50
|
+
for example, with the aiolimiter backend.
|
|
48
51
|
|
|
49
52
|
```sh
|
|
50
|
-
pip install httpx-limiter
|
|
53
|
+
pip install 'httpx-limiter[aiolimiter]'
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
There is also a pyrate-limiter backend available.
|
|
57
|
+
|
|
58
|
+
```sh
|
|
59
|
+
pip install 'httpx-limiter[pyrate]'
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Rate Limiter Backends
|
|
63
|
+
|
|
64
|
+
This package provides two different asynchronous rate limiter implementations to choose
|
|
65
|
+
from:
|
|
66
|
+
|
|
67
|
+
1. [aiolimiter](https://aiolimiter.readthedocs.io)
|
|
68
|
+
2. [pyrate-limiter](https://pyratelimiter.readthedocs.io)
|
|
69
|
+
|
|
70
|
+
> [!IMPORTANT]
|
|
71
|
+
> While both implementations fulfill similar purposes, there are significant differences
|
|
72
|
+
> between them. Please read the descriptions below to make an informed decision about
|
|
73
|
+
> which one best suits your needs.
|
|
74
|
+
|
|
75
|
+
### 1. aiolimiter
|
|
76
|
+
|
|
77
|
+
- **Single rate limit only**: Supports only one rate limit at a time
|
|
78
|
+
- **Lightweight**: Minimal dependencies and simpler configuration
|
|
79
|
+
- **Linear token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
80
|
+
roughly one token will be added every 500 milliseconds
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from httpx_limiter import Rate
|
|
84
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
85
|
+
|
|
86
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### 2. pyrate-limiter
|
|
90
|
+
|
|
91
|
+
- **Multiple rate limits**: Support for multiple concurrent rate limits, for example,
|
|
92
|
+
10 requests per second _and_ 100 requests per minute
|
|
93
|
+
- **Flexible configuration**: Comprehensive configuration options
|
|
94
|
+
- **Multiprocessing dependency**: Current implementation depends on `multiprocessing`
|
|
95
|
+
which may not be available in all environments, such as pyodide
|
|
96
|
+
- **Stepwise token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
97
|
+
two token will be added every second
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from httpx_limiter import Rate
|
|
101
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
102
|
+
|
|
103
|
+
async def main():
|
|
104
|
+
# Single rate limit
|
|
105
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
106
|
+
|
|
107
|
+
# Multiple rate limits
|
|
108
|
+
limiter = PyrateAsyncLimiter.create(
|
|
109
|
+
Rate.create(magnitude=10), # 10 per second
|
|
110
|
+
Rate.create(magnitude=100, duration=60), # 100 per minute
|
|
111
|
+
)
|
|
51
112
|
```
|
|
52
113
|
|
|
53
114
|
## Tutorial
|
|
54
115
|
|
|
55
|
-
You can limit the number of requests made by an HTTPX client using the
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
a rate limit.
|
|
116
|
+
You can limit the number of requests made by an HTTPX client using the transports
|
|
117
|
+
provided in this package. That is useful in situations when you need to make a large
|
|
118
|
+
number of asynchronous requests against endpoints that implement a rate limit.
|
|
59
119
|
|
|
60
120
|
### Single Rate Limit
|
|
61
121
|
|
|
62
|
-
The simplest use case is to apply a single rate limit to all requests. If you
|
|
63
|
-
|
|
64
|
-
following code:
|
|
122
|
+
The simplest use case is to apply a single rate limit to all requests. If you want to be
|
|
123
|
+
able to make twenty requests per second, for example, use the following code:
|
|
65
124
|
|
|
66
125
|
```python
|
|
67
126
|
import httpx
|
|
68
127
|
from httpx_limiter import AsyncRateLimitedTransport, Rate
|
|
128
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
69
129
|
|
|
70
130
|
async def main():
|
|
131
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
132
|
+
|
|
71
133
|
async with httpx.AsyncClient(
|
|
72
|
-
transport=AsyncRateLimitedTransport.create(
|
|
134
|
+
transport=AsyncRateLimitedTransport.create(limiter=limiter),
|
|
73
135
|
) as client:
|
|
74
136
|
response = await client.get("https://httpbin.org")
|
|
75
137
|
```
|
|
@@ -90,11 +152,11 @@ async def main():
|
|
|
90
152
|
|
|
91
153
|
### Multiple Rate Limits
|
|
92
154
|
|
|
93
|
-
For more advanced use cases, you can apply different rate limits based on a
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
155
|
+
For more advanced use cases, you can apply different rate limits based on a concrete
|
|
156
|
+
implementation of the `AbstractRateLimiterRepository`. There are two relevant methods
|
|
157
|
+
that both get passed the current request. One method needs to return an identifier for
|
|
158
|
+
which rate limiter to choose, and the other method creates a rate limiter. See the
|
|
159
|
+
following example:
|
|
98
160
|
|
|
99
161
|
```python
|
|
100
162
|
import httpx
|
|
@@ -103,6 +165,8 @@ from httpx_limiter import (
|
|
|
103
165
|
AsyncMultiRateLimitedTransport,
|
|
104
166
|
Rate
|
|
105
167
|
)
|
|
168
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
169
|
+
|
|
106
170
|
|
|
107
171
|
class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
108
172
|
"""Apply different rate limits based on the domain being requested."""
|
|
@@ -111,15 +175,19 @@ class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
|
111
175
|
"""Return the domain as the identifier for rate limiting."""
|
|
112
176
|
return request.url.host
|
|
113
177
|
|
|
114
|
-
def
|
|
115
|
-
"""
|
|
116
|
-
return Rate.create(magnitude=25)
|
|
178
|
+
def create(self, request: httpx.Request) -> AiolimiterAsyncLimiter:
|
|
179
|
+
"""Create a rate limiter for the domain."""
|
|
180
|
+
return AiolimiterAsyncLimiter.create(Rate.create(magnitude=25))
|
|
117
181
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
)
|
|
182
|
+
|
|
183
|
+
async def main():
|
|
184
|
+
async with httpx.AsyncClient(
|
|
185
|
+
transport=AsyncMultiRateLimitedTransport.create(
|
|
186
|
+
repository=DomainBasedRateLimiterRepository(),
|
|
187
|
+
),
|
|
188
|
+
) as client:
|
|
189
|
+
# Use the client instance as you normally would.
|
|
190
|
+
pass
|
|
123
191
|
```
|
|
124
192
|
|
|
125
193
|
> [!TIP]
|
|
@@ -128,27 +196,28 @@ client = httpx.AsyncClient(
|
|
|
128
196
|
|
|
129
197
|
```python
|
|
130
198
|
from datetime import datetime, timezone
|
|
131
|
-
from collections.abc import Sequence
|
|
132
199
|
|
|
133
200
|
import httpx
|
|
134
201
|
from httpx_limiter import AbstractRateLimiterRepository, Rate
|
|
202
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
203
|
+
|
|
135
204
|
|
|
136
205
|
class DayNightRateLimiterRepository(AbstractRateLimiterRepository):
|
|
137
206
|
"""Apply different rate limits based on the time of day."""
|
|
138
207
|
|
|
139
|
-
def get_identifier(self,
|
|
208
|
+
def get_identifier(self, request: httpx.Request) -> str:
|
|
140
209
|
"""Identify whether it is currently day or night."""
|
|
141
210
|
if 6 <= datetime.now(tz=timezone.utc).hour < 18:
|
|
142
211
|
return "day"
|
|
143
212
|
|
|
144
213
|
return "night"
|
|
145
214
|
|
|
146
|
-
def
|
|
147
|
-
"""
|
|
148
|
-
if self.get_identifier(
|
|
149
|
-
return
|
|
215
|
+
def create(self, request: httpx.Request) -> PyrateAsyncLimiter:
|
|
216
|
+
"""Create a rate limiter based on the time of day."""
|
|
217
|
+
if self.get_identifier(request) == "day":
|
|
218
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=10))
|
|
150
219
|
|
|
151
|
-
return
|
|
220
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=100))
|
|
152
221
|
```
|
|
153
222
|
|
|
154
223
|
## Copyright
|
|
@@ -10,33 +10,92 @@ _A lightweight package that provides rate-limited httpx transports._
|
|
|
10
10
|
|
|
11
11
|
## Installation
|
|
12
12
|
|
|
13
|
-
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/).
|
|
14
|
-
|
|
13
|
+
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). Install it,
|
|
14
|
+
for example, with the aiolimiter backend.
|
|
15
15
|
|
|
16
16
|
```sh
|
|
17
|
-
pip install httpx-limiter
|
|
17
|
+
pip install 'httpx-limiter[aiolimiter]'
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
There is also a pyrate-limiter backend available.
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
pip install 'httpx-limiter[pyrate]'
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Rate Limiter Backends
|
|
27
|
+
|
|
28
|
+
This package provides two different asynchronous rate limiter implementations to choose
|
|
29
|
+
from:
|
|
30
|
+
|
|
31
|
+
1. [aiolimiter](https://aiolimiter.readthedocs.io)
|
|
32
|
+
2. [pyrate-limiter](https://pyratelimiter.readthedocs.io)
|
|
33
|
+
|
|
34
|
+
> [!IMPORTANT]
|
|
35
|
+
> While both implementations fulfill similar purposes, there are significant differences
|
|
36
|
+
> between them. Please read the descriptions below to make an informed decision about
|
|
37
|
+
> which one best suits your needs.
|
|
38
|
+
|
|
39
|
+
### 1. aiolimiter
|
|
40
|
+
|
|
41
|
+
- **Single rate limit only**: Supports only one rate limit at a time
|
|
42
|
+
- **Lightweight**: Minimal dependencies and simpler configuration
|
|
43
|
+
- **Linear token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
44
|
+
roughly one token will be added every 500 milliseconds
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from httpx_limiter import Rate
|
|
48
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
49
|
+
|
|
50
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### 2. pyrate-limiter
|
|
54
|
+
|
|
55
|
+
- **Multiple rate limits**: Support for multiple concurrent rate limits, for example,
|
|
56
|
+
10 requests per second _and_ 100 requests per minute
|
|
57
|
+
- **Flexible configuration**: Comprehensive configuration options
|
|
58
|
+
- **Multiprocessing dependency**: Current implementation depends on `multiprocessing`
|
|
59
|
+
which may not be available in all environments, such as pyodide
|
|
60
|
+
- **Stepwise token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
61
|
+
two token will be added every second
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from httpx_limiter import Rate
|
|
65
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
66
|
+
|
|
67
|
+
async def main():
|
|
68
|
+
# Single rate limit
|
|
69
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
70
|
+
|
|
71
|
+
# Multiple rate limits
|
|
72
|
+
limiter = PyrateAsyncLimiter.create(
|
|
73
|
+
Rate.create(magnitude=10), # 10 per second
|
|
74
|
+
Rate.create(magnitude=100, duration=60), # 100 per minute
|
|
75
|
+
)
|
|
18
76
|
```
|
|
19
77
|
|
|
20
78
|
## Tutorial
|
|
21
79
|
|
|
22
|
-
You can limit the number of requests made by an HTTPX client using the
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
a rate limit.
|
|
80
|
+
You can limit the number of requests made by an HTTPX client using the transports
|
|
81
|
+
provided in this package. That is useful in situations when you need to make a large
|
|
82
|
+
number of asynchronous requests against endpoints that implement a rate limit.
|
|
26
83
|
|
|
27
84
|
### Single Rate Limit
|
|
28
85
|
|
|
29
|
-
The simplest use case is to apply a single rate limit to all requests. If you
|
|
30
|
-
|
|
31
|
-
following code:
|
|
86
|
+
The simplest use case is to apply a single rate limit to all requests. If you want to be
|
|
87
|
+
able to make twenty requests per second, for example, use the following code:
|
|
32
88
|
|
|
33
89
|
```python
|
|
34
90
|
import httpx
|
|
35
91
|
from httpx_limiter import AsyncRateLimitedTransport, Rate
|
|
92
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
36
93
|
|
|
37
94
|
async def main():
|
|
95
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
96
|
+
|
|
38
97
|
async with httpx.AsyncClient(
|
|
39
|
-
transport=AsyncRateLimitedTransport.create(
|
|
98
|
+
transport=AsyncRateLimitedTransport.create(limiter=limiter),
|
|
40
99
|
) as client:
|
|
41
100
|
response = await client.get("https://httpbin.org")
|
|
42
101
|
```
|
|
@@ -57,11 +116,11 @@ async def main():
|
|
|
57
116
|
|
|
58
117
|
### Multiple Rate Limits
|
|
59
118
|
|
|
60
|
-
For more advanced use cases, you can apply different rate limits based on a
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
119
|
+
For more advanced use cases, you can apply different rate limits based on a concrete
|
|
120
|
+
implementation of the `AbstractRateLimiterRepository`. There are two relevant methods
|
|
121
|
+
that both get passed the current request. One method needs to return an identifier for
|
|
122
|
+
which rate limiter to choose, and the other method creates a rate limiter. See the
|
|
123
|
+
following example:
|
|
65
124
|
|
|
66
125
|
```python
|
|
67
126
|
import httpx
|
|
@@ -70,6 +129,8 @@ from httpx_limiter import (
|
|
|
70
129
|
AsyncMultiRateLimitedTransport,
|
|
71
130
|
Rate
|
|
72
131
|
)
|
|
132
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
133
|
+
|
|
73
134
|
|
|
74
135
|
class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
75
136
|
"""Apply different rate limits based on the domain being requested."""
|
|
@@ -78,15 +139,19 @@ class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
|
78
139
|
"""Return the domain as the identifier for rate limiting."""
|
|
79
140
|
return request.url.host
|
|
80
141
|
|
|
81
|
-
def
|
|
82
|
-
"""
|
|
83
|
-
return Rate.create(magnitude=25)
|
|
142
|
+
def create(self, request: httpx.Request) -> AiolimiterAsyncLimiter:
|
|
143
|
+
"""Create a rate limiter for the domain."""
|
|
144
|
+
return AiolimiterAsyncLimiter.create(Rate.create(magnitude=25))
|
|
84
145
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
)
|
|
146
|
+
|
|
147
|
+
async def main():
|
|
148
|
+
async with httpx.AsyncClient(
|
|
149
|
+
transport=AsyncMultiRateLimitedTransport.create(
|
|
150
|
+
repository=DomainBasedRateLimiterRepository(),
|
|
151
|
+
),
|
|
152
|
+
) as client:
|
|
153
|
+
# Use the client instance as you normally would.
|
|
154
|
+
pass
|
|
90
155
|
```
|
|
91
156
|
|
|
92
157
|
> [!TIP]
|
|
@@ -95,27 +160,28 @@ client = httpx.AsyncClient(
|
|
|
95
160
|
|
|
96
161
|
```python
|
|
97
162
|
from datetime import datetime, timezone
|
|
98
|
-
from collections.abc import Sequence
|
|
99
163
|
|
|
100
164
|
import httpx
|
|
101
165
|
from httpx_limiter import AbstractRateLimiterRepository, Rate
|
|
166
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
167
|
+
|
|
102
168
|
|
|
103
169
|
class DayNightRateLimiterRepository(AbstractRateLimiterRepository):
|
|
104
170
|
"""Apply different rate limits based on the time of day."""
|
|
105
171
|
|
|
106
|
-
def get_identifier(self,
|
|
172
|
+
def get_identifier(self, request: httpx.Request) -> str:
|
|
107
173
|
"""Identify whether it is currently day or night."""
|
|
108
174
|
if 6 <= datetime.now(tz=timezone.utc).hour < 18:
|
|
109
175
|
return "day"
|
|
110
176
|
|
|
111
177
|
return "night"
|
|
112
178
|
|
|
113
|
-
def
|
|
114
|
-
"""
|
|
115
|
-
if self.get_identifier(
|
|
116
|
-
return
|
|
179
|
+
def create(self, request: httpx.Request) -> PyrateAsyncLimiter:
|
|
180
|
+
"""Create a rate limiter based on the time of day."""
|
|
181
|
+
if self.get_identifier(request) == "day":
|
|
182
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=10))
|
|
117
183
|
|
|
118
|
-
return
|
|
184
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=100))
|
|
119
185
|
```
|
|
120
186
|
|
|
121
187
|
## Copyright
|
|
@@ -22,7 +22,6 @@ requires-python = ">=3.10"
|
|
|
22
22
|
classifiers = [
|
|
23
23
|
"Development Status :: 2 - Pre-Alpha",
|
|
24
24
|
"Environment :: Web Environment",
|
|
25
|
-
"Framework :: AsyncIO",
|
|
26
25
|
"Intended Audience :: Developers",
|
|
27
26
|
"License :: OSI Approved :: Apache Software License",
|
|
28
27
|
"Natural Language :: English",
|
|
@@ -32,23 +31,38 @@ classifiers = [
|
|
|
32
31
|
"Programming Language :: Python :: 3.11",
|
|
33
32
|
"Programming Language :: Python :: 3.12",
|
|
34
33
|
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Programming Language :: Python :: 3.14",
|
|
35
35
|
"Topic :: Internet :: WWW/HTTP",
|
|
36
36
|
"Typing :: Typed",
|
|
37
37
|
]
|
|
38
38
|
keywords = ["httpx", "limiter", "rate-limit", "leaky bucket"]
|
|
39
39
|
dependencies = [
|
|
40
40
|
"httpx ~=0.25",
|
|
41
|
-
"pyrate-limiter ~=3.9",
|
|
42
41
|
"typing-extensions ~=4.6; python_version < '3.11'",
|
|
43
42
|
]
|
|
44
43
|
|
|
45
44
|
[project.urls]
|
|
46
45
|
Homepage = "https://github.com/Midnighter/httpx-limiter"
|
|
47
|
-
Documentation = "https://httpx-limiter
|
|
46
|
+
Documentation = "https://midnighter.github.io/httpx-limiter/"
|
|
48
47
|
"Source Code" = "https://github.com/Midnighter/httpx-limiter"
|
|
49
48
|
"Bug Tracker" = "https://github.com/Midnighter/httpx-limiter/issues"
|
|
50
49
|
Download = "https://pypi.org/project/httpx-limiter/#files"
|
|
51
50
|
|
|
51
|
+
[project.optional-dependencies]
|
|
52
|
+
aiolimiter = ["aiolimiter ~=1.2"]
|
|
53
|
+
pyrate = ["pyrate-limiter ~=4.0"]
|
|
54
|
+
|
|
55
|
+
[dependency-groups]
|
|
56
|
+
style = ["pydoclint", "ruff"]
|
|
57
|
+
types = ["mypy", "ty"]
|
|
58
|
+
docs = [
|
|
59
|
+
"mkdocs-material ~=9.5",
|
|
60
|
+
"mkdocstrings[python] ~=1.0",
|
|
61
|
+
"mkdocs-awesome-pages-plugin ~=2.9",
|
|
62
|
+
"mike ~=2.1",
|
|
63
|
+
]
|
|
64
|
+
tests = ["anyio", "pytest", "pytest-cov", "pytest-httpx", "pytest-raises"]
|
|
65
|
+
|
|
52
66
|
################################################################################
|
|
53
67
|
# Tool Configuration
|
|
54
68
|
################################################################################
|
|
@@ -109,6 +123,10 @@ exclude = ["src/httpx_limiter/_version.py"]
|
|
|
109
123
|
"ANN201", # 'Missing type annotation for {return}' ignored because all tests return `None`.
|
|
110
124
|
"PLR2004", # Magic numbers are okay in tests.
|
|
111
125
|
]
|
|
126
|
+
"tests/pyodide/**/*.py" = [
|
|
127
|
+
"PLC0415", # We explicitly need imports inside function and ignore the '`import` should be at the top-level of a file' rule.
|
|
128
|
+
"I001", # 'Import block is un-sorted or un-formatted' ignored because we may have to import in a particular, not-alphabetical order.
|
|
129
|
+
]
|
|
112
130
|
|
|
113
131
|
[tool.ruff.lint.isort]
|
|
114
132
|
case-sensitive = true
|
|
@@ -122,38 +140,36 @@ check-return-types = false
|
|
|
122
140
|
check-yield-types = false
|
|
123
141
|
exclude = "_version.py"
|
|
124
142
|
|
|
125
|
-
# TODO: Adjust mypy configuration.
|
|
126
|
-
#[tool.mypy]
|
|
127
|
-
#plugins = [
|
|
128
|
-
# "pydantic.mypy",
|
|
129
|
-
#]
|
|
130
|
-
|
|
131
|
-
# Stop mypy from complaining about missing types from imports.
|
|
132
|
-
[[tool.mypy.overrides]]
|
|
133
|
-
module = ["limiter"]
|
|
134
|
-
ignore_missing_imports = true
|
|
135
|
-
|
|
136
|
-
#[tool.pydantic-mypy]
|
|
137
|
-
#init_forbid_extra = true
|
|
138
|
-
#init_typed = true
|
|
139
|
-
#warn_required_dynamic_aliases = true
|
|
140
|
-
|
|
141
143
|
################################################################################
|
|
142
144
|
# Hatch Environments
|
|
143
145
|
################################################################################
|
|
144
146
|
|
|
147
|
+
[tool.hatch.env]
|
|
148
|
+
requires = ["hatch-pip-compile"]
|
|
149
|
+
|
|
145
150
|
[tool.hatch.envs.default]
|
|
146
|
-
|
|
151
|
+
description = """
|
|
152
|
+
Base environment with default configuration values.
|
|
153
|
+
|
|
154
|
+
All environments inheriting from the default also inherit the configuration values set
|
|
155
|
+
here.
|
|
156
|
+
"""
|
|
157
|
+
type = "pip-compile"
|
|
158
|
+
lock-filename = "requirements/requirements-{env_name}.txt"
|
|
159
|
+
pip-compile-constraint = "dev"
|
|
160
|
+
pip-compile-hashes = true
|
|
161
|
+
pip-compile-resolver = "uv"
|
|
162
|
+
pip-compile-installer = "uv"
|
|
163
|
+
features = ["aiolimiter", "pyrate"]
|
|
164
|
+
|
|
165
|
+
[tool.hatch.envs.no-install]
|
|
166
|
+
description = """A base environment for use cases that do not install the project."""
|
|
167
|
+
skip-install = true
|
|
168
|
+
features = []
|
|
147
169
|
|
|
148
170
|
[tool.hatch.envs.dev]
|
|
149
171
|
description = """Development environment."""
|
|
150
|
-
|
|
151
|
-
"anyio",
|
|
152
|
-
"pytest",
|
|
153
|
-
"pytest-httpx",
|
|
154
|
-
"pytest-watcher",
|
|
155
|
-
"ruff",
|
|
156
|
-
]
|
|
172
|
+
dependency-groups = ["style", "types", "docs", "tests"]
|
|
157
173
|
|
|
158
174
|
[tool.hatch.envs.dev.env-vars]
|
|
159
175
|
PYTHONDEVMODE = "1"
|
|
@@ -161,14 +177,14 @@ PYTHONASYNCIODEBUG = "1"
|
|
|
161
177
|
|
|
162
178
|
[tool.hatch.envs.style]
|
|
163
179
|
description = """Check the style of the codebase."""
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
installer = "uv"
|
|
180
|
+
template = "no-install"
|
|
181
|
+
dependency-groups = ["style"]
|
|
167
182
|
|
|
168
183
|
[tool.hatch.envs.style.scripts]
|
|
169
184
|
docstrings = ["pydoclint src", "pydoclint tests"]
|
|
170
185
|
code = "ruff check {args}"
|
|
171
186
|
format = "ruff format {args}"
|
|
187
|
+
fix = "ruff check --fix {args}"
|
|
172
188
|
check = ["docstrings", "code"]
|
|
173
189
|
|
|
174
190
|
[tool.hatch.envs.audit]
|
|
@@ -180,19 +196,14 @@ check = ["pip-audit"]
|
|
|
180
196
|
|
|
181
197
|
[tool.hatch.envs.types]
|
|
182
198
|
description = """Check the static types of the codebase."""
|
|
183
|
-
|
|
199
|
+
dependency-groups = ["types"]
|
|
184
200
|
|
|
185
201
|
[tool.hatch.envs.types.scripts]
|
|
186
|
-
check = "mypy src/httpx_limiter"
|
|
202
|
+
check = ["mypy src/httpx_limiter", "ty check src/httpx_limiter"]
|
|
187
203
|
|
|
188
204
|
[tool.hatch.envs.docs]
|
|
189
205
|
description = """Build or serve the documentation."""
|
|
190
|
-
|
|
191
|
-
"mkdocs-material ~=9.5",
|
|
192
|
-
"mkdocstrings[python] ~=0.24",
|
|
193
|
-
"mkdocs-awesome-pages-plugin ~=2.9",
|
|
194
|
-
"mike ~=2.1",
|
|
195
|
-
]
|
|
206
|
+
dependency-groups = ["docs"]
|
|
196
207
|
|
|
197
208
|
[tool.hatch.envs.docs.scripts]
|
|
198
209
|
build = "mkdocs build {args:--clean --strict}"
|
|
@@ -201,49 +212,52 @@ deploy = "mike deploy {args}"
|
|
|
201
212
|
|
|
202
213
|
[tool.hatch.envs.distribution]
|
|
203
214
|
description = """Test the source and wheel distribution of the package."""
|
|
215
|
+
template = "no-install"
|
|
204
216
|
dependencies = ["pip", "twine"]
|
|
205
|
-
|
|
206
|
-
installer = "uv"
|
|
217
|
+
builder = true
|
|
207
218
|
|
|
208
219
|
[tool.hatch.envs.distribution.scripts]
|
|
209
220
|
check = ["pip check", "hatch build {args:--clean}", "twine check dist/*"]
|
|
210
221
|
|
|
211
222
|
[tool.hatch.envs.unit-tests]
|
|
212
223
|
description = """Run the unit tests."""
|
|
213
|
-
|
|
214
|
-
"anyio",
|
|
215
|
-
"pytest",
|
|
216
|
-
"pytest-cov",
|
|
217
|
-
"pytest-httpx",
|
|
218
|
-
"pytest-raises",
|
|
219
|
-
]
|
|
224
|
+
dependency-groups = ["tests"]
|
|
220
225
|
|
|
221
226
|
[tool.hatch.envs.unit-tests.env-vars]
|
|
222
227
|
PYTHONDEVMODE = "1"
|
|
223
228
|
PYTHONASYNCIODEBUG = "1"
|
|
224
229
|
|
|
225
230
|
[[tool.hatch.envs.unit-tests.matrix]]
|
|
226
|
-
python = ["3.10", "3.11", "3.12", "3.13"]
|
|
231
|
+
python = ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
227
232
|
|
|
228
233
|
[tool.hatch.envs.unit-tests.scripts]
|
|
229
234
|
run = "pytest {args:--cov=httpx_limiter --cov-report=term-missing} {root}/tests/unit"
|
|
230
235
|
|
|
231
236
|
[tool.hatch.envs.acceptance-tests]
|
|
232
237
|
description = """Run the acceptance tests."""
|
|
233
|
-
|
|
234
|
-
"anyio",
|
|
235
|
-
"pytest",
|
|
236
|
-
"pytest-cov",
|
|
237
|
-
"pytest-httpx",
|
|
238
|
-
"pytest-raises",
|
|
239
|
-
]
|
|
238
|
+
dependency-groups = ["tests"]
|
|
240
239
|
|
|
241
240
|
[tool.hatch.envs.acceptance-tests.env-vars]
|
|
242
241
|
PYTHONDEVMODE = "1"
|
|
243
242
|
PYTHONASYNCIODEBUG = "1"
|
|
244
243
|
|
|
245
244
|
[[tool.hatch.envs.acceptance-tests.matrix]]
|
|
246
|
-
python = ["3.10", "3.11", "3.12", "3.13"]
|
|
245
|
+
python = ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
247
246
|
|
|
248
247
|
[tool.hatch.envs.acceptance-tests.scripts]
|
|
249
248
|
run = "pytest {args:--cov=httpx_limiter --cov-report=term-missing} {root}/tests/acceptance"
|
|
249
|
+
|
|
250
|
+
[tool.hatch.envs.documentation-tests]
|
|
251
|
+
description = """Run the documentation tests."""
|
|
252
|
+
dependency-groups = ["tests"]
|
|
253
|
+
extra-dependencies = ["mktestdocs", "httpx-tenacity ~=0.1"]
|
|
254
|
+
|
|
255
|
+
[tool.hatch.envs.documentation-tests.env-vars]
|
|
256
|
+
PYTHONDEVMODE = "1"
|
|
257
|
+
PYTHONASYNCIODEBUG = "1"
|
|
258
|
+
|
|
259
|
+
[[tool.hatch.envs.documentation-tests.matrix]]
|
|
260
|
+
python = ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
261
|
+
|
|
262
|
+
[tool.hatch.envs.documentation-tests.scripts]
|
|
263
|
+
run = "pytest {args:--cov=httpx_limiter --cov-report=term-missing} {root}/tests/documentation"
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
from .types import HTTPXAsyncHTTPTransportKeywordArguments
|
|
19
19
|
from .rate import Number, Rate
|
|
20
|
-
from .
|
|
20
|
+
from .abstract_async_limiter import AbstractAsyncLimiter
|
|
21
21
|
from .async_rate_limited_transport import AsyncRateLimitedTransport
|
|
22
22
|
from .abstract_rate_limiter_repository import AbstractRateLimiterRepository
|
|
23
23
|
from .async_multi_rate_limited_transport import AsyncMultiRateLimitedTransport
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.6.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 6, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Copyright (c) 2025 Moritz E. Beber
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
4
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
5
|
+
# the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
11
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
12
|
+
# License for the specific language governing permissions and limitations under
|
|
13
|
+
# the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Provide an abstract base class for asynchronous rate limiters."""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from abc import ABC, abstractmethod
|
|
21
|
+
from typing import TYPE_CHECKING
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
25
|
+
from types import TracebackType
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AbstractAsyncLimiter(ABC):
|
|
29
|
+
"""
|
|
30
|
+
Define an abstract base class for asynchronous rate limiters.
|
|
31
|
+
|
|
32
|
+
This interface dictates that all asynchronous rate limiters shall implement the
|
|
33
|
+
context manager protocol.
|
|
34
|
+
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
async def __aenter__(self) -> AbstractAsyncLimiter:
|
|
39
|
+
"""Acquire a rate limit token upon entering the asynchronous context."""
|
|
40
|
+
|
|
41
|
+
async def __aexit__( # noqa: B027
|
|
42
|
+
self,
|
|
43
|
+
exc_type: type[BaseException] | None,
|
|
44
|
+
exc_val: BaseException | None,
|
|
45
|
+
exc_tb: TracebackType | None,
|
|
46
|
+
) -> None:
|
|
47
|
+
"""Exit the asynchronous context."""
|
{httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/abstract_rate_limiter_repository.py
RENAMED
|
@@ -16,12 +16,10 @@
|
|
|
16
16
|
"""Provide an abstract repository for rate limiters."""
|
|
17
17
|
|
|
18
18
|
from abc import ABC, abstractmethod
|
|
19
|
-
from collections.abc import Sequence
|
|
20
19
|
|
|
21
20
|
import httpx
|
|
22
21
|
|
|
23
|
-
from .
|
|
24
|
-
from .rate import Rate
|
|
22
|
+
from .abstract_async_limiter import AbstractAsyncLimiter
|
|
25
23
|
|
|
26
24
|
|
|
27
25
|
class AbstractRateLimiterRepository(ABC):
|
|
@@ -44,31 +42,21 @@ class AbstractRateLimiterRepository(ABC):
|
|
|
44
42
|
|
|
45
43
|
def __init__(self, **kwargs: dict[str, object]) -> None:
|
|
46
44
|
super().__init__(**kwargs)
|
|
47
|
-
self._limiters: dict[str,
|
|
45
|
+
self._limiters: dict[str, AbstractAsyncLimiter] = {}
|
|
48
46
|
|
|
49
47
|
@abstractmethod
|
|
50
48
|
def get_identifier(self, request: httpx.Request) -> str:
|
|
51
49
|
"""Return a request-specific identifier."""
|
|
52
50
|
|
|
53
51
|
@abstractmethod
|
|
54
|
-
def
|
|
55
|
-
"""Return
|
|
56
|
-
|
|
57
|
-
def _get_limiter_kwargs(
|
|
58
|
-
self,
|
|
59
|
-
request: httpx.Request, # noqa: ARG002
|
|
60
|
-
) -> PyRateLimiterKeywordArguments:
|
|
61
|
-
"""Return the keyword arguments for creating a rate limiter."""
|
|
62
|
-
return {}
|
|
52
|
+
def create(self, request: httpx.Request) -> AbstractAsyncLimiter:
|
|
53
|
+
"""Return a request-specific rate limiter."""
|
|
63
54
|
|
|
64
|
-
def get(self, request: httpx.Request) ->
|
|
55
|
+
def get(self, request: httpx.Request) -> AbstractAsyncLimiter:
|
|
65
56
|
"""Return a request-specific rate limiter."""
|
|
66
57
|
identifier = self.get_identifier(request)
|
|
67
58
|
|
|
68
59
|
if identifier not in self._limiters:
|
|
69
|
-
self._limiters[identifier] =
|
|
70
|
-
*self.get_rates(request),
|
|
71
|
-
**self._get_limiter_kwargs(request),
|
|
72
|
-
)
|
|
60
|
+
self._limiters[identifier] = self.create(request)
|
|
73
61
|
|
|
74
62
|
return self._limiters[identifier]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright (c) 2025 Moritz E. Beber
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
4
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
5
|
+
# the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
11
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
12
|
+
# License for the specific language governing permissions and limitations under
|
|
13
|
+
# the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Provide aiolimiter-based rate limiting functionality."""
|
|
17
|
+
|
|
18
|
+
from .aiolimiter_async_limiter import AiolimiterAsyncLimiter
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Copyright (c) 2025 Moritz E. Beber
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
4
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
5
|
+
# the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
11
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
12
|
+
# License for the specific language governing permissions and limitations under
|
|
13
|
+
# the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Provide an asynchronous rate-limited transport using aiolimiter."""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import TYPE_CHECKING
|
|
21
|
+
|
|
22
|
+
from aiolimiter import AsyncLimiter
|
|
23
|
+
|
|
24
|
+
from httpx_limiter import AbstractAsyncLimiter
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
28
|
+
from types import TracebackType
|
|
29
|
+
|
|
30
|
+
from httpx_limiter.rate import Rate
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class AiolimiterAsyncLimiter(AbstractAsyncLimiter):
|
|
34
|
+
"""
|
|
35
|
+
Define an asynchronous limiter that composes the aiolimiter AsyncLimiter.
|
|
36
|
+
|
|
37
|
+
This class encapsulates the creation and configuration of an aiolimiter
|
|
38
|
+
AsyncLimiter with appropriate settings.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
limiter: An instance of an aiolimiter AsyncLimiter.
|
|
42
|
+
**kwargs: Additional keyword arguments for the parent classes.
|
|
43
|
+
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(self, *, limiter: AsyncLimiter, **kwargs: dict[str, object]) -> None:
|
|
47
|
+
super().__init__(**kwargs)
|
|
48
|
+
self._limiter = limiter
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def create(cls, rate: Rate) -> AiolimiterAsyncLimiter:
|
|
52
|
+
"""
|
|
53
|
+
Create an instance of AiolimiterAsyncLimiter.
|
|
54
|
+
|
|
55
|
+
Note: aiolimiter only supports a single rate limit, unlike pyrate-limiter
|
|
56
|
+
which supports multiple rates.
|
|
57
|
+
"""
|
|
58
|
+
limiter = AsyncLimiter(
|
|
59
|
+
max_rate=rate.magnitude,
|
|
60
|
+
time_period=rate.in_seconds(),
|
|
61
|
+
)
|
|
62
|
+
return cls(limiter=limiter)
|
|
63
|
+
|
|
64
|
+
async def __aenter__(self) -> AiolimiterAsyncLimiter: # noqa: PYI034
|
|
65
|
+
"""Acquire a token upon entering the asynchronous context."""
|
|
66
|
+
await self._limiter.__aenter__()
|
|
67
|
+
return self
|
|
68
|
+
|
|
69
|
+
async def __aexit__(
|
|
70
|
+
self,
|
|
71
|
+
exc_type: type[BaseException] | None,
|
|
72
|
+
exc_val: BaseException | None,
|
|
73
|
+
exc_tb: TracebackType | None,
|
|
74
|
+
) -> None:
|
|
75
|
+
"""Exit the asynchronous context."""
|
|
76
|
+
await self._limiter.__aexit__(exc_type=exc_type, exc=exc_val, tb=exc_tb)
|
{httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/async_multi_rate_limited_transport.py
RENAMED
|
@@ -79,7 +79,7 @@ class AsyncMultiRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
79
79
|
"""
|
|
80
80
|
return cls(
|
|
81
81
|
repository=repository,
|
|
82
|
-
transport=httpx.AsyncHTTPTransport(**kwargs),
|
|
82
|
+
transport=httpx.AsyncHTTPTransport(**kwargs),
|
|
83
83
|
)
|
|
84
84
|
|
|
85
85
|
async def handle_async_request(
|
{httpx_limiter-0.4.0 → httpx_limiter-0.6.0}/src/httpx_limiter/async_rate_limited_transport.py
RENAMED
|
@@ -28,11 +28,9 @@ else:
|
|
|
28
28
|
|
|
29
29
|
import httpx
|
|
30
30
|
|
|
31
|
-
from .async_limiter import AsyncLimiter
|
|
32
|
-
|
|
33
31
|
|
|
34
32
|
if TYPE_CHECKING: # pragma: no cover
|
|
35
|
-
from .
|
|
33
|
+
from .abstract_async_limiter import AbstractAsyncLimiter
|
|
36
34
|
from .types import HTTPXAsyncHTTPTransportKeywordArguments
|
|
37
35
|
|
|
38
36
|
|
|
@@ -49,7 +47,7 @@ class AsyncRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
49
47
|
def __init__(
|
|
50
48
|
self,
|
|
51
49
|
*,
|
|
52
|
-
limiter:
|
|
50
|
+
limiter: AbstractAsyncLimiter,
|
|
53
51
|
transport: httpx.AsyncBaseTransport,
|
|
54
52
|
**kwargs: dict[str, object],
|
|
55
53
|
) -> None:
|
|
@@ -60,7 +58,7 @@ class AsyncRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
60
58
|
@classmethod
|
|
61
59
|
def create(
|
|
62
60
|
cls,
|
|
63
|
-
|
|
61
|
+
limiter: AbstractAsyncLimiter,
|
|
64
62
|
**kwargs: Unpack[HTTPXAsyncHTTPTransportKeywordArguments],
|
|
65
63
|
) -> AsyncRateLimitedTransport:
|
|
66
64
|
"""
|
|
@@ -71,7 +69,7 @@ class AsyncRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
71
69
|
That transport is passed any additional keyword arguments.
|
|
72
70
|
|
|
73
71
|
Args:
|
|
74
|
-
|
|
72
|
+
limiter: A concrete instance of an asynchronous limiter.
|
|
75
73
|
**kwargs: Additional keyword arguments are used in the construction of an
|
|
76
74
|
`httpx.AsyncHTTPTransport`.
|
|
77
75
|
|
|
@@ -80,8 +78,8 @@ class AsyncRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
80
78
|
|
|
81
79
|
"""
|
|
82
80
|
return cls(
|
|
83
|
-
limiter=
|
|
84
|
-
transport=httpx.AsyncHTTPTransport(**kwargs),
|
|
81
|
+
limiter=limiter,
|
|
82
|
+
transport=httpx.AsyncHTTPTransport(**kwargs),
|
|
85
83
|
)
|
|
86
84
|
|
|
87
85
|
async def handle_async_request(
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright (c) 2025 Moritz E. Beber
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
4
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
5
|
+
# the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
11
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
12
|
+
# License for the specific language governing permissions and limitations under
|
|
13
|
+
# the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Provide pyrate-based rate limiting functionality."""
|
|
17
|
+
|
|
18
|
+
from .pyrate_async_limiter import PyrateAsyncLimiter, PyRateLimiterKeywordArguments
|
|
@@ -27,34 +27,27 @@ else:
|
|
|
27
27
|
from typing import Unpack
|
|
28
28
|
|
|
29
29
|
from pyrate_limiter import (
|
|
30
|
-
AbstractClock,
|
|
31
30
|
BucketAsyncWrapper,
|
|
32
|
-
Duration,
|
|
33
31
|
InMemoryBucket,
|
|
34
32
|
Limiter,
|
|
35
|
-
TimeAsyncClock,
|
|
36
33
|
validate_rate_list,
|
|
37
34
|
)
|
|
38
35
|
from pyrate_limiter import Rate as PyRate
|
|
39
36
|
|
|
37
|
+
from httpx_limiter import AbstractAsyncLimiter
|
|
40
38
|
|
|
41
|
-
if TYPE_CHECKING: # pragma: no cover
|
|
42
|
-
from types import TracebackType
|
|
43
39
|
|
|
44
|
-
|
|
40
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
41
|
+
from httpx_limiter.rate import Rate
|
|
45
42
|
|
|
46
43
|
|
|
47
44
|
class PyRateLimiterKeywordArguments(TypedDict, total=False):
|
|
48
45
|
"""Keyword arguments for the pyrate limiter."""
|
|
49
46
|
|
|
50
|
-
clock: AbstractClock
|
|
51
|
-
raise_when_fail: bool
|
|
52
|
-
max_delay: int | Duration | None
|
|
53
|
-
retry_until_max_delay: bool
|
|
54
47
|
buffer_ms: int
|
|
55
48
|
|
|
56
49
|
|
|
57
|
-
class
|
|
50
|
+
class PyrateAsyncLimiter(AbstractAsyncLimiter):
|
|
58
51
|
"""
|
|
59
52
|
Define an asynchronous limiter that composes the pyrate limiter.
|
|
60
53
|
|
|
@@ -76,8 +69,8 @@ class AsyncLimiter:
|
|
|
76
69
|
cls,
|
|
77
70
|
*rates: Rate,
|
|
78
71
|
**kwargs: Unpack[PyRateLimiterKeywordArguments],
|
|
79
|
-
) ->
|
|
80
|
-
"""Create an instance of
|
|
72
|
+
) -> PyrateAsyncLimiter:
|
|
73
|
+
"""Create an instance of PyrateAsyncLimiter."""
|
|
81
74
|
if not rates:
|
|
82
75
|
msg = "At least one rate must be provided."
|
|
83
76
|
raise ValueError(msg)
|
|
@@ -95,27 +88,12 @@ class AsyncLimiter:
|
|
|
95
88
|
raise ValueError(msg)
|
|
96
89
|
|
|
97
90
|
limiter = Limiter(
|
|
98
|
-
|
|
99
|
-
clock=kwargs.get("clock", TimeAsyncClock()),
|
|
100
|
-
raise_when_fail=kwargs.get("raise_when_fail", False),
|
|
101
|
-
max_delay=kwargs.get("max_delay", Duration.HOUR),
|
|
102
|
-
retry_until_max_delay=kwargs.get("retry_until_max_delay", False),
|
|
91
|
+
BucketAsyncWrapper(InMemoryBucket(rate_limits)),
|
|
103
92
|
buffer_ms=kwargs.get("buffer_ms", 50),
|
|
104
93
|
)
|
|
105
94
|
return cls(limiter=limiter)
|
|
106
95
|
|
|
107
|
-
async def __aenter__(self) ->
|
|
96
|
+
async def __aenter__(self) -> PyrateAsyncLimiter: # noqa: PYI034
|
|
108
97
|
"""Acquire a token upon entering the asynchronous context."""
|
|
109
|
-
|
|
110
|
-
while not (await self._limiter.try_acquire_async("httpx-limiter")):
|
|
111
|
-
pass # pragma: no cover
|
|
112
|
-
|
|
98
|
+
await self._limiter.try_acquire_async(__name__)
|
|
113
99
|
return self
|
|
114
|
-
|
|
115
|
-
async def __aexit__(
|
|
116
|
-
self,
|
|
117
|
-
_exc_type: type[BaseException] | None,
|
|
118
|
-
_exc_val: BaseException | None,
|
|
119
|
-
_exc_tb: TracebackType | None,
|
|
120
|
-
) -> None:
|
|
121
|
-
"""Exit the asynchronous context."""
|
|
@@ -48,6 +48,10 @@ class Rate(NamedTuple):
|
|
|
48
48
|
|
|
49
49
|
return cls(magnitude=magnitude, duration=duration)
|
|
50
50
|
|
|
51
|
+
def in_seconds(self) -> float:
|
|
52
|
+
"""Return the duration in unit seconds."""
|
|
53
|
+
return self.duration.total_seconds()
|
|
54
|
+
|
|
51
55
|
def in_milliseconds(self) -> int:
|
|
52
56
|
"""Return the duration in unit milliseconds."""
|
|
53
57
|
return int(self.duration.total_seconds() * 1_000)
|
|
@@ -16,23 +16,38 @@
|
|
|
16
16
|
"""Provide additional type definitions."""
|
|
17
17
|
|
|
18
18
|
import ssl
|
|
19
|
-
import
|
|
20
|
-
from typing import TypedDict
|
|
19
|
+
from collections.abc import Iterable
|
|
20
|
+
from typing import TypeAlias, TypedDict
|
|
21
21
|
|
|
22
22
|
import httpx
|
|
23
23
|
|
|
24
24
|
|
|
25
|
+
# Fallback for Pyodide HTTPX. See: https://github.com/Midnighter/httpx-limiter/pull/10#issuecomment-3241566927
|
|
26
|
+
SocketOption: TypeAlias = (
|
|
27
|
+
tuple[int, int, int]
|
|
28
|
+
| tuple[int, int, bytes | bytearray]
|
|
29
|
+
| tuple[int, int, None, int]
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
# We redefine the following types from httpx._types to avoid a dependency on the
|
|
33
|
+
# private module. It would be better if HTTPX exposed these types publicly, but
|
|
34
|
+
# they are unlikely to change often. Thus we consider it acceptable to redefine
|
|
35
|
+
# them here for type hinting purposes.
|
|
36
|
+
CertTypes: TypeAlias = str | tuple[str, str] | tuple[str, str, str]
|
|
37
|
+
ProxyTypes: TypeAlias = httpx.URL | str | httpx.Proxy
|
|
38
|
+
|
|
39
|
+
|
|
25
40
|
class HTTPXAsyncHTTPTransportKeywordArguments(TypedDict, total=False):
|
|
26
41
|
"""Keyword arguments for the httpx.AsyncHTTPTransport constructor."""
|
|
27
42
|
|
|
28
43
|
verify: ssl.SSLContext | str | bool
|
|
29
|
-
cert:
|
|
44
|
+
cert: CertTypes | None
|
|
30
45
|
trust_env: bool
|
|
31
46
|
http1: bool
|
|
32
47
|
http2: bool
|
|
33
|
-
limits: httpx.
|
|
34
|
-
proxy:
|
|
48
|
+
limits: httpx.Limits
|
|
49
|
+
proxy: ProxyTypes | None
|
|
35
50
|
uds: str | None
|
|
36
51
|
local_address: str | None
|
|
37
52
|
retries: int
|
|
38
|
-
socket_options:
|
|
53
|
+
socket_options: Iterable[SocketOption] | None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|