httpx-limiter 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/PKG-INFO +88 -28
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/README.md +83 -26
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/pyproject.toml +11 -2
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/__init__.py +1 -1
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/_version.py +2 -2
- httpx_limiter-0.5.0/src/httpx_limiter/abstract_async_limiter.py +48 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/abstract_rate_limiter_repository.py +6 -18
- httpx_limiter-0.5.0/src/httpx_limiter/aiolimiter/__init__.py +18 -0
- httpx_limiter-0.5.0/src/httpx_limiter/aiolimiter/aiolimiter_async_limiter.py +76 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/async_rate_limited_transport.py +6 -8
- httpx_limiter-0.5.0/src/httpx_limiter/pyrate/__init__.py +18 -0
- httpx_limiter-0.4.0/src/httpx_limiter/async_limiter.py → httpx_limiter-0.5.0/src/httpx_limiter/pyrate/pyrate_async_limiter.py +9 -6
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/rate.py +4 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/types.py +21 -6
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/.gitignore +0 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/LICENSE +0 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/async_multi_rate_limited_transport.py +0 -0
- {httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: httpx-limiter
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A lightweight package that provides rate-limited httpx transports.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Midnighter/httpx-limiter
|
|
6
6
|
Project-URL: Documentation, https://httpx-limiter.readthedocs.io
|
|
@@ -27,8 +27,11 @@ Classifier: Topic :: Internet :: WWW/HTTP
|
|
|
27
27
|
Classifier: Typing :: Typed
|
|
28
28
|
Requires-Python: >=3.10
|
|
29
29
|
Requires-Dist: httpx~=0.25
|
|
30
|
-
Requires-Dist: pyrate-limiter~=3.9
|
|
31
30
|
Requires-Dist: typing-extensions~=4.6; python_version < '3.11'
|
|
31
|
+
Provides-Extra: aiolimiter
|
|
32
|
+
Requires-Dist: aiolimiter~=1.2; extra == 'aiolimiter'
|
|
33
|
+
Provides-Extra: pyrate
|
|
34
|
+
Requires-Dist: pyrate-limiter~=3.9; extra == 'pyrate'
|
|
32
35
|
Description-Content-Type: text/markdown
|
|
33
36
|
|
|
34
37
|
# HTTPX Limiter
|
|
@@ -43,33 +46,89 @@ _A lightweight package that provides rate-limited httpx transports._
|
|
|
43
46
|
|
|
44
47
|
## Installation
|
|
45
48
|
|
|
46
|
-
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/).
|
|
47
|
-
|
|
49
|
+
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). Install it,
|
|
50
|
+
for example, with the aiolimiter backend.
|
|
48
51
|
|
|
49
52
|
```sh
|
|
50
|
-
pip install httpx-limiter
|
|
53
|
+
pip install 'httpx-limiter[aiolimiter]'
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
There is also a pyrate-limiter backend available.
|
|
57
|
+
|
|
58
|
+
```sh
|
|
59
|
+
pip install 'httpx-limiter[pyrate]'
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Rate Limiter Backends
|
|
63
|
+
|
|
64
|
+
This package provides two different asynchronous rate limiter implementations to choose
|
|
65
|
+
from:
|
|
66
|
+
|
|
67
|
+
1. [aiolimiter](https://aiolimiter.readthedocs.io)
|
|
68
|
+
2. [pyrate-limiter](https://pyratelimiter.readthedocs.io)
|
|
69
|
+
|
|
70
|
+
> [!IMPORTANT]
|
|
71
|
+
> While both implementations fulfill similar purposes, there are significant differences
|
|
72
|
+
> between them. Please read the descriptions below to make an informed decision about
|
|
73
|
+
> which one best suits your needs.
|
|
74
|
+
|
|
75
|
+
### 1. aiolimiter
|
|
76
|
+
|
|
77
|
+
- **Single rate limit only**: Supports only one rate limit at a time
|
|
78
|
+
- **Lightweight**: Minimal dependencies and simpler configuration
|
|
79
|
+
- **Linear token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
80
|
+
roughly one token will be added every 500 milliseconds
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
84
|
+
|
|
85
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### 2. pyrate-limiter
|
|
89
|
+
|
|
90
|
+
- **Multiple rate limits**: Support for multiple concurrent rate limits, for example,
|
|
91
|
+
10 requests per second _and_ 100 requests per minute
|
|
92
|
+
- **Flexible configuration**: Comprehensive configuration options
|
|
93
|
+
- **Multiprocessing dependency**: Current implementation depends on `multiprocessing`
|
|
94
|
+
which may not be available in all environments, such as pyodide
|
|
95
|
+
- **Stepwise token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
96
|
+
two token will be added every second
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
100
|
+
|
|
101
|
+
# Single rate limit
|
|
102
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
103
|
+
|
|
104
|
+
# Multiple rate limits
|
|
105
|
+
limiter = PyrateAsyncLimiter.create(
|
|
106
|
+
Rate.create(magnitude=10), # 10 per second
|
|
107
|
+
Rate.create(magnitude=100, duration=60), # 100 per minute
|
|
108
|
+
)
|
|
51
109
|
```
|
|
52
110
|
|
|
53
111
|
## Tutorial
|
|
54
112
|
|
|
55
|
-
You can limit the number of requests made by an HTTPX client using the
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
a rate limit.
|
|
113
|
+
You can limit the number of requests made by an HTTPX client using the transports
|
|
114
|
+
provided in this package. That is useful in situations when you need to make a large
|
|
115
|
+
number of asynchronous requests against endpoints that implement a rate limit.
|
|
59
116
|
|
|
60
117
|
### Single Rate Limit
|
|
61
118
|
|
|
62
|
-
The simplest use case is to apply a single rate limit to all requests. If you
|
|
63
|
-
|
|
64
|
-
following code:
|
|
119
|
+
The simplest use case is to apply a single rate limit to all requests. If you want to be
|
|
120
|
+
able to make twenty requests per second, for example, use the following code:
|
|
65
121
|
|
|
66
122
|
```python
|
|
67
123
|
import httpx
|
|
68
124
|
from httpx_limiter import AsyncRateLimitedTransport, Rate
|
|
125
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
69
126
|
|
|
70
127
|
async def main():
|
|
128
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
129
|
+
|
|
71
130
|
async with httpx.AsyncClient(
|
|
72
|
-
transport=AsyncRateLimitedTransport.create(
|
|
131
|
+
transport=AsyncRateLimitedTransport.create(limiter=limiter),
|
|
73
132
|
) as client:
|
|
74
133
|
response = await client.get("https://httpbin.org")
|
|
75
134
|
```
|
|
@@ -90,11 +149,11 @@ async def main():
|
|
|
90
149
|
|
|
91
150
|
### Multiple Rate Limits
|
|
92
151
|
|
|
93
|
-
For more advanced use cases, you can apply different rate limits based on a
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
152
|
+
For more advanced use cases, you can apply different rate limits based on a concrete
|
|
153
|
+
implementation of the `AbstractRateLimiterRepository`. There are two relevant methods
|
|
154
|
+
that both get passed the current request. One method needs to return an identifier for
|
|
155
|
+
which rate limiter to choose, and the other method creates a rate limiter. See the
|
|
156
|
+
following example:
|
|
98
157
|
|
|
99
158
|
```python
|
|
100
159
|
import httpx
|
|
@@ -103,6 +162,7 @@ from httpx_limiter import (
|
|
|
103
162
|
AsyncMultiRateLimitedTransport,
|
|
104
163
|
Rate
|
|
105
164
|
)
|
|
165
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
106
166
|
|
|
107
167
|
class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
108
168
|
"""Apply different rate limits based on the domain being requested."""
|
|
@@ -111,9 +171,9 @@ class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
|
111
171
|
"""Return the domain as the identifier for rate limiting."""
|
|
112
172
|
return request.url.host
|
|
113
173
|
|
|
114
|
-
def
|
|
115
|
-
"""
|
|
116
|
-
return Rate.create(magnitude=25)
|
|
174
|
+
def create(self, request: httpx.Request) -> AiolimiterAsyncLimiter:
|
|
175
|
+
"""Create a rate limiter for the domain."""
|
|
176
|
+
return AiolimiterAsyncLimiter.create(Rate.create(magnitude=25))
|
|
117
177
|
|
|
118
178
|
client = httpx.AsyncClient(
|
|
119
179
|
transport=AsyncMultiRateLimitedTransport.create(
|
|
@@ -128,27 +188,27 @@ client = httpx.AsyncClient(
|
|
|
128
188
|
|
|
129
189
|
```python
|
|
130
190
|
from datetime import datetime, timezone
|
|
131
|
-
from collections.abc import Sequence
|
|
132
191
|
|
|
133
192
|
import httpx
|
|
134
193
|
from httpx_limiter import AbstractRateLimiterRepository, Rate
|
|
194
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
135
195
|
|
|
136
196
|
class DayNightRateLimiterRepository(AbstractRateLimiterRepository):
|
|
137
197
|
"""Apply different rate limits based on the time of day."""
|
|
138
198
|
|
|
139
|
-
def get_identifier(self,
|
|
199
|
+
def get_identifier(self, request: httpx.Request) -> str:
|
|
140
200
|
"""Identify whether it is currently day or night."""
|
|
141
201
|
if 6 <= datetime.now(tz=timezone.utc).hour < 18:
|
|
142
202
|
return "day"
|
|
143
203
|
|
|
144
204
|
return "night"
|
|
145
205
|
|
|
146
|
-
def
|
|
147
|
-
"""
|
|
148
|
-
if self.get_identifier(
|
|
149
|
-
return
|
|
206
|
+
def create(self, request: httpx.Request) -> PyrateAsyncLimiter:
|
|
207
|
+
"""Create a rate limiter based on the time of day."""
|
|
208
|
+
if self.get_identifier(request) == "day":
|
|
209
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=10))
|
|
150
210
|
|
|
151
|
-
return
|
|
211
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=100))
|
|
152
212
|
```
|
|
153
213
|
|
|
154
214
|
## Copyright
|
|
@@ -10,33 +10,89 @@ _A lightweight package that provides rate-limited httpx transports._
|
|
|
10
10
|
|
|
11
11
|
## Installation
|
|
12
12
|
|
|
13
|
-
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/).
|
|
14
|
-
|
|
13
|
+
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). Install it,
|
|
14
|
+
for example, with the aiolimiter backend.
|
|
15
15
|
|
|
16
16
|
```sh
|
|
17
|
-
pip install httpx-limiter
|
|
17
|
+
pip install 'httpx-limiter[aiolimiter]'
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
There is also a pyrate-limiter backend available.
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
pip install 'httpx-limiter[pyrate]'
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Rate Limiter Backends
|
|
27
|
+
|
|
28
|
+
This package provides two different asynchronous rate limiter implementations to choose
|
|
29
|
+
from:
|
|
30
|
+
|
|
31
|
+
1. [aiolimiter](https://aiolimiter.readthedocs.io)
|
|
32
|
+
2. [pyrate-limiter](https://pyratelimiter.readthedocs.io)
|
|
33
|
+
|
|
34
|
+
> [!IMPORTANT]
|
|
35
|
+
> While both implementations fulfill similar purposes, there are significant differences
|
|
36
|
+
> between them. Please read the descriptions below to make an informed decision about
|
|
37
|
+
> which one best suits your needs.
|
|
38
|
+
|
|
39
|
+
### 1. aiolimiter
|
|
40
|
+
|
|
41
|
+
- **Single rate limit only**: Supports only one rate limit at a time
|
|
42
|
+
- **Lightweight**: Minimal dependencies and simpler configuration
|
|
43
|
+
- **Linear token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
44
|
+
roughly one token will be added every 500 milliseconds
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
48
|
+
|
|
49
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
### 2. pyrate-limiter
|
|
53
|
+
|
|
54
|
+
- **Multiple rate limits**: Support for multiple concurrent rate limits, for example,
|
|
55
|
+
10 requests per second _and_ 100 requests per minute
|
|
56
|
+
- **Flexible configuration**: Comprehensive configuration options
|
|
57
|
+
- **Multiprocessing dependency**: Current implementation depends on `multiprocessing`
|
|
58
|
+
which may not be available in all environments, such as pyodide
|
|
59
|
+
- **Stepwise token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
60
|
+
two token will be added every second
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
64
|
+
|
|
65
|
+
# Single rate limit
|
|
66
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
67
|
+
|
|
68
|
+
# Multiple rate limits
|
|
69
|
+
limiter = PyrateAsyncLimiter.create(
|
|
70
|
+
Rate.create(magnitude=10), # 10 per second
|
|
71
|
+
Rate.create(magnitude=100, duration=60), # 100 per minute
|
|
72
|
+
)
|
|
18
73
|
```
|
|
19
74
|
|
|
20
75
|
## Tutorial
|
|
21
76
|
|
|
22
|
-
You can limit the number of requests made by an HTTPX client using the
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
a rate limit.
|
|
77
|
+
You can limit the number of requests made by an HTTPX client using the transports
|
|
78
|
+
provided in this package. That is useful in situations when you need to make a large
|
|
79
|
+
number of asynchronous requests against endpoints that implement a rate limit.
|
|
26
80
|
|
|
27
81
|
### Single Rate Limit
|
|
28
82
|
|
|
29
|
-
The simplest use case is to apply a single rate limit to all requests. If you
|
|
30
|
-
|
|
31
|
-
following code:
|
|
83
|
+
The simplest use case is to apply a single rate limit to all requests. If you want to be
|
|
84
|
+
able to make twenty requests per second, for example, use the following code:
|
|
32
85
|
|
|
33
86
|
```python
|
|
34
87
|
import httpx
|
|
35
88
|
from httpx_limiter import AsyncRateLimitedTransport, Rate
|
|
89
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
36
90
|
|
|
37
91
|
async def main():
|
|
92
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
93
|
+
|
|
38
94
|
async with httpx.AsyncClient(
|
|
39
|
-
transport=AsyncRateLimitedTransport.create(
|
|
95
|
+
transport=AsyncRateLimitedTransport.create(limiter=limiter),
|
|
40
96
|
) as client:
|
|
41
97
|
response = await client.get("https://httpbin.org")
|
|
42
98
|
```
|
|
@@ -57,11 +113,11 @@ async def main():
|
|
|
57
113
|
|
|
58
114
|
### Multiple Rate Limits
|
|
59
115
|
|
|
60
|
-
For more advanced use cases, you can apply different rate limits based on a
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
116
|
+
For more advanced use cases, you can apply different rate limits based on a concrete
|
|
117
|
+
implementation of the `AbstractRateLimiterRepository`. There are two relevant methods
|
|
118
|
+
that both get passed the current request. One method needs to return an identifier for
|
|
119
|
+
which rate limiter to choose, and the other method creates a rate limiter. See the
|
|
120
|
+
following example:
|
|
65
121
|
|
|
66
122
|
```python
|
|
67
123
|
import httpx
|
|
@@ -70,6 +126,7 @@ from httpx_limiter import (
|
|
|
70
126
|
AsyncMultiRateLimitedTransport,
|
|
71
127
|
Rate
|
|
72
128
|
)
|
|
129
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
73
130
|
|
|
74
131
|
class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
75
132
|
"""Apply different rate limits based on the domain being requested."""
|
|
@@ -78,9 +135,9 @@ class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
|
78
135
|
"""Return the domain as the identifier for rate limiting."""
|
|
79
136
|
return request.url.host
|
|
80
137
|
|
|
81
|
-
def
|
|
82
|
-
"""
|
|
83
|
-
return Rate.create(magnitude=25)
|
|
138
|
+
def create(self, request: httpx.Request) -> AiolimiterAsyncLimiter:
|
|
139
|
+
"""Create a rate limiter for the domain."""
|
|
140
|
+
return AiolimiterAsyncLimiter.create(Rate.create(magnitude=25))
|
|
84
141
|
|
|
85
142
|
client = httpx.AsyncClient(
|
|
86
143
|
transport=AsyncMultiRateLimitedTransport.create(
|
|
@@ -95,27 +152,27 @@ client = httpx.AsyncClient(
|
|
|
95
152
|
|
|
96
153
|
```python
|
|
97
154
|
from datetime import datetime, timezone
|
|
98
|
-
from collections.abc import Sequence
|
|
99
155
|
|
|
100
156
|
import httpx
|
|
101
157
|
from httpx_limiter import AbstractRateLimiterRepository, Rate
|
|
158
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
102
159
|
|
|
103
160
|
class DayNightRateLimiterRepository(AbstractRateLimiterRepository):
|
|
104
161
|
"""Apply different rate limits based on the time of day."""
|
|
105
162
|
|
|
106
|
-
def get_identifier(self,
|
|
163
|
+
def get_identifier(self, request: httpx.Request) -> str:
|
|
107
164
|
"""Identify whether it is currently day or night."""
|
|
108
165
|
if 6 <= datetime.now(tz=timezone.utc).hour < 18:
|
|
109
166
|
return "day"
|
|
110
167
|
|
|
111
168
|
return "night"
|
|
112
169
|
|
|
113
|
-
def
|
|
114
|
-
"""
|
|
115
|
-
if self.get_identifier(
|
|
116
|
-
return
|
|
170
|
+
def create(self, request: httpx.Request) -> PyrateAsyncLimiter:
|
|
171
|
+
"""Create a rate limiter based on the time of day."""
|
|
172
|
+
if self.get_identifier(request) == "day":
|
|
173
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=10))
|
|
117
174
|
|
|
118
|
-
return
|
|
175
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=100))
|
|
119
176
|
```
|
|
120
177
|
|
|
121
178
|
## Copyright
|
|
@@ -38,7 +38,6 @@ classifiers = [
|
|
|
38
38
|
keywords = ["httpx", "limiter", "rate-limit", "leaky bucket"]
|
|
39
39
|
dependencies = [
|
|
40
40
|
"httpx ~=0.25",
|
|
41
|
-
"pyrate-limiter ~=3.9",
|
|
42
41
|
"typing-extensions ~=4.6; python_version < '3.11'",
|
|
43
42
|
]
|
|
44
43
|
|
|
@@ -49,6 +48,10 @@ Documentation = "https://httpx-limiter.readthedocs.io"
|
|
|
49
48
|
"Bug Tracker" = "https://github.com/Midnighter/httpx-limiter/issues"
|
|
50
49
|
Download = "https://pypi.org/project/httpx-limiter/#files"
|
|
51
50
|
|
|
51
|
+
[project.optional-dependencies]
|
|
52
|
+
aiolimiter = ["aiolimiter ~=1.2"]
|
|
53
|
+
pyrate = ["pyrate-limiter ~=3.9"]
|
|
54
|
+
|
|
52
55
|
################################################################################
|
|
53
56
|
# Tool Configuration
|
|
54
57
|
################################################################################
|
|
@@ -109,6 +112,10 @@ exclude = ["src/httpx_limiter/_version.py"]
|
|
|
109
112
|
"ANN201", # 'Missing type annotation for {return}' ignored because all tests return `None`.
|
|
110
113
|
"PLR2004", # Magic numbers are okay in tests.
|
|
111
114
|
]
|
|
115
|
+
"tests/pyodide/**/*.py" = [
|
|
116
|
+
"PLC0415", # We explicitly need imports inside function and ignore the '`import` should be at the top-level of a file' rule.
|
|
117
|
+
"I001", # 'Import block is un-sorted or un-formatted' ignored because we may have to import in a particular, not-alphabetical order.
|
|
118
|
+
]
|
|
112
119
|
|
|
113
120
|
[tool.ruff.lint.isort]
|
|
114
121
|
case-sensitive = true
|
|
@@ -144,6 +151,7 @@ ignore_missing_imports = true
|
|
|
144
151
|
|
|
145
152
|
[tool.hatch.envs.default]
|
|
146
153
|
installer = "uv"
|
|
154
|
+
features = ["aiolimiter", "pyrate"]
|
|
147
155
|
|
|
148
156
|
[tool.hatch.envs.dev]
|
|
149
157
|
description = """Development environment."""
|
|
@@ -180,7 +188,7 @@ check = ["pip-audit"]
|
|
|
180
188
|
|
|
181
189
|
[tool.hatch.envs.types]
|
|
182
190
|
description = """Check the static types of the codebase."""
|
|
183
|
-
dependencies = ["mypy"]
|
|
191
|
+
extra-dependencies = ["mypy"]
|
|
184
192
|
|
|
185
193
|
[tool.hatch.envs.types.scripts]
|
|
186
194
|
check = "mypy src/httpx_limiter"
|
|
@@ -204,6 +212,7 @@ description = """Test the source and wheel distribution of the package."""
|
|
|
204
212
|
dependencies = ["pip", "twine"]
|
|
205
213
|
detached = true
|
|
206
214
|
installer = "uv"
|
|
215
|
+
builder = true
|
|
207
216
|
|
|
208
217
|
[tool.hatch.envs.distribution.scripts]
|
|
209
218
|
check = ["pip check", "hatch build {args:--clean}", "twine check dist/*"]
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
from .types import HTTPXAsyncHTTPTransportKeywordArguments
|
|
19
19
|
from .rate import Number, Rate
|
|
20
|
-
from .
|
|
20
|
+
from .abstract_async_limiter import AbstractAsyncLimiter
|
|
21
21
|
from .async_rate_limited_transport import AsyncRateLimitedTransport
|
|
22
22
|
from .abstract_rate_limiter_repository import AbstractRateLimiterRepository
|
|
23
23
|
from .async_multi_rate_limited_transport import AsyncMultiRateLimitedTransport
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.5.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 5, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Copyright (c) 2025 Moritz E. Beber
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
4
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
5
|
+
# the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
11
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
12
|
+
# License for the specific language governing permissions and limitations under
|
|
13
|
+
# the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Provide an abstract base class for asynchronous rate limiters."""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from abc import ABC, abstractmethod
|
|
21
|
+
from typing import TYPE_CHECKING
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
25
|
+
from types import TracebackType
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AbstractAsyncLimiter(ABC):
|
|
29
|
+
"""
|
|
30
|
+
Define an abstract base class for asynchronous rate limiters.
|
|
31
|
+
|
|
32
|
+
This interface dictates that all asynchronous rate limiters shall implement the
|
|
33
|
+
context manager protocol.
|
|
34
|
+
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
async def __aenter__(self) -> AbstractAsyncLimiter:
|
|
39
|
+
"""Acquire a rate limit token upon entering the asynchronous context."""
|
|
40
|
+
|
|
41
|
+
@abstractmethod
|
|
42
|
+
async def __aexit__(
|
|
43
|
+
self,
|
|
44
|
+
exc_type: type[BaseException] | None,
|
|
45
|
+
exc_val: BaseException | None,
|
|
46
|
+
exc_tb: TracebackType | None,
|
|
47
|
+
) -> None:
|
|
48
|
+
"""Exit the asynchronous context."""
|
{httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/abstract_rate_limiter_repository.py
RENAMED
|
@@ -16,12 +16,10 @@
|
|
|
16
16
|
"""Provide an abstract repository for rate limiters."""
|
|
17
17
|
|
|
18
18
|
from abc import ABC, abstractmethod
|
|
19
|
-
from collections.abc import Sequence
|
|
20
19
|
|
|
21
20
|
import httpx
|
|
22
21
|
|
|
23
|
-
from .
|
|
24
|
-
from .rate import Rate
|
|
22
|
+
from .abstract_async_limiter import AbstractAsyncLimiter
|
|
25
23
|
|
|
26
24
|
|
|
27
25
|
class AbstractRateLimiterRepository(ABC):
|
|
@@ -44,31 +42,21 @@ class AbstractRateLimiterRepository(ABC):
|
|
|
44
42
|
|
|
45
43
|
def __init__(self, **kwargs: dict[str, object]) -> None:
|
|
46
44
|
super().__init__(**kwargs)
|
|
47
|
-
self._limiters: dict[str,
|
|
45
|
+
self._limiters: dict[str, AbstractAsyncLimiter] = {}
|
|
48
46
|
|
|
49
47
|
@abstractmethod
|
|
50
48
|
def get_identifier(self, request: httpx.Request) -> str:
|
|
51
49
|
"""Return a request-specific identifier."""
|
|
52
50
|
|
|
53
51
|
@abstractmethod
|
|
54
|
-
def
|
|
55
|
-
"""Return
|
|
56
|
-
|
|
57
|
-
def _get_limiter_kwargs(
|
|
58
|
-
self,
|
|
59
|
-
request: httpx.Request, # noqa: ARG002
|
|
60
|
-
) -> PyRateLimiterKeywordArguments:
|
|
61
|
-
"""Return the keyword arguments for creating a rate limiter."""
|
|
62
|
-
return {}
|
|
52
|
+
def create(self, request: httpx.Request) -> AbstractAsyncLimiter:
|
|
53
|
+
"""Return a request-specific rate limiter."""
|
|
63
54
|
|
|
64
|
-
def get(self, request: httpx.Request) ->
|
|
55
|
+
def get(self, request: httpx.Request) -> AbstractAsyncLimiter:
|
|
65
56
|
"""Return a request-specific rate limiter."""
|
|
66
57
|
identifier = self.get_identifier(request)
|
|
67
58
|
|
|
68
59
|
if identifier not in self._limiters:
|
|
69
|
-
self._limiters[identifier] =
|
|
70
|
-
*self.get_rates(request),
|
|
71
|
-
**self._get_limiter_kwargs(request),
|
|
72
|
-
)
|
|
60
|
+
self._limiters[identifier] = self.create(request)
|
|
73
61
|
|
|
74
62
|
return self._limiters[identifier]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright (c) 2025 Moritz E. Beber
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
4
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
5
|
+
# the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
11
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
12
|
+
# License for the specific language governing permissions and limitations under
|
|
13
|
+
# the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Provide aiolimiter-based rate limiting functionality."""
|
|
17
|
+
|
|
18
|
+
from .aiolimiter_async_limiter import AiolimiterAsyncLimiter
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Copyright (c) 2025 Moritz E. Beber
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
4
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
5
|
+
# the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
11
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
12
|
+
# License for the specific language governing permissions and limitations under
|
|
13
|
+
# the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Provide an asynchronous rate-limited transport using aiolimiter."""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import TYPE_CHECKING
|
|
21
|
+
|
|
22
|
+
from aiolimiter import AsyncLimiter
|
|
23
|
+
|
|
24
|
+
from httpx_limiter import AbstractAsyncLimiter
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
28
|
+
from types import TracebackType
|
|
29
|
+
|
|
30
|
+
from httpx_limiter.rate import Rate
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class AiolimiterAsyncLimiter(AbstractAsyncLimiter):
|
|
34
|
+
"""
|
|
35
|
+
Define an asynchronous limiter that composes the aiolimiter AsyncLimiter.
|
|
36
|
+
|
|
37
|
+
This class encapsulates the creation and configuration of an aiolimiter
|
|
38
|
+
AsyncLimiter with appropriate settings.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
limiter: An instance of an aiolimiter AsyncLimiter.
|
|
42
|
+
**kwargs: Additional keyword arguments for the parent classes.
|
|
43
|
+
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(self, *, limiter: AsyncLimiter, **kwargs: dict[str, object]) -> None:
|
|
47
|
+
super().__init__(**kwargs)
|
|
48
|
+
self._limiter = limiter
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def create(cls, rate: Rate) -> AiolimiterAsyncLimiter:
|
|
52
|
+
"""
|
|
53
|
+
Create an instance of AiolimiterAsyncLimiter.
|
|
54
|
+
|
|
55
|
+
Note: aiolimiter only supports a single rate limit, unlike pyrate-limiter
|
|
56
|
+
which supports multiple rates.
|
|
57
|
+
"""
|
|
58
|
+
limiter = AsyncLimiter(
|
|
59
|
+
max_rate=rate.magnitude,
|
|
60
|
+
time_period=rate.in_seconds(),
|
|
61
|
+
)
|
|
62
|
+
return cls(limiter=limiter)
|
|
63
|
+
|
|
64
|
+
async def __aenter__(self) -> AiolimiterAsyncLimiter: # noqa: PYI034
|
|
65
|
+
"""Acquire a token upon entering the asynchronous context."""
|
|
66
|
+
await self._limiter.__aenter__()
|
|
67
|
+
return self
|
|
68
|
+
|
|
69
|
+
async def __aexit__(
|
|
70
|
+
self,
|
|
71
|
+
exc_type: type[BaseException] | None,
|
|
72
|
+
exc_val: BaseException | None,
|
|
73
|
+
exc_tb: TracebackType | None,
|
|
74
|
+
) -> None:
|
|
75
|
+
"""Exit the asynchronous context."""
|
|
76
|
+
await self._limiter.__aexit__(exc_type=exc_type, exc=exc_val, tb=exc_tb)
|
{httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/async_rate_limited_transport.py
RENAMED
|
@@ -28,11 +28,9 @@ else:
|
|
|
28
28
|
|
|
29
29
|
import httpx
|
|
30
30
|
|
|
31
|
-
from .async_limiter import AsyncLimiter
|
|
32
|
-
|
|
33
31
|
|
|
34
32
|
if TYPE_CHECKING: # pragma: no cover
|
|
35
|
-
from .
|
|
33
|
+
from .abstract_async_limiter import AbstractAsyncLimiter
|
|
36
34
|
from .types import HTTPXAsyncHTTPTransportKeywordArguments
|
|
37
35
|
|
|
38
36
|
|
|
@@ -49,7 +47,7 @@ class AsyncRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
49
47
|
def __init__(
|
|
50
48
|
self,
|
|
51
49
|
*,
|
|
52
|
-
limiter:
|
|
50
|
+
limiter: AbstractAsyncLimiter,
|
|
53
51
|
transport: httpx.AsyncBaseTransport,
|
|
54
52
|
**kwargs: dict[str, object],
|
|
55
53
|
) -> None:
|
|
@@ -60,7 +58,7 @@ class AsyncRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
60
58
|
@classmethod
|
|
61
59
|
def create(
|
|
62
60
|
cls,
|
|
63
|
-
|
|
61
|
+
limiter: AbstractAsyncLimiter,
|
|
64
62
|
**kwargs: Unpack[HTTPXAsyncHTTPTransportKeywordArguments],
|
|
65
63
|
) -> AsyncRateLimitedTransport:
|
|
66
64
|
"""
|
|
@@ -71,7 +69,7 @@ class AsyncRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
71
69
|
That transport is passed any additional keyword arguments.
|
|
72
70
|
|
|
73
71
|
Args:
|
|
74
|
-
|
|
72
|
+
limiter: A concrete instance of an asynchronous limiter.
|
|
75
73
|
**kwargs: Additional keyword arguments are used in the construction of an
|
|
76
74
|
`httpx.AsyncHTTPTransport`.
|
|
77
75
|
|
|
@@ -80,8 +78,8 @@ class AsyncRateLimitedTransport(httpx.AsyncBaseTransport):
|
|
|
80
78
|
|
|
81
79
|
"""
|
|
82
80
|
return cls(
|
|
83
|
-
limiter=
|
|
84
|
-
transport=httpx.AsyncHTTPTransport(**kwargs),
|
|
81
|
+
limiter=limiter,
|
|
82
|
+
transport=httpx.AsyncHTTPTransport(**kwargs),
|
|
85
83
|
)
|
|
86
84
|
|
|
87
85
|
async def handle_async_request(
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright (c) 2025 Moritz E. Beber
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
4
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
|
5
|
+
# the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
11
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
12
|
+
# License for the specific language governing permissions and limitations under
|
|
13
|
+
# the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""Provide pyrate-based rate limiting functionality."""
|
|
17
|
+
|
|
18
|
+
from .pyrate_async_limiter import PyrateAsyncLimiter, PyRateLimiterKeywordArguments
|
|
@@ -37,11 +37,13 @@ from pyrate_limiter import (
|
|
|
37
37
|
)
|
|
38
38
|
from pyrate_limiter import Rate as PyRate
|
|
39
39
|
|
|
40
|
+
from httpx_limiter import AbstractAsyncLimiter
|
|
41
|
+
|
|
40
42
|
|
|
41
43
|
if TYPE_CHECKING: # pragma: no cover
|
|
42
44
|
from types import TracebackType
|
|
43
45
|
|
|
44
|
-
from .rate import Rate
|
|
46
|
+
from httpx_limiter.rate import Rate
|
|
45
47
|
|
|
46
48
|
|
|
47
49
|
class PyRateLimiterKeywordArguments(TypedDict, total=False):
|
|
@@ -54,7 +56,7 @@ class PyRateLimiterKeywordArguments(TypedDict, total=False):
|
|
|
54
56
|
buffer_ms: int
|
|
55
57
|
|
|
56
58
|
|
|
57
|
-
class
|
|
59
|
+
class PyrateAsyncLimiter(AbstractAsyncLimiter):
|
|
58
60
|
"""
|
|
59
61
|
Define an asynchronous limiter that composes the pyrate limiter.
|
|
60
62
|
|
|
@@ -76,8 +78,8 @@ class AsyncLimiter:
|
|
|
76
78
|
cls,
|
|
77
79
|
*rates: Rate,
|
|
78
80
|
**kwargs: Unpack[PyRateLimiterKeywordArguments],
|
|
79
|
-
) ->
|
|
80
|
-
"""Create an instance of
|
|
81
|
+
) -> PyrateAsyncLimiter:
|
|
82
|
+
"""Create an instance of PyrateAsyncLimiter."""
|
|
81
83
|
if not rates:
|
|
82
84
|
msg = "At least one rate must be provided."
|
|
83
85
|
raise ValueError(msg)
|
|
@@ -94,17 +96,18 @@ class AsyncLimiter:
|
|
|
94
96
|
)
|
|
95
97
|
raise ValueError(msg)
|
|
96
98
|
|
|
99
|
+
# Cast kwargs to proper types for internal use
|
|
97
100
|
limiter = Limiter(
|
|
98
101
|
argument=BucketAsyncWrapper(InMemoryBucket(rate_limits)),
|
|
99
102
|
clock=kwargs.get("clock", TimeAsyncClock()),
|
|
100
103
|
raise_when_fail=kwargs.get("raise_when_fail", False),
|
|
101
104
|
max_delay=kwargs.get("max_delay", Duration.HOUR),
|
|
102
|
-
retry_until_max_delay=kwargs.get("retry_until_max_delay",
|
|
105
|
+
retry_until_max_delay=kwargs.get("retry_until_max_delay", True),
|
|
103
106
|
buffer_ms=kwargs.get("buffer_ms", 50),
|
|
104
107
|
)
|
|
105
108
|
return cls(limiter=limiter)
|
|
106
109
|
|
|
107
|
-
async def __aenter__(self) ->
|
|
110
|
+
async def __aenter__(self) -> PyrateAsyncLimiter: # noqa: PYI034
|
|
108
111
|
"""Acquire a token upon entering the asynchronous context."""
|
|
109
112
|
# Keep trying to acquire, let timeouts be handled externally.
|
|
110
113
|
while not (await self._limiter.try_acquire_async("httpx-limiter")):
|
|
@@ -48,6 +48,10 @@ class Rate(NamedTuple):
|
|
|
48
48
|
|
|
49
49
|
return cls(magnitude=magnitude, duration=duration)
|
|
50
50
|
|
|
51
|
+
def in_seconds(self) -> float:
|
|
52
|
+
"""Return the duration in unit seconds."""
|
|
53
|
+
return self.duration.total_seconds()
|
|
54
|
+
|
|
51
55
|
def in_milliseconds(self) -> int:
|
|
52
56
|
"""Return the duration in unit milliseconds."""
|
|
53
57
|
return int(self.duration.total_seconds() * 1_000)
|
|
@@ -16,23 +16,38 @@
|
|
|
16
16
|
"""Provide additional type definitions."""
|
|
17
17
|
|
|
18
18
|
import ssl
|
|
19
|
-
import
|
|
20
|
-
from typing import TypedDict
|
|
19
|
+
from collections.abc import Iterable
|
|
20
|
+
from typing import TypeAlias, TypedDict
|
|
21
21
|
|
|
22
22
|
import httpx
|
|
23
23
|
|
|
24
24
|
|
|
25
|
+
# Fallback for Pyodide HTTPX. See: https://github.com/Midnighter/httpx-limiter/pull/10#issuecomment-3241566927
|
|
26
|
+
SocketOption: TypeAlias = (
|
|
27
|
+
tuple[int, int, int]
|
|
28
|
+
| tuple[int, int, bytes | bytearray]
|
|
29
|
+
| tuple[int, int, None, int]
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
# We redefine the following types from httpx._types to avoid a dependency on the
|
|
33
|
+
# private module. It would be better if HTTPX exposed these types publicly, but
|
|
34
|
+
# they are unlikely to change often. Thus we consider it acceptable to redefine
|
|
35
|
+
# them here for type hinting purposes.
|
|
36
|
+
CertTypes: TypeAlias = str | tuple[str, str] | tuple[str, str, str]
|
|
37
|
+
ProxyTypes: TypeAlias = httpx.URL | str | httpx.Proxy
|
|
38
|
+
|
|
39
|
+
|
|
25
40
|
class HTTPXAsyncHTTPTransportKeywordArguments(TypedDict, total=False):
|
|
26
41
|
"""Keyword arguments for the httpx.AsyncHTTPTransport constructor."""
|
|
27
42
|
|
|
28
43
|
verify: ssl.SSLContext | str | bool
|
|
29
|
-
cert:
|
|
44
|
+
cert: CertTypes | None
|
|
30
45
|
trust_env: bool
|
|
31
46
|
http1: bool
|
|
32
47
|
http2: bool
|
|
33
|
-
limits: httpx.
|
|
34
|
-
proxy:
|
|
48
|
+
limits: httpx.Limits
|
|
49
|
+
proxy: ProxyTypes | None
|
|
35
50
|
uds: str | None
|
|
36
51
|
local_address: str | None
|
|
37
52
|
retries: int
|
|
38
|
-
socket_options:
|
|
53
|
+
socket_options: Iterable[SocketOption] | None
|
|
File without changes
|
|
File without changes
|
{httpx_limiter-0.4.0 → httpx_limiter-0.5.0}/src/httpx_limiter/async_multi_rate_limited_transport.py
RENAMED
|
File without changes
|
|
File without changes
|