httpx-limiter 0.3.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- httpx_limiter-0.5.0/PKG-INFO +217 -0
- httpx_limiter-0.5.0/README.md +181 -0
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/pyproject.toml +54 -59
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/src/httpx_limiter/__init__.py +2 -0
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/src/httpx_limiter/_version.py +16 -3
- httpx_limiter-0.5.0/src/httpx_limiter/abstract_async_limiter.py +48 -0
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/src/httpx_limiter/abstract_rate_limiter_repository.py +23 -14
- httpx_limiter-0.5.0/src/httpx_limiter/aiolimiter/__init__.py +18 -0
- httpx_limiter-0.5.0/src/httpx_limiter/aiolimiter/aiolimiter_async_limiter.py +76 -0
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/src/httpx_limiter/async_multi_rate_limited_transport.py +10 -2
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/src/httpx_limiter/async_rate_limited_transport.py +16 -13
- httpx_limiter-0.5.0/src/httpx_limiter/pyrate/__init__.py +18 -0
- httpx_limiter-0.5.0/src/httpx_limiter/pyrate/pyrate_async_limiter.py +124 -0
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/src/httpx_limiter/rate.py +15 -3
- httpx_limiter-0.5.0/src/httpx_limiter/types.py +53 -0
- httpx_limiter-0.3.0/PKG-INFO +0 -143
- httpx_limiter-0.3.0/README.md +0 -111
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/.gitignore +0 -0
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/LICENSE +0 -0
- {httpx_limiter-0.3.0 → httpx_limiter-0.5.0}/src/httpx_limiter/py.typed +0 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: httpx-limiter
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: A lightweight package that provides rate-limited httpx transports.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Midnighter/httpx-limiter
|
|
6
|
+
Project-URL: Documentation, https://httpx-limiter.readthedocs.io
|
|
7
|
+
Project-URL: Source Code, https://github.com/Midnighter/httpx-limiter
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/Midnighter/httpx-limiter/issues
|
|
9
|
+
Project-URL: Download, https://pypi.org/project/httpx-limiter/#files
|
|
10
|
+
Author-email: "Moritz E. Beber" <midnighter@posteo.net>
|
|
11
|
+
License-Expression: Apache-2.0
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: httpx,leaky bucket,limiter,rate-limit
|
|
14
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
15
|
+
Classifier: Environment :: Web Environment
|
|
16
|
+
Classifier: Framework :: AsyncIO
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
19
|
+
Classifier: Natural Language :: English
|
|
20
|
+
Classifier: Operating System :: OS Independent
|
|
21
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
26
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.10
|
|
29
|
+
Requires-Dist: httpx~=0.25
|
|
30
|
+
Requires-Dist: typing-extensions~=4.6; python_version < '3.11'
|
|
31
|
+
Provides-Extra: aiolimiter
|
|
32
|
+
Requires-Dist: aiolimiter~=1.2; extra == 'aiolimiter'
|
|
33
|
+
Provides-Extra: pyrate
|
|
34
|
+
Requires-Dist: pyrate-limiter~=3.9; extra == 'pyrate'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# HTTPX Limiter
|
|
38
|
+
|
|
39
|
+
| | |
|
|
40
|
+
| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
41
|
+
| Package | [](https://pypi.org/project/httpx-limiter/) [](https://pypi.org/project/httpx-limiter/) |
|
|
42
|
+
| Meta | [](LICENSE) [](.github/CODE_OF_CONDUCT.md) [](https://mypy-lang.org/) [](https://github.com/ambv/black) [](https://github.com/astral-sh/ruff) |
|
|
43
|
+
| Automation | [](https://github.com/Midnighter/httpx-limiter/actions/workflows/main.yml) |
|
|
44
|
+
|
|
45
|
+
_A lightweight package that provides rate-limited httpx transports._
|
|
46
|
+
|
|
47
|
+
## Installation
|
|
48
|
+
|
|
49
|
+
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). Install it,
|
|
50
|
+
for example, with the aiolimiter backend.
|
|
51
|
+
|
|
52
|
+
```sh
|
|
53
|
+
pip install 'httpx-limiter[aiolimiter]'
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
There is also a pyrate-limiter backend available.
|
|
57
|
+
|
|
58
|
+
```sh
|
|
59
|
+
pip install 'httpx-limiter[pyrate]'
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Rate Limiter Backends
|
|
63
|
+
|
|
64
|
+
This package provides two different asynchronous rate limiter implementations to choose
|
|
65
|
+
from:
|
|
66
|
+
|
|
67
|
+
1. [aiolimiter](https://aiolimiter.readthedocs.io)
|
|
68
|
+
2. [pyrate-limiter](https://pyratelimiter.readthedocs.io)
|
|
69
|
+
|
|
70
|
+
> [!IMPORTANT]
|
|
71
|
+
> While both implementations fulfill similar purposes, there are significant differences
|
|
72
|
+
> between them. Please read the descriptions below to make an informed decision about
|
|
73
|
+
> which one best suits your needs.
|
|
74
|
+
|
|
75
|
+
### 1. aiolimiter
|
|
76
|
+
|
|
77
|
+
- **Single rate limit only**: Supports only one rate limit at a time
|
|
78
|
+
- **Lightweight**: Minimal dependencies and simpler configuration
|
|
79
|
+
- **Linear token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
80
|
+
roughly one token will be added every 500 milliseconds
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
84
|
+
|
|
85
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### 2. pyrate-limiter
|
|
89
|
+
|
|
90
|
+
- **Multiple rate limits**: Support for multiple concurrent rate limits, for example,
|
|
91
|
+
10 requests per second _and_ 100 requests per minute
|
|
92
|
+
- **Flexible configuration**: Comprehensive configuration options
|
|
93
|
+
- **Multiprocessing dependency**: Current implementation depends on `multiprocessing`
|
|
94
|
+
which may not be available in all environments, such as pyodide
|
|
95
|
+
- **Stepwise token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
96
|
+
two token will be added every second
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
100
|
+
|
|
101
|
+
# Single rate limit
|
|
102
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
103
|
+
|
|
104
|
+
# Multiple rate limits
|
|
105
|
+
limiter = PyrateAsyncLimiter.create(
|
|
106
|
+
Rate.create(magnitude=10), # 10 per second
|
|
107
|
+
Rate.create(magnitude=100, duration=60), # 100 per minute
|
|
108
|
+
)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Tutorial
|
|
112
|
+
|
|
113
|
+
You can limit the number of requests made by an HTTPX client using the transports
|
|
114
|
+
provided in this package. That is useful in situations when you need to make a large
|
|
115
|
+
number of asynchronous requests against endpoints that implement a rate limit.
|
|
116
|
+
|
|
117
|
+
### Single Rate Limit
|
|
118
|
+
|
|
119
|
+
The simplest use case is to apply a single rate limit to all requests. If you want to be
|
|
120
|
+
able to make twenty requests per second, for example, use the following code:
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
import httpx
|
|
124
|
+
from httpx_limiter import AsyncRateLimitedTransport, Rate
|
|
125
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
126
|
+
|
|
127
|
+
async def main():
|
|
128
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
129
|
+
|
|
130
|
+
async with httpx.AsyncClient(
|
|
131
|
+
transport=AsyncRateLimitedTransport.create(limiter=limiter),
|
|
132
|
+
) as client:
|
|
133
|
+
response = await client.get("https://httpbin.org")
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
> [!IMPORTANT]
|
|
137
|
+
> Due to limitations in the design of the underlying [leaky
|
|
138
|
+
> bucket](https://en.wikipedia.org/wiki/Leaky_bucket) implementation, which is
|
|
139
|
+
> used to implement the rate limiting, the magnitude of the rate is also the
|
|
140
|
+
> maximum capacity of the bucket. That means, if you set a rate that is larger
|
|
141
|
+
> than one, a burst of requests equal to that capacity will be allowed. If you
|
|
142
|
+
> do not want to allow any bursts, set the magnitude to one, but the duration to
|
|
143
|
+
> the inverse of your desired rate. If you want to allow twenty requests per
|
|
144
|
+
> second, for example, set the magnitude to 1 and the duration to 0.05 seconds.
|
|
145
|
+
>
|
|
146
|
+
> ```python
|
|
147
|
+
> Rate.create(magnitude=1, duration=1/20)
|
|
148
|
+
> ```
|
|
149
|
+
|
|
150
|
+
### Multiple Rate Limits
|
|
151
|
+
|
|
152
|
+
For more advanced use cases, you can apply different rate limits based on a concrete
|
|
153
|
+
implementation of the `AbstractRateLimiterRepository`. There are two relevant methods
|
|
154
|
+
that both get passed the current request. One method needs to return an identifier for
|
|
155
|
+
which rate limiter to choose, and the other method creates a rate limiter. See the
|
|
156
|
+
following example:
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
import httpx
|
|
160
|
+
from httpx_limiter import (
|
|
161
|
+
AbstractRateLimiterRepository,
|
|
162
|
+
AsyncMultiRateLimitedTransport,
|
|
163
|
+
Rate
|
|
164
|
+
)
|
|
165
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
166
|
+
|
|
167
|
+
class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
168
|
+
"""Apply different rate limits based on the domain being requested."""
|
|
169
|
+
|
|
170
|
+
def get_identifier(self, request: httpx.Request) -> str:
|
|
171
|
+
"""Return the domain as the identifier for rate limiting."""
|
|
172
|
+
return request.url.host
|
|
173
|
+
|
|
174
|
+
def create(self, request: httpx.Request) -> AiolimiterAsyncLimiter:
|
|
175
|
+
"""Create a rate limiter for the domain."""
|
|
176
|
+
return AiolimiterAsyncLimiter.create(Rate.create(magnitude=25))
|
|
177
|
+
|
|
178
|
+
client = httpx.AsyncClient(
|
|
179
|
+
transport=AsyncMultiRateLimitedTransport.create(
|
|
180
|
+
repository=DomainBasedRateLimiterRepository(),
|
|
181
|
+
),
|
|
182
|
+
)
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
> [!TIP]
|
|
186
|
+
> You are free to ignore the request parameter and use global information like
|
|
187
|
+
> the time of day to determine the rate limit.
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
from datetime import datetime, timezone
|
|
191
|
+
|
|
192
|
+
import httpx
|
|
193
|
+
from httpx_limiter import AbstractRateLimiterRepository, Rate
|
|
194
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
195
|
+
|
|
196
|
+
class DayNightRateLimiterRepository(AbstractRateLimiterRepository):
|
|
197
|
+
"""Apply different rate limits based on the time of day."""
|
|
198
|
+
|
|
199
|
+
def get_identifier(self, request: httpx.Request) -> str:
|
|
200
|
+
"""Identify whether it is currently day or night."""
|
|
201
|
+
if 6 <= datetime.now(tz=timezone.utc).hour < 18:
|
|
202
|
+
return "day"
|
|
203
|
+
|
|
204
|
+
return "night"
|
|
205
|
+
|
|
206
|
+
def create(self, request: httpx.Request) -> PyrateAsyncLimiter:
|
|
207
|
+
"""Create a rate limiter based on the time of day."""
|
|
208
|
+
if self.get_identifier(request) == "day":
|
|
209
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=10))
|
|
210
|
+
|
|
211
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=100))
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
## Copyright
|
|
215
|
+
|
|
216
|
+
- Copyright © 2024, 2025 Moritz E. Beber.
|
|
217
|
+
- Free software distributed under the [Apache Software License 2.0](./LICENSE).
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# HTTPX Limiter
|
|
2
|
+
|
|
3
|
+
| | |
|
|
4
|
+
| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
5
|
+
| Package | [](https://pypi.org/project/httpx-limiter/) [](https://pypi.org/project/httpx-limiter/) |
|
|
6
|
+
| Meta | [](LICENSE) [](.github/CODE_OF_CONDUCT.md) [](https://mypy-lang.org/) [](https://github.com/ambv/black) [](https://github.com/astral-sh/ruff) |
|
|
7
|
+
| Automation | [](https://github.com/Midnighter/httpx-limiter/actions/workflows/main.yml) |
|
|
8
|
+
|
|
9
|
+
_A lightweight package that provides rate-limited httpx transports._
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). Install it,
|
|
14
|
+
for example, with the aiolimiter backend.
|
|
15
|
+
|
|
16
|
+
```sh
|
|
17
|
+
pip install 'httpx-limiter[aiolimiter]'
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
There is also a pyrate-limiter backend available.
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
pip install 'httpx-limiter[pyrate]'
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Rate Limiter Backends
|
|
27
|
+
|
|
28
|
+
This package provides two different asynchronous rate limiter implementations to choose
|
|
29
|
+
from:
|
|
30
|
+
|
|
31
|
+
1. [aiolimiter](https://aiolimiter.readthedocs.io)
|
|
32
|
+
2. [pyrate-limiter](https://pyratelimiter.readthedocs.io)
|
|
33
|
+
|
|
34
|
+
> [!IMPORTANT]
|
|
35
|
+
> While both implementations fulfill similar purposes, there are significant differences
|
|
36
|
+
> between them. Please read the descriptions below to make an informed decision about
|
|
37
|
+
> which one best suits your needs.
|
|
38
|
+
|
|
39
|
+
### 1. aiolimiter
|
|
40
|
+
|
|
41
|
+
- **Single rate limit only**: Supports only one rate limit at a time
|
|
42
|
+
- **Lightweight**: Minimal dependencies and simpler configuration
|
|
43
|
+
- **Linear token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
44
|
+
roughly one token will be added every 500 milliseconds
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
48
|
+
|
|
49
|
+
limiter = AiolimiterAsyncLimiter.create(Rate.create(magnitude=20))
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
### 2. pyrate-limiter
|
|
53
|
+
|
|
54
|
+
- **Multiple rate limits**: Support for multiple concurrent rate limits, for example,
|
|
55
|
+
10 requests per second _and_ 100 requests per minute
|
|
56
|
+
- **Flexible configuration**: Comprehensive configuration options
|
|
57
|
+
- **Multiprocessing dependency**: Current implementation depends on `multiprocessing`
|
|
58
|
+
which may not be available in all environments, such as pyodide
|
|
59
|
+
- **Stepwise token refresh rate**: As an example, if you set a rate of 2 per second,
|
|
60
|
+
two token will be added every second
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
64
|
+
|
|
65
|
+
# Single rate limit
|
|
66
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
67
|
+
|
|
68
|
+
# Multiple rate limits
|
|
69
|
+
limiter = PyrateAsyncLimiter.create(
|
|
70
|
+
Rate.create(magnitude=10), # 10 per second
|
|
71
|
+
Rate.create(magnitude=100, duration=60), # 100 per minute
|
|
72
|
+
)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Tutorial
|
|
76
|
+
|
|
77
|
+
You can limit the number of requests made by an HTTPX client using the transports
|
|
78
|
+
provided in this package. That is useful in situations when you need to make a large
|
|
79
|
+
number of asynchronous requests against endpoints that implement a rate limit.
|
|
80
|
+
|
|
81
|
+
### Single Rate Limit
|
|
82
|
+
|
|
83
|
+
The simplest use case is to apply a single rate limit to all requests. If you want to be
|
|
84
|
+
able to make twenty requests per second, for example, use the following code:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
import httpx
|
|
88
|
+
from httpx_limiter import AsyncRateLimitedTransport, Rate
|
|
89
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
90
|
+
|
|
91
|
+
async def main():
|
|
92
|
+
limiter = PyrateAsyncLimiter.create(Rate.create(magnitude=20))
|
|
93
|
+
|
|
94
|
+
async with httpx.AsyncClient(
|
|
95
|
+
transport=AsyncRateLimitedTransport.create(limiter=limiter),
|
|
96
|
+
) as client:
|
|
97
|
+
response = await client.get("https://httpbin.org")
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
> [!IMPORTANT]
|
|
101
|
+
> Due to limitations in the design of the underlying [leaky
|
|
102
|
+
> bucket](https://en.wikipedia.org/wiki/Leaky_bucket) implementation, which is
|
|
103
|
+
> used to implement the rate limiting, the magnitude of the rate is also the
|
|
104
|
+
> maximum capacity of the bucket. That means, if you set a rate that is larger
|
|
105
|
+
> than one, a burst of requests equal to that capacity will be allowed. If you
|
|
106
|
+
> do not want to allow any bursts, set the magnitude to one, but the duration to
|
|
107
|
+
> the inverse of your desired rate. If you want to allow twenty requests per
|
|
108
|
+
> second, for example, set the magnitude to 1 and the duration to 0.05 seconds.
|
|
109
|
+
>
|
|
110
|
+
> ```python
|
|
111
|
+
> Rate.create(magnitude=1, duration=1/20)
|
|
112
|
+
> ```
|
|
113
|
+
|
|
114
|
+
### Multiple Rate Limits
|
|
115
|
+
|
|
116
|
+
For more advanced use cases, you can apply different rate limits based on a concrete
|
|
117
|
+
implementation of the `AbstractRateLimiterRepository`. There are two relevant methods
|
|
118
|
+
that both get passed the current request. One method needs to return an identifier for
|
|
119
|
+
which rate limiter to choose, and the other method creates a rate limiter. See the
|
|
120
|
+
following example:
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
import httpx
|
|
124
|
+
from httpx_limiter import (
|
|
125
|
+
AbstractRateLimiterRepository,
|
|
126
|
+
AsyncMultiRateLimitedTransport,
|
|
127
|
+
Rate
|
|
128
|
+
)
|
|
129
|
+
from httpx_limiter.aiolimiter import AiolimiterAsyncLimiter
|
|
130
|
+
|
|
131
|
+
class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository):
|
|
132
|
+
"""Apply different rate limits based on the domain being requested."""
|
|
133
|
+
|
|
134
|
+
def get_identifier(self, request: httpx.Request) -> str:
|
|
135
|
+
"""Return the domain as the identifier for rate limiting."""
|
|
136
|
+
return request.url.host
|
|
137
|
+
|
|
138
|
+
def create(self, request: httpx.Request) -> AiolimiterAsyncLimiter:
|
|
139
|
+
"""Create a rate limiter for the domain."""
|
|
140
|
+
return AiolimiterAsyncLimiter.create(Rate.create(magnitude=25))
|
|
141
|
+
|
|
142
|
+
client = httpx.AsyncClient(
|
|
143
|
+
transport=AsyncMultiRateLimitedTransport.create(
|
|
144
|
+
repository=DomainBasedRateLimiterRepository(),
|
|
145
|
+
),
|
|
146
|
+
)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
> [!TIP]
|
|
150
|
+
> You are free to ignore the request parameter and use global information like
|
|
151
|
+
> the time of day to determine the rate limit.
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
from datetime import datetime, timezone
|
|
155
|
+
|
|
156
|
+
import httpx
|
|
157
|
+
from httpx_limiter import AbstractRateLimiterRepository, Rate
|
|
158
|
+
from httpx_limiter.pyrate import PyrateAsyncLimiter
|
|
159
|
+
|
|
160
|
+
class DayNightRateLimiterRepository(AbstractRateLimiterRepository):
|
|
161
|
+
"""Apply different rate limits based on the time of day."""
|
|
162
|
+
|
|
163
|
+
def get_identifier(self, request: httpx.Request) -> str:
|
|
164
|
+
"""Identify whether it is currently day or night."""
|
|
165
|
+
if 6 <= datetime.now(tz=timezone.utc).hour < 18:
|
|
166
|
+
return "day"
|
|
167
|
+
|
|
168
|
+
return "night"
|
|
169
|
+
|
|
170
|
+
def create(self, request: httpx.Request) -> PyrateAsyncLimiter:
|
|
171
|
+
"""Create a rate limiter based on the time of day."""
|
|
172
|
+
if self.get_identifier(request) == "day":
|
|
173
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=10))
|
|
174
|
+
|
|
175
|
+
return PyrateAsyncLimiter.create(Rate.create(magnitude=100))
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
## Copyright
|
|
179
|
+
|
|
180
|
+
- Copyright © 2024, 2025 Moritz E. Beber.
|
|
181
|
+
- Free software distributed under the [Apache Software License 2.0](./LICENSE).
|
|
@@ -14,11 +14,9 @@ requires = ["hatchling", "hatch-vcs"]
|
|
|
14
14
|
name = "httpx-limiter"
|
|
15
15
|
dynamic = ["version"]
|
|
16
16
|
description = "A lightweight package that provides rate-limited httpx transports."
|
|
17
|
-
authors = [
|
|
18
|
-
{ name = "Moritz E. Beber", email = "midnighter@posteo.net" },
|
|
19
|
-
]
|
|
17
|
+
authors = [{ name = "Moritz E. Beber", email = "midnighter@posteo.net" }]
|
|
20
18
|
license = "Apache-2.0"
|
|
21
|
-
readme = {"file" = "README.md", "content-type" = "text/markdown"}
|
|
19
|
+
readme = { "file" = "README.md", "content-type" = "text/markdown" }
|
|
22
20
|
requires-python = ">=3.10"
|
|
23
21
|
# Please consult https://pypi.org/classifiers/ for a full list.
|
|
24
22
|
classifiers = [
|
|
@@ -37,15 +35,10 @@ classifiers = [
|
|
|
37
35
|
"Topic :: Internet :: WWW/HTTP",
|
|
38
36
|
"Typing :: Typed",
|
|
39
37
|
]
|
|
40
|
-
keywords = [
|
|
41
|
-
"httpx",
|
|
42
|
-
"limiter",
|
|
43
|
-
"rate-limit",
|
|
44
|
-
"leaky bucket",
|
|
45
|
-
]
|
|
38
|
+
keywords = ["httpx", "limiter", "rate-limit", "leaky bucket"]
|
|
46
39
|
dependencies = [
|
|
47
40
|
"httpx ~=0.25",
|
|
48
|
-
"
|
|
41
|
+
"typing-extensions ~=4.6; python_version < '3.11'",
|
|
49
42
|
]
|
|
50
43
|
|
|
51
44
|
[project.urls]
|
|
@@ -55,6 +48,10 @@ Documentation = "https://httpx-limiter.readthedocs.io"
|
|
|
55
48
|
"Bug Tracker" = "https://github.com/Midnighter/httpx-limiter/issues"
|
|
56
49
|
Download = "https://pypi.org/project/httpx-limiter/#files"
|
|
57
50
|
|
|
51
|
+
[project.optional-dependencies]
|
|
52
|
+
aiolimiter = ["aiolimiter ~=1.2"]
|
|
53
|
+
pyrate = ["pyrate-limiter ~=3.9"]
|
|
54
|
+
|
|
58
55
|
################################################################################
|
|
59
56
|
# Tool Configuration
|
|
60
57
|
################################################################################
|
|
@@ -76,17 +73,12 @@ testpaths = ["tests"]
|
|
|
76
73
|
markers = ["raises"]
|
|
77
74
|
|
|
78
75
|
[tool.coverage.paths]
|
|
79
|
-
source = [
|
|
80
|
-
"src/httpx_limiter",
|
|
81
|
-
"*/site-packages/httpx_limiter",
|
|
82
|
-
]
|
|
76
|
+
source = ["src/httpx_limiter", "*/site-packages/httpx_limiter"]
|
|
83
77
|
|
|
84
78
|
[tool.coverage.run]
|
|
85
79
|
branch = true
|
|
86
80
|
parallel = true
|
|
87
|
-
omit = [
|
|
88
|
-
"src/httpx_limiter/_version.py",
|
|
89
|
-
]
|
|
81
|
+
omit = ["src/httpx_limiter/_version.py"]
|
|
90
82
|
|
|
91
83
|
[tool.coverage.report]
|
|
92
84
|
exclude_lines = ["pragma: no cover"]
|
|
@@ -98,29 +90,31 @@ line-length = 88
|
|
|
98
90
|
[tool.ruff.lint]
|
|
99
91
|
select = ["ALL"]
|
|
100
92
|
ignore = [
|
|
101
|
-
"D107",
|
|
102
|
-
"D203",
|
|
103
|
-
"D212",
|
|
104
|
-
"D407",
|
|
105
|
-
"ANN002",
|
|
106
|
-
"ANN003",
|
|
107
|
-
]
|
|
108
|
-
exclude = [
|
|
109
|
-
"src/httpx_limiter/_version.py",
|
|
93
|
+
"D107", # 'Missing docstring in __init__' ignored because pydoclint wants us to document the class instead.
|
|
94
|
+
"D203", # '1 blank line required before class docstring' ignored because we want no blank line.
|
|
95
|
+
"D212", # 'Multi-line docstring summary should start at the first line' ignored because we want the summary to start on the second line.
|
|
96
|
+
"D407", # 'Missing dashed underline after section' ignored because Google style docstrings don't underline.
|
|
97
|
+
"ANN002", # 'Missing type annotation for {*args} in method'.
|
|
98
|
+
"ANN003", # 'Missing type annotation for {*kwargs} in method'.
|
|
110
99
|
]
|
|
100
|
+
exclude = ["src/httpx_limiter/_version.py"]
|
|
111
101
|
|
|
112
102
|
[tool.ruff.lint.extend-per-file-ignores]
|
|
113
103
|
"__init__.py" = [
|
|
114
|
-
"E401",
|
|
115
|
-
"E402",
|
|
116
|
-
"F401",
|
|
117
|
-
"I001",
|
|
104
|
+
"E401", # 'Multiple imports on one line'
|
|
105
|
+
"E402", # 'Module level import not at top of file'
|
|
106
|
+
"F401", # 'Imported but unused'
|
|
107
|
+
"I001", # 'Import block is un-sorted or un-formatted' ignored because we may have to import in a particular, not-alphabetical order.
|
|
118
108
|
]
|
|
119
109
|
"tests/**/*.py" = [
|
|
120
|
-
"S101",
|
|
110
|
+
"S101", # 'Use of assert detected' ignored because we are using pytest.
|
|
121
111
|
"INP001", # 'Insecure input' ignored because we are testing.
|
|
122
112
|
"ANN201", # 'Missing type annotation for {return}' ignored because all tests return `None`.
|
|
123
|
-
"PLR2004",
|
|
113
|
+
"PLR2004", # Magic numbers are okay in tests.
|
|
114
|
+
]
|
|
115
|
+
"tests/pyodide/**/*.py" = [
|
|
116
|
+
"PLC0415", # We explicitly need imports inside function and ignore the '`import` should be at the top-level of a file' rule.
|
|
117
|
+
"I001", # 'Import block is un-sorted or un-formatted' ignored because we may have to import in a particular, not-alphabetical order.
|
|
124
118
|
]
|
|
125
119
|
|
|
126
120
|
[tool.ruff.lint.isort]
|
|
@@ -143,9 +137,7 @@ exclude = "_version.py"
|
|
|
143
137
|
|
|
144
138
|
# Stop mypy from complaining about missing types from imports.
|
|
145
139
|
[[tool.mypy.overrides]]
|
|
146
|
-
module = [
|
|
147
|
-
"limiter",
|
|
148
|
-
]
|
|
140
|
+
module = ["limiter"]
|
|
149
141
|
ignore_missing_imports = true
|
|
150
142
|
|
|
151
143
|
#[tool.pydantic-mypy]
|
|
@@ -159,6 +151,7 @@ ignore_missing_imports = true
|
|
|
159
151
|
|
|
160
152
|
[tool.hatch.envs.default]
|
|
161
153
|
installer = "uv"
|
|
154
|
+
features = ["aiolimiter", "pyrate"]
|
|
162
155
|
|
|
163
156
|
[tool.hatch.envs.dev]
|
|
164
157
|
description = """Development environment."""
|
|
@@ -170,12 +163,13 @@ extra-dependencies = [
|
|
|
170
163
|
"ruff",
|
|
171
164
|
]
|
|
172
165
|
|
|
166
|
+
[tool.hatch.envs.dev.env-vars]
|
|
167
|
+
PYTHONDEVMODE = "1"
|
|
168
|
+
PYTHONASYNCIODEBUG = "1"
|
|
169
|
+
|
|
173
170
|
[tool.hatch.envs.style]
|
|
174
171
|
description = """Check the style of the codebase."""
|
|
175
|
-
dependencies = [
|
|
176
|
-
"pydoclint ~=0.6",
|
|
177
|
-
"ruff",
|
|
178
|
-
]
|
|
172
|
+
dependencies = ["pydoclint", "ruff"]
|
|
179
173
|
detached = true
|
|
180
174
|
installer = "uv"
|
|
181
175
|
|
|
@@ -187,18 +181,14 @@ check = ["docstrings", "code"]
|
|
|
187
181
|
|
|
188
182
|
[tool.hatch.envs.audit]
|
|
189
183
|
description = """Check dependencies for security vulnerabilities."""
|
|
190
|
-
extra-dependencies = [
|
|
191
|
-
"pip-audit",
|
|
192
|
-
]
|
|
184
|
+
extra-dependencies = ["pip-audit"]
|
|
193
185
|
|
|
194
186
|
[tool.hatch.envs.audit.scripts]
|
|
195
187
|
check = ["pip-audit"]
|
|
196
188
|
|
|
197
189
|
[tool.hatch.envs.types]
|
|
198
190
|
description = """Check the static types of the codebase."""
|
|
199
|
-
dependencies = [
|
|
200
|
-
"mypy",
|
|
201
|
-
]
|
|
191
|
+
extra-dependencies = ["mypy"]
|
|
202
192
|
|
|
203
193
|
[tool.hatch.envs.types.scripts]
|
|
204
194
|
check = "mypy src/httpx_limiter"
|
|
@@ -209,26 +199,23 @@ extra-dependencies = [
|
|
|
209
199
|
"mkdocs-material ~=9.5",
|
|
210
200
|
"mkdocstrings[python] ~=0.24",
|
|
211
201
|
"mkdocs-awesome-pages-plugin ~=2.9",
|
|
202
|
+
"mike ~=2.1",
|
|
212
203
|
]
|
|
213
204
|
|
|
214
205
|
[tool.hatch.envs.docs.scripts]
|
|
215
206
|
build = "mkdocs build {args:--clean --strict}"
|
|
216
207
|
serve = "mkdocs serve {args}"
|
|
208
|
+
deploy = "mike deploy {args}"
|
|
217
209
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
#detached = true
|
|
210
|
+
[tool.hatch.envs.distribution]
|
|
211
|
+
description = """Test the source and wheel distribution of the package."""
|
|
212
|
+
dependencies = ["pip", "twine"]
|
|
213
|
+
detached = true
|
|
214
|
+
installer = "uv"
|
|
215
|
+
builder = true
|
|
225
216
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
# "pip check",
|
|
229
|
-
# "hatch build {args:--clean}",
|
|
230
|
-
# "twine check dist/*",
|
|
231
|
-
#]
|
|
217
|
+
[tool.hatch.envs.distribution.scripts]
|
|
218
|
+
check = ["pip check", "hatch build {args:--clean}", "twine check dist/*"]
|
|
232
219
|
|
|
233
220
|
[tool.hatch.envs.unit-tests]
|
|
234
221
|
description = """Run the unit tests."""
|
|
@@ -240,6 +227,10 @@ extra-dependencies = [
|
|
|
240
227
|
"pytest-raises",
|
|
241
228
|
]
|
|
242
229
|
|
|
230
|
+
[tool.hatch.envs.unit-tests.env-vars]
|
|
231
|
+
PYTHONDEVMODE = "1"
|
|
232
|
+
PYTHONASYNCIODEBUG = "1"
|
|
233
|
+
|
|
243
234
|
[[tool.hatch.envs.unit-tests.matrix]]
|
|
244
235
|
python = ["3.10", "3.11", "3.12", "3.13"]
|
|
245
236
|
|
|
@@ -256,6 +247,10 @@ extra-dependencies = [
|
|
|
256
247
|
"pytest-raises",
|
|
257
248
|
]
|
|
258
249
|
|
|
250
|
+
[tool.hatch.envs.acceptance-tests.env-vars]
|
|
251
|
+
PYTHONDEVMODE = "1"
|
|
252
|
+
PYTHONASYNCIODEBUG = "1"
|
|
253
|
+
|
|
259
254
|
[[tool.hatch.envs.acceptance-tests.matrix]]
|
|
260
255
|
python = ["3.10", "3.11", "3.12", "3.13"]
|
|
261
256
|
|
|
@@ -15,7 +15,9 @@
|
|
|
15
15
|
|
|
16
16
|
"""Provide top level symbols."""
|
|
17
17
|
|
|
18
|
+
from .types import HTTPXAsyncHTTPTransportKeywordArguments
|
|
18
19
|
from .rate import Number, Rate
|
|
20
|
+
from .abstract_async_limiter import AbstractAsyncLimiter
|
|
19
21
|
from .async_rate_limited_transport import AsyncRateLimitedTransport
|
|
20
22
|
from .abstract_rate_limiter_repository import AbstractRateLimiterRepository
|
|
21
23
|
from .async_multi_rate_limited_transport import AsyncMultiRateLimitedTransport
|
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
# file generated by setuptools-scm
|
|
2
2
|
# don't change, don't track in version control
|
|
3
3
|
|
|
4
|
-
__all__ = [
|
|
4
|
+
__all__ = [
|
|
5
|
+
"__version__",
|
|
6
|
+
"__version_tuple__",
|
|
7
|
+
"version",
|
|
8
|
+
"version_tuple",
|
|
9
|
+
"__commit_id__",
|
|
10
|
+
"commit_id",
|
|
11
|
+
]
|
|
5
12
|
|
|
6
13
|
TYPE_CHECKING = False
|
|
7
14
|
if TYPE_CHECKING:
|
|
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
|
|
|
9
16
|
from typing import Union
|
|
10
17
|
|
|
11
18
|
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
+
COMMIT_ID = Union[str, None]
|
|
12
20
|
else:
|
|
13
21
|
VERSION_TUPLE = object
|
|
22
|
+
COMMIT_ID = object
|
|
14
23
|
|
|
15
24
|
version: str
|
|
16
25
|
__version__: str
|
|
17
26
|
__version_tuple__: VERSION_TUPLE
|
|
18
27
|
version_tuple: VERSION_TUPLE
|
|
28
|
+
commit_id: COMMIT_ID
|
|
29
|
+
__commit_id__: COMMIT_ID
|
|
19
30
|
|
|
20
|
-
__version__ = version = '0.
|
|
21
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.5.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 5, 0)
|
|
33
|
+
|
|
34
|
+
__commit_id__ = commit_id = None
|