geventhttpclient 2.2.0__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geventhttpclient-2.3.0/LICENSE-MIT +22 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/MANIFEST.in +2 -3
- geventhttpclient-2.3.0/PKG-INFO +248 -0
- geventhttpclient-2.3.0/README.md +185 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/ext/_parser.c +17 -17
- geventhttpclient-2.3.0/geventhttpclient/__init__.py +9 -0
- geventhttpclient-2.3.0/geventhttpclient/api.py +157 -0
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient/client.py +6 -10
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient/connectionpool.py +11 -35
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient/header.py +22 -4
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient/httplib.py +33 -9
- geventhttpclient-2.3.0/geventhttpclient/httplib2.py +69 -0
- geventhttpclient-2.3.0/geventhttpclient/requests.py +252 -0
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient/response.py +2 -1
- geventhttpclient-2.3.0/geventhttpclient/url.py +160 -0
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient/useragent.py +265 -198
- geventhttpclient-2.3.0/geventhttpclient.egg-info/PKG-INFO +248 -0
- geventhttpclient-2.3.0/geventhttpclient.egg-info/SOURCES.txt +28 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/pyproject.toml +32 -22
- geventhttpclient-2.2.0/LICENSE.txt +0 -27
- geventhttpclient-2.2.0/PKG-INFO +0 -243
- geventhttpclient-2.2.0/README.md +0 -176
- geventhttpclient-2.2.0/src/geventhttpclient/__init__.py +0 -7
- geventhttpclient-2.2.0/src/geventhttpclient/httplib2.py +0 -37
- geventhttpclient-2.2.0/src/geventhttpclient/tests/conftest.py +0 -7
- geventhttpclient-2.2.0/src/geventhttpclient/tests/oncert.pem +0 -20
- geventhttpclient-2.2.0/src/geventhttpclient/tests/server.crt +0 -22
- geventhttpclient-2.2.0/src/geventhttpclient/tests/server.key +0 -28
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_client.py +0 -381
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_headers.py +0 -227
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_http_host.py +0 -14
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_httplib.py +0 -75
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_httplib2.py +0 -20
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_keep_alive.py +0 -95
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_network_failures.py +0 -167
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_no_module_ssl.py +0 -36
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_parser.py +0 -148
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_ssl.py +0 -258
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_url.py +0 -141
- geventhttpclient-2.2.0/src/geventhttpclient/tests/test_useragent.py +0 -247
- geventhttpclient-2.2.0/src/geventhttpclient/url.py +0 -246
- geventhttpclient-2.2.0/src/geventhttpclient.egg-info/PKG-INFO +0 -243
- geventhttpclient-2.2.0/src/geventhttpclient.egg-info/SOURCES.txt +0 -42
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/ext/Python_compat.h +0 -0
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient.egg-info/dependency_links.txt +0 -0
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient.egg-info/requires.txt +0 -0
- {geventhttpclient-2.2.0/src → geventhttpclient-2.3.0}/geventhttpclient.egg-info/top_level.txt +0 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/llhttp/LICENSE-MIT +0 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/llhttp/include/llhttp.h +0 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/llhttp/src/api.c +0 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/llhttp/src/http.c +0 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/llhttp/src/llhttp.c +0 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/setup.cfg +0 -0
- {geventhttpclient-2.2.0 → geventhttpclient-2.3.0}/setup.py +0 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
This software is licensed under the MIT License.
|
|
2
|
+
|
|
3
|
+
Copyright Antonin Amand <antonin.amand@gmail.com>, 2018.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a
|
|
6
|
+
copy of this software and associated documentation files (the
|
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to permit
|
|
10
|
+
persons to whom the Software is furnished to do so, subject to the
|
|
11
|
+
following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included
|
|
14
|
+
in all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
17
|
+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
|
|
19
|
+
NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
|
20
|
+
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
21
|
+
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
22
|
+
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
include
|
|
1
|
+
include geventhttpclient/*.py
|
|
2
2
|
include ext/*.c
|
|
3
3
|
include ext/*.h
|
|
4
4
|
recursive-include llhttp *.h *.c LICENSE-MIT
|
|
5
|
-
recursive-
|
|
5
|
+
recursive-exclude tests *
|
|
6
6
|
global-exclude __pycache__
|
|
7
7
|
global-exclude *.py[co]
|
|
8
8
|
include pyproject.toml
|
|
9
|
-
include LICENSE.txt
|
|
10
9
|
include README.md
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: geventhttpclient
|
|
3
|
+
Version: 2.3.0
|
|
4
|
+
Summary: HTTP client library for gevent
|
|
5
|
+
Author-email: Antonin Amand <antonin.amand@gmail.com>
|
|
6
|
+
License: This software is licensed under the MIT License.
|
|
7
|
+
|
|
8
|
+
Copyright Antonin Amand <antonin.amand@gmail.com>, 2018.
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a
|
|
11
|
+
copy of this software and associated documentation files (the
|
|
12
|
+
"Software"), to deal in the Software without restriction, including
|
|
13
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
14
|
+
distribute, sublicense, and/or sell copies of the Software, and to permit
|
|
15
|
+
persons to whom the Software is furnished to do so, subject to the
|
|
16
|
+
following conditions:
|
|
17
|
+
|
|
18
|
+
The above copyright notice and this permission notice shall be included
|
|
19
|
+
in all copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
22
|
+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
23
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
|
|
24
|
+
NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
|
25
|
+
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
26
|
+
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
27
|
+
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
28
|
+
|
|
29
|
+
Project-URL: homepage, http://github.com/geventhttpclient/geventhttpclient
|
|
30
|
+
Project-URL: issues, http://github.com/geventhttpclient/geventhttpclient/issues
|
|
31
|
+
Project-URL: download, https://pypi.org/project/geventhttpclient/#files
|
|
32
|
+
Keywords: http,gevent,client
|
|
33
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
34
|
+
Classifier: Intended Audience :: Developers
|
|
35
|
+
Classifier: Intended Audience :: Information Technology
|
|
36
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
37
|
+
Classifier: Operating System :: OS Independent
|
|
38
|
+
Classifier: Programming Language :: Python :: 3
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
40
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
41
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
42
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
43
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
44
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
45
|
+
Requires-Python: >=3.9
|
|
46
|
+
Description-Content-Type: text/markdown
|
|
47
|
+
License-File: LICENSE-MIT
|
|
48
|
+
Requires-Dist: gevent
|
|
49
|
+
Requires-Dist: certifi
|
|
50
|
+
Requires-Dist: brotli
|
|
51
|
+
Requires-Dist: urllib3
|
|
52
|
+
Provides-Extra: dev
|
|
53
|
+
Requires-Dist: pytest; extra == "dev"
|
|
54
|
+
Requires-Dist: dpkt; extra == "dev"
|
|
55
|
+
Requires-Dist: requests; extra == "dev"
|
|
56
|
+
Provides-Extra: benchmarks
|
|
57
|
+
Requires-Dist: requests; extra == "benchmarks"
|
|
58
|
+
Requires-Dist: httpx; extra == "benchmarks"
|
|
59
|
+
Requires-Dist: urllib3; extra == "benchmarks"
|
|
60
|
+
Requires-Dist: httplib2; extra == "benchmarks"
|
|
61
|
+
Provides-Extra: examples
|
|
62
|
+
Requires-Dist: oauth2; extra == "examples"
|
|
63
|
+
|
|
64
|
+
[](https://github.com/geventhttpclient/geventhttpclient/actions)
|
|
65
|
+
[](https://pypi.org/project/geventhttpclient/)
|
|
66
|
+

|
|
67
|
+

|
|
68
|
+
|
|
69
|
+
# geventhttpclient
|
|
70
|
+
|
|
71
|
+
A [high performance](https://github.com/geventhttpclient/geventhttpclient/blob/master/README.md#Benchmarks),
|
|
72
|
+
concurrent HTTP client library for python using
|
|
73
|
+
[gevent](http://gevent.org).
|
|
74
|
+
|
|
75
|
+
`gevent.httplib` support for patching `http.client` was removed in
|
|
76
|
+
[gevent 1.0](https://github.com/surfly/gevent/commit/b45b83b1bc4de14e3c4859362825044b8e3df7d6),
|
|
77
|
+
`geventhttpclient` now provides that missing functionality.
|
|
78
|
+
|
|
79
|
+
`geventhttpclient` uses a fast [http parser](https://github.com/nodejs/llhttp),
|
|
80
|
+
written in C.
|
|
81
|
+
|
|
82
|
+
`geventhttpclient` has been specifically designed for high concurrency,
|
|
83
|
+
streaming and support HTTP 1.1 persistent connections. More generally it is
|
|
84
|
+
designed for efficiently pulling from REST APIs and streaming APIs
|
|
85
|
+
like Twitter's.
|
|
86
|
+
|
|
87
|
+
Safe SSL support is provided by default. `geventhttpclient` depends on
|
|
88
|
+
the certifi CA Bundle. This is the same CA Bundle which ships with the
|
|
89
|
+
Requests codebase, and is derived from Mozilla Firefox's canonical set.
|
|
90
|
+
|
|
91
|
+
Since version 2.3, `geventhttpclient` features a largely `requests`
|
|
92
|
+
compatible interface. It covers basic HTTP usage including cookie
|
|
93
|
+
management, form data encoding or decoding of compressed data,
|
|
94
|
+
but otherwise isn't as feature rich as the original `requests`. For
|
|
95
|
+
simple use-cases, it can serve as a drop-in replacement.
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
import geventhttpclient as requests
|
|
99
|
+
requests.get("https://github.com").text
|
|
100
|
+
requests.post("http://httpbingo.org/post", data="asdfasd").json()
|
|
101
|
+
|
|
102
|
+
from geventhttpclient import Session
|
|
103
|
+
s = Session()
|
|
104
|
+
s.get("http://httpbingo.org/headers").json()
|
|
105
|
+
s.get("https://github.com").content
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
This interface builds on top of the lower level `HTTPClient`.
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from geventhttpclient import HTTPClient
|
|
112
|
+
from geventhttpclient.url import URL
|
|
113
|
+
|
|
114
|
+
url = URL("http://gevent.org/")
|
|
115
|
+
client = HTTPClient(url.host)
|
|
116
|
+
response = client.get(url.request_uri)
|
|
117
|
+
response.status_code
|
|
118
|
+
body = response.read()
|
|
119
|
+
client.close()
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## httplib compatibility and monkey patch
|
|
123
|
+
|
|
124
|
+
`geventhttpclient.httplib` module contains classes for drop in
|
|
125
|
+
replacement of `http.client` connection and response objects.
|
|
126
|
+
If you use http.client directly you can replace the `httplib` imports
|
|
127
|
+
by `geventhttpclient.httplib`.
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
# from http.client import HTTPConnection
|
|
131
|
+
from geventhttpclient.httplib import HTTPConnection
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
If you use `httplib2`, `urllib` or `urllib2`; you can patch `httplib` to
|
|
135
|
+
use the wrappers from `geventhttpclient`. For `httplib2`, make sure you
|
|
136
|
+
patch before you import or the `super()` calls will fail.
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import geventhttpclient.httplib
|
|
140
|
+
geventhttpclient.httplib.patch()
|
|
141
|
+
|
|
142
|
+
import httplib2
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## High Concurrency
|
|
146
|
+
|
|
147
|
+
`HTTPClient` has a connection pool built in and is greenlet safe by design.
|
|
148
|
+
You can use the same instance among several greenlets. It is the low
|
|
149
|
+
level building block of this library.
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
import gevent.pool
|
|
153
|
+
import json
|
|
154
|
+
|
|
155
|
+
from geventhttpclient import HTTPClient
|
|
156
|
+
from geventhttpclient.url import URL
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
# go to http://developers.facebook.com/tools/explorer and copy the access token
|
|
160
|
+
TOKEN = "<MY_DEV_TOKEN>"
|
|
161
|
+
|
|
162
|
+
url = URL("https://graph.facebook.com/me/friends", params={"access_token": TOKEN})
|
|
163
|
+
|
|
164
|
+
# setting the concurrency to 10 allow to create 10 connections and
|
|
165
|
+
# reuse them.
|
|
166
|
+
client = HTTPClient.from_url(url, concurrency=10)
|
|
167
|
+
|
|
168
|
+
response = client.get(url.request_uri)
|
|
169
|
+
assert response.status_code == 200
|
|
170
|
+
|
|
171
|
+
# response comply to the read protocol. It passes the stream to
|
|
172
|
+
# the json parser as it's being read.
|
|
173
|
+
data = json.load(response)["data"]
|
|
174
|
+
|
|
175
|
+
def print_friend_username(client, friend_id):
|
|
176
|
+
friend_url = URL(f"/{friend_id}", params={"access_token": TOKEN})
|
|
177
|
+
# the greenlet will block until a connection is available
|
|
178
|
+
response = client.get(friend_url.request_uri)
|
|
179
|
+
assert response.status_code == 200
|
|
180
|
+
friend = json.load(response)
|
|
181
|
+
if "username" in friend:
|
|
182
|
+
print(f"{friend['username']}: {friend['name']}")
|
|
183
|
+
else:
|
|
184
|
+
print(f"{friend['name']} has no username.")
|
|
185
|
+
|
|
186
|
+
# allow to run 20 greenlet at a time, this is more than concurrency
|
|
187
|
+
# of the http client but isn't a problem since the client has its own
|
|
188
|
+
# connection pool.
|
|
189
|
+
pool = gevent.pool.Pool(20)
|
|
190
|
+
for item in data:
|
|
191
|
+
friend_id = item["id"]
|
|
192
|
+
pool.spawn(print_friend_username, client, friend_id)
|
|
193
|
+
|
|
194
|
+
pool.join()
|
|
195
|
+
client.close()
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
## Streaming
|
|
199
|
+
|
|
200
|
+
`geventhttpclient` supports streaming. Response objects have a `read(n)` and
|
|
201
|
+
`readline()` method that read the stream incrementally.
|
|
202
|
+
See [examples/twitter_streaming.py](https://github.com/geventhttpclient/geventhttpclient/blob/master/examples/twitter_streaming.py)
|
|
203
|
+
for pulling twitter stream API.
|
|
204
|
+
|
|
205
|
+
Here is an example on how to download a big file chunk by chunk to save memory:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
from geventhttpclient import HTTPClient, URL
|
|
209
|
+
|
|
210
|
+
url = URL("http://127.0.0.1:80/100.dat")
|
|
211
|
+
client = HTTPClient.from_url(url)
|
|
212
|
+
response = client.get(url.query_string)
|
|
213
|
+
assert response.status_code == 200
|
|
214
|
+
|
|
215
|
+
CHUNK_SIZE = 1024 * 16 # 16KB
|
|
216
|
+
with open("/tmp/100.dat", "w") as f:
|
|
217
|
+
data = response.read(CHUNK_SIZE)
|
|
218
|
+
while data:
|
|
219
|
+
f.write(data)
|
|
220
|
+
data = response.read(CHUNK_SIZE)
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## Benchmarks
|
|
224
|
+
|
|
225
|
+
The benchmark runs 10000 `GET` requests against a local nginx server in the default
|
|
226
|
+
configuration with a concurrency of 10. See `benchmarks` folder. The requests per
|
|
227
|
+
second for a couple of popular clients is given in the table below. Please read
|
|
228
|
+
[benchmarks/README.md](https://github.com/geventhttpclient/geventhttpclient/blob/master/benchmarks/README.md)
|
|
229
|
+
for more details. Also note, [HTTPX](https://www.python-httpx.org/) is better be
|
|
230
|
+
used with `asyncio`, not `gevent`.
|
|
231
|
+
|
|
232
|
+
| HTTP Client | RPS |
|
|
233
|
+
|--------------------|--------|
|
|
234
|
+
| GeventHTTPClient | 7268.9 |
|
|
235
|
+
| Httplib2 (patched) | 2323.9 |
|
|
236
|
+
| Urllib3 | 2242.5 |
|
|
237
|
+
| Requests | 1046.1 |
|
|
238
|
+
| Httpx | 770.3 |
|
|
239
|
+
|
|
240
|
+
*Linux(x86_64), Python 3.11.6 @ Intel i7-7560U*
|
|
241
|
+
|
|
242
|
+
## License
|
|
243
|
+
|
|
244
|
+
This package is distributed under the [MIT license](https://github.com/geventhttpclient/geventhttpclient/blob/master/LICENSE-MIT).
|
|
245
|
+
Previous versions of geventhttpclient used `http_parser.c`, which in turn was
|
|
246
|
+
based on `http/ngx_http_parse.c` from [NGINX](https://nginx.org), copyright Igor
|
|
247
|
+
Sysoev, Joyent, Inc., and other Node contributors. For more information, see
|
|
248
|
+
http://github.com/joyent/http-parser
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
[](https://github.com/geventhttpclient/geventhttpclient/actions)
|
|
2
|
+
[](https://pypi.org/project/geventhttpclient/)
|
|
3
|
+

|
|
4
|
+

|
|
5
|
+
|
|
6
|
+
# geventhttpclient
|
|
7
|
+
|
|
8
|
+
A [high performance](https://github.com/geventhttpclient/geventhttpclient/blob/master/README.md#Benchmarks),
|
|
9
|
+
concurrent HTTP client library for python using
|
|
10
|
+
[gevent](http://gevent.org).
|
|
11
|
+
|
|
12
|
+
`gevent.httplib` support for patching `http.client` was removed in
|
|
13
|
+
[gevent 1.0](https://github.com/surfly/gevent/commit/b45b83b1bc4de14e3c4859362825044b8e3df7d6),
|
|
14
|
+
`geventhttpclient` now provides that missing functionality.
|
|
15
|
+
|
|
16
|
+
`geventhttpclient` uses a fast [http parser](https://github.com/nodejs/llhttp),
|
|
17
|
+
written in C.
|
|
18
|
+
|
|
19
|
+
`geventhttpclient` has been specifically designed for high concurrency,
|
|
20
|
+
streaming and support HTTP 1.1 persistent connections. More generally it is
|
|
21
|
+
designed for efficiently pulling from REST APIs and streaming APIs
|
|
22
|
+
like Twitter's.
|
|
23
|
+
|
|
24
|
+
Safe SSL support is provided by default. `geventhttpclient` depends on
|
|
25
|
+
the certifi CA Bundle. This is the same CA Bundle which ships with the
|
|
26
|
+
Requests codebase, and is derived from Mozilla Firefox's canonical set.
|
|
27
|
+
|
|
28
|
+
Since version 2.3, `geventhttpclient` features a largely `requests`
|
|
29
|
+
compatible interface. It covers basic HTTP usage including cookie
|
|
30
|
+
management, form data encoding or decoding of compressed data,
|
|
31
|
+
but otherwise isn't as feature rich as the original `requests`. For
|
|
32
|
+
simple use-cases, it can serve as a drop-in replacement.
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
import geventhttpclient as requests
|
|
36
|
+
requests.get("https://github.com").text
|
|
37
|
+
requests.post("http://httpbingo.org/post", data="asdfasd").json()
|
|
38
|
+
|
|
39
|
+
from geventhttpclient import Session
|
|
40
|
+
s = Session()
|
|
41
|
+
s.get("http://httpbingo.org/headers").json()
|
|
42
|
+
s.get("https://github.com").content
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
This interface builds on top of the lower level `HTTPClient`.
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from geventhttpclient import HTTPClient
|
|
49
|
+
from geventhttpclient.url import URL
|
|
50
|
+
|
|
51
|
+
url = URL("http://gevent.org/")
|
|
52
|
+
client = HTTPClient(url.host)
|
|
53
|
+
response = client.get(url.request_uri)
|
|
54
|
+
response.status_code
|
|
55
|
+
body = response.read()
|
|
56
|
+
client.close()
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## httplib compatibility and monkey patch
|
|
60
|
+
|
|
61
|
+
`geventhttpclient.httplib` module contains classes for drop in
|
|
62
|
+
replacement of `http.client` connection and response objects.
|
|
63
|
+
If you use http.client directly you can replace the `httplib` imports
|
|
64
|
+
by `geventhttpclient.httplib`.
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
# from http.client import HTTPConnection
|
|
68
|
+
from geventhttpclient.httplib import HTTPConnection
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
If you use `httplib2`, `urllib` or `urllib2`; you can patch `httplib` to
|
|
72
|
+
use the wrappers from `geventhttpclient`. For `httplib2`, make sure you
|
|
73
|
+
patch before you import or the `super()` calls will fail.
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
import geventhttpclient.httplib
|
|
77
|
+
geventhttpclient.httplib.patch()
|
|
78
|
+
|
|
79
|
+
import httplib2
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## High Concurrency
|
|
83
|
+
|
|
84
|
+
`HTTPClient` has a connection pool built in and is greenlet safe by design.
|
|
85
|
+
You can use the same instance among several greenlets. It is the low
|
|
86
|
+
level building block of this library.
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
import gevent.pool
|
|
90
|
+
import json
|
|
91
|
+
|
|
92
|
+
from geventhttpclient import HTTPClient
|
|
93
|
+
from geventhttpclient.url import URL
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# go to http://developers.facebook.com/tools/explorer and copy the access token
|
|
97
|
+
TOKEN = "<MY_DEV_TOKEN>"
|
|
98
|
+
|
|
99
|
+
url = URL("https://graph.facebook.com/me/friends", params={"access_token": TOKEN})
|
|
100
|
+
|
|
101
|
+
# setting the concurrency to 10 allow to create 10 connections and
|
|
102
|
+
# reuse them.
|
|
103
|
+
client = HTTPClient.from_url(url, concurrency=10)
|
|
104
|
+
|
|
105
|
+
response = client.get(url.request_uri)
|
|
106
|
+
assert response.status_code == 200
|
|
107
|
+
|
|
108
|
+
# response comply to the read protocol. It passes the stream to
|
|
109
|
+
# the json parser as it's being read.
|
|
110
|
+
data = json.load(response)["data"]
|
|
111
|
+
|
|
112
|
+
def print_friend_username(client, friend_id):
|
|
113
|
+
friend_url = URL(f"/{friend_id}", params={"access_token": TOKEN})
|
|
114
|
+
# the greenlet will block until a connection is available
|
|
115
|
+
response = client.get(friend_url.request_uri)
|
|
116
|
+
assert response.status_code == 200
|
|
117
|
+
friend = json.load(response)
|
|
118
|
+
if "username" in friend:
|
|
119
|
+
print(f"{friend['username']}: {friend['name']}")
|
|
120
|
+
else:
|
|
121
|
+
print(f"{friend['name']} has no username.")
|
|
122
|
+
|
|
123
|
+
# allow to run 20 greenlet at a time, this is more than concurrency
|
|
124
|
+
# of the http client but isn't a problem since the client has its own
|
|
125
|
+
# connection pool.
|
|
126
|
+
pool = gevent.pool.Pool(20)
|
|
127
|
+
for item in data:
|
|
128
|
+
friend_id = item["id"]
|
|
129
|
+
pool.spawn(print_friend_username, client, friend_id)
|
|
130
|
+
|
|
131
|
+
pool.join()
|
|
132
|
+
client.close()
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## Streaming
|
|
136
|
+
|
|
137
|
+
`geventhttpclient` supports streaming. Response objects have a `read(n)` and
|
|
138
|
+
`readline()` method that read the stream incrementally.
|
|
139
|
+
See [examples/twitter_streaming.py](https://github.com/geventhttpclient/geventhttpclient/blob/master/examples/twitter_streaming.py)
|
|
140
|
+
for pulling twitter stream API.
|
|
141
|
+
|
|
142
|
+
Here is an example on how to download a big file chunk by chunk to save memory:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from geventhttpclient import HTTPClient, URL
|
|
146
|
+
|
|
147
|
+
url = URL("http://127.0.0.1:80/100.dat")
|
|
148
|
+
client = HTTPClient.from_url(url)
|
|
149
|
+
response = client.get(url.query_string)
|
|
150
|
+
assert response.status_code == 200
|
|
151
|
+
|
|
152
|
+
CHUNK_SIZE = 1024 * 16 # 16KB
|
|
153
|
+
with open("/tmp/100.dat", "w") as f:
|
|
154
|
+
data = response.read(CHUNK_SIZE)
|
|
155
|
+
while data:
|
|
156
|
+
f.write(data)
|
|
157
|
+
data = response.read(CHUNK_SIZE)
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
## Benchmarks
|
|
161
|
+
|
|
162
|
+
The benchmark runs 10000 `GET` requests against a local nginx server in the default
|
|
163
|
+
configuration with a concurrency of 10. See `benchmarks` folder. The requests per
|
|
164
|
+
second for a couple of popular clients is given in the table below. Please read
|
|
165
|
+
[benchmarks/README.md](https://github.com/geventhttpclient/geventhttpclient/blob/master/benchmarks/README.md)
|
|
166
|
+
for more details. Also note, [HTTPX](https://www.python-httpx.org/) is better be
|
|
167
|
+
used with `asyncio`, not `gevent`.
|
|
168
|
+
|
|
169
|
+
| HTTP Client | RPS |
|
|
170
|
+
|--------------------|--------|
|
|
171
|
+
| GeventHTTPClient | 7268.9 |
|
|
172
|
+
| Httplib2 (patched) | 2323.9 |
|
|
173
|
+
| Urllib3 | 2242.5 |
|
|
174
|
+
| Requests | 1046.1 |
|
|
175
|
+
| Httpx | 770.3 |
|
|
176
|
+
|
|
177
|
+
*Linux(x86_64), Python 3.11.6 @ Intel i7-7560U*
|
|
178
|
+
|
|
179
|
+
## License
|
|
180
|
+
|
|
181
|
+
This package is distributed under the [MIT license](https://github.com/geventhttpclient/geventhttpclient/blob/master/LICENSE-MIT).
|
|
182
|
+
Previous versions of geventhttpclient used `http_parser.c`, which in turn was
|
|
183
|
+
based on `http/ngx_http_parse.c` from [NGINX](https://nginx.org), copyright Igor
|
|
184
|
+
Sysoev, Joyent, Inc., and other Node contributors. For more information, see
|
|
185
|
+
http://github.com/joyent/http-parser
|
|
@@ -23,9 +23,9 @@ typedef struct {
|
|
|
23
23
|
static int on_message_begin(llhttp_t* parser)
|
|
24
24
|
{
|
|
25
25
|
int fail = 0;
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
PyHTTPResponseParser *self = (PyHTTPResponseParser*) parser->data;
|
|
27
|
+
PyObject* callable = PyObject_GetAttrString((PyObject*)self, "_on_message_begin");
|
|
28
|
+
if (callable) {
|
|
29
29
|
PyObject* result = PyObject_CallObject(callable, NULL);
|
|
30
30
|
PyObject* exception = PyErr_Occurred();
|
|
31
31
|
if (exception != NULL) {
|
|
@@ -45,8 +45,8 @@ static int on_message_complete(llhttp_t* parser)
|
|
|
45
45
|
int fail = 0;
|
|
46
46
|
PyHTTPResponseParser *self = (PyHTTPResponseParser*) parser->data;
|
|
47
47
|
self->should_keep_alive = llhttp_should_keep_alive(parser) ? KA_TRUE : KA_FALSE;
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
PyObject* callable = PyObject_GetAttrString((PyObject*)self, "_on_message_complete");
|
|
49
|
+
if (callable) {
|
|
50
50
|
PyObject* result = PyObject_CallObject(callable, NULL);
|
|
51
51
|
PyObject* exception = PyErr_Occurred();
|
|
52
52
|
if (exception != NULL) {
|
|
@@ -65,9 +65,9 @@ static int on_headers_complete(llhttp_t* parser)
|
|
|
65
65
|
{
|
|
66
66
|
/* 1 => skip body, 2 => upgrade, 0 => continue, -1 => error */
|
|
67
67
|
int skip_body = 0;
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
68
|
+
PyHTTPResponseParser *self = (PyHTTPResponseParser*) parser->data;
|
|
69
|
+
PyObject* callable = PyObject_GetAttrString((PyObject*)self, "_on_headers_complete");
|
|
70
|
+
if (callable) {
|
|
71
71
|
PyObject* result = PyObject_CallObject(callable, NULL);
|
|
72
72
|
PyObject* exception = PyErr_Occurred();
|
|
73
73
|
if (exception != NULL) {
|
|
@@ -84,9 +84,9 @@ static int on_headers_complete(llhttp_t* parser)
|
|
|
84
84
|
static int on_http_data_cb(llhttp_t* parser, const char *at, size_t length, const char * python_cb)
|
|
85
85
|
{
|
|
86
86
|
int fail = 0;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
87
|
+
PyHTTPResponseParser *self = (PyHTTPResponseParser*) parser->data;
|
|
88
|
+
PyObject* callable = PyObject_GetAttrString((PyObject*)self, python_cb);
|
|
89
|
+
if (callable) {
|
|
90
90
|
PyObject* args = Py_BuildValue("(s#)", at, length);
|
|
91
91
|
PyObject* result = PyObject_CallObject(callable, args);
|
|
92
92
|
PyObject* exception = PyErr_Occurred();
|
|
@@ -100,7 +100,7 @@ static int on_http_data_cb(llhttp_t* parser, const char *at, size_t length, cons
|
|
|
100
100
|
Py_DECREF(callable);
|
|
101
101
|
Py_DECREF(args);
|
|
102
102
|
}
|
|
103
|
-
return fail;
|
|
103
|
+
return fail;
|
|
104
104
|
}
|
|
105
105
|
|
|
106
106
|
static int on_status(llhttp_t* parser, const char *at, size_t length)
|
|
@@ -121,9 +121,9 @@ static int on_header_value(llhttp_t* parser, const char *at, size_t length)
|
|
|
121
121
|
static int on_body(llhttp_t* parser, const char *at, size_t length)
|
|
122
122
|
{
|
|
123
123
|
int fail = 0;
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
124
|
+
PyHTTPResponseParser *self = (PyHTTPResponseParser*) parser->data;
|
|
125
|
+
PyObject* callable = PyObject_GetAttrString((PyObject*)self, "_on_body");
|
|
126
|
+
if (callable) {
|
|
127
127
|
PyObject* bytearray = PyByteArray_FromStringAndSize(at, length);
|
|
128
128
|
PyObject* result = PyObject_CallFunctionObjArgs(
|
|
129
129
|
callable, bytearray, NULL);
|
|
@@ -269,7 +269,7 @@ PyHTTPResponseParser_should_keep_alive(PyHTTPResponseParser* self)
|
|
|
269
269
|
if (self->error != HPE_OK) {
|
|
270
270
|
Py_RETURN_FALSE;
|
|
271
271
|
}
|
|
272
|
-
int should_keep_alive;
|
|
272
|
+
int should_keep_alive = 0;
|
|
273
273
|
switch (self->should_keep_alive) {
|
|
274
274
|
case KA_INCOMPLETE:
|
|
275
275
|
should_keep_alive = llhttp_should_keep_alive(self->parser);
|
|
@@ -305,7 +305,7 @@ static PyMethodDef PyHTTPResponseParser_methods[] = {
|
|
|
305
305
|
"Get remaining content length to read"},
|
|
306
306
|
{"should_keep_alive", (PyCFunction)PyHTTPResponseParser_should_keep_alive,
|
|
307
307
|
METH_NOARGS,
|
|
308
|
-
"Tell
|
|
308
|
+
"Tell whether the connection should stay connected (HTTP 1.1)"},
|
|
309
309
|
{"parser_failed", (PyCFunction)PyHTTPResponseParser_parser_failed,
|
|
310
310
|
METH_NOARGS,
|
|
311
311
|
"Tell if parser have failed."},
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# package
|
|
2
|
+
|
|
3
|
+
__version__ = "2.3.0" # dont forget to update version in pyproject.toml as well
|
|
4
|
+
|
|
5
|
+
from geventhttpclient.api import delete, get, head, options, patch, post, put, request
|
|
6
|
+
from geventhttpclient.client import HTTPClient
|
|
7
|
+
from geventhttpclient.requests import Session
|
|
8
|
+
from geventhttpclient.url import URL
|
|
9
|
+
from geventhttpclient.useragent import UserAgent
|