appier 1.31.4__py2.py3-none-any.whl → 1.32.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- appier/__init__.py +333 -52
- appier/amqp.py +29 -30
- appier/api.py +214 -212
- appier/asgi.py +54 -55
- appier/async_neo.py +46 -35
- appier/async_old.py +55 -42
- appier/asynchronous.py +7 -13
- appier/base.py +1762 -1429
- appier/bus.py +51 -52
- appier/cache.py +99 -84
- appier/common.py +9 -11
- appier/component.py +17 -19
- appier/compress.py +25 -28
- appier/config.py +96 -73
- appier/controller.py +9 -15
- appier/crypt.py +25 -21
- appier/data.py +73 -57
- appier/defines.py +191 -226
- appier/exceptions.py +103 -63
- appier/execution.py +94 -88
- appier/export.py +90 -88
- appier/extra.py +6 -13
- appier/extra_neo.py +8 -11
- appier/extra_old.py +18 -16
- appier/geo.py +57 -47
- appier/git.py +101 -90
- appier/graph.py +23 -24
- appier/http.py +520 -398
- appier/legacy.py +373 -180
- appier/log.py +90 -97
- appier/meta.py +42 -42
- appier/mock.py +32 -34
- appier/model.py +793 -681
- appier/model_a.py +208 -183
- appier/mongo.py +183 -107
- appier/observer.py +39 -31
- appier/part.py +23 -24
- appier/preferences.py +44 -47
- appier/queuing.py +78 -96
- appier/redisdb.py +40 -35
- appier/request.py +227 -175
- appier/scheduler.py +13 -18
- appier/serialize.py +37 -31
- appier/session.py +161 -147
- appier/settings.py +2 -11
- appier/smtp.py +53 -49
- appier/storage.py +39 -33
- appier/structures.py +50 -45
- appier/test/__init__.py +2 -11
- appier/test/base.py +111 -108
- appier/test/cache.py +28 -35
- appier/test/config.py +10 -19
- appier/test/crypt.py +3 -12
- appier/test/data.py +3 -12
- appier/test/exceptions.py +8 -17
- appier/test/export.py +16 -33
- appier/test/graph.py +27 -60
- appier/test/http.py +42 -54
- appier/test/legacy.py +20 -30
- appier/test/log.py +14 -35
- appier/test/mock.py +27 -123
- appier/test/model.py +79 -91
- appier/test/part.py +5 -14
- appier/test/preferences.py +5 -13
- appier/test/queuing.py +29 -37
- appier/test/request.py +61 -73
- appier/test/serialize.py +12 -23
- appier/test/session.py +10 -19
- appier/test/smtp.py +8 -14
- appier/test/structures.py +20 -24
- appier/test/typesf.py +14 -28
- appier/test/util.py +480 -438
- appier/typesf.py +251 -171
- appier/util.py +578 -407
- appier/validation.py +280 -143
- {appier-1.31.4.dist-info → appier-1.32.0.dist-info}/METADATA +6 -1
- appier-1.32.0.dist-info/RECORD +86 -0
- appier-1.31.4.dist-info/RECORD +0 -86
- {appier-1.31.4.dist-info → appier-1.32.0.dist-info}/LICENSE +0 -0
- {appier-1.31.4.dist-info → appier-1.32.0.dist-info}/WHEEL +0 -0
- {appier-1.31.4.dist-info → appier-1.32.0.dist-info}/top_level.txt +0 -0
appier/http.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
# -*- coding: utf-8 -*-
|
|
3
3
|
|
|
4
4
|
# Hive Appier Framework
|
|
5
|
-
# Copyright (c) 2008-
|
|
5
|
+
# Copyright (c) 2008-2024 Hive Solutions Lda.
|
|
6
6
|
#
|
|
7
7
|
# This file is part of Hive Appier Framework.
|
|
8
8
|
#
|
|
@@ -22,16 +22,7 @@
|
|
|
22
22
|
__author__ = "João Magalhães <joamag@hive.pt>"
|
|
23
23
|
""" The author(s) of the module """
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
""" The version of the module """
|
|
27
|
-
|
|
28
|
-
__revision__ = "$LastChangedRevision$"
|
|
29
|
-
""" The revision number of the module """
|
|
30
|
-
|
|
31
|
-
__date__ = "$LastChangedDate$"
|
|
32
|
-
""" The last change date of the module """
|
|
33
|
-
|
|
34
|
-
__copyright__ = "Copyright (c) 2008-2022 Hive Solutions Lda."
|
|
25
|
+
__copyright__ = "Copyright (c) 2008-2024 Hive Solutions Lda."
|
|
35
26
|
""" The copyright for the module """
|
|
36
27
|
|
|
37
28
|
__license__ = "Apache License, Version 2.0"
|
|
@@ -75,17 +66,20 @@ ACCESS_LOCK = threading.RLock()
|
|
|
75
66
|
""" Global access lock used for locking global operations
|
|
76
67
|
that require thread safety under the HTTP infra-structure """
|
|
77
68
|
|
|
78
|
-
|
|
69
|
+
|
|
70
|
+
def file_g(path, chunk=40960):
|
|
79
71
|
yield os.path.getsize(path)
|
|
80
72
|
file = open(path, "rb")
|
|
81
73
|
try:
|
|
82
74
|
while True:
|
|
83
75
|
data = file.read(chunk)
|
|
84
|
-
if not data:
|
|
76
|
+
if not data:
|
|
77
|
+
break
|
|
85
78
|
yield data
|
|
86
79
|
finally:
|
|
87
80
|
file.close()
|
|
88
81
|
|
|
82
|
+
|
|
89
83
|
def get_f(*args, **kwargs):
|
|
90
84
|
name = kwargs.pop("name", "default")
|
|
91
85
|
kwargs["handle"] = kwargs.get("handle", True)
|
|
@@ -96,160 +90,171 @@ def get_f(*args, **kwargs):
|
|
|
96
90
|
file_tuple = util.FileTuple((name, mime, data))
|
|
97
91
|
return typesf.File(file_tuple)
|
|
98
92
|
|
|
93
|
+
|
|
99
94
|
def get(
|
|
100
95
|
url,
|
|
101
|
-
params
|
|
102
|
-
headers
|
|
103
|
-
handle
|
|
104
|
-
silent
|
|
105
|
-
redirect
|
|
106
|
-
timeout
|
|
107
|
-
auth_callback
|
|
96
|
+
params=None,
|
|
97
|
+
headers=None,
|
|
98
|
+
handle=None,
|
|
99
|
+
silent=None,
|
|
100
|
+
redirect=None,
|
|
101
|
+
timeout=None,
|
|
102
|
+
auth_callback=None,
|
|
108
103
|
**kwargs
|
|
109
104
|
):
|
|
110
105
|
return _method(
|
|
111
106
|
_get,
|
|
112
107
|
url,
|
|
113
|
-
params
|
|
114
|
-
headers
|
|
115
|
-
handle
|
|
116
|
-
silent
|
|
117
|
-
redirect
|
|
118
|
-
timeout
|
|
119
|
-
auth_callback
|
|
108
|
+
params=params,
|
|
109
|
+
headers=headers,
|
|
110
|
+
handle=handle,
|
|
111
|
+
silent=silent,
|
|
112
|
+
redirect=redirect,
|
|
113
|
+
timeout=timeout,
|
|
114
|
+
auth_callback=auth_callback,
|
|
120
115
|
**kwargs
|
|
121
116
|
)
|
|
122
117
|
|
|
118
|
+
|
|
123
119
|
def post(
|
|
124
120
|
url,
|
|
125
|
-
params
|
|
126
|
-
data
|
|
127
|
-
data_j
|
|
128
|
-
data_m
|
|
129
|
-
headers
|
|
130
|
-
mime
|
|
131
|
-
handle
|
|
132
|
-
silent
|
|
133
|
-
redirect
|
|
134
|
-
timeout
|
|
135
|
-
auth_callback
|
|
121
|
+
params=None,
|
|
122
|
+
data=None,
|
|
123
|
+
data_j=None,
|
|
124
|
+
data_m=None,
|
|
125
|
+
headers=None,
|
|
126
|
+
mime=None,
|
|
127
|
+
handle=None,
|
|
128
|
+
silent=None,
|
|
129
|
+
redirect=None,
|
|
130
|
+
timeout=None,
|
|
131
|
+
auth_callback=None,
|
|
136
132
|
**kwargs
|
|
137
133
|
):
|
|
138
134
|
return _method(
|
|
139
135
|
_post,
|
|
140
136
|
url,
|
|
141
|
-
params
|
|
142
|
-
data
|
|
143
|
-
data_j
|
|
144
|
-
data_m
|
|
145
|
-
headers
|
|
146
|
-
mime
|
|
147
|
-
handle
|
|
148
|
-
silent
|
|
149
|
-
redirect
|
|
150
|
-
timeout
|
|
151
|
-
auth_callback
|
|
137
|
+
params=params,
|
|
138
|
+
data=data,
|
|
139
|
+
data_j=data_j,
|
|
140
|
+
data_m=data_m,
|
|
141
|
+
headers=headers,
|
|
142
|
+
mime=mime,
|
|
143
|
+
handle=handle,
|
|
144
|
+
silent=silent,
|
|
145
|
+
redirect=redirect,
|
|
146
|
+
timeout=timeout,
|
|
147
|
+
auth_callback=auth_callback,
|
|
152
148
|
**kwargs
|
|
153
149
|
)
|
|
154
150
|
|
|
151
|
+
|
|
155
152
|
def put(
|
|
156
153
|
url,
|
|
157
|
-
params
|
|
158
|
-
data
|
|
159
|
-
data_j
|
|
160
|
-
data_m
|
|
161
|
-
headers
|
|
162
|
-
mime
|
|
163
|
-
handle
|
|
164
|
-
silent
|
|
165
|
-
redirect
|
|
166
|
-
timeout
|
|
167
|
-
auth_callback
|
|
154
|
+
params=None,
|
|
155
|
+
data=None,
|
|
156
|
+
data_j=None,
|
|
157
|
+
data_m=None,
|
|
158
|
+
headers=None,
|
|
159
|
+
mime=None,
|
|
160
|
+
handle=None,
|
|
161
|
+
silent=None,
|
|
162
|
+
redirect=None,
|
|
163
|
+
timeout=None,
|
|
164
|
+
auth_callback=None,
|
|
168
165
|
**kwargs
|
|
169
166
|
):
|
|
170
167
|
return _method(
|
|
171
168
|
_put,
|
|
172
169
|
url,
|
|
173
|
-
params
|
|
174
|
-
data
|
|
175
|
-
data_j
|
|
176
|
-
data_m
|
|
177
|
-
headers
|
|
178
|
-
mime
|
|
179
|
-
handle
|
|
180
|
-
silent
|
|
181
|
-
redirect
|
|
182
|
-
timeout
|
|
183
|
-
auth_callback
|
|
170
|
+
params=params,
|
|
171
|
+
data=data,
|
|
172
|
+
data_j=data_j,
|
|
173
|
+
data_m=data_m,
|
|
174
|
+
headers=headers,
|
|
175
|
+
mime=mime,
|
|
176
|
+
handle=handle,
|
|
177
|
+
silent=silent,
|
|
178
|
+
redirect=redirect,
|
|
179
|
+
timeout=timeout,
|
|
180
|
+
auth_callback=auth_callback,
|
|
184
181
|
**kwargs
|
|
185
182
|
)
|
|
186
183
|
|
|
184
|
+
|
|
187
185
|
def delete(
|
|
188
186
|
url,
|
|
189
|
-
params
|
|
190
|
-
headers
|
|
191
|
-
handle
|
|
192
|
-
silent
|
|
193
|
-
redirect
|
|
194
|
-
timeout
|
|
195
|
-
auth_callback
|
|
187
|
+
params=None,
|
|
188
|
+
headers=None,
|
|
189
|
+
handle=None,
|
|
190
|
+
silent=None,
|
|
191
|
+
redirect=None,
|
|
192
|
+
timeout=None,
|
|
193
|
+
auth_callback=None,
|
|
196
194
|
**kwargs
|
|
197
195
|
):
|
|
198
196
|
return _method(
|
|
199
197
|
_delete,
|
|
200
198
|
url,
|
|
201
|
-
params
|
|
202
|
-
headers
|
|
203
|
-
handle
|
|
204
|
-
silent
|
|
205
|
-
redirect
|
|
206
|
-
timeout
|
|
207
|
-
auth_callback
|
|
199
|
+
params=params,
|
|
200
|
+
headers=headers,
|
|
201
|
+
handle=handle,
|
|
202
|
+
silent=silent,
|
|
203
|
+
redirect=redirect,
|
|
204
|
+
timeout=timeout,
|
|
205
|
+
auth_callback=auth_callback,
|
|
208
206
|
**kwargs
|
|
209
207
|
)
|
|
210
208
|
|
|
209
|
+
|
|
211
210
|
def patch(
|
|
212
211
|
url,
|
|
213
|
-
params
|
|
214
|
-
data
|
|
215
|
-
data_j
|
|
216
|
-
data_m
|
|
217
|
-
headers
|
|
218
|
-
mime
|
|
219
|
-
handle
|
|
220
|
-
silent
|
|
221
|
-
redirect
|
|
222
|
-
timeout
|
|
223
|
-
auth_callback
|
|
212
|
+
params=None,
|
|
213
|
+
data=None,
|
|
214
|
+
data_j=None,
|
|
215
|
+
data_m=None,
|
|
216
|
+
headers=None,
|
|
217
|
+
mime=None,
|
|
218
|
+
handle=None,
|
|
219
|
+
silent=None,
|
|
220
|
+
redirect=None,
|
|
221
|
+
timeout=None,
|
|
222
|
+
auth_callback=None,
|
|
224
223
|
**kwargs
|
|
225
224
|
):
|
|
226
225
|
return _method(
|
|
227
226
|
_patch,
|
|
228
227
|
url,
|
|
229
|
-
params
|
|
230
|
-
data
|
|
231
|
-
data_j
|
|
232
|
-
data_m
|
|
233
|
-
headers
|
|
234
|
-
mime
|
|
235
|
-
handle
|
|
236
|
-
silent
|
|
237
|
-
redirect
|
|
238
|
-
timeout
|
|
239
|
-
auth_callback
|
|
228
|
+
params=params,
|
|
229
|
+
data=data,
|
|
230
|
+
data_j=data_j,
|
|
231
|
+
data_m=data_m,
|
|
232
|
+
headers=headers,
|
|
233
|
+
mime=mime,
|
|
234
|
+
handle=handle,
|
|
235
|
+
silent=silent,
|
|
236
|
+
redirect=redirect,
|
|
237
|
+
timeout=timeout,
|
|
238
|
+
auth_callback=auth_callback,
|
|
240
239
|
**kwargs
|
|
241
240
|
)
|
|
242
241
|
|
|
243
|
-
|
|
244
|
-
|
|
242
|
+
|
|
243
|
+
def basic_auth(username, password=None):
|
|
244
|
+
if not password:
|
|
245
|
+
password = username
|
|
245
246
|
authorization = _authorization(username, password)
|
|
246
247
|
return "Basic %s" % authorization
|
|
247
248
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
if
|
|
249
|
+
|
|
250
|
+
def _try_auth(auth_callback, params, headers=None):
|
|
251
|
+
if not auth_callback:
|
|
252
|
+
raise
|
|
253
|
+
if headers == None:
|
|
254
|
+
headers = dict()
|
|
251
255
|
auth_callback(params, headers)
|
|
252
256
|
|
|
257
|
+
|
|
253
258
|
def _method(method, *args, **kwargs):
|
|
254
259
|
try:
|
|
255
260
|
auth_callback = kwargs.pop("auth_callback", None)
|
|
@@ -258,7 +263,8 @@ def _method(method, *args, **kwargs):
|
|
|
258
263
|
try:
|
|
259
264
|
params = kwargs.get("params", None)
|
|
260
265
|
headers = kwargs.get("headers", None)
|
|
261
|
-
if not error.code in AUTH_ERRORS
|
|
266
|
+
if not error.code in AUTH_ERRORS:
|
|
267
|
+
raise
|
|
262
268
|
_try_auth(auth_callback, params, headers)
|
|
263
269
|
result = method(*args, **kwargs)
|
|
264
270
|
except legacy.HTTPError as error:
|
|
@@ -267,167 +273,181 @@ def _method(method, *args, **kwargs):
|
|
|
267
273
|
|
|
268
274
|
return result
|
|
269
275
|
|
|
276
|
+
|
|
270
277
|
def _get(
|
|
271
278
|
url,
|
|
272
|
-
params
|
|
273
|
-
headers
|
|
274
|
-
handle
|
|
275
|
-
silent
|
|
276
|
-
redirect
|
|
277
|
-
timeout
|
|
279
|
+
params=None,
|
|
280
|
+
headers=None,
|
|
281
|
+
handle=None,
|
|
282
|
+
silent=None,
|
|
283
|
+
redirect=None,
|
|
284
|
+
timeout=None,
|
|
278
285
|
**kwargs
|
|
279
286
|
):
|
|
280
287
|
return _method_empty(
|
|
281
288
|
"GET",
|
|
282
289
|
url,
|
|
283
|
-
params
|
|
284
|
-
headers
|
|
285
|
-
handle
|
|
286
|
-
silent
|
|
287
|
-
redirect
|
|
288
|
-
timeout
|
|
290
|
+
params=params,
|
|
291
|
+
headers=headers,
|
|
292
|
+
handle=handle,
|
|
293
|
+
silent=silent,
|
|
294
|
+
redirect=redirect,
|
|
295
|
+
timeout=timeout,
|
|
289
296
|
**kwargs
|
|
290
297
|
)
|
|
291
298
|
|
|
299
|
+
|
|
292
300
|
def _post(
|
|
293
301
|
url,
|
|
294
|
-
params
|
|
295
|
-
data
|
|
296
|
-
data_j
|
|
297
|
-
data_m
|
|
298
|
-
headers
|
|
299
|
-
mime
|
|
300
|
-
handle
|
|
301
|
-
silent
|
|
302
|
-
redirect
|
|
303
|
-
timeout
|
|
302
|
+
params=None,
|
|
303
|
+
data=None,
|
|
304
|
+
data_j=None,
|
|
305
|
+
data_m=None,
|
|
306
|
+
headers=None,
|
|
307
|
+
mime=None,
|
|
308
|
+
handle=None,
|
|
309
|
+
silent=None,
|
|
310
|
+
redirect=None,
|
|
311
|
+
timeout=None,
|
|
304
312
|
**kwargs
|
|
305
313
|
):
|
|
306
314
|
return _method_payload(
|
|
307
315
|
"POST",
|
|
308
316
|
url,
|
|
309
|
-
params
|
|
310
|
-
data
|
|
311
|
-
data_j
|
|
312
|
-
data_m
|
|
313
|
-
headers
|
|
314
|
-
mime
|
|
315
|
-
handle
|
|
316
|
-
silent
|
|
317
|
-
redirect
|
|
318
|
-
timeout
|
|
317
|
+
params=params,
|
|
318
|
+
data=data,
|
|
319
|
+
data_j=data_j,
|
|
320
|
+
data_m=data_m,
|
|
321
|
+
headers=headers,
|
|
322
|
+
mime=mime,
|
|
323
|
+
handle=handle,
|
|
324
|
+
silent=silent,
|
|
325
|
+
redirect=redirect,
|
|
326
|
+
timeout=timeout,
|
|
319
327
|
**kwargs
|
|
320
328
|
)
|
|
321
329
|
|
|
330
|
+
|
|
322
331
|
def _put(
|
|
323
332
|
url,
|
|
324
|
-
params
|
|
325
|
-
data
|
|
326
|
-
data_j
|
|
327
|
-
data_m
|
|
328
|
-
headers
|
|
329
|
-
mime
|
|
330
|
-
handle
|
|
331
|
-
silent
|
|
332
|
-
redirect
|
|
333
|
-
timeout
|
|
333
|
+
params=None,
|
|
334
|
+
data=None,
|
|
335
|
+
data_j=None,
|
|
336
|
+
data_m=None,
|
|
337
|
+
headers=None,
|
|
338
|
+
mime=None,
|
|
339
|
+
handle=None,
|
|
340
|
+
silent=None,
|
|
341
|
+
redirect=None,
|
|
342
|
+
timeout=None,
|
|
334
343
|
**kwargs
|
|
335
344
|
):
|
|
336
345
|
return _method_payload(
|
|
337
346
|
"PUT",
|
|
338
347
|
url,
|
|
339
|
-
params
|
|
340
|
-
data
|
|
341
|
-
data_j
|
|
342
|
-
data_m
|
|
343
|
-
headers
|
|
344
|
-
mime
|
|
345
|
-
handle
|
|
346
|
-
silent
|
|
347
|
-
redirect
|
|
348
|
-
timeout
|
|
348
|
+
params=params,
|
|
349
|
+
data=data,
|
|
350
|
+
data_j=data_j,
|
|
351
|
+
data_m=data_m,
|
|
352
|
+
headers=headers,
|
|
353
|
+
mime=mime,
|
|
354
|
+
handle=handle,
|
|
355
|
+
silent=silent,
|
|
356
|
+
redirect=redirect,
|
|
357
|
+
timeout=timeout,
|
|
349
358
|
**kwargs
|
|
350
359
|
)
|
|
351
360
|
|
|
361
|
+
|
|
352
362
|
def _delete(
|
|
353
363
|
url,
|
|
354
|
-
params
|
|
355
|
-
headers
|
|
356
|
-
handle
|
|
357
|
-
silent
|
|
358
|
-
redirect
|
|
359
|
-
timeout
|
|
364
|
+
params=None,
|
|
365
|
+
headers=None,
|
|
366
|
+
handle=None,
|
|
367
|
+
silent=None,
|
|
368
|
+
redirect=None,
|
|
369
|
+
timeout=None,
|
|
360
370
|
**kwargs
|
|
361
371
|
):
|
|
362
372
|
return _method_empty(
|
|
363
373
|
"DELETE",
|
|
364
374
|
url,
|
|
365
|
-
params
|
|
366
|
-
headers
|
|
367
|
-
handle
|
|
368
|
-
silent
|
|
369
|
-
redirect
|
|
370
|
-
timeout
|
|
375
|
+
params=params,
|
|
376
|
+
headers=headers,
|
|
377
|
+
handle=handle,
|
|
378
|
+
silent=silent,
|
|
379
|
+
redirect=redirect,
|
|
380
|
+
timeout=timeout,
|
|
371
381
|
**kwargs
|
|
372
382
|
)
|
|
373
383
|
|
|
384
|
+
|
|
374
385
|
def _patch(
|
|
375
386
|
url,
|
|
376
|
-
params
|
|
377
|
-
data
|
|
378
|
-
data_j
|
|
379
|
-
data_m
|
|
380
|
-
headers
|
|
381
|
-
mime
|
|
382
|
-
handle
|
|
383
|
-
silent
|
|
384
|
-
redirect
|
|
385
|
-
timeout
|
|
387
|
+
params=None,
|
|
388
|
+
data=None,
|
|
389
|
+
data_j=None,
|
|
390
|
+
data_m=None,
|
|
391
|
+
headers=None,
|
|
392
|
+
mime=None,
|
|
393
|
+
handle=None,
|
|
394
|
+
silent=None,
|
|
395
|
+
redirect=None,
|
|
396
|
+
timeout=None,
|
|
386
397
|
**kwargs
|
|
387
398
|
):
|
|
388
399
|
return _method_payload(
|
|
389
400
|
"PATCH",
|
|
390
401
|
url,
|
|
391
|
-
params
|
|
392
|
-
data
|
|
393
|
-
data_j
|
|
394
|
-
data_m
|
|
395
|
-
headers
|
|
396
|
-
mime
|
|
397
|
-
handle
|
|
398
|
-
silent
|
|
399
|
-
redirect
|
|
400
|
-
timeout
|
|
402
|
+
params=params,
|
|
403
|
+
data=data,
|
|
404
|
+
data_j=data_j,
|
|
405
|
+
data_m=data_m,
|
|
406
|
+
headers=headers,
|
|
407
|
+
mime=mime,
|
|
408
|
+
handle=handle,
|
|
409
|
+
silent=silent,
|
|
410
|
+
redirect=redirect,
|
|
411
|
+
timeout=timeout,
|
|
401
412
|
**kwargs
|
|
402
413
|
)
|
|
403
414
|
|
|
415
|
+
|
|
404
416
|
def _method_empty(
|
|
405
417
|
name,
|
|
406
418
|
url,
|
|
407
|
-
params
|
|
408
|
-
headers
|
|
409
|
-
handle
|
|
410
|
-
silent
|
|
411
|
-
redirect
|
|
412
|
-
timeout
|
|
419
|
+
params=None,
|
|
420
|
+
headers=None,
|
|
421
|
+
handle=None,
|
|
422
|
+
silent=None,
|
|
423
|
+
redirect=None,
|
|
424
|
+
timeout=None,
|
|
413
425
|
**kwargs
|
|
414
426
|
):
|
|
415
|
-
if handle == None:
|
|
416
|
-
|
|
417
|
-
if
|
|
418
|
-
|
|
427
|
+
if handle == None:
|
|
428
|
+
handle = False
|
|
429
|
+
if silent == None:
|
|
430
|
+
silent = config.conf("HTTP_SILENT", False, cast=bool)
|
|
431
|
+
if redirect == None:
|
|
432
|
+
redirect = config.conf("HTTP_REDIRECT", False, cast=bool)
|
|
433
|
+
if timeout == None:
|
|
434
|
+
timeout = config.conf("HTTP_TIMEOUT", TIMEOUT, cast=int)
|
|
419
435
|
values = params or dict()
|
|
420
436
|
|
|
421
437
|
values_s = " with '%s'" % str(values) if values else ""
|
|
422
|
-
if not silent:
|
|
438
|
+
if not silent:
|
|
439
|
+
logging.debug("%s %s%s" % (name, url, values_s))
|
|
423
440
|
|
|
424
441
|
url, scheme, host, authorization, extra = _parse_url(url)
|
|
425
|
-
if extra:
|
|
442
|
+
if extra:
|
|
443
|
+
values.update(extra)
|
|
426
444
|
data = _urlencode(values)
|
|
427
445
|
|
|
428
446
|
headers = dict(headers) if headers else dict()
|
|
429
|
-
if host:
|
|
430
|
-
|
|
447
|
+
if host:
|
|
448
|
+
headers["Host"] = host
|
|
449
|
+
if authorization:
|
|
450
|
+
headers["Authorization"] = "Basic %s" % authorization
|
|
431
451
|
url = url + "?" + data if data else url
|
|
432
452
|
url = str(url)
|
|
433
453
|
|
|
@@ -440,58 +460,71 @@ def _method_empty(
|
|
|
440
460
|
# verifies if the resulting "file" from the resolution process is either
|
|
441
461
|
# an invalid or tuple value and if that's the case as the request has
|
|
442
462
|
# probably been deferred for asynchronous execution
|
|
443
|
-
if file == None:
|
|
444
|
-
|
|
463
|
+
if file == None:
|
|
464
|
+
return file
|
|
465
|
+
if isinstance(file, tuple):
|
|
466
|
+
return file
|
|
445
467
|
|
|
446
|
-
try:
|
|
447
|
-
|
|
468
|
+
try:
|
|
469
|
+
result = file.read()
|
|
470
|
+
finally:
|
|
471
|
+
file.close()
|
|
448
472
|
|
|
449
473
|
code = file.getcode()
|
|
450
474
|
info = file.info()
|
|
451
475
|
|
|
452
476
|
location = info.get("Location", None) if redirect else None
|
|
453
|
-
if location:
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
477
|
+
if location:
|
|
478
|
+
return _redirect(
|
|
479
|
+
location,
|
|
480
|
+
scheme,
|
|
481
|
+
host,
|
|
482
|
+
handle=handle,
|
|
483
|
+
silent=silent,
|
|
484
|
+
redirect=redirect,
|
|
485
|
+
timeout=timeout,
|
|
486
|
+
**kwargs
|
|
487
|
+
)
|
|
463
488
|
|
|
464
|
-
if not silent:
|
|
489
|
+
if not silent:
|
|
490
|
+
logging.debug("%s %s returned '%d'" % (name, url, code))
|
|
465
491
|
|
|
466
492
|
result = _result(result, info)
|
|
467
493
|
return (result, file) if handle else result
|
|
468
494
|
|
|
495
|
+
|
|
469
496
|
def _method_payload(
|
|
470
497
|
name,
|
|
471
498
|
url,
|
|
472
|
-
params
|
|
473
|
-
data
|
|
474
|
-
data_j
|
|
475
|
-
data_m
|
|
476
|
-
headers
|
|
477
|
-
mime
|
|
478
|
-
handle
|
|
479
|
-
silent
|
|
480
|
-
redirect
|
|
481
|
-
timeout
|
|
499
|
+
params=None,
|
|
500
|
+
data=None,
|
|
501
|
+
data_j=None,
|
|
502
|
+
data_m=None,
|
|
503
|
+
headers=None,
|
|
504
|
+
mime=None,
|
|
505
|
+
handle=None,
|
|
506
|
+
silent=None,
|
|
507
|
+
redirect=None,
|
|
508
|
+
timeout=None,
|
|
482
509
|
**kwargs
|
|
483
510
|
):
|
|
484
|
-
if handle == None:
|
|
485
|
-
|
|
486
|
-
if
|
|
487
|
-
|
|
511
|
+
if handle == None:
|
|
512
|
+
handle = False
|
|
513
|
+
if silent == None:
|
|
514
|
+
silent = config.conf("HTTP_SILENT", False, cast=bool)
|
|
515
|
+
if redirect == None:
|
|
516
|
+
redirect = config.conf("HTTP_REDIRECT", False, cast=bool)
|
|
517
|
+
if timeout == None:
|
|
518
|
+
timeout = config.conf("HTTP_TIMEOUT", TIMEOUT, cast=int)
|
|
488
519
|
values = params or dict()
|
|
489
520
|
|
|
490
521
|
values_s = " with '%s'" % str(values) if values else ""
|
|
491
|
-
if not silent:
|
|
522
|
+
if not silent:
|
|
523
|
+
logging.debug("%s %s%s" % (name, url, values_s))
|
|
492
524
|
|
|
493
525
|
url, scheme, host, authorization, extra = _parse_url(url)
|
|
494
|
-
if extra:
|
|
526
|
+
if extra:
|
|
527
|
+
values.update(extra)
|
|
495
528
|
data_e = _urlencode(values)
|
|
496
529
|
|
|
497
530
|
if not data == None:
|
|
@@ -502,59 +535,72 @@ def _method_payload(
|
|
|
502
535
|
mime = mime or "application/json"
|
|
503
536
|
elif not data_m == None:
|
|
504
537
|
url = url + "?" + data_e if data_e else url
|
|
505
|
-
content_type, data = _encode_multipart(
|
|
506
|
-
data_m, mime = mime, doseq = True
|
|
507
|
-
)
|
|
538
|
+
content_type, data = _encode_multipart(data_m, mime=mime, doseq=True)
|
|
508
539
|
mime = content_type
|
|
509
540
|
elif data_e:
|
|
510
541
|
data = data_e
|
|
511
542
|
mime = mime or "application/x-www-form-urlencoded"
|
|
512
543
|
|
|
513
|
-
if legacy.is_unicode(data):
|
|
544
|
+
if legacy.is_unicode(data):
|
|
545
|
+
data = legacy.bytes(data, force=True)
|
|
514
546
|
|
|
515
|
-
if not data:
|
|
516
|
-
|
|
517
|
-
|
|
547
|
+
if not data:
|
|
548
|
+
length = 0
|
|
549
|
+
elif legacy.is_bytes(data):
|
|
550
|
+
length = len(data)
|
|
551
|
+
else:
|
|
552
|
+
length = -1
|
|
518
553
|
|
|
519
554
|
headers = dict(headers) if headers else dict()
|
|
520
|
-
if not length == -1:
|
|
521
|
-
|
|
522
|
-
if
|
|
523
|
-
|
|
555
|
+
if not length == -1:
|
|
556
|
+
headers["Content-Length"] = str(length)
|
|
557
|
+
if mime:
|
|
558
|
+
headers["Content-Type"] = mime
|
|
559
|
+
if host:
|
|
560
|
+
headers["Host"] = host
|
|
561
|
+
if authorization:
|
|
562
|
+
headers["Authorization"] = "Basic %s" % authorization
|
|
524
563
|
url = str(url)
|
|
525
564
|
|
|
526
565
|
_method_callback(handle, kwargs)
|
|
527
566
|
file = _resolve(url, name, headers, data, silent, timeout, **kwargs)
|
|
528
|
-
if file == None:
|
|
567
|
+
if file == None:
|
|
568
|
+
return file
|
|
529
569
|
|
|
530
|
-
try:
|
|
531
|
-
|
|
570
|
+
try:
|
|
571
|
+
result = file.read()
|
|
572
|
+
finally:
|
|
573
|
+
file.close()
|
|
532
574
|
|
|
533
575
|
code = file.getcode()
|
|
534
576
|
info = file.info()
|
|
535
577
|
|
|
536
578
|
location = info.get("Location", None) if redirect else None
|
|
537
|
-
if location:
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
579
|
+
if location:
|
|
580
|
+
return _redirect(
|
|
581
|
+
location,
|
|
582
|
+
scheme,
|
|
583
|
+
host,
|
|
584
|
+
handle=handle,
|
|
585
|
+
silent=silent,
|
|
586
|
+
redirect=redirect,
|
|
587
|
+
timeout=timeout,
|
|
588
|
+
)
|
|
546
589
|
|
|
547
|
-
if not silent:
|
|
590
|
+
if not silent:
|
|
591
|
+
logging.debug("%s %s returned '%d'" % (name, url, code))
|
|
548
592
|
|
|
549
593
|
result = _result(result, info)
|
|
550
594
|
return (result, file) if handle else result
|
|
551
595
|
|
|
596
|
+
|
|
552
597
|
def _method_callback(handle, kwargs):
|
|
553
598
|
# tries to determine if a callback value has been registered
|
|
554
599
|
# in the set of keyword argument and if that's not the case
|
|
555
600
|
# returns immediately (nothing to be done)
|
|
556
601
|
callback = kwargs.get("callback", None)
|
|
557
|
-
if not callback:
|
|
602
|
+
if not callback:
|
|
603
|
+
return
|
|
558
604
|
|
|
559
605
|
def callback_wrap(file):
|
|
560
606
|
# determines if the received file is valid (no error)
|
|
@@ -562,8 +608,10 @@ def _method_callback(handle, kwargs):
|
|
|
562
608
|
# and an invalid/unset value has been provided
|
|
563
609
|
if file:
|
|
564
610
|
info = file.info()
|
|
565
|
-
try:
|
|
566
|
-
|
|
611
|
+
try:
|
|
612
|
+
result = file.read()
|
|
613
|
+
finally:
|
|
614
|
+
file.close()
|
|
567
615
|
result = _result(result, info)
|
|
568
616
|
else:
|
|
569
617
|
result = None
|
|
@@ -578,28 +626,31 @@ def _method_callback(handle, kwargs):
|
|
|
578
626
|
# so that this new callback is called instead of the original
|
|
579
627
|
kwargs["callback"] = callback_wrap
|
|
580
628
|
|
|
629
|
+
|
|
581
630
|
def _redirect(
|
|
582
631
|
location,
|
|
583
632
|
scheme,
|
|
584
633
|
host,
|
|
585
|
-
handle
|
|
586
|
-
silent
|
|
587
|
-
redirect
|
|
588
|
-
timeout
|
|
634
|
+
handle=None,
|
|
635
|
+
silent=None,
|
|
636
|
+
redirect=None,
|
|
637
|
+
timeout=None,
|
|
589
638
|
**kwargs
|
|
590
639
|
):
|
|
591
640
|
is_relative = location.startswith("/")
|
|
592
|
-
if is_relative:
|
|
641
|
+
if is_relative:
|
|
642
|
+
location = scheme + "://" + host + location
|
|
593
643
|
logging.debug("Redirecting to %s" % location)
|
|
594
644
|
return get(
|
|
595
645
|
location,
|
|
596
|
-
handle
|
|
597
|
-
silent
|
|
598
|
-
redirect
|
|
599
|
-
timeout
|
|
646
|
+
handle=handle,
|
|
647
|
+
silent=silent,
|
|
648
|
+
redirect=redirect,
|
|
649
|
+
timeout=timeout,
|
|
600
650
|
**kwargs
|
|
601
651
|
)
|
|
602
652
|
|
|
653
|
+
|
|
603
654
|
def _resolve(*args, **kwargs):
|
|
604
655
|
# obtains the reference to the global set of variables, so
|
|
605
656
|
# that it's possible to obtain the proper resolver method
|
|
@@ -609,7 +660,7 @@ def _resolve(*args, **kwargs):
|
|
|
609
660
|
# tries to retrieve the global configuration values that
|
|
610
661
|
# will condition the way the request is going to be performed
|
|
611
662
|
client = config.conf("HTTP_CLIENT", "netius")
|
|
612
|
-
reuse = config.conf("HTTP_REUSE", True, cast
|
|
663
|
+
reuse = config.conf("HTTP_REUSE", True, cast=bool)
|
|
613
664
|
|
|
614
665
|
# tries to determine the set of configurations requested on
|
|
615
666
|
# a request basis (not global) these have priority when
|
|
@@ -626,23 +677,31 @@ def _resolve(*args, **kwargs):
|
|
|
626
677
|
# current client and then runs it, retrieve then the final result,
|
|
627
678
|
# note that the result structure may be engine dependent
|
|
628
679
|
resolver = _global.get("_resolve_" + client, _resolve_legacy)
|
|
629
|
-
try:
|
|
630
|
-
|
|
680
|
+
try:
|
|
681
|
+
result = resolver(*args, **kwargs)
|
|
682
|
+
except ImportError:
|
|
683
|
+
result = _resolve_legacy(*args, **kwargs)
|
|
631
684
|
return result
|
|
632
685
|
|
|
686
|
+
|
|
633
687
|
def _resolve_legacy(url, method, headers, data, silent, timeout, **kwargs):
|
|
634
688
|
is_generator = not data == None and legacy.is_generator(data)
|
|
635
|
-
if is_generator:
|
|
689
|
+
if is_generator:
|
|
690
|
+
next(data)
|
|
691
|
+
data = b"".join(data)
|
|
636
692
|
is_file = hasattr(data, "tell")
|
|
637
|
-
if is_file:
|
|
693
|
+
if is_file:
|
|
694
|
+
data = data.read()
|
|
638
695
|
opener = legacy.build_opener(legacy.HTTPHandler)
|
|
639
|
-
request = legacy.Request(url, data
|
|
696
|
+
request = legacy.Request(url, data=data, headers=headers)
|
|
640
697
|
request.get_method = lambda: method
|
|
641
|
-
return opener.open(request, timeout
|
|
698
|
+
return opener.open(request, timeout=timeout)
|
|
699
|
+
|
|
642
700
|
|
|
643
701
|
def _resolve_requests(url, method, headers, data, silent, timeout, **kwargs):
|
|
644
702
|
util.ensure_pip("requests")
|
|
645
703
|
import requests
|
|
704
|
+
|
|
646
705
|
global _requests_session
|
|
647
706
|
|
|
648
707
|
# retrieves the various dynamic parameters for the HTTP client
|
|
@@ -656,7 +715,8 @@ def _resolve_requests(url, method, headers, data, silent, timeout, **kwargs):
|
|
|
656
715
|
# into a generator based file-like object so that it can be used inside
|
|
657
716
|
# the request infra-structure (as it accepts only file objects)
|
|
658
717
|
is_generator = not data == None and legacy.is_generator(data)
|
|
659
|
-
if is_generator:
|
|
718
|
+
if is_generator:
|
|
719
|
+
data = structures.GeneratorFile(data)
|
|
660
720
|
|
|
661
721
|
# verifies if the session for the requests infra-structure is
|
|
662
722
|
# already created and if that's not the case and the re-use
|
|
@@ -665,15 +725,16 @@ def _resolve_requests(url, method, headers, data, silent, timeout, **kwargs):
|
|
|
665
725
|
if not registered and reuse:
|
|
666
726
|
_requests_session = requests.Session()
|
|
667
727
|
adapter = requests.adapters.HTTPAdapter(
|
|
668
|
-
pool_connections =
|
|
669
|
-
pool_maxsize = connections
|
|
728
|
+
pool_connections=connections, pool_maxsize=connections
|
|
670
729
|
)
|
|
671
730
|
_requests_session.mount("", adapter)
|
|
672
731
|
|
|
673
732
|
# determines the based object from which the concrete methods
|
|
674
733
|
# are going to be loaded by inspecting the re-use flag
|
|
675
|
-
if reuse:
|
|
676
|
-
|
|
734
|
+
if reuse:
|
|
735
|
+
base = _requests_session
|
|
736
|
+
else:
|
|
737
|
+
base = requests
|
|
677
738
|
|
|
678
739
|
# converts the string based method value into a lower cased value
|
|
679
740
|
# and then uses it to retrieve the method object method (callable)
|
|
@@ -683,11 +744,9 @@ def _resolve_requests(url, method, headers, data, silent, timeout, **kwargs):
|
|
|
683
744
|
|
|
684
745
|
# runs the caller method (according to selected method) and waits for
|
|
685
746
|
# the result object converting it then to the target response object
|
|
686
|
-
result = caller(url, headers
|
|
747
|
+
result = caller(url, headers=headers, data=data, timeout=timeout)
|
|
687
748
|
response = HTTPResponse(
|
|
688
|
-
data = result.
|
|
689
|
-
code = result.status_code,
|
|
690
|
-
headers = result.headers
|
|
749
|
+
data=result.content, code=result.status_code, headers=result.headers
|
|
691
750
|
)
|
|
692
751
|
|
|
693
752
|
# retrieves the response code of the created response and verifies if
|
|
@@ -695,14 +754,14 @@ def _resolve_requests(url, method, headers, data, silent, timeout, **kwargs):
|
|
|
695
754
|
# to the upper layers to break the current execution logic properly
|
|
696
755
|
code = response.getcode()
|
|
697
756
|
is_error = _is_error(code)
|
|
698
|
-
if is_error:
|
|
699
|
-
url, code, "HTTP retrieval problem", None, response
|
|
700
|
-
)
|
|
757
|
+
if is_error:
|
|
758
|
+
raise legacy.HTTPError(url, code, "HTTP retrieval problem", None, response)
|
|
701
759
|
|
|
702
760
|
# returns the final response object to the caller method, this object
|
|
703
761
|
# should comply with the proper upper layers structure
|
|
704
762
|
return response
|
|
705
763
|
|
|
764
|
+
|
|
706
765
|
def _resolve_netius(url, method, headers, data, silent, timeout, **kwargs):
|
|
707
766
|
util.ensure_pip("netius")
|
|
708
767
|
import netius.clients
|
|
@@ -744,41 +803,46 @@ def _resolve_netius(url, method, headers, data, silent, timeout, **kwargs):
|
|
|
744
803
|
|
|
745
804
|
# creates the proper set of extra parameters to be sent to the
|
|
746
805
|
# HTTP client taking into account a possible async method request
|
|
747
|
-
extra =
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
806
|
+
extra = (
|
|
807
|
+
_async_netius(
|
|
808
|
+
callback=callback,
|
|
809
|
+
callback_init=callback_init,
|
|
810
|
+
callback_open=callback_open,
|
|
811
|
+
callback_headers=callback_headers,
|
|
812
|
+
callback_data=callback_data,
|
|
813
|
+
callback_result=callback_result,
|
|
814
|
+
)
|
|
815
|
+
if asynchronous
|
|
816
|
+
else dict(
|
|
817
|
+
on_init=lambda c: callback_init and callback_init(c),
|
|
818
|
+
on_open=lambda c: callback_open and callback_open(c),
|
|
819
|
+
on_headers=lambda c, p: callback_headers and callback_headers(p.headers),
|
|
820
|
+
on_data=lambda c, p, d: callback_data and callback_data(d),
|
|
821
|
+
on_result=lambda c, p, r: callback_result and callback_result(r),
|
|
822
|
+
)
|
|
760
823
|
)
|
|
761
824
|
|
|
762
825
|
# verifies if client re-usage must be enforced and if that's the
|
|
763
826
|
# case the global client object is requested (singleton) otherwise
|
|
764
827
|
# the client should be created inside the HTTP client static method
|
|
765
|
-
http_client = _client_netius(level
|
|
828
|
+
http_client = _client_netius(level=level) if reuse else None
|
|
766
829
|
result = netius.clients.HTTPClient.method_s(
|
|
767
830
|
method,
|
|
768
831
|
url,
|
|
769
|
-
headers
|
|
770
|
-
data
|
|
771
|
-
asynchronous
|
|
772
|
-
timeout
|
|
773
|
-
use_file
|
|
774
|
-
http_client
|
|
775
|
-
level
|
|
832
|
+
headers=headers,
|
|
833
|
+
data=data,
|
|
834
|
+
asynchronous=asynchronous,
|
|
835
|
+
timeout=timeout,
|
|
836
|
+
use_file=use_file,
|
|
837
|
+
http_client=http_client,
|
|
838
|
+
level=level,
|
|
776
839
|
**extra
|
|
777
840
|
)
|
|
778
841
|
|
|
779
842
|
# if the async mode is defined the result (tuple) is returned immediately
|
|
780
843
|
# as the processing will be taking place latter (on callback)
|
|
781
|
-
if asynchronous:
|
|
844
|
+
if asynchronous:
|
|
845
|
+
return result
|
|
782
846
|
|
|
783
847
|
# tries to retrieve any possible error coming from the result object
|
|
784
848
|
# if this happens it means an exception has been raised internally and
|
|
@@ -788,9 +852,7 @@ def _resolve_netius(url, method, headers, data, silent, timeout, **kwargs):
|
|
|
788
852
|
error = result.get("error", None)
|
|
789
853
|
if error == "closed" and retry > 0:
|
|
790
854
|
kwargs["retry"] = retry - 1
|
|
791
|
-
return _resolve_netius(
|
|
792
|
-
url, method, headers, data, silent, timeout, **kwargs
|
|
793
|
-
)
|
|
855
|
+
return _resolve_netius(url, method, headers, data, silent, timeout, **kwargs)
|
|
794
856
|
|
|
795
857
|
# converts the netius specific result map into a response compatible
|
|
796
858
|
# object (equivalent to the urllib one) to be used by the upper layers
|
|
@@ -803,16 +865,17 @@ def _resolve_netius(url, method, headers, data, silent, timeout, **kwargs):
|
|
|
803
865
|
# to the upper layers to break the current execution logic properly
|
|
804
866
|
code = response.getcode()
|
|
805
867
|
is_error = _is_error(code)
|
|
806
|
-
if is_error:
|
|
807
|
-
url, code, "HTTP retrieval problem", None, response
|
|
808
|
-
)
|
|
868
|
+
if is_error:
|
|
869
|
+
raise legacy.HTTPError(url, code, "HTTP retrieval problem", None, response)
|
|
809
870
|
|
|
810
871
|
# returns the final response object to the upper layers, this object
|
|
811
872
|
# may be used freely under the compatibility interface it provides
|
|
812
873
|
return response
|
|
813
874
|
|
|
814
|
-
|
|
875
|
+
|
|
876
|
+
def _client_netius(level=logging.CRITICAL):
|
|
815
877
|
import netius.clients
|
|
878
|
+
|
|
816
879
|
global _netius_clients
|
|
817
880
|
|
|
818
881
|
# retrieves the reference to the current thread and uses the value
|
|
@@ -833,26 +896,29 @@ def _client_netius(level = logging.CRITICAL):
|
|
|
833
896
|
|
|
834
897
|
# in case a previously created netius client has been retrieved
|
|
835
898
|
# returns it to the caller method for proper re-usage
|
|
836
|
-
if netius_client:
|
|
899
|
+
if netius_client:
|
|
900
|
+
return netius_client
|
|
837
901
|
|
|
838
902
|
# creates the "new" HTTP client for the current thread and registers
|
|
839
903
|
# it under the netius client structure so that it may be re-used
|
|
840
|
-
netius_client = netius.clients.HTTPClient(auto_release
|
|
904
|
+
netius_client = netius.clients.HTTPClient(auto_release=False)
|
|
841
905
|
_netius_clients[tid] = netius_client
|
|
842
906
|
|
|
843
907
|
# in case this is the first registration of the dictionary a new on
|
|
844
908
|
# exit callback is registered to cleanup the netius infra-structure
|
|
845
909
|
# then the final client is returned to the caller of the method
|
|
846
|
-
if not registered:
|
|
910
|
+
if not registered:
|
|
911
|
+
common.base().on_exit(_cleanup_netius)
|
|
847
912
|
return netius_client
|
|
848
913
|
|
|
914
|
+
|
|
849
915
|
def _async_netius(
|
|
850
|
-
callback
|
|
851
|
-
callback_init
|
|
852
|
-
callback_open
|
|
853
|
-
callback_headers
|
|
854
|
-
callback_data
|
|
855
|
-
callback_result
|
|
916
|
+
callback=None,
|
|
917
|
+
callback_init=None,
|
|
918
|
+
callback_open=None,
|
|
919
|
+
callback_headers=None,
|
|
920
|
+
callback_data=None,
|
|
921
|
+
callback_result=None,
|
|
856
922
|
):
|
|
857
923
|
import netius.clients
|
|
858
924
|
|
|
@@ -886,20 +952,27 @@ def _async_netius(
|
|
|
886
952
|
|
|
887
953
|
extra["callback"] = _callback
|
|
888
954
|
extra["on_data"] = _on_data
|
|
889
|
-
if callback_init:
|
|
890
|
-
|
|
891
|
-
if
|
|
892
|
-
|
|
893
|
-
if
|
|
955
|
+
if callback_init:
|
|
956
|
+
extra["_on_init"] = _on_init
|
|
957
|
+
if callback_open:
|
|
958
|
+
extra["on_open"] = _on_open
|
|
959
|
+
if callback:
|
|
960
|
+
extra["on_close"] = _on_close
|
|
961
|
+
if callback_headers:
|
|
962
|
+
extra["on_headers"] = _on_headers
|
|
963
|
+
if callback_result:
|
|
964
|
+
extra["on_result"] = _on_result
|
|
894
965
|
|
|
895
966
|
return extra
|
|
896
967
|
|
|
968
|
+
|
|
897
969
|
def _cleanup_netius():
|
|
898
970
|
global _netius_clients
|
|
899
971
|
for netius_client in _netius_clients.values():
|
|
900
972
|
netius_client.cleanup()
|
|
901
973
|
del _netius_clients
|
|
902
974
|
|
|
975
|
+
|
|
903
976
|
def _parse_url(url):
|
|
904
977
|
parse = legacy.urlparse(url)
|
|
905
978
|
scheme = parse.scheme
|
|
@@ -907,37 +980,42 @@ def _parse_url(url):
|
|
|
907
980
|
default = 443 if secure else 80
|
|
908
981
|
port = parse.port or default
|
|
909
982
|
url = parse.scheme + "://" + parse.hostname + ":" + str(port) + parse.path
|
|
910
|
-
if port in (80, 443):
|
|
911
|
-
|
|
983
|
+
if port in (80, 443):
|
|
984
|
+
host = parse.hostname
|
|
985
|
+
else:
|
|
986
|
+
host = parse.hostname + ":" + str(port)
|
|
912
987
|
authorization = _authorization(parse.username, parse.password)
|
|
913
988
|
params = _params(parse.query)
|
|
914
989
|
return (url, scheme, host, authorization, params)
|
|
915
990
|
|
|
916
|
-
|
|
991
|
+
|
|
992
|
+
def _result(data, info={}, force=False, strict=False):
|
|
917
993
|
# tries to retrieve the content type value from the headers
|
|
918
994
|
# info and verifies if the current data is JSON encoded, so
|
|
919
995
|
# that it gets automatically decoded for such cases
|
|
920
996
|
content_type = info.get("Content-Type", None) or ""
|
|
921
|
-
is_json =
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
"application/json",
|
|
925
|
-
"text/json",
|
|
926
|
-
"text/javascript"
|
|
997
|
+
is_json = (
|
|
998
|
+
util.is_content_type(
|
|
999
|
+
content_type, ("application/json", "text/json", "text/javascript")
|
|
927
1000
|
)
|
|
928
|
-
|
|
1001
|
+
or force
|
|
1002
|
+
)
|
|
929
1003
|
|
|
930
1004
|
# verifies if the current result set is JSON encoded and in
|
|
931
1005
|
# case it's decodes it and loads it as JSON otherwise returns
|
|
932
1006
|
# the "raw" data to the caller method as expected, note that
|
|
933
1007
|
# the strict flag is used to determine if the exception should
|
|
934
1008
|
# be re-raised to the upper level in case of value error
|
|
935
|
-
if is_json and legacy.is_bytes(data):
|
|
936
|
-
|
|
1009
|
+
if is_json and legacy.is_bytes(data):
|
|
1010
|
+
data = data.decode("utf-8")
|
|
1011
|
+
try:
|
|
1012
|
+
data = json.loads(data) if is_json else data
|
|
937
1013
|
except ValueError:
|
|
938
|
-
if strict:
|
|
1014
|
+
if strict:
|
|
1015
|
+
raise
|
|
939
1016
|
return data
|
|
940
1017
|
|
|
1018
|
+
|
|
941
1019
|
def _params(query):
|
|
942
1020
|
# creates the dictionary that is going to be used to store the
|
|
943
1021
|
# complete information regarding the parameters in query
|
|
@@ -946,7 +1024,8 @@ def _params(query):
|
|
|
946
1024
|
# validates that the provided query value is valid and if
|
|
947
1025
|
# that's not the case returns the created parameters immediately
|
|
948
1026
|
# (empty parameters are returned)
|
|
949
|
-
if not query:
|
|
1027
|
+
if not query:
|
|
1028
|
+
return params
|
|
950
1029
|
|
|
951
1030
|
# splits the query value around the initial parameter separator
|
|
952
1031
|
# symbol and iterates over each of them to parse them and create
|
|
@@ -954,8 +1033,10 @@ def _params(query):
|
|
|
954
1033
|
query_s = query.split("&")
|
|
955
1034
|
for part in query_s:
|
|
956
1035
|
parts = part.split("=", 1)
|
|
957
|
-
if len(parts) == 1:
|
|
958
|
-
|
|
1036
|
+
if len(parts) == 1:
|
|
1037
|
+
value = ""
|
|
1038
|
+
else:
|
|
1039
|
+
value = parts[1]
|
|
959
1040
|
key = parts[0]
|
|
960
1041
|
key = legacy.unquote_plus(key)
|
|
961
1042
|
value = legacy.unquote_plus(value)
|
|
@@ -967,7 +1048,8 @@ def _params(query):
|
|
|
967
1048
|
# so that it may be used as a proper structure representation
|
|
968
1049
|
return params
|
|
969
1050
|
|
|
970
|
-
|
|
1051
|
+
|
|
1052
|
+
def _urlencode(values, as_string=True):
|
|
971
1053
|
# creates the list that will hold the final tuple of values
|
|
972
1054
|
# (without the unset and invalid values)
|
|
973
1055
|
final = []
|
|
@@ -975,7 +1057,8 @@ def _urlencode(values, as_string = True):
|
|
|
975
1057
|
# verifies if the provided value is a sequence and in case it's
|
|
976
1058
|
# not converts it into a sequence (assuming a map)
|
|
977
1059
|
is_sequence = isinstance(values, (list, tuple))
|
|
978
|
-
if not is_sequence:
|
|
1060
|
+
if not is_sequence:
|
|
1061
|
+
values = values.items()
|
|
979
1062
|
|
|
980
1063
|
# iterates over all the items in the values sequence to
|
|
981
1064
|
# try to filter the values that are not valid
|
|
@@ -987,26 +1070,32 @@ def _urlencode(values, as_string = True):
|
|
|
987
1070
|
# in case the current data type of the key is unicode
|
|
988
1071
|
# the value must be converted into a string using the
|
|
989
1072
|
# default utf encoding strategy (as defined)
|
|
990
|
-
if type(key) == legacy.UNICODE:
|
|
1073
|
+
if type(key) == legacy.UNICODE:
|
|
1074
|
+
key = key.encode("utf-8")
|
|
991
1075
|
|
|
992
1076
|
# verifies the type of the current value and in case
|
|
993
1077
|
# it's sequence based converts it into a list using
|
|
994
1078
|
# the conversion method otherwise creates a new list
|
|
995
1079
|
# and includes the value in it
|
|
996
1080
|
value_t = type(value)
|
|
997
|
-
if value_t in SEQUENCE_TYPES:
|
|
998
|
-
|
|
1081
|
+
if value_t in SEQUENCE_TYPES:
|
|
1082
|
+
value = list(value)
|
|
1083
|
+
else:
|
|
1084
|
+
value = [value]
|
|
999
1085
|
|
|
1000
1086
|
# iterates over all the values in the current sequence
|
|
1001
1087
|
# and adds the valid values to the sanitized sequence,
|
|
1002
1088
|
# this includes the conversion from unicode string into
|
|
1003
1089
|
# a simple string using the default utf encoder
|
|
1004
1090
|
for _value in value:
|
|
1005
|
-
if _value == None:
|
|
1091
|
+
if _value == None:
|
|
1092
|
+
continue
|
|
1006
1093
|
is_string = type(_value) in legacy.STRINGS
|
|
1007
|
-
if not is_string:
|
|
1094
|
+
if not is_string:
|
|
1095
|
+
_value = str(_value)
|
|
1008
1096
|
is_unicode = type(_value) == legacy.UNICODE
|
|
1009
|
-
if is_unicode:
|
|
1097
|
+
if is_unicode:
|
|
1098
|
+
_value = _value.encode("utf-8")
|
|
1010
1099
|
_values.append(_value)
|
|
1011
1100
|
|
|
1012
1101
|
# sets the sanitized list of values as the new value for
|
|
@@ -1016,38 +1105,44 @@ def _urlencode(values, as_string = True):
|
|
|
1016
1105
|
# in case the "as string" flag is not set the ended key to value
|
|
1017
1106
|
# dictionary should be returned to the called method and not the
|
|
1018
1107
|
# "default" linear and string based value
|
|
1019
|
-
if not as_string:
|
|
1108
|
+
if not as_string:
|
|
1109
|
+
return final
|
|
1020
1110
|
|
|
1021
1111
|
# runs the encoding with sequence support on the final map
|
|
1022
1112
|
# of sanitized values and returns the encoded result to the
|
|
1023
1113
|
# caller method as the encoded value
|
|
1024
|
-
return legacy.urlencode(final, doseq
|
|
1114
|
+
return legacy.urlencode(final, doseq=True)
|
|
1115
|
+
|
|
1025
1116
|
|
|
1026
|
-
def _quote(values, plus
|
|
1117
|
+
def _quote(values, plus=False, safe="/"):
|
|
1027
1118
|
method = legacy.quote_plus if plus else legacy.quote
|
|
1028
|
-
values = _urlencode(values, as_string
|
|
1119
|
+
values = _urlencode(values, as_string=False)
|
|
1029
1120
|
|
|
1030
1121
|
final = dict()
|
|
1031
1122
|
|
|
1032
1123
|
for key, value in values.items():
|
|
1033
|
-
key = method(key, safe
|
|
1034
|
-
value = method(value[0], safe
|
|
1124
|
+
key = method(key, safe=safe)
|
|
1125
|
+
value = method(value[0], safe=safe)
|
|
1035
1126
|
final[key] = value
|
|
1036
1127
|
|
|
1037
1128
|
return final
|
|
1038
1129
|
|
|
1130
|
+
|
|
1039
1131
|
def _authorization(username, password):
|
|
1040
|
-
if not username:
|
|
1041
|
-
|
|
1132
|
+
if not username:
|
|
1133
|
+
return None
|
|
1134
|
+
if not password:
|
|
1135
|
+
return None
|
|
1042
1136
|
payload = "%s:%s" % (username, password)
|
|
1043
1137
|
payload = legacy.bytes(payload)
|
|
1044
1138
|
authorization = base64.b64encode(payload)
|
|
1045
1139
|
authorization = legacy.str(authorization)
|
|
1046
1140
|
return authorization
|
|
1047
1141
|
|
|
1048
|
-
|
|
1142
|
+
|
|
1143
|
+
def _encode_multipart(fields, mime=None, doseq=False):
|
|
1049
1144
|
mime = mime or "multipart/form-data"
|
|
1050
|
-
boundary = _create_boundary(fields, doseq
|
|
1145
|
+
boundary = _create_boundary(fields, doseq=doseq)
|
|
1051
1146
|
boundary_b = legacy.bytes(boundary)
|
|
1052
1147
|
|
|
1053
1148
|
buffer = []
|
|
@@ -1057,26 +1152,34 @@ def _encode_multipart(fields, mime = None, doseq = False):
|
|
|
1057
1152
|
values = values if is_list else [values]
|
|
1058
1153
|
|
|
1059
1154
|
for value in values:
|
|
1060
|
-
if value == None:
|
|
1155
|
+
if value == None:
|
|
1156
|
+
continue
|
|
1061
1157
|
|
|
1062
1158
|
if isinstance(value, dict):
|
|
1063
1159
|
header_l = []
|
|
1064
1160
|
data = None
|
|
1065
1161
|
for key, item in value.items():
|
|
1066
|
-
if key == "data":
|
|
1067
|
-
|
|
1162
|
+
if key == "data":
|
|
1163
|
+
data = item
|
|
1164
|
+
else:
|
|
1165
|
+
header_l.append("%s: %s" % (key, item))
|
|
1068
1166
|
value = data
|
|
1069
1167
|
header = "\r\n".join(header_l)
|
|
1070
1168
|
elif isinstance(value, tuple):
|
|
1071
1169
|
content_type = None
|
|
1072
|
-
if len(value) == 2:
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1170
|
+
if len(value) == 2:
|
|
1171
|
+
name, contents = value
|
|
1172
|
+
else:
|
|
1173
|
+
name, content_type, contents = value
|
|
1174
|
+
header = 'Content-Disposition: form-data; name="%s"; filename="%s"' % (
|
|
1175
|
+
key,
|
|
1176
|
+
name,
|
|
1177
|
+
)
|
|
1178
|
+
if content_type:
|
|
1179
|
+
header += "\r\nContent-Type: %s" % content_type
|
|
1077
1180
|
value = contents
|
|
1078
1181
|
else:
|
|
1079
|
-
header =
|
|
1182
|
+
header = 'Content-Disposition: form-data; name="%s"' % key
|
|
1080
1183
|
value = _encode(value)
|
|
1081
1184
|
|
|
1082
1185
|
header = _encode(header)
|
|
@@ -1094,16 +1197,19 @@ def _encode_multipart(fields, mime = None, doseq = False):
|
|
|
1094
1197
|
|
|
1095
1198
|
return content_type, body
|
|
1096
1199
|
|
|
1097
|
-
|
|
1200
|
+
|
|
1201
|
+
def _create_boundary(fields, size=32, doseq=False):
|
|
1098
1202
|
while True:
|
|
1099
1203
|
base = "".join(random.choice(RANGE) for _value in range(size))
|
|
1100
1204
|
boundary = "----------" + base
|
|
1101
|
-
result = _try_boundary(fields, boundary, doseq
|
|
1102
|
-
if result:
|
|
1205
|
+
result = _try_boundary(fields, boundary, doseq=doseq)
|
|
1206
|
+
if result:
|
|
1207
|
+
break
|
|
1103
1208
|
|
|
1104
1209
|
return boundary
|
|
1105
1210
|
|
|
1106
|
-
|
|
1211
|
+
|
|
1212
|
+
def _try_boundary(fields, boundary, doseq=False):
|
|
1107
1213
|
boundary_b = legacy.bytes(boundary)
|
|
1108
1214
|
|
|
1109
1215
|
for key, values in fields.items():
|
|
@@ -1111,27 +1217,43 @@ def _try_boundary(fields, boundary, doseq = False):
|
|
|
1111
1217
|
values = values if is_list else [values]
|
|
1112
1218
|
|
|
1113
1219
|
for value in values:
|
|
1114
|
-
if isinstance(value, dict):
|
|
1220
|
+
if isinstance(value, dict):
|
|
1221
|
+
name = ""
|
|
1222
|
+
value = value.get("data", b"")
|
|
1115
1223
|
elif isinstance(value, tuple):
|
|
1116
|
-
if len(value) == 2:
|
|
1117
|
-
|
|
1118
|
-
|
|
1224
|
+
if len(value) == 2:
|
|
1225
|
+
name = value[0] or ""
|
|
1226
|
+
value = value[1] or b""
|
|
1227
|
+
else:
|
|
1228
|
+
name = value[0] or ""
|
|
1229
|
+
value = value[2] or b""
|
|
1230
|
+
else:
|
|
1231
|
+
name = ""
|
|
1232
|
+
value = _encode(value)
|
|
1119
1233
|
|
|
1120
|
-
if not key.find(boundary) == -1:
|
|
1121
|
-
|
|
1122
|
-
if not
|
|
1234
|
+
if not key.find(boundary) == -1:
|
|
1235
|
+
return False
|
|
1236
|
+
if not name.find(boundary) == -1:
|
|
1237
|
+
return False
|
|
1238
|
+
if not value.find(boundary_b) == -1:
|
|
1239
|
+
return False
|
|
1123
1240
|
|
|
1124
1241
|
return True
|
|
1125
1242
|
|
|
1243
|
+
|
|
1126
1244
|
def _is_error(code):
|
|
1127
1245
|
return code // 100 in (4, 5) if code else True
|
|
1128
1246
|
|
|
1129
|
-
|
|
1247
|
+
|
|
1248
|
+
def _encode(value, encoding="utf-8"):
|
|
1130
1249
|
value_t = type(value)
|
|
1131
|
-
if value_t == legacy.BYTES:
|
|
1132
|
-
|
|
1250
|
+
if value_t == legacy.BYTES:
|
|
1251
|
+
return value
|
|
1252
|
+
elif value_t == legacy.UNICODE:
|
|
1253
|
+
return value.encode(encoding)
|
|
1133
1254
|
return legacy.bytes(str(value))
|
|
1134
1255
|
|
|
1256
|
+
|
|
1135
1257
|
class HTTPResponse(object):
|
|
1136
1258
|
"""
|
|
1137
1259
|
Compatibility object to be used by HTTP libraries that do
|
|
@@ -1139,7 +1261,7 @@ class HTTPResponse(object):
|
|
|
1139
1261
|
for any of their structures.
|
|
1140
1262
|
"""
|
|
1141
1263
|
|
|
1142
|
-
def __init__(self, data
|
|
1264
|
+
def __init__(self, data=None, code=200, status=None, headers=None):
|
|
1143
1265
|
self.data = data
|
|
1144
1266
|
self.code = code
|
|
1145
1267
|
self.status = status
|