pangea-sdk 3.1.0__py3-none-any.whl → 3.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +1 -1
- pangea/asyncio/request.py +122 -88
- pangea/asyncio/services/audit.py +82 -3
- pangea/asyncio/services/base.py +20 -3
- pangea/asyncio/services/file_scan.py +75 -5
- pangea/asyncio/services/redact.py +2 -7
- pangea/config.py +2 -2
- pangea/exceptions.py +1 -1
- pangea/request.py +131 -91
- pangea/response.py +10 -4
- pangea/services/audit/audit.py +163 -62
- pangea/services/audit/models.py +32 -0
- pangea/services/authn/authn.py +4 -1
- pangea/services/authn/models.py +2 -1
- pangea/services/base.py +17 -4
- pangea/services/file_scan.py +59 -3
- pangea/services/redact.py +2 -7
- pangea/services/vault/vault.py +2 -2
- pangea/utils.py +9 -2
- {pangea_sdk-3.1.0.dist-info → pangea_sdk-3.3.0.dist-info}/METADATA +13 -13
- {pangea_sdk-3.1.0.dist-info → pangea_sdk-3.3.0.dist-info}/RECORD +22 -22
- {pangea_sdk-3.1.0.dist-info → pangea_sdk-3.3.0.dist-info}/WHEEL +0 -0
pangea/request.py
CHANGED
@@ -12,7 +12,7 @@ import pangea
|
|
12
12
|
import pangea.exceptions as pe
|
13
13
|
import requests
|
14
14
|
from pangea.config import PangeaConfig
|
15
|
-
from pangea.response import
|
15
|
+
from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
16
16
|
from pangea.utils import default_encoder
|
17
17
|
from requests.adapters import HTTPAdapter, Retry
|
18
18
|
|
@@ -203,12 +203,16 @@ class PangeaRequest(PangeaRequestBase):
|
|
203
203
|
if self.config_id and data.get("config_id", None) is None:
|
204
204
|
data["config_id"] = self.config_id
|
205
205
|
|
206
|
+
transfer_method = data.get("transfer_method", None)
|
207
|
+
|
206
208
|
if (
|
207
209
|
files is not None
|
208
210
|
and type(data) is dict
|
209
|
-
and
|
211
|
+
and (transfer_method == TransferMethod.DIRECT.value or transfer_method == TransferMethod.POST_URL.value)
|
210
212
|
):
|
211
|
-
requests_response = self.
|
213
|
+
requests_response = self._full_post_presigned_url(
|
214
|
+
endpoint, result_class=result_class, data=data, files=files
|
215
|
+
)
|
212
216
|
else:
|
213
217
|
requests_response = self._http_post(
|
214
218
|
url, headers=self._headers(), data=data, files=files, multipart_post=True
|
@@ -220,13 +224,106 @@ class PangeaRequest(PangeaRequestBase):
|
|
220
224
|
|
221
225
|
return self._check_response(pangea_response)
|
222
226
|
|
227
|
+
def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
|
228
|
+
"""Makes the GET call to a Pangea Service endpoint.
|
229
|
+
|
230
|
+
Args:
|
231
|
+
endpoint(str): The Pangea Service API endpoint.
|
232
|
+
path(str): Additional URL path
|
233
|
+
|
234
|
+
Returns:
|
235
|
+
PangeaResponse which contains the response in its entirety and
|
236
|
+
various properties to retrieve individual fields
|
237
|
+
"""
|
238
|
+
|
239
|
+
url = self._url(path)
|
240
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
241
|
+
requests_response = self.session.get(url, headers=self._headers())
|
242
|
+
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
|
243
|
+
|
244
|
+
self.logger.debug(
|
245
|
+
json.dumps(
|
246
|
+
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
247
|
+
default=default_encoder,
|
248
|
+
)
|
249
|
+
)
|
250
|
+
|
251
|
+
if check_response is False:
|
252
|
+
return pangea_response
|
253
|
+
|
254
|
+
return self._check_response(pangea_response)
|
255
|
+
|
256
|
+
def poll_result_by_id(
|
257
|
+
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
258
|
+
):
|
259
|
+
path = self._get_poll_path(request_id)
|
260
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
261
|
+
return self.get(path, result_class, check_response=check_response)
|
262
|
+
|
263
|
+
def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
264
|
+
request_id = response.request_id
|
265
|
+
if not request_id:
|
266
|
+
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
267
|
+
|
268
|
+
if response.status != ResponseStatus.ACCEPTED.value:
|
269
|
+
raise pe.PangeaException("Response already proccesed")
|
270
|
+
|
271
|
+
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
272
|
+
|
273
|
+
def request_presigned_url(
|
274
|
+
self,
|
275
|
+
endpoint: str,
|
276
|
+
result_class: Type[PangeaResponseResult],
|
277
|
+
data: Union[str, Dict] = {},
|
278
|
+
) -> PangeaResponse:
|
279
|
+
# Send request
|
280
|
+
try:
|
281
|
+
# This should return 202 (AcceptedRequestException)
|
282
|
+
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
283
|
+
raise pe.PresignedURLException("Should return 202", resp)
|
284
|
+
|
285
|
+
except pe.AcceptedRequestException as e:
|
286
|
+
accepted_exception = e
|
287
|
+
except Exception as e:
|
288
|
+
raise e
|
289
|
+
|
290
|
+
# Receive 202 with accepted_status
|
291
|
+
return self._poll_presigned_url(accepted_exception.response)
|
292
|
+
|
293
|
+
def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
|
294
|
+
# Send form request with file and upload_details as body
|
295
|
+
resp = self._http_post(url=url, data=data, files=files, multipart_post=False)
|
296
|
+
self.logger.debug(
|
297
|
+
json.dumps(
|
298
|
+
{"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
|
299
|
+
default=default_encoder,
|
300
|
+
)
|
301
|
+
)
|
302
|
+
|
303
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
304
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
|
305
|
+
|
306
|
+
def put_presigned_url(self, url: str, files: List[Tuple]):
|
307
|
+
# Send put request with file as body
|
308
|
+
resp = self._http_put(url=url, files=files)
|
309
|
+
self.logger.debug(
|
310
|
+
json.dumps(
|
311
|
+
{"service": self.service, "action": "put presigned", "url": url, "response": resp.text},
|
312
|
+
default=default_encoder,
|
313
|
+
)
|
314
|
+
)
|
315
|
+
|
316
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
317
|
+
raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status_code}", resp.text)
|
318
|
+
|
319
|
+
# Start internal methods
|
223
320
|
def _http_post(
|
224
321
|
self,
|
225
322
|
url: str,
|
226
323
|
headers: Dict = {},
|
227
324
|
data: Union[str, Dict] = {},
|
228
325
|
files: Optional[List[Tuple]] = None,
|
229
|
-
multipart_post: bool = True,
|
326
|
+
multipart_post: bool = True, # Multipart or form post
|
230
327
|
) -> requests.Response:
|
231
328
|
self.logger.debug(
|
232
329
|
json.dumps(
|
@@ -237,6 +334,17 @@ class PangeaRequest(PangeaRequestBase):
|
|
237
334
|
data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
|
238
335
|
return self.session.post(url, headers=headers, data=data_send, files=files)
|
239
336
|
|
337
|
+
def _http_put(
|
338
|
+
self,
|
339
|
+
url: str,
|
340
|
+
files: List[Tuple],
|
341
|
+
headers: Dict = {},
|
342
|
+
) -> requests.Response:
|
343
|
+
self.logger.debug(
|
344
|
+
json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
|
345
|
+
)
|
346
|
+
return self.session.put(url, headers=headers, files=files)
|
347
|
+
|
240
348
|
def _http_post_process(
|
241
349
|
self, data: Union[str, Dict] = {}, files: Optional[List[Tuple]] = None, multipart_post: bool = True
|
242
350
|
):
|
@@ -263,7 +371,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
263
371
|
|
264
372
|
return data, files
|
265
373
|
|
266
|
-
def
|
374
|
+
def _full_post_presigned_url(
|
267
375
|
self,
|
268
376
|
endpoint: str,
|
269
377
|
result_class: Type[PangeaResponseResult],
|
@@ -273,35 +381,12 @@ class PangeaRequest(PangeaRequestBase):
|
|
273
381
|
if len(files) == 0:
|
274
382
|
raise AttributeError("files attribute should have at least 1 file")
|
275
383
|
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
280
|
-
raise pe.PresignedURLException("Should return 202", resp)
|
281
|
-
|
282
|
-
except pe.AcceptedRequestException as e:
|
283
|
-
accepted_exception = e
|
284
|
-
except Exception as e:
|
285
|
-
raise e
|
286
|
-
|
287
|
-
# Receive 202 with accepted_status
|
288
|
-
result = self._poll_presigned_url(accepted_exception)
|
289
|
-
data_to_presigned = result.accepted_status.upload_details
|
290
|
-
presigned_url = result.accepted_status.upload_url
|
291
|
-
|
292
|
-
# Send multipart request with file and upload_details as body
|
293
|
-
resp = self._http_post(url=presigned_url, data=data_to_presigned, files=files, multipart_post=False)
|
294
|
-
self.logger.debug(
|
295
|
-
json.dumps(
|
296
|
-
{"service": self.service, "action": "post presigned", "url": presigned_url, "response": resp.text},
|
297
|
-
default=default_encoder,
|
298
|
-
)
|
299
|
-
)
|
300
|
-
|
301
|
-
if resp.status_code < 200 or resp.status_code >= 300:
|
302
|
-
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
|
384
|
+
response = self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
|
385
|
+
data_to_presigned = response.accepted_result.accepted_status.upload_details
|
386
|
+
presigned_url = response.accepted_result.accepted_status.upload_url
|
303
387
|
|
304
|
-
|
388
|
+
self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
389
|
+
return response.raw_response
|
305
390
|
|
306
391
|
def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
|
307
392
|
if self._queued_retry_enabled and response.raw_response.status_code == 202:
|
@@ -315,52 +400,6 @@ class PangeaRequest(PangeaRequestBase):
|
|
315
400
|
|
316
401
|
return response
|
317
402
|
|
318
|
-
def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
|
319
|
-
"""Makes the GET call to a Pangea Service endpoint.
|
320
|
-
|
321
|
-
Args:
|
322
|
-
endpoint(str): The Pangea Service API endpoint.
|
323
|
-
path(str): Additional URL path
|
324
|
-
|
325
|
-
Returns:
|
326
|
-
PangeaResponse which contains the response in its entirety and
|
327
|
-
various properties to retrieve individual fields
|
328
|
-
"""
|
329
|
-
|
330
|
-
url = self._url(path)
|
331
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
332
|
-
requests_response = self.session.get(url, headers=self._headers())
|
333
|
-
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
|
334
|
-
|
335
|
-
self.logger.debug(
|
336
|
-
json.dumps(
|
337
|
-
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
338
|
-
default=default_encoder,
|
339
|
-
)
|
340
|
-
)
|
341
|
-
|
342
|
-
if check_response is False:
|
343
|
-
return pangea_response
|
344
|
-
|
345
|
-
return self._check_response(pangea_response)
|
346
|
-
|
347
|
-
def poll_result_by_id(
|
348
|
-
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
349
|
-
):
|
350
|
-
path = self._get_poll_path(request_id)
|
351
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
352
|
-
return self.get(path, result_class, check_response=check_response)
|
353
|
-
|
354
|
-
def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
355
|
-
request_id = response.request_id
|
356
|
-
if not request_id:
|
357
|
-
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
358
|
-
|
359
|
-
if response.status != ResponseStatus.ACCEPTED.value:
|
360
|
-
raise pe.PangeaException("Response already proccesed")
|
361
|
-
|
362
|
-
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
363
|
-
|
364
403
|
def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
|
365
404
|
retry_count = 1
|
366
405
|
start = time.time()
|
@@ -373,26 +412,26 @@ class PangeaRequest(PangeaRequestBase):
|
|
373
412
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
|
374
413
|
return self._check_response(response)
|
375
414
|
|
376
|
-
def _poll_presigned_url(self,
|
377
|
-
if
|
378
|
-
raise AttributeError("
|
415
|
+
def _poll_presigned_url(self, response: PangeaResponse) -> PangeaResponse:
|
416
|
+
if response.http_status != 202:
|
417
|
+
raise AttributeError("Response should be 202")
|
379
418
|
|
380
|
-
if
|
381
|
-
return
|
419
|
+
if response.accepted_result.accepted_status.upload_url:
|
420
|
+
return response
|
382
421
|
|
383
422
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
384
423
|
retry_count = 1
|
385
424
|
start = time.time()
|
386
|
-
|
425
|
+
loop_resp = response
|
387
426
|
|
388
427
|
while (
|
389
|
-
|
390
|
-
and not
|
428
|
+
loop_resp.accepted_result is not None
|
429
|
+
and not loop_resp.accepted_result.accepted_status.upload_url
|
391
430
|
and not self._reach_timeout(start)
|
392
431
|
):
|
393
432
|
time.sleep(self._get_delay(retry_count, start))
|
394
433
|
try:
|
395
|
-
self.poll_result_once(
|
434
|
+
self.poll_result_once(loop_resp, check_response=False)
|
396
435
|
msg = "Polling presigned url return 200 instead of 202"
|
397
436
|
self.logger.debug(
|
398
437
|
json.dumps(
|
@@ -402,6 +441,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
402
441
|
raise pe.PangeaException(msg)
|
403
442
|
except pe.AcceptedRequestException as e:
|
404
443
|
retry_count += 1
|
444
|
+
loop_resp = e.response
|
405
445
|
loop_exc = e
|
406
446
|
except Exception as e:
|
407
447
|
self.logger.debug(
|
@@ -409,12 +449,12 @@ class PangeaRequest(PangeaRequestBase):
|
|
409
449
|
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
|
410
450
|
)
|
411
451
|
)
|
412
|
-
raise pe.PresignedURLException("Failed to pull Presigned URL",
|
452
|
+
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_resp, e)
|
413
453
|
|
414
454
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
415
455
|
|
416
|
-
if
|
417
|
-
return
|
456
|
+
if loop_resp.accepted_result is not None and not loop_resp.accepted_result.accepted_status.upload_url:
|
457
|
+
return loop_resp
|
418
458
|
else:
|
419
459
|
raise loop_exc
|
420
460
|
|
pangea/response.py
CHANGED
@@ -13,8 +13,11 @@ T = TypeVar("T")
|
|
13
13
|
|
14
14
|
|
15
15
|
class TransferMethod(str, enum.Enum):
|
16
|
-
DIRECT = "
|
16
|
+
DIRECT = "post-url" # deprecated, use POST_URL instead
|
17
17
|
MULTIPART = "multipart"
|
18
|
+
POST_URL = "post-url"
|
19
|
+
PUT_URL = "put-url"
|
20
|
+
# URL = "url"
|
18
21
|
|
19
22
|
def __str__(self):
|
20
23
|
return str(self.value)
|
@@ -76,7 +79,7 @@ class AcceptedStatus(APIResponseModel):
|
|
76
79
|
|
77
80
|
|
78
81
|
class AcceptedResult(PangeaResponseResult):
|
79
|
-
accepted_status: AcceptedStatus
|
82
|
+
accepted_status: Optional[AcceptedStatus] = None
|
80
83
|
|
81
84
|
|
82
85
|
class PangeaError(PangeaResponseResult):
|
@@ -144,8 +147,11 @@ class PangeaResponse(Generic[T], ResponseHeader):
|
|
144
147
|
if self.raw_result is not None and issubclass(self.result_class, PangeaResponseResult) and self.success
|
145
148
|
else None
|
146
149
|
)
|
147
|
-
if not self.success
|
148
|
-
|
150
|
+
if not self.success:
|
151
|
+
if self.http_status == 202:
|
152
|
+
self.accepted_result = AcceptedResult(**self.raw_result) if self.raw_result is not None else None
|
153
|
+
else:
|
154
|
+
self.pangea_error = PangeaError(**self.raw_result) if self.raw_result is not None else None
|
149
155
|
|
150
156
|
@property
|
151
157
|
def success(self) -> bool:
|