pangea-sdk 3.0.0__py3-none-any.whl → 3.2.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- pangea/__init__.py +1 -1
- pangea/asyncio/request.py +138 -22
- pangea/asyncio/services/audit.py +82 -3
- pangea/asyncio/services/file_scan.py +6 -2
- pangea/exceptions.py +20 -2
- pangea/request.py +158 -35
- pangea/response.py +26 -4
- pangea/services/audit/audit.py +163 -62
- pangea/services/audit/models.py +32 -0
- pangea/services/authn/models.py +2 -2
- pangea/services/file_scan.py +21 -3
- pangea/services/vault/models/common.py +11 -0
- pangea/utils.py +24 -1
- {pangea_sdk-3.0.0.dist-info → pangea_sdk-3.2.0.dist-info}/METADATA +2 -1
- {pangea_sdk-3.0.0.dist-info → pangea_sdk-3.2.0.dist-info}/RECORD +16 -16
- {pangea_sdk-3.0.0.dist-info → pangea_sdk-3.2.0.dist-info}/WHEEL +0 -0
pangea/__init__.py
CHANGED
pangea/asyncio/request.py
CHANGED
@@ -7,12 +7,12 @@ import time
|
|
7
7
|
from typing import Dict, List, Optional, Tuple, Type, Union
|
8
8
|
|
9
9
|
import aiohttp
|
10
|
+
import pangea.exceptions as pe
|
10
11
|
from aiohttp import FormData
|
11
|
-
from pangea import exceptions
|
12
12
|
|
13
13
|
# from requests.adapters import HTTPAdapter, Retry
|
14
14
|
from pangea.request import PangeaRequestBase
|
15
|
-
from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus
|
15
|
+
from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
16
16
|
from pangea.utils import default_encoder
|
17
17
|
|
18
18
|
|
@@ -32,6 +32,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
32
32
|
data: Union[str, Dict] = {},
|
33
33
|
files: Optional[List[Tuple]] = None,
|
34
34
|
poll_result: bool = True,
|
35
|
+
url: Optional[str] = None,
|
35
36
|
) -> PangeaResponse:
|
36
37
|
"""Makes the POST call to a Pangea Service endpoint.
|
37
38
|
|
@@ -43,41 +44,156 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
43
44
|
PangeaResponse which contains the response in its entirety and
|
44
45
|
various properties to retrieve individual fields
|
45
46
|
"""
|
46
|
-
url
|
47
|
+
if url is None:
|
48
|
+
url = self._url(endpoint)
|
49
|
+
|
47
50
|
# Set config ID if available
|
48
|
-
if self.config_id and data.
|
51
|
+
if self.config_id and data.get("config_id", None) is None:
|
49
52
|
data["config_id"] = self.config_id
|
50
53
|
|
51
|
-
|
54
|
+
if (
|
55
|
+
files is not None
|
56
|
+
and type(data) is dict
|
57
|
+
and data.get("transfer_method", None) == TransferMethod.DIRECT.value
|
58
|
+
):
|
59
|
+
requests_response = await self._post_presigned_url(
|
60
|
+
endpoint, result_class=result_class, data=data, files=files
|
61
|
+
)
|
62
|
+
else:
|
63
|
+
requests_response = await self._http_post(
|
64
|
+
url, headers=self._headers(), data=data, files=files, presigned_url_post=False
|
65
|
+
)
|
66
|
+
|
67
|
+
pangea_response = PangeaResponse(
|
68
|
+
requests_response, result_class=result_class, json=await requests_response.json()
|
69
|
+
)
|
70
|
+
if poll_result:
|
71
|
+
pangea_response = await self._handle_queued_result(pangea_response)
|
72
|
+
|
73
|
+
return self._check_response(pangea_response)
|
74
|
+
|
75
|
+
async def _http_post(
|
76
|
+
self,
|
77
|
+
url: str,
|
78
|
+
headers: Dict = {},
|
79
|
+
data: Union[str, Dict] = {},
|
80
|
+
files: Optional[List[Tuple]] = None,
|
81
|
+
presigned_url_post: bool = False,
|
82
|
+
) -> aiohttp.ClientResponse:
|
52
83
|
self.logger.debug(
|
53
|
-
json.dumps(
|
84
|
+
json.dumps(
|
85
|
+
{"service": self.service, "action": "http_post", "url": url, "data": data}, default=default_encoder
|
86
|
+
)
|
54
87
|
)
|
55
88
|
|
56
|
-
# FIXME: use FormData to send multipart request
|
57
89
|
if files:
|
58
90
|
form = FormData()
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
91
|
+
if presigned_url_post:
|
92
|
+
for k, v in data.items():
|
93
|
+
form.add_field(k, v)
|
94
|
+
for name, value in files:
|
95
|
+
form.add_field("file", value[1], filename=value[0], content_type=value[2])
|
96
|
+
else:
|
97
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
98
|
+
form.add_field("request", data_send, content_type="application/json")
|
99
|
+
for name, value in files:
|
100
|
+
form.add_field(name, value[1], filename=value[0], content_type=value[2])
|
63
101
|
|
64
102
|
data_send = form
|
103
|
+
else:
|
104
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
65
105
|
|
66
|
-
|
67
|
-
pangea_response = PangeaResponse(
|
68
|
-
requests_response, result_class=result_class, json=await requests_response.json()
|
69
|
-
)
|
70
|
-
|
71
|
-
if poll_result:
|
72
|
-
pangea_response = await self._handle_queued_result(pangea_response)
|
106
|
+
return await self.session.post(url, headers=headers, data=data_send)
|
73
107
|
|
108
|
+
async def _post_presigned_url(
|
109
|
+
self,
|
110
|
+
endpoint: str,
|
111
|
+
result_class: Type[PangeaResponseResult],
|
112
|
+
data: Union[str, Dict] = {},
|
113
|
+
files: Optional[List[Tuple]] = None,
|
114
|
+
):
|
115
|
+
if len(files) == 0:
|
116
|
+
raise AttributeError("files attribute should have at least 1 file")
|
117
|
+
|
118
|
+
# Send request
|
119
|
+
try:
|
120
|
+
# This should return 202 (AcceptedRequestException)
|
121
|
+
resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
122
|
+
raise pe.PresignedURLException("Should return 202", resp)
|
123
|
+
|
124
|
+
except pe.AcceptedRequestException as e:
|
125
|
+
accepted_exception = e
|
126
|
+
except Exception as e:
|
127
|
+
raise e
|
128
|
+
|
129
|
+
# Receive 202 with accepted_status
|
130
|
+
result = await self._poll_presigned_url(accepted_exception)
|
131
|
+
data_to_presigned = result.accepted_status.upload_details
|
132
|
+
presigned_url = result.accepted_status.upload_url
|
133
|
+
|
134
|
+
# Send multipart request with file and upload_details as body
|
135
|
+
resp = await self._http_post(url=presigned_url, data=data_to_presigned, files=files, presigned_url_post=True)
|
74
136
|
self.logger.debug(
|
75
137
|
json.dumps(
|
76
|
-
{
|
138
|
+
{
|
139
|
+
"service": self.service,
|
140
|
+
"action": "post presigned",
|
141
|
+
"url": presigned_url,
|
142
|
+
"response": await resp.text(),
|
143
|
+
},
|
77
144
|
default=default_encoder,
|
78
145
|
)
|
79
146
|
)
|
80
|
-
|
147
|
+
|
148
|
+
if resp.status < 200 or resp.status >= 300:
|
149
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
|
150
|
+
|
151
|
+
return accepted_exception.response.raw_response
|
152
|
+
|
153
|
+
async def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
|
154
|
+
if type(initial_exc) is not pe.AcceptedRequestException:
|
155
|
+
raise AttributeError("Exception should be of type AcceptedRequestException")
|
156
|
+
|
157
|
+
if initial_exc.accepted_result.accepted_status.upload_url:
|
158
|
+
return initial_exc.accepted_result
|
159
|
+
|
160
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
161
|
+
retry_count = 1
|
162
|
+
start = time.time()
|
163
|
+
loop_exc = initial_exc
|
164
|
+
|
165
|
+
while (
|
166
|
+
loop_exc.accepted_result is not None
|
167
|
+
and not loop_exc.accepted_result.accepted_status.upload_url
|
168
|
+
and not self._reach_timeout(start)
|
169
|
+
):
|
170
|
+
await asyncio.sleep(self._get_delay(retry_count, start))
|
171
|
+
try:
|
172
|
+
await self.poll_result_once(initial_exc.response, check_response=False)
|
173
|
+
msg = "Polling presigned url return 200 instead of 202"
|
174
|
+
self.logger.debug(
|
175
|
+
json.dumps(
|
176
|
+
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {msg}}
|
177
|
+
)
|
178
|
+
)
|
179
|
+
raise pe.PangeaException(msg)
|
180
|
+
except pe.AcceptedRequestException as e:
|
181
|
+
retry_count += 1
|
182
|
+
loop_exc = e
|
183
|
+
except Exception as e:
|
184
|
+
self.logger.debug(
|
185
|
+
json.dumps(
|
186
|
+
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
|
187
|
+
)
|
188
|
+
)
|
189
|
+
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
|
190
|
+
|
191
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
192
|
+
|
193
|
+
if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
|
194
|
+
return loop_exc.accepted_result
|
195
|
+
else:
|
196
|
+
raise loop_exc
|
81
197
|
|
82
198
|
async def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
|
83
199
|
if self._queued_retry_enabled and response.http_status == 202:
|
@@ -135,10 +251,10 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
135
251
|
async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
136
252
|
request_id = response.request_id
|
137
253
|
if not request_id:
|
138
|
-
raise
|
254
|
+
raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
|
139
255
|
|
140
256
|
if response.status != ResponseStatus.ACCEPTED.value:
|
141
|
-
raise
|
257
|
+
raise pe.PangeaException("Response already proccesed")
|
142
258
|
|
143
259
|
return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
144
260
|
|
pangea/asyncio/services/audit.py
CHANGED
@@ -1,13 +1,15 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import datetime
|
4
|
-
from typing import Any, Dict, Optional, Union
|
4
|
+
from typing import Any, Dict, List, Optional, Union
|
5
5
|
|
6
|
+
import pangea.exceptions as pexc
|
6
7
|
from pangea.response import PangeaResponse
|
7
8
|
from pangea.services.audit.audit import AuditBase
|
8
9
|
from pangea.services.audit.exceptions import AuditException
|
9
10
|
from pangea.services.audit.models import (
|
10
11
|
Event,
|
12
|
+
LogBulkResult,
|
11
13
|
LogResult,
|
12
14
|
RootRequest,
|
13
15
|
RootResult,
|
@@ -172,9 +174,86 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
172
174
|
print(f"\\t{err.detail} \\n")
|
173
175
|
"""
|
174
176
|
|
175
|
-
input = self.
|
177
|
+
input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
|
176
178
|
response = await self.request.post("v1/log", LogResult, data=input.dict(exclude_none=True))
|
177
|
-
|
179
|
+
if response.success:
|
180
|
+
self._process_log_result(response.result, verify=verify)
|
181
|
+
return response
|
182
|
+
|
183
|
+
async def log_bulk(
|
184
|
+
self,
|
185
|
+
events: List[Dict[str, Any]],
|
186
|
+
sign_local: bool = False,
|
187
|
+
verbose: Optional[bool] = None,
|
188
|
+
) -> PangeaResponse[LogBulkResult]:
|
189
|
+
"""
|
190
|
+
Log an entry
|
191
|
+
|
192
|
+
Create a log entry in the Secure Audit Log.
|
193
|
+
Args:
|
194
|
+
events (List[dict[str, Any]]): events to be logged
|
195
|
+
verify (bool, optional): True to verify logs consistency after response.
|
196
|
+
sign_local (bool, optional): True to sign event with local key.
|
197
|
+
verbose (bool, optional): True to get a more verbose response.
|
198
|
+
Raises:
|
199
|
+
AuditException: If an audit based api exception happens
|
200
|
+
PangeaAPIException: If an API Error happens
|
201
|
+
|
202
|
+
Returns:
|
203
|
+
A PangeaResponse where the hash of event data and optional verbose
|
204
|
+
results are returned in the response.result field.
|
205
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
|
206
|
+
|
207
|
+
Examples:
|
208
|
+
FIXME:
|
209
|
+
"""
|
210
|
+
|
211
|
+
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
212
|
+
response = await self.request.post("v2/log", LogBulkResult, data=input.dict(exclude_none=True))
|
213
|
+
if response.success:
|
214
|
+
for result in response.result.results:
|
215
|
+
self._process_log_result(result, verify=True)
|
216
|
+
return response
|
217
|
+
|
218
|
+
async def log_bulk_async(
|
219
|
+
self,
|
220
|
+
events: List[Dict[str, Any]],
|
221
|
+
sign_local: bool = False,
|
222
|
+
verbose: Optional[bool] = None,
|
223
|
+
) -> PangeaResponse[LogBulkResult]:
|
224
|
+
"""
|
225
|
+
Log an entry
|
226
|
+
|
227
|
+
Create a log entry in the Secure Audit Log.
|
228
|
+
Args:
|
229
|
+
events (List[dict[str, Any]]): events to be logged
|
230
|
+
verify (bool, optional): True to verify logs consistency after response.
|
231
|
+
sign_local (bool, optional): True to sign event with local key.
|
232
|
+
verbose (bool, optional): True to get a more verbose response.
|
233
|
+
Raises:
|
234
|
+
AuditException: If an audit based api exception happens
|
235
|
+
PangeaAPIException: If an API Error happens
|
236
|
+
|
237
|
+
Returns:
|
238
|
+
A PangeaResponse where the hash of event data and optional verbose
|
239
|
+
results are returned in the response.result field.
|
240
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
|
241
|
+
|
242
|
+
Examples:
|
243
|
+
FIXME:
|
244
|
+
"""
|
245
|
+
|
246
|
+
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
247
|
+
try:
|
248
|
+
response = await self.request.post(
|
249
|
+
"v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
|
250
|
+
)
|
251
|
+
except pexc.AcceptedRequestException as e:
|
252
|
+
return e.response
|
253
|
+
if response.success:
|
254
|
+
for result in response.result.results:
|
255
|
+
self._process_log_result(result, verify=True)
|
256
|
+
return response
|
178
257
|
|
179
258
|
async def search(
|
180
259
|
self,
|
@@ -5,6 +5,7 @@ from typing import Optional
|
|
5
5
|
|
6
6
|
import pangea.services.file_scan as m
|
7
7
|
from pangea.response import PangeaResponse
|
8
|
+
from pangea.utils import get_presigned_url_upload_params
|
8
9
|
|
9
10
|
from .base import ServiceBaseAsync
|
10
11
|
|
@@ -79,14 +80,17 @@ class FileScanAsync(ServiceBaseAsync):
|
|
79
80
|
for err in e.errors:
|
80
81
|
print(f"\\t{err.detail} \\n")
|
81
82
|
"""
|
82
|
-
input = m.FileScanRequest(verbose=verbose, raw=raw, provider=provider)
|
83
83
|
|
84
84
|
if file or file_path:
|
85
85
|
if file_path:
|
86
86
|
file = open(file_path, "rb")
|
87
|
-
|
87
|
+
crc, sha, size, _ = get_presigned_url_upload_params(file)
|
88
|
+
files = [("upload", ("filename", file, "application/octet-stream"))]
|
88
89
|
else:
|
89
90
|
raise ValueError("Need to set file_path or file arguments")
|
90
91
|
|
92
|
+
input = m.FileScanRequest(
|
93
|
+
verbose=verbose, raw=raw, provider=provider, transfer_crc32c=crc, transfer_sha256=sha, transfer_size=size
|
94
|
+
)
|
91
95
|
data = input.dict(exclude_none=True)
|
92
96
|
return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
|
pangea/exceptions.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
|
4
|
-
from typing import List
|
4
|
+
from typing import List, Optional
|
5
5
|
|
6
|
-
from pangea.response import ErrorField, PangeaResponse
|
6
|
+
from pangea.response import AcceptedResult, ErrorField, PangeaResponse
|
7
7
|
|
8
8
|
|
9
9
|
class PangeaException(Exception):
|
@@ -14,6 +14,14 @@ class PangeaException(Exception):
|
|
14
14
|
self.message = message
|
15
15
|
|
16
16
|
|
17
|
+
class PresignedUploadError(PangeaException):
|
18
|
+
body: str
|
19
|
+
|
20
|
+
def __init__(self, message: str, body: str):
|
21
|
+
super().__init__(message)
|
22
|
+
self.body = body
|
23
|
+
|
24
|
+
|
17
25
|
class PangeaAPIException(PangeaException):
|
18
26
|
"""Exceptions raised during API calls"""
|
19
27
|
|
@@ -43,6 +51,14 @@ class PangeaAPIException(PangeaException):
|
|
43
51
|
return self.__repr__()
|
44
52
|
|
45
53
|
|
54
|
+
class PresignedURLException(PangeaAPIException):
|
55
|
+
cause: Optional[Exception] = None
|
56
|
+
|
57
|
+
def __init__(self, message: str, response: PangeaResponse, cause: Optional[Exception] = None):
|
58
|
+
super().__init__(message, response)
|
59
|
+
self.cause = cause
|
60
|
+
|
61
|
+
|
46
62
|
class ValidationException(PangeaAPIException):
|
47
63
|
"""Pangea Validation Errors denoting issues with an API request"""
|
48
64
|
|
@@ -103,11 +119,13 @@ class AcceptedRequestException(PangeaAPIException):
|
|
103
119
|
"""Accepted request exception. Async response"""
|
104
120
|
|
105
121
|
request_id: str
|
122
|
+
accepted_result: Optional[AcceptedResult] = None
|
106
123
|
|
107
124
|
def __init__(self, response: PangeaResponse):
|
108
125
|
message = f"summary: {response.summary}. request_id: {response.request_id}."
|
109
126
|
super().__init__(message, response)
|
110
127
|
self.request_id = response.request_id
|
128
|
+
self.accepted_result = AcceptedResult(**response.raw_result) if response.raw_result is not None else None
|
111
129
|
|
112
130
|
|
113
131
|
class ServiceNotAvailableException(PangeaAPIException):
|
pangea/request.py
CHANGED
@@ -9,10 +9,10 @@ from typing import Dict, List, Optional, Tuple, Type, Union
|
|
9
9
|
|
10
10
|
import aiohttp
|
11
11
|
import pangea
|
12
|
+
import pangea.exceptions as pe
|
12
13
|
import requests
|
13
|
-
from pangea import exceptions
|
14
14
|
from pangea.config import PangeaConfig
|
15
|
-
from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus
|
15
|
+
from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
16
16
|
from pangea.utils import default_encoder
|
17
17
|
from requests.adapters import HTTPAdapter, Retry
|
18
18
|
|
@@ -131,38 +131,38 @@ class PangeaRequestBase(object):
|
|
131
131
|
)
|
132
132
|
|
133
133
|
if status == ResponseStatus.VALIDATION_ERR.value:
|
134
|
-
raise
|
134
|
+
raise pe.ValidationException(summary, response)
|
135
135
|
elif status == ResponseStatus.TOO_MANY_REQUESTS.value:
|
136
|
-
raise
|
136
|
+
raise pe.RateLimitException(summary, response)
|
137
137
|
elif status == ResponseStatus.NO_CREDIT.value:
|
138
|
-
raise
|
138
|
+
raise pe.NoCreditException(summary, response)
|
139
139
|
elif status == ResponseStatus.UNAUTHORIZED.value:
|
140
|
-
raise
|
140
|
+
raise pe.UnauthorizedException(self.service, response)
|
141
141
|
elif status == ResponseStatus.SERVICE_NOT_ENABLED.value:
|
142
|
-
raise
|
142
|
+
raise pe.ServiceNotEnabledException(self.service, response)
|
143
143
|
elif status == ResponseStatus.PROVIDER_ERR.value:
|
144
|
-
raise
|
144
|
+
raise pe.ProviderErrorException(summary, response)
|
145
145
|
elif status in (ResponseStatus.MISSING_CONFIG_ID_SCOPE.value, ResponseStatus.MISSING_CONFIG_ID.value):
|
146
|
-
raise
|
146
|
+
raise pe.MissingConfigID(self.service, response)
|
147
147
|
elif status == ResponseStatus.SERVICE_NOT_AVAILABLE.value:
|
148
|
-
raise
|
148
|
+
raise pe.ServiceNotAvailableException(summary, response)
|
149
149
|
elif status == ResponseStatus.TREE_NOT_FOUND.value:
|
150
|
-
raise
|
150
|
+
raise pe.TreeNotFoundException(summary, response)
|
151
151
|
elif status == ResponseStatus.IP_NOT_FOUND.value:
|
152
|
-
raise
|
152
|
+
raise pe.IPNotFoundException(summary)
|
153
153
|
elif status == ResponseStatus.BAD_OFFSET.value:
|
154
|
-
raise
|
154
|
+
raise pe.BadOffsetException(summary, response)
|
155
155
|
elif status == ResponseStatus.FORBIDDEN_VAULT_OPERATION.value:
|
156
|
-
raise
|
156
|
+
raise pe.ForbiddenVaultOperation(summary, response)
|
157
157
|
elif status == ResponseStatus.VAULT_ITEM_NOT_FOUND.value:
|
158
|
-
raise
|
158
|
+
raise pe.VaultItemNotFound(summary, response)
|
159
159
|
elif status == ResponseStatus.NOT_FOUND.value:
|
160
|
-
raise
|
160
|
+
raise pe.NotFound(response.raw_response.url if response.raw_response is not None else "", response)
|
161
161
|
elif status == ResponseStatus.INTERNAL_SERVER_ERROR.value:
|
162
|
-
raise
|
162
|
+
raise pe.InternalServerError(response)
|
163
163
|
elif status == ResponseStatus.ACCEPTED.value:
|
164
|
-
raise
|
165
|
-
raise
|
164
|
+
raise pe.AcceptedRequestException(response)
|
165
|
+
raise pe.PangeaAPIException(f"{summary} ", response)
|
166
166
|
|
167
167
|
|
168
168
|
class PangeaRequest(PangeaRequestBase):
|
@@ -184,6 +184,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
184
184
|
data: Union[str, Dict] = {},
|
185
185
|
files: Optional[List[Tuple]] = None,
|
186
186
|
poll_result: bool = True,
|
187
|
+
url: Optional[str] = None,
|
187
188
|
) -> PangeaResponse:
|
188
189
|
"""Makes the POST call to a Pangea Service endpoint.
|
189
190
|
|
@@ -195,35 +196,112 @@ class PangeaRequest(PangeaRequestBase):
|
|
195
196
|
PangeaResponse which contains the response in its entirety and
|
196
197
|
various properties to retrieve individual fields
|
197
198
|
"""
|
198
|
-
url
|
199
|
+
if url is None:
|
200
|
+
url = self._url(endpoint)
|
201
|
+
|
199
202
|
# Set config ID if available
|
200
|
-
if self.config_id and data.
|
203
|
+
if self.config_id and data.get("config_id", None) is None:
|
201
204
|
data["config_id"] = self.config_id
|
202
205
|
|
203
|
-
|
206
|
+
if (
|
207
|
+
files is not None
|
208
|
+
and type(data) is dict
|
209
|
+
and data.get("transfer_method", None) == TransferMethod.DIRECT.value
|
210
|
+
):
|
211
|
+
requests_response = self._post_presigned_url(endpoint, result_class=result_class, data=data, files=files)
|
212
|
+
else:
|
213
|
+
requests_response = self._http_post(
|
214
|
+
url, headers=self._headers(), data=data, files=files, multipart_post=True
|
215
|
+
)
|
216
|
+
|
217
|
+
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
|
218
|
+
if poll_result:
|
219
|
+
pangea_response = self._handle_queued_result(pangea_response)
|
220
|
+
|
221
|
+
return self._check_response(pangea_response)
|
222
|
+
|
223
|
+
def _http_post(
|
224
|
+
self,
|
225
|
+
url: str,
|
226
|
+
headers: Dict = {},
|
227
|
+
data: Union[str, Dict] = {},
|
228
|
+
files: Optional[List[Tuple]] = None,
|
229
|
+
multipart_post: bool = True,
|
230
|
+
) -> requests.Response:
|
204
231
|
self.logger.debug(
|
205
|
-
json.dumps(
|
232
|
+
json.dumps(
|
233
|
+
{"service": self.service, "action": "http_post", "url": url, "data": data}, default=default_encoder
|
234
|
+
)
|
206
235
|
)
|
207
236
|
|
237
|
+
data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
|
238
|
+
return self.session.post(url, headers=headers, data=data_send, files=files)
|
239
|
+
|
240
|
+
def _http_post_process(
|
241
|
+
self, data: Union[str, Dict] = {}, files: Optional[List[Tuple]] = None, multipart_post: bool = True
|
242
|
+
):
|
208
243
|
if files:
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
244
|
+
if multipart_post is True:
|
245
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
246
|
+
multi = [("request", (None, data_send, "application/json"))]
|
247
|
+
multi.extend(files)
|
248
|
+
files = multi
|
249
|
+
return None, files
|
250
|
+
else:
|
251
|
+
# Post to presigned url as form
|
252
|
+
data_send = []
|
253
|
+
for k, v in data.items():
|
254
|
+
data_send.append((k, v))
|
255
|
+
# When posting to presigned url, file key should be 'file'
|
256
|
+
files = {
|
257
|
+
"file": files[0][1],
|
258
|
+
}
|
259
|
+
return data_send, files
|
260
|
+
else:
|
261
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
262
|
+
return data_send, None
|
213
263
|
|
214
|
-
|
264
|
+
return data, files
|
265
|
+
|
266
|
+
def _post_presigned_url(
|
267
|
+
self,
|
268
|
+
endpoint: str,
|
269
|
+
result_class: Type[PangeaResponseResult],
|
270
|
+
data: Union[str, Dict] = {},
|
271
|
+
files: Optional[List[Tuple]] = None,
|
272
|
+
):
|
273
|
+
if len(files) == 0:
|
274
|
+
raise AttributeError("files attribute should have at least 1 file")
|
275
|
+
|
276
|
+
# Send request
|
277
|
+
try:
|
278
|
+
# This should return 202 (AcceptedRequestException)
|
279
|
+
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
280
|
+
raise pe.PresignedURLException("Should return 202", resp)
|
281
|
+
|
282
|
+
except pe.AcceptedRequestException as e:
|
283
|
+
accepted_exception = e
|
284
|
+
except Exception as e:
|
285
|
+
raise e
|
286
|
+
|
287
|
+
# Receive 202 with accepted_status
|
288
|
+
result = self._poll_presigned_url(accepted_exception)
|
289
|
+
data_to_presigned = result.accepted_status.upload_details
|
290
|
+
presigned_url = result.accepted_status.upload_url
|
291
|
+
|
292
|
+
# Send multipart request with file and upload_details as body
|
293
|
+
resp = self._http_post(url=presigned_url, data=data_to_presigned, files=files, multipart_post=False)
|
215
294
|
self.logger.debug(
|
216
295
|
json.dumps(
|
217
|
-
{"service": self.service, "action": "post", "url":
|
296
|
+
{"service": self.service, "action": "post presigned", "url": presigned_url, "response": resp.text},
|
218
297
|
default=default_encoder,
|
219
298
|
)
|
220
299
|
)
|
221
300
|
|
222
|
-
|
223
|
-
|
224
|
-
pangea_response = self._handle_queued_result(pangea_response)
|
301
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
302
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
|
225
303
|
|
226
|
-
return
|
304
|
+
return accepted_exception.response.raw_response
|
227
305
|
|
228
306
|
def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
|
229
307
|
if self._queued_retry_enabled and response.raw_response.status_code == 202:
|
@@ -276,10 +354,10 @@ class PangeaRequest(PangeaRequestBase):
|
|
276
354
|
def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
277
355
|
request_id = response.request_id
|
278
356
|
if not request_id:
|
279
|
-
raise
|
357
|
+
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
280
358
|
|
281
359
|
if response.status != ResponseStatus.ACCEPTED.value:
|
282
|
-
raise
|
360
|
+
raise pe.PangeaException("Response already proccesed")
|
283
361
|
|
284
362
|
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
285
363
|
|
@@ -295,6 +373,51 @@ class PangeaRequest(PangeaRequestBase):
|
|
295
373
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
|
296
374
|
return self._check_response(response)
|
297
375
|
|
376
|
+
def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
|
377
|
+
if type(initial_exc) is not pe.AcceptedRequestException:
|
378
|
+
raise AttributeError("Exception should be of type AcceptedRequestException")
|
379
|
+
|
380
|
+
if initial_exc.accepted_result.accepted_status.upload_url:
|
381
|
+
return initial_exc.accepted_result
|
382
|
+
|
383
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
384
|
+
retry_count = 1
|
385
|
+
start = time.time()
|
386
|
+
loop_exc = initial_exc
|
387
|
+
|
388
|
+
while (
|
389
|
+
loop_exc.accepted_result is not None
|
390
|
+
and not loop_exc.accepted_result.accepted_status.upload_url
|
391
|
+
and not self._reach_timeout(start)
|
392
|
+
):
|
393
|
+
time.sleep(self._get_delay(retry_count, start))
|
394
|
+
try:
|
395
|
+
self.poll_result_once(initial_exc.response, check_response=False)
|
396
|
+
msg = "Polling presigned url return 200 instead of 202"
|
397
|
+
self.logger.debug(
|
398
|
+
json.dumps(
|
399
|
+
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {msg}}
|
400
|
+
)
|
401
|
+
)
|
402
|
+
raise pe.PangeaException(msg)
|
403
|
+
except pe.AcceptedRequestException as e:
|
404
|
+
retry_count += 1
|
405
|
+
loop_exc = e
|
406
|
+
except Exception as e:
|
407
|
+
self.logger.debug(
|
408
|
+
json.dumps(
|
409
|
+
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
|
410
|
+
)
|
411
|
+
)
|
412
|
+
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
|
413
|
+
|
414
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
415
|
+
|
416
|
+
if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
|
417
|
+
return loop_exc.accepted_result
|
418
|
+
else:
|
419
|
+
raise loop_exc
|
420
|
+
|
298
421
|
def _init_session(self) -> requests.Session:
|
299
422
|
retry_config = Retry(
|
300
423
|
total=self.config.request_retries,
|