pangea-sdk 3.0.0__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +1 -1
- pangea/asyncio/request.py +138 -22
- pangea/asyncio/services/audit.py +82 -3
- pangea/asyncio/services/file_scan.py +6 -2
- pangea/exceptions.py +20 -2
- pangea/request.py +158 -35
- pangea/response.py +26 -4
- pangea/services/audit/audit.py +163 -62
- pangea/services/audit/models.py +32 -0
- pangea/services/authn/models.py +2 -2
- pangea/services/file_scan.py +21 -3
- pangea/services/vault/models/common.py +11 -0
- pangea/utils.py +24 -1
- {pangea_sdk-3.0.0.dist-info → pangea_sdk-3.2.0.dist-info}/METADATA +2 -1
- {pangea_sdk-3.0.0.dist-info → pangea_sdk-3.2.0.dist-info}/RECORD +16 -16
- {pangea_sdk-3.0.0.dist-info → pangea_sdk-3.2.0.dist-info}/WHEEL +0 -0
pangea/__init__.py
CHANGED
pangea/asyncio/request.py
CHANGED
@@ -7,12 +7,12 @@ import time
|
|
7
7
|
from typing import Dict, List, Optional, Tuple, Type, Union
|
8
8
|
|
9
9
|
import aiohttp
|
10
|
+
import pangea.exceptions as pe
|
10
11
|
from aiohttp import FormData
|
11
|
-
from pangea import exceptions
|
12
12
|
|
13
13
|
# from requests.adapters import HTTPAdapter, Retry
|
14
14
|
from pangea.request import PangeaRequestBase
|
15
|
-
from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus
|
15
|
+
from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
16
16
|
from pangea.utils import default_encoder
|
17
17
|
|
18
18
|
|
@@ -32,6 +32,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
32
32
|
data: Union[str, Dict] = {},
|
33
33
|
files: Optional[List[Tuple]] = None,
|
34
34
|
poll_result: bool = True,
|
35
|
+
url: Optional[str] = None,
|
35
36
|
) -> PangeaResponse:
|
36
37
|
"""Makes the POST call to a Pangea Service endpoint.
|
37
38
|
|
@@ -43,41 +44,156 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
43
44
|
PangeaResponse which contains the response in its entirety and
|
44
45
|
various properties to retrieve individual fields
|
45
46
|
"""
|
46
|
-
url
|
47
|
+
if url is None:
|
48
|
+
url = self._url(endpoint)
|
49
|
+
|
47
50
|
# Set config ID if available
|
48
|
-
if self.config_id and data.
|
51
|
+
if self.config_id and data.get("config_id", None) is None:
|
49
52
|
data["config_id"] = self.config_id
|
50
53
|
|
51
|
-
|
54
|
+
if (
|
55
|
+
files is not None
|
56
|
+
and type(data) is dict
|
57
|
+
and data.get("transfer_method", None) == TransferMethod.DIRECT.value
|
58
|
+
):
|
59
|
+
requests_response = await self._post_presigned_url(
|
60
|
+
endpoint, result_class=result_class, data=data, files=files
|
61
|
+
)
|
62
|
+
else:
|
63
|
+
requests_response = await self._http_post(
|
64
|
+
url, headers=self._headers(), data=data, files=files, presigned_url_post=False
|
65
|
+
)
|
66
|
+
|
67
|
+
pangea_response = PangeaResponse(
|
68
|
+
requests_response, result_class=result_class, json=await requests_response.json()
|
69
|
+
)
|
70
|
+
if poll_result:
|
71
|
+
pangea_response = await self._handle_queued_result(pangea_response)
|
72
|
+
|
73
|
+
return self._check_response(pangea_response)
|
74
|
+
|
75
|
+
async def _http_post(
|
76
|
+
self,
|
77
|
+
url: str,
|
78
|
+
headers: Dict = {},
|
79
|
+
data: Union[str, Dict] = {},
|
80
|
+
files: Optional[List[Tuple]] = None,
|
81
|
+
presigned_url_post: bool = False,
|
82
|
+
) -> aiohttp.ClientResponse:
|
52
83
|
self.logger.debug(
|
53
|
-
json.dumps(
|
84
|
+
json.dumps(
|
85
|
+
{"service": self.service, "action": "http_post", "url": url, "data": data}, default=default_encoder
|
86
|
+
)
|
54
87
|
)
|
55
88
|
|
56
|
-
# FIXME: use FormData to send multipart request
|
57
89
|
if files:
|
58
90
|
form = FormData()
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
91
|
+
if presigned_url_post:
|
92
|
+
for k, v in data.items():
|
93
|
+
form.add_field(k, v)
|
94
|
+
for name, value in files:
|
95
|
+
form.add_field("file", value[1], filename=value[0], content_type=value[2])
|
96
|
+
else:
|
97
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
98
|
+
form.add_field("request", data_send, content_type="application/json")
|
99
|
+
for name, value in files:
|
100
|
+
form.add_field(name, value[1], filename=value[0], content_type=value[2])
|
63
101
|
|
64
102
|
data_send = form
|
103
|
+
else:
|
104
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
65
105
|
|
66
|
-
|
67
|
-
pangea_response = PangeaResponse(
|
68
|
-
requests_response, result_class=result_class, json=await requests_response.json()
|
69
|
-
)
|
70
|
-
|
71
|
-
if poll_result:
|
72
|
-
pangea_response = await self._handle_queued_result(pangea_response)
|
106
|
+
return await self.session.post(url, headers=headers, data=data_send)
|
73
107
|
|
108
|
+
async def _post_presigned_url(
|
109
|
+
self,
|
110
|
+
endpoint: str,
|
111
|
+
result_class: Type[PangeaResponseResult],
|
112
|
+
data: Union[str, Dict] = {},
|
113
|
+
files: Optional[List[Tuple]] = None,
|
114
|
+
):
|
115
|
+
if len(files) == 0:
|
116
|
+
raise AttributeError("files attribute should have at least 1 file")
|
117
|
+
|
118
|
+
# Send request
|
119
|
+
try:
|
120
|
+
# This should return 202 (AcceptedRequestException)
|
121
|
+
resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
122
|
+
raise pe.PresignedURLException("Should return 202", resp)
|
123
|
+
|
124
|
+
except pe.AcceptedRequestException as e:
|
125
|
+
accepted_exception = e
|
126
|
+
except Exception as e:
|
127
|
+
raise e
|
128
|
+
|
129
|
+
# Receive 202 with accepted_status
|
130
|
+
result = await self._poll_presigned_url(accepted_exception)
|
131
|
+
data_to_presigned = result.accepted_status.upload_details
|
132
|
+
presigned_url = result.accepted_status.upload_url
|
133
|
+
|
134
|
+
# Send multipart request with file and upload_details as body
|
135
|
+
resp = await self._http_post(url=presigned_url, data=data_to_presigned, files=files, presigned_url_post=True)
|
74
136
|
self.logger.debug(
|
75
137
|
json.dumps(
|
76
|
-
{
|
138
|
+
{
|
139
|
+
"service": self.service,
|
140
|
+
"action": "post presigned",
|
141
|
+
"url": presigned_url,
|
142
|
+
"response": await resp.text(),
|
143
|
+
},
|
77
144
|
default=default_encoder,
|
78
145
|
)
|
79
146
|
)
|
80
|
-
|
147
|
+
|
148
|
+
if resp.status < 200 or resp.status >= 300:
|
149
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
|
150
|
+
|
151
|
+
return accepted_exception.response.raw_response
|
152
|
+
|
153
|
+
async def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
|
154
|
+
if type(initial_exc) is not pe.AcceptedRequestException:
|
155
|
+
raise AttributeError("Exception should be of type AcceptedRequestException")
|
156
|
+
|
157
|
+
if initial_exc.accepted_result.accepted_status.upload_url:
|
158
|
+
return initial_exc.accepted_result
|
159
|
+
|
160
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
161
|
+
retry_count = 1
|
162
|
+
start = time.time()
|
163
|
+
loop_exc = initial_exc
|
164
|
+
|
165
|
+
while (
|
166
|
+
loop_exc.accepted_result is not None
|
167
|
+
and not loop_exc.accepted_result.accepted_status.upload_url
|
168
|
+
and not self._reach_timeout(start)
|
169
|
+
):
|
170
|
+
await asyncio.sleep(self._get_delay(retry_count, start))
|
171
|
+
try:
|
172
|
+
await self.poll_result_once(initial_exc.response, check_response=False)
|
173
|
+
msg = "Polling presigned url return 200 instead of 202"
|
174
|
+
self.logger.debug(
|
175
|
+
json.dumps(
|
176
|
+
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {msg}}
|
177
|
+
)
|
178
|
+
)
|
179
|
+
raise pe.PangeaException(msg)
|
180
|
+
except pe.AcceptedRequestException as e:
|
181
|
+
retry_count += 1
|
182
|
+
loop_exc = e
|
183
|
+
except Exception as e:
|
184
|
+
self.logger.debug(
|
185
|
+
json.dumps(
|
186
|
+
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
|
187
|
+
)
|
188
|
+
)
|
189
|
+
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
|
190
|
+
|
191
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
192
|
+
|
193
|
+
if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
|
194
|
+
return loop_exc.accepted_result
|
195
|
+
else:
|
196
|
+
raise loop_exc
|
81
197
|
|
82
198
|
async def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
|
83
199
|
if self._queued_retry_enabled and response.http_status == 202:
|
@@ -135,10 +251,10 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
135
251
|
async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
136
252
|
request_id = response.request_id
|
137
253
|
if not request_id:
|
138
|
-
raise
|
254
|
+
raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
|
139
255
|
|
140
256
|
if response.status != ResponseStatus.ACCEPTED.value:
|
141
|
-
raise
|
257
|
+
raise pe.PangeaException("Response already proccesed")
|
142
258
|
|
143
259
|
return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
144
260
|
|
pangea/asyncio/services/audit.py
CHANGED
@@ -1,13 +1,15 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import datetime
|
4
|
-
from typing import Any, Dict, Optional, Union
|
4
|
+
from typing import Any, Dict, List, Optional, Union
|
5
5
|
|
6
|
+
import pangea.exceptions as pexc
|
6
7
|
from pangea.response import PangeaResponse
|
7
8
|
from pangea.services.audit.audit import AuditBase
|
8
9
|
from pangea.services.audit.exceptions import AuditException
|
9
10
|
from pangea.services.audit.models import (
|
10
11
|
Event,
|
12
|
+
LogBulkResult,
|
11
13
|
LogResult,
|
12
14
|
RootRequest,
|
13
15
|
RootResult,
|
@@ -172,9 +174,86 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
172
174
|
print(f"\\t{err.detail} \\n")
|
173
175
|
"""
|
174
176
|
|
175
|
-
input = self.
|
177
|
+
input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
|
176
178
|
response = await self.request.post("v1/log", LogResult, data=input.dict(exclude_none=True))
|
177
|
-
|
179
|
+
if response.success:
|
180
|
+
self._process_log_result(response.result, verify=verify)
|
181
|
+
return response
|
182
|
+
|
183
|
+
async def log_bulk(
|
184
|
+
self,
|
185
|
+
events: List[Dict[str, Any]],
|
186
|
+
sign_local: bool = False,
|
187
|
+
verbose: Optional[bool] = None,
|
188
|
+
) -> PangeaResponse[LogBulkResult]:
|
189
|
+
"""
|
190
|
+
Log an entry
|
191
|
+
|
192
|
+
Create a log entry in the Secure Audit Log.
|
193
|
+
Args:
|
194
|
+
events (List[dict[str, Any]]): events to be logged
|
195
|
+
verify (bool, optional): True to verify logs consistency after response.
|
196
|
+
sign_local (bool, optional): True to sign event with local key.
|
197
|
+
verbose (bool, optional): True to get a more verbose response.
|
198
|
+
Raises:
|
199
|
+
AuditException: If an audit based api exception happens
|
200
|
+
PangeaAPIException: If an API Error happens
|
201
|
+
|
202
|
+
Returns:
|
203
|
+
A PangeaResponse where the hash of event data and optional verbose
|
204
|
+
results are returned in the response.result field.
|
205
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
|
206
|
+
|
207
|
+
Examples:
|
208
|
+
FIXME:
|
209
|
+
"""
|
210
|
+
|
211
|
+
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
212
|
+
response = await self.request.post("v2/log", LogBulkResult, data=input.dict(exclude_none=True))
|
213
|
+
if response.success:
|
214
|
+
for result in response.result.results:
|
215
|
+
self._process_log_result(result, verify=True)
|
216
|
+
return response
|
217
|
+
|
218
|
+
async def log_bulk_async(
|
219
|
+
self,
|
220
|
+
events: List[Dict[str, Any]],
|
221
|
+
sign_local: bool = False,
|
222
|
+
verbose: Optional[bool] = None,
|
223
|
+
) -> PangeaResponse[LogBulkResult]:
|
224
|
+
"""
|
225
|
+
Log an entry
|
226
|
+
|
227
|
+
Create a log entry in the Secure Audit Log.
|
228
|
+
Args:
|
229
|
+
events (List[dict[str, Any]]): events to be logged
|
230
|
+
verify (bool, optional): True to verify logs consistency after response.
|
231
|
+
sign_local (bool, optional): True to sign event with local key.
|
232
|
+
verbose (bool, optional): True to get a more verbose response.
|
233
|
+
Raises:
|
234
|
+
AuditException: If an audit based api exception happens
|
235
|
+
PangeaAPIException: If an API Error happens
|
236
|
+
|
237
|
+
Returns:
|
238
|
+
A PangeaResponse where the hash of event data and optional verbose
|
239
|
+
results are returned in the response.result field.
|
240
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
|
241
|
+
|
242
|
+
Examples:
|
243
|
+
FIXME:
|
244
|
+
"""
|
245
|
+
|
246
|
+
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
247
|
+
try:
|
248
|
+
response = await self.request.post(
|
249
|
+
"v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
|
250
|
+
)
|
251
|
+
except pexc.AcceptedRequestException as e:
|
252
|
+
return e.response
|
253
|
+
if response.success:
|
254
|
+
for result in response.result.results:
|
255
|
+
self._process_log_result(result, verify=True)
|
256
|
+
return response
|
178
257
|
|
179
258
|
async def search(
|
180
259
|
self,
|
@@ -5,6 +5,7 @@ from typing import Optional
|
|
5
5
|
|
6
6
|
import pangea.services.file_scan as m
|
7
7
|
from pangea.response import PangeaResponse
|
8
|
+
from pangea.utils import get_presigned_url_upload_params
|
8
9
|
|
9
10
|
from .base import ServiceBaseAsync
|
10
11
|
|
@@ -79,14 +80,17 @@ class FileScanAsync(ServiceBaseAsync):
|
|
79
80
|
for err in e.errors:
|
80
81
|
print(f"\\t{err.detail} \\n")
|
81
82
|
"""
|
82
|
-
input = m.FileScanRequest(verbose=verbose, raw=raw, provider=provider)
|
83
83
|
|
84
84
|
if file or file_path:
|
85
85
|
if file_path:
|
86
86
|
file = open(file_path, "rb")
|
87
|
-
|
87
|
+
crc, sha, size, _ = get_presigned_url_upload_params(file)
|
88
|
+
files = [("upload", ("filename", file, "application/octet-stream"))]
|
88
89
|
else:
|
89
90
|
raise ValueError("Need to set file_path or file arguments")
|
90
91
|
|
92
|
+
input = m.FileScanRequest(
|
93
|
+
verbose=verbose, raw=raw, provider=provider, transfer_crc32c=crc, transfer_sha256=sha, transfer_size=size
|
94
|
+
)
|
91
95
|
data = input.dict(exclude_none=True)
|
92
96
|
return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
|
pangea/exceptions.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
|
4
|
-
from typing import List
|
4
|
+
from typing import List, Optional
|
5
5
|
|
6
|
-
from pangea.response import ErrorField, PangeaResponse
|
6
|
+
from pangea.response import AcceptedResult, ErrorField, PangeaResponse
|
7
7
|
|
8
8
|
|
9
9
|
class PangeaException(Exception):
|
@@ -14,6 +14,14 @@ class PangeaException(Exception):
|
|
14
14
|
self.message = message
|
15
15
|
|
16
16
|
|
17
|
+
class PresignedUploadError(PangeaException):
|
18
|
+
body: str
|
19
|
+
|
20
|
+
def __init__(self, message: str, body: str):
|
21
|
+
super().__init__(message)
|
22
|
+
self.body = body
|
23
|
+
|
24
|
+
|
17
25
|
class PangeaAPIException(PangeaException):
|
18
26
|
"""Exceptions raised during API calls"""
|
19
27
|
|
@@ -43,6 +51,14 @@ class PangeaAPIException(PangeaException):
|
|
43
51
|
return self.__repr__()
|
44
52
|
|
45
53
|
|
54
|
+
class PresignedURLException(PangeaAPIException):
|
55
|
+
cause: Optional[Exception] = None
|
56
|
+
|
57
|
+
def __init__(self, message: str, response: PangeaResponse, cause: Optional[Exception] = None):
|
58
|
+
super().__init__(message, response)
|
59
|
+
self.cause = cause
|
60
|
+
|
61
|
+
|
46
62
|
class ValidationException(PangeaAPIException):
|
47
63
|
"""Pangea Validation Errors denoting issues with an API request"""
|
48
64
|
|
@@ -103,11 +119,13 @@ class AcceptedRequestException(PangeaAPIException):
|
|
103
119
|
"""Accepted request exception. Async response"""
|
104
120
|
|
105
121
|
request_id: str
|
122
|
+
accepted_result: Optional[AcceptedResult] = None
|
106
123
|
|
107
124
|
def __init__(self, response: PangeaResponse):
|
108
125
|
message = f"summary: {response.summary}. request_id: {response.request_id}."
|
109
126
|
super().__init__(message, response)
|
110
127
|
self.request_id = response.request_id
|
128
|
+
self.accepted_result = AcceptedResult(**response.raw_result) if response.raw_result is not None else None
|
111
129
|
|
112
130
|
|
113
131
|
class ServiceNotAvailableException(PangeaAPIException):
|
pangea/request.py
CHANGED
@@ -9,10 +9,10 @@ from typing import Dict, List, Optional, Tuple, Type, Union
|
|
9
9
|
|
10
10
|
import aiohttp
|
11
11
|
import pangea
|
12
|
+
import pangea.exceptions as pe
|
12
13
|
import requests
|
13
|
-
from pangea import exceptions
|
14
14
|
from pangea.config import PangeaConfig
|
15
|
-
from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus
|
15
|
+
from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
16
16
|
from pangea.utils import default_encoder
|
17
17
|
from requests.adapters import HTTPAdapter, Retry
|
18
18
|
|
@@ -131,38 +131,38 @@ class PangeaRequestBase(object):
|
|
131
131
|
)
|
132
132
|
|
133
133
|
if status == ResponseStatus.VALIDATION_ERR.value:
|
134
|
-
raise
|
134
|
+
raise pe.ValidationException(summary, response)
|
135
135
|
elif status == ResponseStatus.TOO_MANY_REQUESTS.value:
|
136
|
-
raise
|
136
|
+
raise pe.RateLimitException(summary, response)
|
137
137
|
elif status == ResponseStatus.NO_CREDIT.value:
|
138
|
-
raise
|
138
|
+
raise pe.NoCreditException(summary, response)
|
139
139
|
elif status == ResponseStatus.UNAUTHORIZED.value:
|
140
|
-
raise
|
140
|
+
raise pe.UnauthorizedException(self.service, response)
|
141
141
|
elif status == ResponseStatus.SERVICE_NOT_ENABLED.value:
|
142
|
-
raise
|
142
|
+
raise pe.ServiceNotEnabledException(self.service, response)
|
143
143
|
elif status == ResponseStatus.PROVIDER_ERR.value:
|
144
|
-
raise
|
144
|
+
raise pe.ProviderErrorException(summary, response)
|
145
145
|
elif status in (ResponseStatus.MISSING_CONFIG_ID_SCOPE.value, ResponseStatus.MISSING_CONFIG_ID.value):
|
146
|
-
raise
|
146
|
+
raise pe.MissingConfigID(self.service, response)
|
147
147
|
elif status == ResponseStatus.SERVICE_NOT_AVAILABLE.value:
|
148
|
-
raise
|
148
|
+
raise pe.ServiceNotAvailableException(summary, response)
|
149
149
|
elif status == ResponseStatus.TREE_NOT_FOUND.value:
|
150
|
-
raise
|
150
|
+
raise pe.TreeNotFoundException(summary, response)
|
151
151
|
elif status == ResponseStatus.IP_NOT_FOUND.value:
|
152
|
-
raise
|
152
|
+
raise pe.IPNotFoundException(summary)
|
153
153
|
elif status == ResponseStatus.BAD_OFFSET.value:
|
154
|
-
raise
|
154
|
+
raise pe.BadOffsetException(summary, response)
|
155
155
|
elif status == ResponseStatus.FORBIDDEN_VAULT_OPERATION.value:
|
156
|
-
raise
|
156
|
+
raise pe.ForbiddenVaultOperation(summary, response)
|
157
157
|
elif status == ResponseStatus.VAULT_ITEM_NOT_FOUND.value:
|
158
|
-
raise
|
158
|
+
raise pe.VaultItemNotFound(summary, response)
|
159
159
|
elif status == ResponseStatus.NOT_FOUND.value:
|
160
|
-
raise
|
160
|
+
raise pe.NotFound(response.raw_response.url if response.raw_response is not None else "", response)
|
161
161
|
elif status == ResponseStatus.INTERNAL_SERVER_ERROR.value:
|
162
|
-
raise
|
162
|
+
raise pe.InternalServerError(response)
|
163
163
|
elif status == ResponseStatus.ACCEPTED.value:
|
164
|
-
raise
|
165
|
-
raise
|
164
|
+
raise pe.AcceptedRequestException(response)
|
165
|
+
raise pe.PangeaAPIException(f"{summary} ", response)
|
166
166
|
|
167
167
|
|
168
168
|
class PangeaRequest(PangeaRequestBase):
|
@@ -184,6 +184,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
184
184
|
data: Union[str, Dict] = {},
|
185
185
|
files: Optional[List[Tuple]] = None,
|
186
186
|
poll_result: bool = True,
|
187
|
+
url: Optional[str] = None,
|
187
188
|
) -> PangeaResponse:
|
188
189
|
"""Makes the POST call to a Pangea Service endpoint.
|
189
190
|
|
@@ -195,35 +196,112 @@ class PangeaRequest(PangeaRequestBase):
|
|
195
196
|
PangeaResponse which contains the response in its entirety and
|
196
197
|
various properties to retrieve individual fields
|
197
198
|
"""
|
198
|
-
url
|
199
|
+
if url is None:
|
200
|
+
url = self._url(endpoint)
|
201
|
+
|
199
202
|
# Set config ID if available
|
200
|
-
if self.config_id and data.
|
203
|
+
if self.config_id and data.get("config_id", None) is None:
|
201
204
|
data["config_id"] = self.config_id
|
202
205
|
|
203
|
-
|
206
|
+
if (
|
207
|
+
files is not None
|
208
|
+
and type(data) is dict
|
209
|
+
and data.get("transfer_method", None) == TransferMethod.DIRECT.value
|
210
|
+
):
|
211
|
+
requests_response = self._post_presigned_url(endpoint, result_class=result_class, data=data, files=files)
|
212
|
+
else:
|
213
|
+
requests_response = self._http_post(
|
214
|
+
url, headers=self._headers(), data=data, files=files, multipart_post=True
|
215
|
+
)
|
216
|
+
|
217
|
+
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
|
218
|
+
if poll_result:
|
219
|
+
pangea_response = self._handle_queued_result(pangea_response)
|
220
|
+
|
221
|
+
return self._check_response(pangea_response)
|
222
|
+
|
223
|
+
def _http_post(
|
224
|
+
self,
|
225
|
+
url: str,
|
226
|
+
headers: Dict = {},
|
227
|
+
data: Union[str, Dict] = {},
|
228
|
+
files: Optional[List[Tuple]] = None,
|
229
|
+
multipart_post: bool = True,
|
230
|
+
) -> requests.Response:
|
204
231
|
self.logger.debug(
|
205
|
-
json.dumps(
|
232
|
+
json.dumps(
|
233
|
+
{"service": self.service, "action": "http_post", "url": url, "data": data}, default=default_encoder
|
234
|
+
)
|
206
235
|
)
|
207
236
|
|
237
|
+
data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
|
238
|
+
return self.session.post(url, headers=headers, data=data_send, files=files)
|
239
|
+
|
240
|
+
def _http_post_process(
|
241
|
+
self, data: Union[str, Dict] = {}, files: Optional[List[Tuple]] = None, multipart_post: bool = True
|
242
|
+
):
|
208
243
|
if files:
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
244
|
+
if multipart_post is True:
|
245
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
246
|
+
multi = [("request", (None, data_send, "application/json"))]
|
247
|
+
multi.extend(files)
|
248
|
+
files = multi
|
249
|
+
return None, files
|
250
|
+
else:
|
251
|
+
# Post to presigned url as form
|
252
|
+
data_send = []
|
253
|
+
for k, v in data.items():
|
254
|
+
data_send.append((k, v))
|
255
|
+
# When posting to presigned url, file key should be 'file'
|
256
|
+
files = {
|
257
|
+
"file": files[0][1],
|
258
|
+
}
|
259
|
+
return data_send, files
|
260
|
+
else:
|
261
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
262
|
+
return data_send, None
|
213
263
|
|
214
|
-
|
264
|
+
return data, files
|
265
|
+
|
266
|
+
def _post_presigned_url(
|
267
|
+
self,
|
268
|
+
endpoint: str,
|
269
|
+
result_class: Type[PangeaResponseResult],
|
270
|
+
data: Union[str, Dict] = {},
|
271
|
+
files: Optional[List[Tuple]] = None,
|
272
|
+
):
|
273
|
+
if len(files) == 0:
|
274
|
+
raise AttributeError("files attribute should have at least 1 file")
|
275
|
+
|
276
|
+
# Send request
|
277
|
+
try:
|
278
|
+
# This should return 202 (AcceptedRequestException)
|
279
|
+
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
280
|
+
raise pe.PresignedURLException("Should return 202", resp)
|
281
|
+
|
282
|
+
except pe.AcceptedRequestException as e:
|
283
|
+
accepted_exception = e
|
284
|
+
except Exception as e:
|
285
|
+
raise e
|
286
|
+
|
287
|
+
# Receive 202 with accepted_status
|
288
|
+
result = self._poll_presigned_url(accepted_exception)
|
289
|
+
data_to_presigned = result.accepted_status.upload_details
|
290
|
+
presigned_url = result.accepted_status.upload_url
|
291
|
+
|
292
|
+
# Send multipart request with file and upload_details as body
|
293
|
+
resp = self._http_post(url=presigned_url, data=data_to_presigned, files=files, multipart_post=False)
|
215
294
|
self.logger.debug(
|
216
295
|
json.dumps(
|
217
|
-
{"service": self.service, "action": "post", "url":
|
296
|
+
{"service": self.service, "action": "post presigned", "url": presigned_url, "response": resp.text},
|
218
297
|
default=default_encoder,
|
219
298
|
)
|
220
299
|
)
|
221
300
|
|
222
|
-
|
223
|
-
|
224
|
-
pangea_response = self._handle_queued_result(pangea_response)
|
301
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
302
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
|
225
303
|
|
226
|
-
return
|
304
|
+
return accepted_exception.response.raw_response
|
227
305
|
|
228
306
|
def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
|
229
307
|
if self._queued_retry_enabled and response.raw_response.status_code == 202:
|
@@ -276,10 +354,10 @@ class PangeaRequest(PangeaRequestBase):
|
|
276
354
|
def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
277
355
|
request_id = response.request_id
|
278
356
|
if not request_id:
|
279
|
-
raise
|
357
|
+
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
280
358
|
|
281
359
|
if response.status != ResponseStatus.ACCEPTED.value:
|
282
|
-
raise
|
360
|
+
raise pe.PangeaException("Response already proccesed")
|
283
361
|
|
284
362
|
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
285
363
|
|
@@ -295,6 +373,51 @@ class PangeaRequest(PangeaRequestBase):
|
|
295
373
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
|
296
374
|
return self._check_response(response)
|
297
375
|
|
376
|
+
def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
|
377
|
+
if type(initial_exc) is not pe.AcceptedRequestException:
|
378
|
+
raise AttributeError("Exception should be of type AcceptedRequestException")
|
379
|
+
|
380
|
+
if initial_exc.accepted_result.accepted_status.upload_url:
|
381
|
+
return initial_exc.accepted_result
|
382
|
+
|
383
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
384
|
+
retry_count = 1
|
385
|
+
start = time.time()
|
386
|
+
loop_exc = initial_exc
|
387
|
+
|
388
|
+
while (
|
389
|
+
loop_exc.accepted_result is not None
|
390
|
+
and not loop_exc.accepted_result.accepted_status.upload_url
|
391
|
+
and not self._reach_timeout(start)
|
392
|
+
):
|
393
|
+
time.sleep(self._get_delay(retry_count, start))
|
394
|
+
try:
|
395
|
+
self.poll_result_once(initial_exc.response, check_response=False)
|
396
|
+
msg = "Polling presigned url return 200 instead of 202"
|
397
|
+
self.logger.debug(
|
398
|
+
json.dumps(
|
399
|
+
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {msg}}
|
400
|
+
)
|
401
|
+
)
|
402
|
+
raise pe.PangeaException(msg)
|
403
|
+
except pe.AcceptedRequestException as e:
|
404
|
+
retry_count += 1
|
405
|
+
loop_exc = e
|
406
|
+
except Exception as e:
|
407
|
+
self.logger.debug(
|
408
|
+
json.dumps(
|
409
|
+
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
|
410
|
+
)
|
411
|
+
)
|
412
|
+
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
|
413
|
+
|
414
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
415
|
+
|
416
|
+
if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
|
417
|
+
return loop_exc.accepted_result
|
418
|
+
else:
|
419
|
+
raise loop_exc
|
420
|
+
|
298
421
|
def _init_session(self) -> requests.Session:
|
299
422
|
retry_config = Retry(
|
300
423
|
total=self.config.request_retries,
|