pangea-sdk 3.0.0__py3-none-any.whl → 3.2.0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
pangea/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- __version__ = "3.0.0"
1
+ __version__ = "3.2.0"
2
2
 
3
3
  from pangea.asyncio.request import PangeaRequestAsync
4
4
  from pangea.config import PangeaConfig
pangea/asyncio/request.py CHANGED
@@ -7,12 +7,12 @@ import time
7
7
  from typing import Dict, List, Optional, Tuple, Type, Union
8
8
 
9
9
  import aiohttp
10
+ import pangea.exceptions as pe
10
11
  from aiohttp import FormData
11
- from pangea import exceptions
12
12
 
13
13
  # from requests.adapters import HTTPAdapter, Retry
14
14
  from pangea.request import PangeaRequestBase
15
- from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus
15
+ from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
16
16
  from pangea.utils import default_encoder
17
17
 
18
18
 
@@ -32,6 +32,7 @@ class PangeaRequestAsync(PangeaRequestBase):
32
32
  data: Union[str, Dict] = {},
33
33
  files: Optional[List[Tuple]] = None,
34
34
  poll_result: bool = True,
35
+ url: Optional[str] = None,
35
36
  ) -> PangeaResponse:
36
37
  """Makes the POST call to a Pangea Service endpoint.
37
38
 
@@ -43,41 +44,156 @@ class PangeaRequestAsync(PangeaRequestBase):
43
44
  PangeaResponse which contains the response in its entirety and
44
45
  various properties to retrieve individual fields
45
46
  """
46
- url = self._url(endpoint)
47
+ if url is None:
48
+ url = self._url(endpoint)
49
+
47
50
  # Set config ID if available
48
- if self.config_id and data.pop("config_id", None) is None:
51
+ if self.config_id and data.get("config_id", None) is None:
49
52
  data["config_id"] = self.config_id
50
53
 
51
- data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
54
+ if (
55
+ files is not None
56
+ and type(data) is dict
57
+ and data.get("transfer_method", None) == TransferMethod.DIRECT.value
58
+ ):
59
+ requests_response = await self._post_presigned_url(
60
+ endpoint, result_class=result_class, data=data, files=files
61
+ )
62
+ else:
63
+ requests_response = await self._http_post(
64
+ url, headers=self._headers(), data=data, files=files, presigned_url_post=False
65
+ )
66
+
67
+ pangea_response = PangeaResponse(
68
+ requests_response, result_class=result_class, json=await requests_response.json()
69
+ )
70
+ if poll_result:
71
+ pangea_response = await self._handle_queued_result(pangea_response)
72
+
73
+ return self._check_response(pangea_response)
74
+
75
+ async def _http_post(
76
+ self,
77
+ url: str,
78
+ headers: Dict = {},
79
+ data: Union[str, Dict] = {},
80
+ files: Optional[List[Tuple]] = None,
81
+ presigned_url_post: bool = False,
82
+ ) -> aiohttp.ClientResponse:
52
83
  self.logger.debug(
53
- json.dumps({"service": self.service, "action": "post", "url": url, "data": data}, default=default_encoder)
84
+ json.dumps(
85
+ {"service": self.service, "action": "http_post", "url": url, "data": data}, default=default_encoder
86
+ )
54
87
  )
55
88
 
56
- # FIXME: use FormData to send multipart request
57
89
  if files:
58
90
  form = FormData()
59
- form.add_field("request", data_send, content_type="application/json")
60
- for name, value in files:
61
- # [("upload", ("filename.exe", file, "application/octet-stream"))]
62
- form.add_field(name, value[1], filename=value[0], content_type=value[2])
91
+ if presigned_url_post:
92
+ for k, v in data.items():
93
+ form.add_field(k, v)
94
+ for name, value in files:
95
+ form.add_field("file", value[1], filename=value[0], content_type=value[2])
96
+ else:
97
+ data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
98
+ form.add_field("request", data_send, content_type="application/json")
99
+ for name, value in files:
100
+ form.add_field(name, value[1], filename=value[0], content_type=value[2])
63
101
 
64
102
  data_send = form
103
+ else:
104
+ data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
65
105
 
66
- async with self.session.post(url, headers=self._headers(), data=data_send) as requests_response:
67
- pangea_response = PangeaResponse(
68
- requests_response, result_class=result_class, json=await requests_response.json()
69
- )
70
-
71
- if poll_result:
72
- pangea_response = await self._handle_queued_result(pangea_response)
106
+ return await self.session.post(url, headers=headers, data=data_send)
73
107
 
108
+ async def _post_presigned_url(
109
+ self,
110
+ endpoint: str,
111
+ result_class: Type[PangeaResponseResult],
112
+ data: Union[str, Dict] = {},
113
+ files: Optional[List[Tuple]] = None,
114
+ ):
115
+ if len(files) == 0:
116
+ raise AttributeError("files attribute should have at least 1 file")
117
+
118
+ # Send request
119
+ try:
120
+ # This should return 202 (AcceptedRequestException)
121
+ resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
122
+ raise pe.PresignedURLException("Should return 202", resp)
123
+
124
+ except pe.AcceptedRequestException as e:
125
+ accepted_exception = e
126
+ except Exception as e:
127
+ raise e
128
+
129
+ # Receive 202 with accepted_status
130
+ result = await self._poll_presigned_url(accepted_exception)
131
+ data_to_presigned = result.accepted_status.upload_details
132
+ presigned_url = result.accepted_status.upload_url
133
+
134
+ # Send multipart request with file and upload_details as body
135
+ resp = await self._http_post(url=presigned_url, data=data_to_presigned, files=files, presigned_url_post=True)
74
136
  self.logger.debug(
75
137
  json.dumps(
76
- {"service": self.service, "action": "post", "url": url, "response": pangea_response.json},
138
+ {
139
+ "service": self.service,
140
+ "action": "post presigned",
141
+ "url": presigned_url,
142
+ "response": await resp.text(),
143
+ },
77
144
  default=default_encoder,
78
145
  )
79
146
  )
80
- return self._check_response(pangea_response)
147
+
148
+ if resp.status < 200 or resp.status >= 300:
149
+ raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
150
+
151
+ return accepted_exception.response.raw_response
152
+
153
+ async def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
154
+ if type(initial_exc) is not pe.AcceptedRequestException:
155
+ raise AttributeError("Exception should be of type AcceptedRequestException")
156
+
157
+ if initial_exc.accepted_result.accepted_status.upload_url:
158
+ return initial_exc.accepted_result
159
+
160
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
161
+ retry_count = 1
162
+ start = time.time()
163
+ loop_exc = initial_exc
164
+
165
+ while (
166
+ loop_exc.accepted_result is not None
167
+ and not loop_exc.accepted_result.accepted_status.upload_url
168
+ and not self._reach_timeout(start)
169
+ ):
170
+ await asyncio.sleep(self._get_delay(retry_count, start))
171
+ try:
172
+ await self.poll_result_once(initial_exc.response, check_response=False)
173
+ msg = "Polling presigned url return 200 instead of 202"
174
+ self.logger.debug(
175
+ json.dumps(
176
+ {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {msg}}
177
+ )
178
+ )
179
+ raise pe.PangeaException(msg)
180
+ except pe.AcceptedRequestException as e:
181
+ retry_count += 1
182
+ loop_exc = e
183
+ except Exception as e:
184
+ self.logger.debug(
185
+ json.dumps(
186
+ {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
187
+ )
188
+ )
189
+ raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
190
+
191
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
192
+
193
+ if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
194
+ return loop_exc.accepted_result
195
+ else:
196
+ raise loop_exc
81
197
 
82
198
  async def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
83
199
  if self._queued_retry_enabled and response.http_status == 202:
@@ -135,10 +251,10 @@ class PangeaRequestAsync(PangeaRequestBase):
135
251
  async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
136
252
  request_id = response.request_id
137
253
  if not request_id:
138
- raise exceptions.PangeaException("Poll result error error: response did not include a 'request_id'")
254
+ raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
139
255
 
140
256
  if response.status != ResponseStatus.ACCEPTED.value:
141
- raise exceptions.PangeaException("Response already proccesed")
257
+ raise pe.PangeaException("Response already proccesed")
142
258
 
143
259
  return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
144
260
 
@@ -1,13 +1,15 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
3
  import datetime
4
- from typing import Any, Dict, Optional, Union
4
+ from typing import Any, Dict, List, Optional, Union
5
5
 
6
+ import pangea.exceptions as pexc
6
7
  from pangea.response import PangeaResponse
7
8
  from pangea.services.audit.audit import AuditBase
8
9
  from pangea.services.audit.exceptions import AuditException
9
10
  from pangea.services.audit.models import (
10
11
  Event,
12
+ LogBulkResult,
11
13
  LogResult,
12
14
  RootRequest,
13
15
  RootResult,
@@ -172,9 +174,86 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
172
174
  print(f"\\t{err.detail} \\n")
173
175
  """
174
176
 
175
- input = self._pre_log_process(event, sign_local=sign_local, verify=verify, verbose=verbose)
177
+ input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
176
178
  response = await self.request.post("v1/log", LogResult, data=input.dict(exclude_none=True))
177
- return self.handle_log_response(response, verify=verify)
179
+ if response.success:
180
+ self._process_log_result(response.result, verify=verify)
181
+ return response
182
+
183
+ async def log_bulk(
184
+ self,
185
+ events: List[Dict[str, Any]],
186
+ sign_local: bool = False,
187
+ verbose: Optional[bool] = None,
188
+ ) -> PangeaResponse[LogBulkResult]:
189
+ """
190
+ Log an entry
191
+
192
+ Create a log entry in the Secure Audit Log.
193
+ Args:
194
+ events (List[dict[str, Any]]): events to be logged
195
+ verify (bool, optional): True to verify logs consistency after response.
196
+ sign_local (bool, optional): True to sign event with local key.
197
+ verbose (bool, optional): True to get a more verbose response.
198
+ Raises:
199
+ AuditException: If an audit based api exception happens
200
+ PangeaAPIException: If an API Error happens
201
+
202
+ Returns:
203
+ A PangeaResponse where the hash of event data and optional verbose
204
+ results are returned in the response.result field.
205
+ Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
206
+
207
+ Examples:
208
+ FIXME:
209
+ """
210
+
211
+ input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
212
+ response = await self.request.post("v2/log", LogBulkResult, data=input.dict(exclude_none=True))
213
+ if response.success:
214
+ for result in response.result.results:
215
+ self._process_log_result(result, verify=True)
216
+ return response
217
+
218
+ async def log_bulk_async(
219
+ self,
220
+ events: List[Dict[str, Any]],
221
+ sign_local: bool = False,
222
+ verbose: Optional[bool] = None,
223
+ ) -> PangeaResponse[LogBulkResult]:
224
+ """
225
+ Log an entry
226
+
227
+ Create a log entry in the Secure Audit Log.
228
+ Args:
229
+ events (List[dict[str, Any]]): events to be logged
230
+ verify (bool, optional): True to verify logs consistency after response.
231
+ sign_local (bool, optional): True to sign event with local key.
232
+ verbose (bool, optional): True to get a more verbose response.
233
+ Raises:
234
+ AuditException: If an audit based api exception happens
235
+ PangeaAPIException: If an API Error happens
236
+
237
+ Returns:
238
+ A PangeaResponse where the hash of event data and optional verbose
239
+ results are returned in the response.result field.
240
+ Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
241
+
242
+ Examples:
243
+ FIXME:
244
+ """
245
+
246
+ input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
247
+ try:
248
+ response = await self.request.post(
249
+ "v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
250
+ )
251
+ except pexc.AcceptedRequestException as e:
252
+ return e.response
253
+ if response.success:
254
+ for result in response.result.results:
255
+ self._process_log_result(result, verify=True)
256
+ return response
178
257
 
179
258
  async def search(
180
259
  self,
@@ -5,6 +5,7 @@ from typing import Optional
5
5
 
6
6
  import pangea.services.file_scan as m
7
7
  from pangea.response import PangeaResponse
8
+ from pangea.utils import get_presigned_url_upload_params
8
9
 
9
10
  from .base import ServiceBaseAsync
10
11
 
@@ -79,14 +80,17 @@ class FileScanAsync(ServiceBaseAsync):
79
80
  for err in e.errors:
80
81
  print(f"\\t{err.detail} \\n")
81
82
  """
82
- input = m.FileScanRequest(verbose=verbose, raw=raw, provider=provider)
83
83
 
84
84
  if file or file_path:
85
85
  if file_path:
86
86
  file = open(file_path, "rb")
87
- files = [("upload", ("filename.exe", file, "application/octet-stream"))]
87
+ crc, sha, size, _ = get_presigned_url_upload_params(file)
88
+ files = [("upload", ("filename", file, "application/octet-stream"))]
88
89
  else:
89
90
  raise ValueError("Need to set file_path or file arguments")
90
91
 
92
+ input = m.FileScanRequest(
93
+ verbose=verbose, raw=raw, provider=provider, transfer_crc32c=crc, transfer_sha256=sha, transfer_size=size
94
+ )
91
95
  data = input.dict(exclude_none=True)
92
96
  return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
pangea/exceptions.py CHANGED
@@ -1,9 +1,9 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
3
 
4
- from typing import List
4
+ from typing import List, Optional
5
5
 
6
- from pangea.response import ErrorField, PangeaResponse
6
+ from pangea.response import AcceptedResult, ErrorField, PangeaResponse
7
7
 
8
8
 
9
9
  class PangeaException(Exception):
@@ -14,6 +14,14 @@ class PangeaException(Exception):
14
14
  self.message = message
15
15
 
16
16
 
17
+ class PresignedUploadError(PangeaException):
18
+ body: str
19
+
20
+ def __init__(self, message: str, body: str):
21
+ super().__init__(message)
22
+ self.body = body
23
+
24
+
17
25
  class PangeaAPIException(PangeaException):
18
26
  """Exceptions raised during API calls"""
19
27
 
@@ -43,6 +51,14 @@ class PangeaAPIException(PangeaException):
43
51
  return self.__repr__()
44
52
 
45
53
 
54
+ class PresignedURLException(PangeaAPIException):
55
+ cause: Optional[Exception] = None
56
+
57
+ def __init__(self, message: str, response: PangeaResponse, cause: Optional[Exception] = None):
58
+ super().__init__(message, response)
59
+ self.cause = cause
60
+
61
+
46
62
  class ValidationException(PangeaAPIException):
47
63
  """Pangea Validation Errors denoting issues with an API request"""
48
64
 
@@ -103,11 +119,13 @@ class AcceptedRequestException(PangeaAPIException):
103
119
  """Accepted request exception. Async response"""
104
120
 
105
121
  request_id: str
122
+ accepted_result: Optional[AcceptedResult] = None
106
123
 
107
124
  def __init__(self, response: PangeaResponse):
108
125
  message = f"summary: {response.summary}. request_id: {response.request_id}."
109
126
  super().__init__(message, response)
110
127
  self.request_id = response.request_id
128
+ self.accepted_result = AcceptedResult(**response.raw_result) if response.raw_result is not None else None
111
129
 
112
130
 
113
131
  class ServiceNotAvailableException(PangeaAPIException):
pangea/request.py CHANGED
@@ -9,10 +9,10 @@ from typing import Dict, List, Optional, Tuple, Type, Union
9
9
 
10
10
  import aiohttp
11
11
  import pangea
12
+ import pangea.exceptions as pe
12
13
  import requests
13
- from pangea import exceptions
14
14
  from pangea.config import PangeaConfig
15
- from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus
15
+ from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
16
16
  from pangea.utils import default_encoder
17
17
  from requests.adapters import HTTPAdapter, Retry
18
18
 
@@ -131,38 +131,38 @@ class PangeaRequestBase(object):
131
131
  )
132
132
 
133
133
  if status == ResponseStatus.VALIDATION_ERR.value:
134
- raise exceptions.ValidationException(summary, response)
134
+ raise pe.ValidationException(summary, response)
135
135
  elif status == ResponseStatus.TOO_MANY_REQUESTS.value:
136
- raise exceptions.RateLimitException(summary, response)
136
+ raise pe.RateLimitException(summary, response)
137
137
  elif status == ResponseStatus.NO_CREDIT.value:
138
- raise exceptions.NoCreditException(summary, response)
138
+ raise pe.NoCreditException(summary, response)
139
139
  elif status == ResponseStatus.UNAUTHORIZED.value:
140
- raise exceptions.UnauthorizedException(self.service, response)
140
+ raise pe.UnauthorizedException(self.service, response)
141
141
  elif status == ResponseStatus.SERVICE_NOT_ENABLED.value:
142
- raise exceptions.ServiceNotEnabledException(self.service, response)
142
+ raise pe.ServiceNotEnabledException(self.service, response)
143
143
  elif status == ResponseStatus.PROVIDER_ERR.value:
144
- raise exceptions.ProviderErrorException(summary, response)
144
+ raise pe.ProviderErrorException(summary, response)
145
145
  elif status in (ResponseStatus.MISSING_CONFIG_ID_SCOPE.value, ResponseStatus.MISSING_CONFIG_ID.value):
146
- raise exceptions.MissingConfigID(self.service, response)
146
+ raise pe.MissingConfigID(self.service, response)
147
147
  elif status == ResponseStatus.SERVICE_NOT_AVAILABLE.value:
148
- raise exceptions.ServiceNotAvailableException(summary, response)
148
+ raise pe.ServiceNotAvailableException(summary, response)
149
149
  elif status == ResponseStatus.TREE_NOT_FOUND.value:
150
- raise exceptions.TreeNotFoundException(summary, response)
150
+ raise pe.TreeNotFoundException(summary, response)
151
151
  elif status == ResponseStatus.IP_NOT_FOUND.value:
152
- raise exceptions.IPNotFoundException(summary)
152
+ raise pe.IPNotFoundException(summary)
153
153
  elif status == ResponseStatus.BAD_OFFSET.value:
154
- raise exceptions.BadOffsetException(summary, response)
154
+ raise pe.BadOffsetException(summary, response)
155
155
  elif status == ResponseStatus.FORBIDDEN_VAULT_OPERATION.value:
156
- raise exceptions.ForbiddenVaultOperation(summary, response)
156
+ raise pe.ForbiddenVaultOperation(summary, response)
157
157
  elif status == ResponseStatus.VAULT_ITEM_NOT_FOUND.value:
158
- raise exceptions.VaultItemNotFound(summary, response)
158
+ raise pe.VaultItemNotFound(summary, response)
159
159
  elif status == ResponseStatus.NOT_FOUND.value:
160
- raise exceptions.NotFound(response.raw_response.url if response.raw_response is not None else "", response)
160
+ raise pe.NotFound(response.raw_response.url if response.raw_response is not None else "", response)
161
161
  elif status == ResponseStatus.INTERNAL_SERVER_ERROR.value:
162
- raise exceptions.InternalServerError(response)
162
+ raise pe.InternalServerError(response)
163
163
  elif status == ResponseStatus.ACCEPTED.value:
164
- raise exceptions.AcceptedRequestException(response)
165
- raise exceptions.PangeaAPIException(f"{summary} ", response)
164
+ raise pe.AcceptedRequestException(response)
165
+ raise pe.PangeaAPIException(f"{summary} ", response)
166
166
 
167
167
 
168
168
  class PangeaRequest(PangeaRequestBase):
@@ -184,6 +184,7 @@ class PangeaRequest(PangeaRequestBase):
184
184
  data: Union[str, Dict] = {},
185
185
  files: Optional[List[Tuple]] = None,
186
186
  poll_result: bool = True,
187
+ url: Optional[str] = None,
187
188
  ) -> PangeaResponse:
188
189
  """Makes the POST call to a Pangea Service endpoint.
189
190
 
@@ -195,35 +196,112 @@ class PangeaRequest(PangeaRequestBase):
195
196
  PangeaResponse which contains the response in its entirety and
196
197
  various properties to retrieve individual fields
197
198
  """
198
- url = self._url(endpoint)
199
+ if url is None:
200
+ url = self._url(endpoint)
201
+
199
202
  # Set config ID if available
200
- if self.config_id and data.pop("config_id", None) is None:
203
+ if self.config_id and data.get("config_id", None) is None:
201
204
  data["config_id"] = self.config_id
202
205
 
203
- data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
206
+ if (
207
+ files is not None
208
+ and type(data) is dict
209
+ and data.get("transfer_method", None) == TransferMethod.DIRECT.value
210
+ ):
211
+ requests_response = self._post_presigned_url(endpoint, result_class=result_class, data=data, files=files)
212
+ else:
213
+ requests_response = self._http_post(
214
+ url, headers=self._headers(), data=data, files=files, multipart_post=True
215
+ )
216
+
217
+ pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
218
+ if poll_result:
219
+ pangea_response = self._handle_queued_result(pangea_response)
220
+
221
+ return self._check_response(pangea_response)
222
+
223
+ def _http_post(
224
+ self,
225
+ url: str,
226
+ headers: Dict = {},
227
+ data: Union[str, Dict] = {},
228
+ files: Optional[List[Tuple]] = None,
229
+ multipart_post: bool = True,
230
+ ) -> requests.Response:
204
231
  self.logger.debug(
205
- json.dumps({"service": self.service, "action": "post", "url": url, "data": data}, default=default_encoder)
232
+ json.dumps(
233
+ {"service": self.service, "action": "http_post", "url": url, "data": data}, default=default_encoder
234
+ )
206
235
  )
207
236
 
237
+ data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
238
+ return self.session.post(url, headers=headers, data=data_send, files=files)
239
+
240
+ def _http_post_process(
241
+ self, data: Union[str, Dict] = {}, files: Optional[List[Tuple]] = None, multipart_post: bool = True
242
+ ):
208
243
  if files:
209
- multi = [("request", (None, data_send, "application/json"))]
210
- multi.extend(files)
211
- files = multi
212
- data_send = None
244
+ if multipart_post is True:
245
+ data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
246
+ multi = [("request", (None, data_send, "application/json"))]
247
+ multi.extend(files)
248
+ files = multi
249
+ return None, files
250
+ else:
251
+ # Post to presigned url as form
252
+ data_send = []
253
+ for k, v in data.items():
254
+ data_send.append((k, v))
255
+ # When posting to presigned url, file key should be 'file'
256
+ files = {
257
+ "file": files[0][1],
258
+ }
259
+ return data_send, files
260
+ else:
261
+ data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
262
+ return data_send, None
213
263
 
214
- requests_response = self.session.post(url, headers=self._headers(), data=data_send, files=files)
264
+ return data, files
265
+
266
+ def _post_presigned_url(
267
+ self,
268
+ endpoint: str,
269
+ result_class: Type[PangeaResponseResult],
270
+ data: Union[str, Dict] = {},
271
+ files: Optional[List[Tuple]] = None,
272
+ ):
273
+ if len(files) == 0:
274
+ raise AttributeError("files attribute should have at least 1 file")
275
+
276
+ # Send request
277
+ try:
278
+ # This should return 202 (AcceptedRequestException)
279
+ resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
280
+ raise pe.PresignedURLException("Should return 202", resp)
281
+
282
+ except pe.AcceptedRequestException as e:
283
+ accepted_exception = e
284
+ except Exception as e:
285
+ raise e
286
+
287
+ # Receive 202 with accepted_status
288
+ result = self._poll_presigned_url(accepted_exception)
289
+ data_to_presigned = result.accepted_status.upload_details
290
+ presigned_url = result.accepted_status.upload_url
291
+
292
+ # Send multipart request with file and upload_details as body
293
+ resp = self._http_post(url=presigned_url, data=data_to_presigned, files=files, multipart_post=False)
215
294
  self.logger.debug(
216
295
  json.dumps(
217
- {"service": self.service, "action": "post", "url": url, "response": requests_response.json()},
296
+ {"service": self.service, "action": "post presigned", "url": presigned_url, "response": resp.text},
218
297
  default=default_encoder,
219
298
  )
220
299
  )
221
300
 
222
- pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
223
- if poll_result:
224
- pangea_response = self._handle_queued_result(pangea_response)
301
+ if resp.status_code < 200 or resp.status_code >= 300:
302
+ raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
225
303
 
226
- return self._check_response(pangea_response)
304
+ return accepted_exception.response.raw_response
227
305
 
228
306
  def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
229
307
  if self._queued_retry_enabled and response.raw_response.status_code == 202:
@@ -276,10 +354,10 @@ class PangeaRequest(PangeaRequestBase):
276
354
  def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
277
355
  request_id = response.request_id
278
356
  if not request_id:
279
- raise exceptions.PangeaException("Poll result error error: response did not include a 'request_id'")
357
+ raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
280
358
 
281
359
  if response.status != ResponseStatus.ACCEPTED.value:
282
- raise exceptions.PangeaException("Response already proccesed")
360
+ raise pe.PangeaException("Response already proccesed")
283
361
 
284
362
  return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
285
363
 
@@ -295,6 +373,51 @@ class PangeaRequest(PangeaRequestBase):
295
373
  self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
296
374
  return self._check_response(response)
297
375
 
376
+ def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
377
+ if type(initial_exc) is not pe.AcceptedRequestException:
378
+ raise AttributeError("Exception should be of type AcceptedRequestException")
379
+
380
+ if initial_exc.accepted_result.accepted_status.upload_url:
381
+ return initial_exc.accepted_result
382
+
383
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
384
+ retry_count = 1
385
+ start = time.time()
386
+ loop_exc = initial_exc
387
+
388
+ while (
389
+ loop_exc.accepted_result is not None
390
+ and not loop_exc.accepted_result.accepted_status.upload_url
391
+ and not self._reach_timeout(start)
392
+ ):
393
+ time.sleep(self._get_delay(retry_count, start))
394
+ try:
395
+ self.poll_result_once(initial_exc.response, check_response=False)
396
+ msg = "Polling presigned url return 200 instead of 202"
397
+ self.logger.debug(
398
+ json.dumps(
399
+ {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {msg}}
400
+ )
401
+ )
402
+ raise pe.PangeaException(msg)
403
+ except pe.AcceptedRequestException as e:
404
+ retry_count += 1
405
+ loop_exc = e
406
+ except Exception as e:
407
+ self.logger.debug(
408
+ json.dumps(
409
+ {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
410
+ )
411
+ )
412
+ raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
413
+
414
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
415
+
416
+ if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
417
+ return loop_exc.accepted_result
418
+ else:
419
+ raise loop_exc
420
+
298
421
  def _init_session(self) -> requests.Session:
299
422
  retry_config = Retry(
300
423
  total=self.config.request_retries,