pangea-sdk 3.0.0__py3-none-any.whl → 3.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pangea/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- __version__ = "3.0.0"
1
+ __version__ = "3.2.0"
2
2
 
3
3
  from pangea.asyncio.request import PangeaRequestAsync
4
4
  from pangea.config import PangeaConfig
pangea/asyncio/request.py CHANGED
@@ -7,12 +7,12 @@ import time
7
7
  from typing import Dict, List, Optional, Tuple, Type, Union
8
8
 
9
9
  import aiohttp
10
+ import pangea.exceptions as pe
10
11
  from aiohttp import FormData
11
- from pangea import exceptions
12
12
 
13
13
  # from requests.adapters import HTTPAdapter, Retry
14
14
  from pangea.request import PangeaRequestBase
15
- from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus
15
+ from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
16
16
  from pangea.utils import default_encoder
17
17
 
18
18
 
@@ -32,6 +32,7 @@ class PangeaRequestAsync(PangeaRequestBase):
32
32
  data: Union[str, Dict] = {},
33
33
  files: Optional[List[Tuple]] = None,
34
34
  poll_result: bool = True,
35
+ url: Optional[str] = None,
35
36
  ) -> PangeaResponse:
36
37
  """Makes the POST call to a Pangea Service endpoint.
37
38
 
@@ -43,41 +44,156 @@ class PangeaRequestAsync(PangeaRequestBase):
43
44
  PangeaResponse which contains the response in its entirety and
44
45
  various properties to retrieve individual fields
45
46
  """
46
- url = self._url(endpoint)
47
+ if url is None:
48
+ url = self._url(endpoint)
49
+
47
50
  # Set config ID if available
48
- if self.config_id and data.pop("config_id", None) is None:
51
+ if self.config_id and data.get("config_id", None) is None:
49
52
  data["config_id"] = self.config_id
50
53
 
51
- data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
54
+ if (
55
+ files is not None
56
+ and type(data) is dict
57
+ and data.get("transfer_method", None) == TransferMethod.DIRECT.value
58
+ ):
59
+ requests_response = await self._post_presigned_url(
60
+ endpoint, result_class=result_class, data=data, files=files
61
+ )
62
+ else:
63
+ requests_response = await self._http_post(
64
+ url, headers=self._headers(), data=data, files=files, presigned_url_post=False
65
+ )
66
+
67
+ pangea_response = PangeaResponse(
68
+ requests_response, result_class=result_class, json=await requests_response.json()
69
+ )
70
+ if poll_result:
71
+ pangea_response = await self._handle_queued_result(pangea_response)
72
+
73
+ return self._check_response(pangea_response)
74
+
75
+ async def _http_post(
76
+ self,
77
+ url: str,
78
+ headers: Dict = {},
79
+ data: Union[str, Dict] = {},
80
+ files: Optional[List[Tuple]] = None,
81
+ presigned_url_post: bool = False,
82
+ ) -> aiohttp.ClientResponse:
52
83
  self.logger.debug(
53
- json.dumps({"service": self.service, "action": "post", "url": url, "data": data}, default=default_encoder)
84
+ json.dumps(
85
+ {"service": self.service, "action": "http_post", "url": url, "data": data}, default=default_encoder
86
+ )
54
87
  )
55
88
 
56
- # FIXME: use FormData to send multipart request
57
89
  if files:
58
90
  form = FormData()
59
- form.add_field("request", data_send, content_type="application/json")
60
- for name, value in files:
61
- # [("upload", ("filename.exe", file, "application/octet-stream"))]
62
- form.add_field(name, value[1], filename=value[0], content_type=value[2])
91
+ if presigned_url_post:
92
+ for k, v in data.items():
93
+ form.add_field(k, v)
94
+ for name, value in files:
95
+ form.add_field("file", value[1], filename=value[0], content_type=value[2])
96
+ else:
97
+ data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
98
+ form.add_field("request", data_send, content_type="application/json")
99
+ for name, value in files:
100
+ form.add_field(name, value[1], filename=value[0], content_type=value[2])
63
101
 
64
102
  data_send = form
103
+ else:
104
+ data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
65
105
 
66
- async with self.session.post(url, headers=self._headers(), data=data_send) as requests_response:
67
- pangea_response = PangeaResponse(
68
- requests_response, result_class=result_class, json=await requests_response.json()
69
- )
70
-
71
- if poll_result:
72
- pangea_response = await self._handle_queued_result(pangea_response)
106
+ return await self.session.post(url, headers=headers, data=data_send)
73
107
 
108
+ async def _post_presigned_url(
109
+ self,
110
+ endpoint: str,
111
+ result_class: Type[PangeaResponseResult],
112
+ data: Union[str, Dict] = {},
113
+ files: Optional[List[Tuple]] = None,
114
+ ):
115
+ if len(files) == 0:
116
+ raise AttributeError("files attribute should have at least 1 file")
117
+
118
+ # Send request
119
+ try:
120
+ # This should return 202 (AcceptedRequestException)
121
+ resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
122
+ raise pe.PresignedURLException("Should return 202", resp)
123
+
124
+ except pe.AcceptedRequestException as e:
125
+ accepted_exception = e
126
+ except Exception as e:
127
+ raise e
128
+
129
+ # Receive 202 with accepted_status
130
+ result = await self._poll_presigned_url(accepted_exception)
131
+ data_to_presigned = result.accepted_status.upload_details
132
+ presigned_url = result.accepted_status.upload_url
133
+
134
+ # Send multipart request with file and upload_details as body
135
+ resp = await self._http_post(url=presigned_url, data=data_to_presigned, files=files, presigned_url_post=True)
74
136
  self.logger.debug(
75
137
  json.dumps(
76
- {"service": self.service, "action": "post", "url": url, "response": pangea_response.json},
138
+ {
139
+ "service": self.service,
140
+ "action": "post presigned",
141
+ "url": presigned_url,
142
+ "response": await resp.text(),
143
+ },
77
144
  default=default_encoder,
78
145
  )
79
146
  )
80
- return self._check_response(pangea_response)
147
+
148
+ if resp.status < 200 or resp.status >= 300:
149
+ raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
150
+
151
+ return accepted_exception.response.raw_response
152
+
153
+ async def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
154
+ if type(initial_exc) is not pe.AcceptedRequestException:
155
+ raise AttributeError("Exception should be of type AcceptedRequestException")
156
+
157
+ if initial_exc.accepted_result.accepted_status.upload_url:
158
+ return initial_exc.accepted_result
159
+
160
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
161
+ retry_count = 1
162
+ start = time.time()
163
+ loop_exc = initial_exc
164
+
165
+ while (
166
+ loop_exc.accepted_result is not None
167
+ and not loop_exc.accepted_result.accepted_status.upload_url
168
+ and not self._reach_timeout(start)
169
+ ):
170
+ await asyncio.sleep(self._get_delay(retry_count, start))
171
+ try:
172
+ await self.poll_result_once(initial_exc.response, check_response=False)
173
+ msg = "Polling presigned url return 200 instead of 202"
174
+ self.logger.debug(
175
+ json.dumps(
176
+ {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {msg}}
177
+ )
178
+ )
179
+ raise pe.PangeaException(msg)
180
+ except pe.AcceptedRequestException as e:
181
+ retry_count += 1
182
+ loop_exc = e
183
+ except Exception as e:
184
+ self.logger.debug(
185
+ json.dumps(
186
+ {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
187
+ )
188
+ )
189
+ raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
190
+
191
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
192
+
193
+ if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
194
+ return loop_exc.accepted_result
195
+ else:
196
+ raise loop_exc
81
197
 
82
198
  async def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
83
199
  if self._queued_retry_enabled and response.http_status == 202:
@@ -135,10 +251,10 @@ class PangeaRequestAsync(PangeaRequestBase):
135
251
  async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
136
252
  request_id = response.request_id
137
253
  if not request_id:
138
- raise exceptions.PangeaException("Poll result error error: response did not include a 'request_id'")
254
+ raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
139
255
 
140
256
  if response.status != ResponseStatus.ACCEPTED.value:
141
- raise exceptions.PangeaException("Response already proccesed")
257
+ raise pe.PangeaException("Response already proccesed")
142
258
 
143
259
  return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
144
260
 
@@ -1,13 +1,15 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
3
  import datetime
4
- from typing import Any, Dict, Optional, Union
4
+ from typing import Any, Dict, List, Optional, Union
5
5
 
6
+ import pangea.exceptions as pexc
6
7
  from pangea.response import PangeaResponse
7
8
  from pangea.services.audit.audit import AuditBase
8
9
  from pangea.services.audit.exceptions import AuditException
9
10
  from pangea.services.audit.models import (
10
11
  Event,
12
+ LogBulkResult,
11
13
  LogResult,
12
14
  RootRequest,
13
15
  RootResult,
@@ -172,9 +174,86 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
172
174
  print(f"\\t{err.detail} \\n")
173
175
  """
174
176
 
175
- input = self._pre_log_process(event, sign_local=sign_local, verify=verify, verbose=verbose)
177
+ input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
176
178
  response = await self.request.post("v1/log", LogResult, data=input.dict(exclude_none=True))
177
- return self.handle_log_response(response, verify=verify)
179
+ if response.success:
180
+ self._process_log_result(response.result, verify=verify)
181
+ return response
182
+
183
+ async def log_bulk(
184
+ self,
185
+ events: List[Dict[str, Any]],
186
+ sign_local: bool = False,
187
+ verbose: Optional[bool] = None,
188
+ ) -> PangeaResponse[LogBulkResult]:
189
+ """
190
+ Log an entry
191
+
192
+ Create a log entry in the Secure Audit Log.
193
+ Args:
194
+ events (List[dict[str, Any]]): events to be logged
195
+ verify (bool, optional): True to verify logs consistency after response.
196
+ sign_local (bool, optional): True to sign event with local key.
197
+ verbose (bool, optional): True to get a more verbose response.
198
+ Raises:
199
+ AuditException: If an audit based api exception happens
200
+ PangeaAPIException: If an API Error happens
201
+
202
+ Returns:
203
+ A PangeaResponse where the hash of event data and optional verbose
204
+ results are returned in the response.result field.
205
+ Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
206
+
207
+ Examples:
208
+ FIXME:
209
+ """
210
+
211
+ input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
212
+ response = await self.request.post("v2/log", LogBulkResult, data=input.dict(exclude_none=True))
213
+ if response.success:
214
+ for result in response.result.results:
215
+ self._process_log_result(result, verify=True)
216
+ return response
217
+
218
+ async def log_bulk_async(
219
+ self,
220
+ events: List[Dict[str, Any]],
221
+ sign_local: bool = False,
222
+ verbose: Optional[bool] = None,
223
+ ) -> PangeaResponse[LogBulkResult]:
224
+ """
225
+ Log an entry
226
+
227
+ Create a log entry in the Secure Audit Log.
228
+ Args:
229
+ events (List[dict[str, Any]]): events to be logged
230
+ verify (bool, optional): True to verify logs consistency after response.
231
+ sign_local (bool, optional): True to sign event with local key.
232
+ verbose (bool, optional): True to get a more verbose response.
233
+ Raises:
234
+ AuditException: If an audit based api exception happens
235
+ PangeaAPIException: If an API Error happens
236
+
237
+ Returns:
238
+ A PangeaResponse where the hash of event data and optional verbose
239
+ results are returned in the response.result field.
240
+ Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
241
+
242
+ Examples:
243
+ FIXME:
244
+ """
245
+
246
+ input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
247
+ try:
248
+ response = await self.request.post(
249
+ "v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
250
+ )
251
+ except pexc.AcceptedRequestException as e:
252
+ return e.response
253
+ if response.success:
254
+ for result in response.result.results:
255
+ self._process_log_result(result, verify=True)
256
+ return response
178
257
 
179
258
  async def search(
180
259
  self,
@@ -5,6 +5,7 @@ from typing import Optional
5
5
 
6
6
  import pangea.services.file_scan as m
7
7
  from pangea.response import PangeaResponse
8
+ from pangea.utils import get_presigned_url_upload_params
8
9
 
9
10
  from .base import ServiceBaseAsync
10
11
 
@@ -79,14 +80,17 @@ class FileScanAsync(ServiceBaseAsync):
79
80
  for err in e.errors:
80
81
  print(f"\\t{err.detail} \\n")
81
82
  """
82
- input = m.FileScanRequest(verbose=verbose, raw=raw, provider=provider)
83
83
 
84
84
  if file or file_path:
85
85
  if file_path:
86
86
  file = open(file_path, "rb")
87
- files = [("upload", ("filename.exe", file, "application/octet-stream"))]
87
+ crc, sha, size, _ = get_presigned_url_upload_params(file)
88
+ files = [("upload", ("filename", file, "application/octet-stream"))]
88
89
  else:
89
90
  raise ValueError("Need to set file_path or file arguments")
90
91
 
92
+ input = m.FileScanRequest(
93
+ verbose=verbose, raw=raw, provider=provider, transfer_crc32c=crc, transfer_sha256=sha, transfer_size=size
94
+ )
91
95
  data = input.dict(exclude_none=True)
92
96
  return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
pangea/exceptions.py CHANGED
@@ -1,9 +1,9 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
3
 
4
- from typing import List
4
+ from typing import List, Optional
5
5
 
6
- from pangea.response import ErrorField, PangeaResponse
6
+ from pangea.response import AcceptedResult, ErrorField, PangeaResponse
7
7
 
8
8
 
9
9
  class PangeaException(Exception):
@@ -14,6 +14,14 @@ class PangeaException(Exception):
14
14
  self.message = message
15
15
 
16
16
 
17
+ class PresignedUploadError(PangeaException):
18
+ body: str
19
+
20
+ def __init__(self, message: str, body: str):
21
+ super().__init__(message)
22
+ self.body = body
23
+
24
+
17
25
  class PangeaAPIException(PangeaException):
18
26
  """Exceptions raised during API calls"""
19
27
 
@@ -43,6 +51,14 @@ class PangeaAPIException(PangeaException):
43
51
  return self.__repr__()
44
52
 
45
53
 
54
+ class PresignedURLException(PangeaAPIException):
55
+ cause: Optional[Exception] = None
56
+
57
+ def __init__(self, message: str, response: PangeaResponse, cause: Optional[Exception] = None):
58
+ super().__init__(message, response)
59
+ self.cause = cause
60
+
61
+
46
62
  class ValidationException(PangeaAPIException):
47
63
  """Pangea Validation Errors denoting issues with an API request"""
48
64
 
@@ -103,11 +119,13 @@ class AcceptedRequestException(PangeaAPIException):
103
119
  """Accepted request exception. Async response"""
104
120
 
105
121
  request_id: str
122
+ accepted_result: Optional[AcceptedResult] = None
106
123
 
107
124
  def __init__(self, response: PangeaResponse):
108
125
  message = f"summary: {response.summary}. request_id: {response.request_id}."
109
126
  super().__init__(message, response)
110
127
  self.request_id = response.request_id
128
+ self.accepted_result = AcceptedResult(**response.raw_result) if response.raw_result is not None else None
111
129
 
112
130
 
113
131
  class ServiceNotAvailableException(PangeaAPIException):
pangea/request.py CHANGED
@@ -9,10 +9,10 @@ from typing import Dict, List, Optional, Tuple, Type, Union
9
9
 
10
10
  import aiohttp
11
11
  import pangea
12
+ import pangea.exceptions as pe
12
13
  import requests
13
- from pangea import exceptions
14
14
  from pangea.config import PangeaConfig
15
- from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus
15
+ from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
16
16
  from pangea.utils import default_encoder
17
17
  from requests.adapters import HTTPAdapter, Retry
18
18
 
@@ -131,38 +131,38 @@ class PangeaRequestBase(object):
131
131
  )
132
132
 
133
133
  if status == ResponseStatus.VALIDATION_ERR.value:
134
- raise exceptions.ValidationException(summary, response)
134
+ raise pe.ValidationException(summary, response)
135
135
  elif status == ResponseStatus.TOO_MANY_REQUESTS.value:
136
- raise exceptions.RateLimitException(summary, response)
136
+ raise pe.RateLimitException(summary, response)
137
137
  elif status == ResponseStatus.NO_CREDIT.value:
138
- raise exceptions.NoCreditException(summary, response)
138
+ raise pe.NoCreditException(summary, response)
139
139
  elif status == ResponseStatus.UNAUTHORIZED.value:
140
- raise exceptions.UnauthorizedException(self.service, response)
140
+ raise pe.UnauthorizedException(self.service, response)
141
141
  elif status == ResponseStatus.SERVICE_NOT_ENABLED.value:
142
- raise exceptions.ServiceNotEnabledException(self.service, response)
142
+ raise pe.ServiceNotEnabledException(self.service, response)
143
143
  elif status == ResponseStatus.PROVIDER_ERR.value:
144
- raise exceptions.ProviderErrorException(summary, response)
144
+ raise pe.ProviderErrorException(summary, response)
145
145
  elif status in (ResponseStatus.MISSING_CONFIG_ID_SCOPE.value, ResponseStatus.MISSING_CONFIG_ID.value):
146
- raise exceptions.MissingConfigID(self.service, response)
146
+ raise pe.MissingConfigID(self.service, response)
147
147
  elif status == ResponseStatus.SERVICE_NOT_AVAILABLE.value:
148
- raise exceptions.ServiceNotAvailableException(summary, response)
148
+ raise pe.ServiceNotAvailableException(summary, response)
149
149
  elif status == ResponseStatus.TREE_NOT_FOUND.value:
150
- raise exceptions.TreeNotFoundException(summary, response)
150
+ raise pe.TreeNotFoundException(summary, response)
151
151
  elif status == ResponseStatus.IP_NOT_FOUND.value:
152
- raise exceptions.IPNotFoundException(summary)
152
+ raise pe.IPNotFoundException(summary)
153
153
  elif status == ResponseStatus.BAD_OFFSET.value:
154
- raise exceptions.BadOffsetException(summary, response)
154
+ raise pe.BadOffsetException(summary, response)
155
155
  elif status == ResponseStatus.FORBIDDEN_VAULT_OPERATION.value:
156
- raise exceptions.ForbiddenVaultOperation(summary, response)
156
+ raise pe.ForbiddenVaultOperation(summary, response)
157
157
  elif status == ResponseStatus.VAULT_ITEM_NOT_FOUND.value:
158
- raise exceptions.VaultItemNotFound(summary, response)
158
+ raise pe.VaultItemNotFound(summary, response)
159
159
  elif status == ResponseStatus.NOT_FOUND.value:
160
- raise exceptions.NotFound(response.raw_response.url if response.raw_response is not None else "", response)
160
+ raise pe.NotFound(response.raw_response.url if response.raw_response is not None else "", response)
161
161
  elif status == ResponseStatus.INTERNAL_SERVER_ERROR.value:
162
- raise exceptions.InternalServerError(response)
162
+ raise pe.InternalServerError(response)
163
163
  elif status == ResponseStatus.ACCEPTED.value:
164
- raise exceptions.AcceptedRequestException(response)
165
- raise exceptions.PangeaAPIException(f"{summary} ", response)
164
+ raise pe.AcceptedRequestException(response)
165
+ raise pe.PangeaAPIException(f"{summary} ", response)
166
166
 
167
167
 
168
168
  class PangeaRequest(PangeaRequestBase):
@@ -184,6 +184,7 @@ class PangeaRequest(PangeaRequestBase):
184
184
  data: Union[str, Dict] = {},
185
185
  files: Optional[List[Tuple]] = None,
186
186
  poll_result: bool = True,
187
+ url: Optional[str] = None,
187
188
  ) -> PangeaResponse:
188
189
  """Makes the POST call to a Pangea Service endpoint.
189
190
 
@@ -195,35 +196,112 @@ class PangeaRequest(PangeaRequestBase):
195
196
  PangeaResponse which contains the response in its entirety and
196
197
  various properties to retrieve individual fields
197
198
  """
198
- url = self._url(endpoint)
199
+ if url is None:
200
+ url = self._url(endpoint)
201
+
199
202
  # Set config ID if available
200
- if self.config_id and data.pop("config_id", None) is None:
203
+ if self.config_id and data.get("config_id", None) is None:
201
204
  data["config_id"] = self.config_id
202
205
 
203
- data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
206
+ if (
207
+ files is not None
208
+ and type(data) is dict
209
+ and data.get("transfer_method", None) == TransferMethod.DIRECT.value
210
+ ):
211
+ requests_response = self._post_presigned_url(endpoint, result_class=result_class, data=data, files=files)
212
+ else:
213
+ requests_response = self._http_post(
214
+ url, headers=self._headers(), data=data, files=files, multipart_post=True
215
+ )
216
+
217
+ pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
218
+ if poll_result:
219
+ pangea_response = self._handle_queued_result(pangea_response)
220
+
221
+ return self._check_response(pangea_response)
222
+
223
+ def _http_post(
224
+ self,
225
+ url: str,
226
+ headers: Dict = {},
227
+ data: Union[str, Dict] = {},
228
+ files: Optional[List[Tuple]] = None,
229
+ multipart_post: bool = True,
230
+ ) -> requests.Response:
204
231
  self.logger.debug(
205
- json.dumps({"service": self.service, "action": "post", "url": url, "data": data}, default=default_encoder)
232
+ json.dumps(
233
+ {"service": self.service, "action": "http_post", "url": url, "data": data}, default=default_encoder
234
+ )
206
235
  )
207
236
 
237
+ data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
238
+ return self.session.post(url, headers=headers, data=data_send, files=files)
239
+
240
+ def _http_post_process(
241
+ self, data: Union[str, Dict] = {}, files: Optional[List[Tuple]] = None, multipart_post: bool = True
242
+ ):
208
243
  if files:
209
- multi = [("request", (None, data_send, "application/json"))]
210
- multi.extend(files)
211
- files = multi
212
- data_send = None
244
+ if multipart_post is True:
245
+ data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
246
+ multi = [("request", (None, data_send, "application/json"))]
247
+ multi.extend(files)
248
+ files = multi
249
+ return None, files
250
+ else:
251
+ # Post to presigned url as form
252
+ data_send = []
253
+ for k, v in data.items():
254
+ data_send.append((k, v))
255
+ # When posting to presigned url, file key should be 'file'
256
+ files = {
257
+ "file": files[0][1],
258
+ }
259
+ return data_send, files
260
+ else:
261
+ data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
262
+ return data_send, None
213
263
 
214
- requests_response = self.session.post(url, headers=self._headers(), data=data_send, files=files)
264
+ return data, files
265
+
266
+ def _post_presigned_url(
267
+ self,
268
+ endpoint: str,
269
+ result_class: Type[PangeaResponseResult],
270
+ data: Union[str, Dict] = {},
271
+ files: Optional[List[Tuple]] = None,
272
+ ):
273
+ if len(files) == 0:
274
+ raise AttributeError("files attribute should have at least 1 file")
275
+
276
+ # Send request
277
+ try:
278
+ # This should return 202 (AcceptedRequestException)
279
+ resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
280
+ raise pe.PresignedURLException("Should return 202", resp)
281
+
282
+ except pe.AcceptedRequestException as e:
283
+ accepted_exception = e
284
+ except Exception as e:
285
+ raise e
286
+
287
+ # Receive 202 with accepted_status
288
+ result = self._poll_presigned_url(accepted_exception)
289
+ data_to_presigned = result.accepted_status.upload_details
290
+ presigned_url = result.accepted_status.upload_url
291
+
292
+ # Send multipart request with file and upload_details as body
293
+ resp = self._http_post(url=presigned_url, data=data_to_presigned, files=files, multipart_post=False)
215
294
  self.logger.debug(
216
295
  json.dumps(
217
- {"service": self.service, "action": "post", "url": url, "response": requests_response.json()},
296
+ {"service": self.service, "action": "post presigned", "url": presigned_url, "response": resp.text},
218
297
  default=default_encoder,
219
298
  )
220
299
  )
221
300
 
222
- pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
223
- if poll_result:
224
- pangea_response = self._handle_queued_result(pangea_response)
301
+ if resp.status_code < 200 or resp.status_code >= 300:
302
+ raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
225
303
 
226
- return self._check_response(pangea_response)
304
+ return accepted_exception.response.raw_response
227
305
 
228
306
  def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
229
307
  if self._queued_retry_enabled and response.raw_response.status_code == 202:
@@ -276,10 +354,10 @@ class PangeaRequest(PangeaRequestBase):
276
354
  def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
277
355
  request_id = response.request_id
278
356
  if not request_id:
279
- raise exceptions.PangeaException("Poll result error error: response did not include a 'request_id'")
357
+ raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
280
358
 
281
359
  if response.status != ResponseStatus.ACCEPTED.value:
282
- raise exceptions.PangeaException("Response already proccesed")
360
+ raise pe.PangeaException("Response already proccesed")
283
361
 
284
362
  return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
285
363
 
@@ -295,6 +373,51 @@ class PangeaRequest(PangeaRequestBase):
295
373
  self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
296
374
  return self._check_response(response)
297
375
 
376
+ def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
377
+ if type(initial_exc) is not pe.AcceptedRequestException:
378
+ raise AttributeError("Exception should be of type AcceptedRequestException")
379
+
380
+ if initial_exc.accepted_result.accepted_status.upload_url:
381
+ return initial_exc.accepted_result
382
+
383
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
384
+ retry_count = 1
385
+ start = time.time()
386
+ loop_exc = initial_exc
387
+
388
+ while (
389
+ loop_exc.accepted_result is not None
390
+ and not loop_exc.accepted_result.accepted_status.upload_url
391
+ and not self._reach_timeout(start)
392
+ ):
393
+ time.sleep(self._get_delay(retry_count, start))
394
+ try:
395
+ self.poll_result_once(initial_exc.response, check_response=False)
396
+ msg = "Polling presigned url return 200 instead of 202"
397
+ self.logger.debug(
398
+ json.dumps(
399
+ {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {msg}}
400
+ )
401
+ )
402
+ raise pe.PangeaException(msg)
403
+ except pe.AcceptedRequestException as e:
404
+ retry_count += 1
405
+ loop_exc = e
406
+ except Exception as e:
407
+ self.logger.debug(
408
+ json.dumps(
409
+ {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
410
+ )
411
+ )
412
+ raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
413
+
414
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
415
+
416
+ if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
417
+ return loop_exc.accepted_result
418
+ else:
419
+ raise loop_exc
420
+
298
421
  def _init_session(self) -> requests.Session:
299
422
  retry_config = Retry(
300
423
  total=self.config.request_retries,