pangea-sdk 3.1.0__py3-none-any.whl → 3.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pangea/request.py CHANGED
@@ -12,7 +12,7 @@ import pangea
12
12
  import pangea.exceptions as pe
13
13
  import requests
14
14
  from pangea.config import PangeaConfig
15
- from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
15
+ from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
16
16
  from pangea.utils import default_encoder
17
17
  from requests.adapters import HTTPAdapter, Retry
18
18
 
@@ -203,12 +203,16 @@ class PangeaRequest(PangeaRequestBase):
203
203
  if self.config_id and data.get("config_id", None) is None:
204
204
  data["config_id"] = self.config_id
205
205
 
206
+ transfer_method = data.get("transfer_method", None)
207
+
206
208
  if (
207
209
  files is not None
208
210
  and type(data) is dict
209
- and data.get("transfer_method", None) == TransferMethod.DIRECT.value
211
+ and (transfer_method == TransferMethod.DIRECT.value or transfer_method == TransferMethod.POST_URL.value)
210
212
  ):
211
- requests_response = self._post_presigned_url(endpoint, result_class=result_class, data=data, files=files)
213
+ requests_response = self._full_post_presigned_url(
214
+ endpoint, result_class=result_class, data=data, files=files
215
+ )
212
216
  else:
213
217
  requests_response = self._http_post(
214
218
  url, headers=self._headers(), data=data, files=files, multipart_post=True
@@ -220,13 +224,106 @@ class PangeaRequest(PangeaRequestBase):
220
224
 
221
225
  return self._check_response(pangea_response)
222
226
 
227
+ def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
228
+ """Makes the GET call to a Pangea Service endpoint.
229
+
230
+ Args:
231
+ endpoint(str): The Pangea Service API endpoint.
232
+ path(str): Additional URL path
233
+
234
+ Returns:
235
+ PangeaResponse which contains the response in its entirety and
236
+ various properties to retrieve individual fields
237
+ """
238
+
239
+ url = self._url(path)
240
+ self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
241
+ requests_response = self.session.get(url, headers=self._headers())
242
+ pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
243
+
244
+ self.logger.debug(
245
+ json.dumps(
246
+ {"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
247
+ default=default_encoder,
248
+ )
249
+ )
250
+
251
+ if check_response is False:
252
+ return pangea_response
253
+
254
+ return self._check_response(pangea_response)
255
+
256
+ def poll_result_by_id(
257
+ self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
258
+ ):
259
+ path = self._get_poll_path(request_id)
260
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
261
+ return self.get(path, result_class, check_response=check_response)
262
+
263
+ def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
264
+ request_id = response.request_id
265
+ if not request_id:
266
+ raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
267
+
268
+ if response.status != ResponseStatus.ACCEPTED.value:
269
+ raise pe.PangeaException("Response already proccesed")
270
+
271
+ return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
272
+
273
+ def request_presigned_url(
274
+ self,
275
+ endpoint: str,
276
+ result_class: Type[PangeaResponseResult],
277
+ data: Union[str, Dict] = {},
278
+ ) -> PangeaResponse:
279
+ # Send request
280
+ try:
281
+ # This should return 202 (AcceptedRequestException)
282
+ resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
283
+ raise pe.PresignedURLException("Should return 202", resp)
284
+
285
+ except pe.AcceptedRequestException as e:
286
+ accepted_exception = e
287
+ except Exception as e:
288
+ raise e
289
+
290
+ # Receive 202 with accepted_status
291
+ return self._poll_presigned_url(accepted_exception.response)
292
+
293
+ def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
294
+ # Send form request with file and upload_details as body
295
+ resp = self._http_post(url=url, data=data, files=files, multipart_post=False)
296
+ self.logger.debug(
297
+ json.dumps(
298
+ {"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
299
+ default=default_encoder,
300
+ )
301
+ )
302
+
303
+ if resp.status_code < 200 or resp.status_code >= 300:
304
+ raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
305
+
306
+ def put_presigned_url(self, url: str, files: List[Tuple]):
307
+ # Send put request with file as body
308
+ resp = self._http_put(url=url, files=files)
309
+ self.logger.debug(
310
+ json.dumps(
311
+ {"service": self.service, "action": "put presigned", "url": url, "response": resp.text},
312
+ default=default_encoder,
313
+ )
314
+ )
315
+
316
+ if resp.status_code < 200 or resp.status_code >= 300:
317
+ raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status_code}", resp.text)
318
+
319
+ # Start internal methods
223
320
  def _http_post(
224
321
  self,
225
322
  url: str,
226
323
  headers: Dict = {},
227
324
  data: Union[str, Dict] = {},
228
325
  files: Optional[List[Tuple]] = None,
229
- multipart_post: bool = True,
326
+ multipart_post: bool = True, # Multipart or form post
230
327
  ) -> requests.Response:
231
328
  self.logger.debug(
232
329
  json.dumps(
@@ -237,6 +334,17 @@ class PangeaRequest(PangeaRequestBase):
237
334
  data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
238
335
  return self.session.post(url, headers=headers, data=data_send, files=files)
239
336
 
337
+ def _http_put(
338
+ self,
339
+ url: str,
340
+ files: List[Tuple],
341
+ headers: Dict = {},
342
+ ) -> requests.Response:
343
+ self.logger.debug(
344
+ json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
345
+ )
346
+ return self.session.put(url, headers=headers, files=files)
347
+
240
348
  def _http_post_process(
241
349
  self, data: Union[str, Dict] = {}, files: Optional[List[Tuple]] = None, multipart_post: bool = True
242
350
  ):
@@ -263,7 +371,7 @@ class PangeaRequest(PangeaRequestBase):
263
371
 
264
372
  return data, files
265
373
 
266
- def _post_presigned_url(
374
+ def _full_post_presigned_url(
267
375
  self,
268
376
  endpoint: str,
269
377
  result_class: Type[PangeaResponseResult],
@@ -273,35 +381,12 @@ class PangeaRequest(PangeaRequestBase):
273
381
  if len(files) == 0:
274
382
  raise AttributeError("files attribute should have at least 1 file")
275
383
 
276
- # Send request
277
- try:
278
- # This should return 202 (AcceptedRequestException)
279
- resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
280
- raise pe.PresignedURLException("Should return 202", resp)
281
-
282
- except pe.AcceptedRequestException as e:
283
- accepted_exception = e
284
- except Exception as e:
285
- raise e
286
-
287
- # Receive 202 with accepted_status
288
- result = self._poll_presigned_url(accepted_exception)
289
- data_to_presigned = result.accepted_status.upload_details
290
- presigned_url = result.accepted_status.upload_url
291
-
292
- # Send multipart request with file and upload_details as body
293
- resp = self._http_post(url=presigned_url, data=data_to_presigned, files=files, multipart_post=False)
294
- self.logger.debug(
295
- json.dumps(
296
- {"service": self.service, "action": "post presigned", "url": presigned_url, "response": resp.text},
297
- default=default_encoder,
298
- )
299
- )
300
-
301
- if resp.status_code < 200 or resp.status_code >= 300:
302
- raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
384
+ response = self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
385
+ data_to_presigned = response.accepted_result.accepted_status.upload_details
386
+ presigned_url = response.accepted_result.accepted_status.upload_url
303
387
 
304
- return accepted_exception.response.raw_response
388
+ self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
389
+ return response.raw_response
305
390
 
306
391
  def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
307
392
  if self._queued_retry_enabled and response.raw_response.status_code == 202:
@@ -315,52 +400,6 @@ class PangeaRequest(PangeaRequestBase):
315
400
 
316
401
  return response
317
402
 
318
- def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
319
- """Makes the GET call to a Pangea Service endpoint.
320
-
321
- Args:
322
- endpoint(str): The Pangea Service API endpoint.
323
- path(str): Additional URL path
324
-
325
- Returns:
326
- PangeaResponse which contains the response in its entirety and
327
- various properties to retrieve individual fields
328
- """
329
-
330
- url = self._url(path)
331
- self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
332
- requests_response = self.session.get(url, headers=self._headers())
333
- pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
334
-
335
- self.logger.debug(
336
- json.dumps(
337
- {"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
338
- default=default_encoder,
339
- )
340
- )
341
-
342
- if check_response is False:
343
- return pangea_response
344
-
345
- return self._check_response(pangea_response)
346
-
347
- def poll_result_by_id(
348
- self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
349
- ):
350
- path = self._get_poll_path(request_id)
351
- self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
352
- return self.get(path, result_class, check_response=check_response)
353
-
354
- def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
355
- request_id = response.request_id
356
- if not request_id:
357
- raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
358
-
359
- if response.status != ResponseStatus.ACCEPTED.value:
360
- raise pe.PangeaException("Response already proccesed")
361
-
362
- return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
363
-
364
403
  def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
365
404
  retry_count = 1
366
405
  start = time.time()
@@ -373,26 +412,26 @@ class PangeaRequest(PangeaRequestBase):
373
412
  self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
374
413
  return self._check_response(response)
375
414
 
376
- def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
377
- if type(initial_exc) is not pe.AcceptedRequestException:
378
- raise AttributeError("Exception should be of type AcceptedRequestException")
415
+ def _poll_presigned_url(self, response: PangeaResponse) -> PangeaResponse:
416
+ if response.http_status != 202:
417
+ raise AttributeError("Response should be 202")
379
418
 
380
- if initial_exc.accepted_result.accepted_status.upload_url:
381
- return initial_exc.accepted_result
419
+ if response.accepted_result.accepted_status.upload_url:
420
+ return response
382
421
 
383
422
  self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
384
423
  retry_count = 1
385
424
  start = time.time()
386
- loop_exc = initial_exc
425
+ loop_resp = response
387
426
 
388
427
  while (
389
- loop_exc.accepted_result is not None
390
- and not loop_exc.accepted_result.accepted_status.upload_url
428
+ loop_resp.accepted_result is not None
429
+ and not loop_resp.accepted_result.accepted_status.upload_url
391
430
  and not self._reach_timeout(start)
392
431
  ):
393
432
  time.sleep(self._get_delay(retry_count, start))
394
433
  try:
395
- self.poll_result_once(initial_exc.response, check_response=False)
434
+ self.poll_result_once(loop_resp, check_response=False)
396
435
  msg = "Polling presigned url return 200 instead of 202"
397
436
  self.logger.debug(
398
437
  json.dumps(
@@ -402,6 +441,7 @@ class PangeaRequest(PangeaRequestBase):
402
441
  raise pe.PangeaException(msg)
403
442
  except pe.AcceptedRequestException as e:
404
443
  retry_count += 1
444
+ loop_resp = e.response
405
445
  loop_exc = e
406
446
  except Exception as e:
407
447
  self.logger.debug(
@@ -409,12 +449,12 @@ class PangeaRequest(PangeaRequestBase):
409
449
  {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
410
450
  )
411
451
  )
412
- raise pe.PresignedURLException("Failed to pull Presigned URL", loop_exc.response, e)
452
+ raise pe.PresignedURLException("Failed to pull Presigned URL", loop_resp, e)
413
453
 
414
454
  self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
415
455
 
416
- if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
417
- return loop_exc.accepted_result
456
+ if loop_resp.accepted_result is not None and not loop_resp.accepted_result.accepted_status.upload_url:
457
+ return loop_resp
418
458
  else:
419
459
  raise loop_exc
420
460
 
pangea/response.py CHANGED
@@ -13,8 +13,11 @@ T = TypeVar("T")
13
13
 
14
14
 
15
15
  class TransferMethod(str, enum.Enum):
16
- DIRECT = "direct"
16
+ DIRECT = "post-url" # deprecated, use POST_URL instead
17
17
  MULTIPART = "multipart"
18
+ POST_URL = "post-url"
19
+ PUT_URL = "put-url"
20
+ # URL = "url"
18
21
 
19
22
  def __str__(self):
20
23
  return str(self.value)
@@ -76,7 +79,7 @@ class AcceptedStatus(APIResponseModel):
76
79
 
77
80
 
78
81
  class AcceptedResult(PangeaResponseResult):
79
- accepted_status: AcceptedStatus
82
+ accepted_status: Optional[AcceptedStatus] = None
80
83
 
81
84
 
82
85
  class PangeaError(PangeaResponseResult):
@@ -144,8 +147,11 @@ class PangeaResponse(Generic[T], ResponseHeader):
144
147
  if self.raw_result is not None and issubclass(self.result_class, PangeaResponseResult) and self.success
145
148
  else None
146
149
  )
147
- if not self.success and self.http_status != 202:
148
- self.pangea_error = PangeaError(**self.raw_result) if self.raw_result is not None else None
150
+ if not self.success:
151
+ if self.http_status == 202:
152
+ self.accepted_result = AcceptedResult(**self.raw_result) if self.raw_result is not None else None
153
+ else:
154
+ self.pangea_error = PangeaError(**self.raw_result) if self.raw_result is not None else None
149
155
 
150
156
  @property
151
157
  def success(self) -> bool: