pangea-sdk 3.2.0__tar.gz → 3.3.0__tar.gz

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/PKG-INFO +13 -13
  2. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/README.md +10 -10
  3. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/__init__.py +1 -1
  4. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/request.py +122 -88
  5. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/base.py +20 -3
  6. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/file_scan.py +75 -5
  7. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/redact.py +2 -7
  8. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/config.py +2 -2
  9. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/request.py +131 -91
  10. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/response.py +4 -1
  11. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/audit.py +7 -7
  12. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/authn/authn.py +4 -1
  13. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/authn/models.py +2 -1
  14. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/base.py +17 -4
  15. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/file_scan.py +59 -3
  16. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/redact.py +2 -7
  17. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/utils.py +9 -2
  18. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pyproject.toml +3 -3
  19. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/__init__.py +0 -0
  20. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/audit.py +0 -0
  21. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/authn.py +0 -0
  22. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/embargo.py +0 -0
  23. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/intel.py +0 -0
  24. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/vault.py +0 -0
  25. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/audit_logger.py +0 -0
  26. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/deep_verify.py +0 -0
  27. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/deprecated.py +0 -0
  28. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/dump_audit.py +0 -0
  29. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/exceptions.py +0 -0
  30. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/__init__.py +0 -0
  31. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/exceptions.py +0 -0
  32. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/models.py +0 -0
  33. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/signing.py +0 -0
  34. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/util.py +0 -0
  35. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/embargo.py +0 -0
  36. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/intel.py +0 -0
  37. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/models/asymmetric.py +0 -0
  38. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/models/common.py +0 -0
  39. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/models/secret.py +0 -0
  40. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/models/symmetric.py +0 -0
  41. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/vault.py +2 -2
  42. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/tools.py +0 -0
  43. {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/verify_audit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pangea-sdk
3
- Version: 3.2.0
3
+ Version: 3.3.0
4
4
  Summary: Pangea API SDK
5
5
  License: MIT
6
6
  Keywords: Pangea,SDK,Audit
@@ -15,10 +15,10 @@ Classifier: Programming Language :: Python :: 3.10
15
15
  Classifier: Programming Language :: Python :: 3.11
16
16
  Classifier: Topic :: Software Development
17
17
  Classifier: Topic :: Software Development :: Libraries
18
- Requires-Dist: aiohttp (>=3.8.5,<4.0.0)
18
+ Requires-Dist: aiohttp (>=3.8.6,<4.0.0)
19
19
  Requires-Dist: alive-progress (>=2.4.1,<3.0.0)
20
20
  Requires-Dist: asyncio (>=3.4.3,<4.0.0)
21
- Requires-Dist: cryptography (==41.0.3)
21
+ Requires-Dist: cryptography (==41.0.5)
22
22
  Requires-Dist: deprecated (>=1.2.13,<2.0.0)
23
23
  Requires-Dist: google-crc32c (>=1.5.0,<2.0.0)
24
24
  Requires-Dist: pydantic (>=1.10.2,<2.0.0)
@@ -59,18 +59,18 @@ poetry add pangea-sdk
59
59
 
60
60
  ## Usage
61
61
 
62
- For samples apps look at [/examples](https://github.com/pangeacyber/pangea-python/tree/main/examples) folder in this repository. There you will find basic samples apps for each services supported on this SDK. Each service folder has a README.md with intructions to install, setup and run.
62
+ For sample apps, look at the [/examples](https://github.com/pangeacyber/pangea-python/tree/main/examples) folder in this repository. There you will find basic sample apps for each of the services supported on this SDK. Each service folder has a README.md with instructions to install, setup, and run the sample app.
63
63
 
64
64
 
65
65
  ## Asyncio support
66
66
 
67
- We have added support to asyncio library using aiohttp in order to support async/await calls to all our services.
67
+ We have added support to the asyncio library using aiohttp in order to support async/await calls to all our services.
68
68
  Async services classes are inside [pangea/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk/pangea/asyncio) folder, and examples about how to use them are in [/examples/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/examples/asyncio).
69
69
 
70
70
 
71
71
  ### Secure Audit Service - Integrity Tools
72
72
 
73
- Python Pangea SDK include also some extra features to validate Audit Service log's integrity. Here we explain how to run them.
73
+ The Python Pangea SDK also includes some extra features to validate Audit Service log's integrity. Here we explain how to run them.
74
74
 
75
75
  #### Verify audit data
76
76
 
@@ -101,7 +101,7 @@ curl -H "Authorization: Bearer ${PANGEA_TOKEN}" -X POST -H 'Content-Type: applic
101
101
 
102
102
  Download all audit logs for a given time range. Start and end date should be provided,
103
103
  a variety of formats is supported, including ISO-8601. The result is stored in a
104
- jsonl file (one json per line)
104
+ json file (one json per line).
105
105
 
106
106
  ```
107
107
  usage: python -m pangea.dump_audit [-h] [--token TOKEN] [--domain DOMAIN] [--output OUTPUT] start end
@@ -126,8 +126,8 @@ options:
126
126
 
127
127
  #### Perform Exhaustive Verification of Audit Data
128
128
 
129
- This script performs extensive verification on a range of events of the log stream. Appart from verifying the hash
130
- and the membership proof, it checks that there is no omissions in the stream, i.e. all the events are present and properly located.
129
+ This script performs extensive verification on a range of events of the log stream. Apart from verifying the hash
130
+ and the membership proof, it checks that there are no omissions in the stream, i.e. all the events are present and properly located.
131
131
 
132
132
  ```
133
133
  usage: python -m pangea.deep_verify [-h] [--token TOKEN] [--domain DOMAIN] --file FILE
@@ -153,9 +153,9 @@ It accepts multiple file formats:
153
153
 
154
154
  ## Reporting issues and new features
155
155
 
156
- If faced some issue using or testing this SDK or a new feature request feel free to open an issue [clicking here](https://github.com/pangeacyber/pangea-python/issues).
157
- We would need you to provide some basic information like what SDK's version you are using, stack trace if you got it, framework used, and steps to reproduce the issue.
158
- Also feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/)
156
+ If you run into an issue using or testing this SDK or if you have a new feature request, feel free to open an issue by [clicking here](https://github.com/pangeacyber/pangea-python/issues).
157
+ We would need you to provide some basic information, such as what SDK version you are using, the stack trace if you got it, the framework used, and steps to reproduce the issue.
158
+ Also, feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/).
159
159
 
160
160
 
161
161
  ## Contributing
@@ -168,5 +168,5 @@ These linters will run on every `git commit` operation.
168
168
 
169
169
  ### Send a PR
170
170
 
171
- If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in code or an error in documents we will really appreciate it and after review and approval you will be included in our [contributors list](https://github.com/pangeacyber/pangea-python/blob/main/packages/pangea-sdk/CONTRIBUTING.md)
171
+ If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in the code or an error in documents, we really appreciate it and after review and approval, you will be included in our [contributors list](https://github.com/pangeacyber/pangea-python/blob/main/packages/pangea-sdk/CONTRIBUTING.md).
172
172
 
@@ -29,18 +29,18 @@ poetry add pangea-sdk
29
29
 
30
30
  ## Usage
31
31
 
32
- For samples apps look at [/examples](https://github.com/pangeacyber/pangea-python/tree/main/examples) folder in this repository. There you will find basic samples apps for each services supported on this SDK. Each service folder has a README.md with intructions to install, setup and run.
32
+ For sample apps, look at the [/examples](https://github.com/pangeacyber/pangea-python/tree/main/examples) folder in this repository. There you will find basic sample apps for each of the services supported on this SDK. Each service folder has a README.md with instructions to install, setup, and run the sample app.
33
33
 
34
34
 
35
35
  ## Asyncio support
36
36
 
37
- We have added support to asyncio library using aiohttp in order to support async/await calls to all our services.
37
+ We have added support to the asyncio library using aiohttp in order to support async/await calls to all our services.
38
38
  Async services classes are inside [pangea/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk/pangea/asyncio) folder, and examples about how to use them are in [/examples/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/examples/asyncio).
39
39
 
40
40
 
41
41
  ### Secure Audit Service - Integrity Tools
42
42
 
43
- Python Pangea SDK include also some extra features to validate Audit Service log's integrity. Here we explain how to run them.
43
+ The Python Pangea SDK also includes some extra features to validate Audit Service log's integrity. Here we explain how to run them.
44
44
 
45
45
  #### Verify audit data
46
46
 
@@ -71,7 +71,7 @@ curl -H "Authorization: Bearer ${PANGEA_TOKEN}" -X POST -H 'Content-Type: applic
71
71
 
72
72
  Download all audit logs for a given time range. Start and end date should be provided,
73
73
  a variety of formats is supported, including ISO-8601. The result is stored in a
74
- jsonl file (one json per line)
74
+ json file (one json per line).
75
75
 
76
76
  ```
77
77
  usage: python -m pangea.dump_audit [-h] [--token TOKEN] [--domain DOMAIN] [--output OUTPUT] start end
@@ -96,8 +96,8 @@ options:
96
96
 
97
97
  #### Perform Exhaustive Verification of Audit Data
98
98
 
99
- This script performs extensive verification on a range of events of the log stream. Appart from verifying the hash
100
- and the membership proof, it checks that there is no omissions in the stream, i.e. all the events are present and properly located.
99
+ This script performs extensive verification on a range of events of the log stream. Apart from verifying the hash
100
+ and the membership proof, it checks that there are no omissions in the stream, i.e. all the events are present and properly located.
101
101
 
102
102
  ```
103
103
  usage: python -m pangea.deep_verify [-h] [--token TOKEN] [--domain DOMAIN] --file FILE
@@ -123,9 +123,9 @@ It accepts multiple file formats:
123
123
 
124
124
  ## Reporting issues and new features
125
125
 
126
- If faced some issue using or testing this SDK or a new feature request feel free to open an issue [clicking here](https://github.com/pangeacyber/pangea-python/issues).
127
- We would need you to provide some basic information like what SDK's version you are using, stack trace if you got it, framework used, and steps to reproduce the issue.
128
- Also feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/)
126
+ If you run into an issue using or testing this SDK or if you have a new feature request, feel free to open an issue by [clicking here](https://github.com/pangeacyber/pangea-python/issues).
127
+ We would need you to provide some basic information, such as what SDK version you are using, the stack trace if you got it, the framework used, and steps to reproduce the issue.
128
+ Also, feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/).
129
129
 
130
130
 
131
131
  ## Contributing
@@ -138,4 +138,4 @@ These linters will run on every `git commit` operation.
138
138
 
139
139
  ### Send a PR
140
140
 
141
- If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in code or an error in documents we will really appreciate it and after review and approval you will be included in our [contributors list](https://github.com/pangeacyber/pangea-python/blob/main/packages/pangea-sdk/CONTRIBUTING.md)
141
+ If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in the code or an error in documents, we really appreciate it and after review and approval, you will be included in our [contributors list](https://github.com/pangeacyber/pangea-python/blob/main/packages/pangea-sdk/CONTRIBUTING.md).
@@ -1,4 +1,4 @@
1
- __version__ = "3.2.0"
1
+ __version__ = "3.3.0"
2
2
 
3
3
  from pangea.asyncio.request import PangeaRequestAsync
4
4
  from pangea.config import PangeaConfig
@@ -51,12 +51,14 @@ class PangeaRequestAsync(PangeaRequestBase):
51
51
  if self.config_id and data.get("config_id", None) is None:
52
52
  data["config_id"] = self.config_id
53
53
 
54
+ transfer_method = data.get("transfer_method", None)
55
+
54
56
  if (
55
57
  files is not None
56
58
  and type(data) is dict
57
- and data.get("transfer_method", None) == TransferMethod.DIRECT.value
59
+ and (transfer_method == TransferMethod.DIRECT.value or transfer_method == TransferMethod.POST_URL.value)
58
60
  ):
59
- requests_response = await self._post_presigned_url(
61
+ requests_response = await self._full_post_presigned_url(
60
62
  endpoint, result_class=result_class, data=data, files=files
61
63
  )
62
64
  else:
@@ -72,6 +74,83 @@ class PangeaRequestAsync(PangeaRequestBase):
72
74
 
73
75
  return self._check_response(pangea_response)
74
76
 
77
+ async def get(
78
+ self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
79
+ ) -> PangeaResponse:
80
+ """Makes the GET call to a Pangea Service endpoint.
81
+
82
+ Args:
83
+ endpoint(str): The Pangea Service API endpoint.
84
+ path(str): Additional URL path
85
+
86
+ Returns:
87
+ PangeaResponse which contains the response in its entirety and
88
+ various properties to retrieve individual fields
89
+ """
90
+
91
+ url = self._url(path)
92
+ self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
93
+
94
+ async with self.session.get(url, headers=self._headers()) as requests_response:
95
+ pangea_response = PangeaResponse(
96
+ requests_response, result_class=result_class, json=await requests_response.json()
97
+ )
98
+
99
+ self.logger.debug(
100
+ json.dumps(
101
+ {"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
102
+ default=default_encoder,
103
+ )
104
+ )
105
+
106
+ if check_response is False:
107
+ return pangea_response
108
+
109
+ return self._check_response(pangea_response)
110
+
111
+ async def poll_result_by_id(
112
+ self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
113
+ ):
114
+ path = self._get_poll_path(request_id)
115
+ self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
116
+ return await self.get(path, result_class, check_response=check_response)
117
+
118
+ async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
119
+ request_id = response.request_id
120
+ if not request_id:
121
+ raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
122
+
123
+ if response.status != ResponseStatus.ACCEPTED.value:
124
+ raise pe.PangeaException("Response already proccesed")
125
+
126
+ return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
127
+
128
+ async def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
129
+ # Send form request with file and upload_details as body
130
+ resp = await self._http_post(url=url, data=data, files=files, presigned_url_post=True)
131
+ self.logger.debug(
132
+ json.dumps(
133
+ {"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
134
+ default=default_encoder,
135
+ )
136
+ )
137
+
138
+ if resp.status < 200 or resp.status >= 300:
139
+ raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", resp.text)
140
+
141
+ async def put_presigned_url(self, url: str, files: List[Tuple]):
142
+ # Send put request with file as body
143
+ resp = await self._http_put(url=url, files=files)
144
+ self.logger.debug(
145
+ json.dumps(
146
+ {"service": self.service, "action": "put presigned", "url": url, "response": resp.text},
147
+ default=default_encoder,
148
+ )
149
+ )
150
+
151
+ if resp.status_code < 200 or resp.status_code >= 300:
152
+ raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status_code}", resp.text)
153
+
75
154
  async def _http_post(
76
155
  self,
77
156
  url: str,
@@ -105,7 +184,21 @@ class PangeaRequestAsync(PangeaRequestBase):
105
184
 
106
185
  return await self.session.post(url, headers=headers, data=data_send)
107
186
 
108
- async def _post_presigned_url(
187
+ async def _http_put(
188
+ self,
189
+ url: str,
190
+ files: List[Tuple],
191
+ headers: Dict = {},
192
+ ) -> aiohttp.ClientResponse:
193
+ self.logger.debug(
194
+ json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
195
+ )
196
+ form = FormData()
197
+ name, value = files[0]
198
+ form.add_field(name, value[1], filename=value[0], content_type=value[2])
199
+ return self.session.put(url, headers=headers, data=form)
200
+
201
+ async def _full_post_presigned_url(
109
202
  self,
110
203
  endpoint: str,
111
204
  result_class: Type[PangeaResponseResult],
@@ -115,61 +208,52 @@ class PangeaRequestAsync(PangeaRequestBase):
115
208
  if len(files) == 0:
116
209
  raise AttributeError("files attribute should have at least 1 file")
117
210
 
211
+ response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
212
+ data_to_presigned = response.accepted_result.accepted_status.upload_details
213
+ presigned_url = response.accepted_result.accepted_status.upload_url
214
+
215
+ await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
216
+ return response.raw_response
217
+
218
+ async def request_presigned_url(
219
+ self,
220
+ endpoint: str,
221
+ result_class: Type[PangeaResponseResult],
222
+ data: Union[str, Dict] = {},
223
+ ) -> PangeaResponse:
118
224
  # Send request
119
225
  try:
120
226
  # This should return 202 (AcceptedRequestException)
121
227
  resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
122
228
  raise pe.PresignedURLException("Should return 202", resp)
123
-
124
229
  except pe.AcceptedRequestException as e:
125
230
  accepted_exception = e
126
231
  except Exception as e:
127
232
  raise e
128
233
 
129
234
  # Receive 202 with accepted_status
130
- result = await self._poll_presigned_url(accepted_exception)
131
- data_to_presigned = result.accepted_status.upload_details
132
- presigned_url = result.accepted_status.upload_url
133
-
134
- # Send multipart request with file and upload_details as body
135
- resp = await self._http_post(url=presigned_url, data=data_to_presigned, files=files, presigned_url_post=True)
136
- self.logger.debug(
137
- json.dumps(
138
- {
139
- "service": self.service,
140
- "action": "post presigned",
141
- "url": presigned_url,
142
- "response": await resp.text(),
143
- },
144
- default=default_encoder,
145
- )
146
- )
147
-
148
- if resp.status < 200 or resp.status >= 300:
149
- raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
150
-
151
- return accepted_exception.response.raw_response
235
+ return await self._poll_presigned_url(accepted_exception.response)
152
236
 
153
- async def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
154
- if type(initial_exc) is not pe.AcceptedRequestException:
155
- raise AttributeError("Exception should be of type AcceptedRequestException")
237
+ async def _poll_presigned_url(self, response: PangeaResponse) -> AcceptedResult:
238
+ if response.http_status != 202:
239
+ raise AttributeError("Response should be 202")
156
240
 
157
- if initial_exc.accepted_result.accepted_status.upload_url:
158
- return initial_exc.accepted_result
241
+ if response.accepted_result.accepted_status.upload_url:
242
+ return response
159
243
 
160
244
  self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
161
245
  retry_count = 1
162
246
  start = time.time()
163
- loop_exc = initial_exc
247
+ loop_resp = response
164
248
 
165
249
  while (
166
- loop_exc.accepted_result is not None
167
- and not loop_exc.accepted_result.accepted_status.upload_url
250
+ loop_resp.accepted_result is not None
251
+ and not loop_resp.accepted_result.accepted_status.upload_url
168
252
  and not self._reach_timeout(start)
169
253
  ):
170
254
  await asyncio.sleep(self._get_delay(retry_count, start))
171
255
  try:
172
- await self.poll_result_once(initial_exc.response, check_response=False)
256
+ await self.poll_result_once(response, check_response=False)
173
257
  msg = "Polling presigned url return 200 instead of 202"
174
258
  self.logger.debug(
175
259
  json.dumps(
@@ -179,6 +263,7 @@ class PangeaRequestAsync(PangeaRequestBase):
179
263
  raise pe.PangeaException(msg)
180
264
  except pe.AcceptedRequestException as e:
181
265
  retry_count += 1
266
+ loop_resp = e.response
182
267
  loop_exc = e
183
268
  except Exception as e:
184
269
  self.logger.debug(
@@ -190,8 +275,8 @@ class PangeaRequestAsync(PangeaRequestBase):
190
275
 
191
276
  self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
192
277
 
193
- if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
194
- return loop_exc.accepted_result
278
+ if loop_resp.accepted_result is not None and not loop_resp.accepted_result.accepted_status.upload_url:
279
+ return loop_resp
195
280
  else:
196
281
  raise loop_exc
197
282
 
@@ -207,57 +292,6 @@ class PangeaRequestAsync(PangeaRequestBase):
207
292
 
208
293
  return response
209
294
 
210
- async def get(
211
- self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
212
- ) -> PangeaResponse:
213
- """Makes the GET call to a Pangea Service endpoint.
214
-
215
- Args:
216
- endpoint(str): The Pangea Service API endpoint.
217
- path(str): Additional URL path
218
-
219
- Returns:
220
- PangeaResponse which contains the response in its entirety and
221
- various properties to retrieve individual fields
222
- """
223
-
224
- url = self._url(path)
225
- self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
226
-
227
- async with self.session.get(url, headers=self._headers()) as requests_response:
228
- pangea_response = PangeaResponse(
229
- requests_response, result_class=result_class, json=await requests_response.json()
230
- )
231
-
232
- self.logger.debug(
233
- json.dumps(
234
- {"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
235
- default=default_encoder,
236
- )
237
- )
238
-
239
- if check_response is False:
240
- return pangea_response
241
-
242
- return self._check_response(pangea_response)
243
-
244
- async def poll_result_by_id(
245
- self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
246
- ):
247
- path = self._get_poll_path(request_id)
248
- self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
249
- return await self.get(path, result_class, check_response=check_response)
250
-
251
- async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
252
- request_id = response.request_id
253
- if not request_id:
254
- raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
255
-
256
- if response.status != ResponseStatus.ACCEPTED.value:
257
- raise pe.PangeaException("Response already proccesed")
258
-
259
- return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
260
-
261
295
  async def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
262
296
  retry_count = 1
263
297
  start = time.time()
@@ -1,9 +1,11 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
3
 
4
+ from typing import Optional, Type, Union
5
+
4
6
  from pangea.asyncio.request import PangeaRequestAsync
5
7
  from pangea.exceptions import AcceptedRequestException
6
- from pangea.response import PangeaResponse
8
+ from pangea.response import PangeaResponse, PangeaResponseResult
7
9
  from pangea.services.base import ServiceBase
8
10
 
9
11
 
@@ -21,7 +23,13 @@ class ServiceBaseAsync(ServiceBase):
21
23
 
22
24
  return self._request
23
25
 
24
- async def poll_result(self, exception: AcceptedRequestException) -> PangeaResponse:
26
+ async def poll_result(
27
+ self,
28
+ exception: Optional[AcceptedRequestException] = None,
29
+ response: Optional[PangeaResponse] = None,
30
+ request_id: Optional[str] = None,
31
+ result_class: Union[Type[PangeaResponseResult], dict] = dict,
32
+ ) -> PangeaResponse:
25
33
  """
26
34
  Poll result
27
35
 
@@ -39,7 +47,16 @@ class ServiceBaseAsync(ServiceBase):
39
47
  Examples:
40
48
  response = service.poll_result(exception)
41
49
  """
42
- return await self.request.poll_result_once(exception.response, check_response=True)
50
+ if exception is not None:
51
+ return await self.request.poll_result_once(exception.response, check_response=True)
52
+ elif response is not None:
53
+ return await self.request.poll_result_once(response, check_response=True)
54
+ elif request_id is not None:
55
+ return await self.request.poll_result_by_id(
56
+ request_id=request_id, result_class=result_class, check_response=True
57
+ )
58
+ else:
59
+ raise AttributeError("Need to set exception, response or request_id")
43
60
 
44
61
  async def close(self):
45
62
  await self.request.session.close()
@@ -1,11 +1,14 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
3
  import io
4
- from typing import Optional
4
+ import logging
5
+ from typing import Dict, Optional
5
6
 
6
7
  import pangea.services.file_scan as m
7
- from pangea.response import PangeaResponse
8
- from pangea.utils import get_presigned_url_upload_params
8
+ from pangea.asyncio.request import PangeaRequestAsync
9
+ from pangea.request import PangeaConfig
10
+ from pangea.response import PangeaResponse, TransferMethod
11
+ from pangea.utils import FileUploadParams, get_file_upload_params
9
12
 
10
13
  from .base import ServiceBaseAsync
11
14
 
@@ -46,6 +49,7 @@ class FileScanAsync(ServiceBaseAsync):
46
49
  raw: Optional[bool] = None,
47
50
  provider: Optional[str] = None,
48
51
  sync_call: bool = True,
52
+ transfer_method: TransferMethod = TransferMethod.DIRECT,
49
53
  ) -> PangeaResponse[m.FileScanResult]:
50
54
  """
51
55
  Scan
@@ -84,13 +88,79 @@ class FileScanAsync(ServiceBaseAsync):
84
88
  if file or file_path:
85
89
  if file_path:
86
90
  file = open(file_path, "rb")
87
- crc, sha, size, _ = get_presigned_url_upload_params(file)
91
+ if transfer_method == TransferMethod.DIRECT or transfer_method == TransferMethod.POST_URL:
92
+ params = get_file_upload_params(file)
93
+ crc = params.crc_hex
94
+ sha = params.sha256_hex
95
+ size = params.size
96
+ else:
97
+ crc, sha, size = None, None, None
88
98
  files = [("upload", ("filename", file, "application/octet-stream"))]
89
99
  else:
90
100
  raise ValueError("Need to set file_path or file arguments")
91
101
 
92
102
  input = m.FileScanRequest(
93
- verbose=verbose, raw=raw, provider=provider, transfer_crc32c=crc, transfer_sha256=sha, transfer_size=size
103
+ verbose=verbose,
104
+ raw=raw,
105
+ provider=provider,
106
+ transfer_crc32c=crc,
107
+ transfer_sha256=sha,
108
+ transfer_size=size,
109
+ transfer_method=transfer_method,
94
110
  )
95
111
  data = input.dict(exclude_none=True)
96
112
  return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
113
+
114
+ async def request_upload_url(
115
+ self,
116
+ transfer_method: TransferMethod = TransferMethod.PUT_URL,
117
+ params: Optional[FileUploadParams] = None,
118
+ verbose: Optional[bool] = None,
119
+ raw: Optional[bool] = None,
120
+ provider: Optional[str] = None,
121
+ ) -> PangeaResponse[m.FileScanResult]:
122
+ input = m.FileScanRequest(
123
+ verbose=verbose,
124
+ raw=raw,
125
+ provider=provider,
126
+ transfer_method=transfer_method,
127
+ )
128
+ if params is not None and (
129
+ transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT
130
+ ):
131
+ input.transfer_crc32c = params.crc_hex
132
+ input.transfer_sha256 = params.sha256_hex
133
+ input.transfer_size = params.size
134
+
135
+ data = input.dict(exclude_none=True)
136
+ return await self.request.request_presigned_url("v1/scan", m.FileScanResult, data=data)
137
+
138
+
139
+ class FileUploaderAsync:
140
+ def __init__(self):
141
+ self.logger = logging.getLogger("pangea")
142
+ self._request: PangeaRequestAsync = PangeaRequestAsync(
143
+ config=PangeaConfig(),
144
+ token="",
145
+ service="FileScanUploader",
146
+ logger=self.logger,
147
+ )
148
+
149
+ async def upload_file(
150
+ self,
151
+ url: str,
152
+ file: io.BufferedReader,
153
+ transfer_method: TransferMethod = TransferMethod.PUT_URL,
154
+ file_details: Optional[Dict] = None,
155
+ ):
156
+ if transfer_method == TransferMethod.PUT_URL:
157
+ files = [("file", ("filename", file, "application/octet-stream"))]
158
+ await self._request.put_presigned_url(url=url, files=files)
159
+ elif transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT:
160
+ files = [("file", ("filename", file, "application/octet-stream"))]
161
+ await self._request.post_presigned_url(url=url, data=file_details, files=files)
162
+ else:
163
+ raise ValueError(f"Transfer method not supported: {transfer_method}")
164
+
165
+ async def close(self):
166
+ await self._request.session.close()
@@ -36,13 +36,8 @@ class RedactAsync(ServiceBaseAsync):
36
36
 
37
37
  service_name = "redact"
38
38
 
39
- def __init__(
40
- self,
41
- token,
42
- config=None,
43
- logger_name="pangea",
44
- ):
45
- super().__init__(token, config, logger_name)
39
+ def __init__(self, token, config=None, logger_name="pangea", config_id: Optional[str] = None):
40
+ super().__init__(token, config, logger_name, config_id=config_id)
46
41
 
47
42
  async def redact(
48
43
  self,
@@ -10,7 +10,7 @@ class PangeaConfig:
10
10
  """Holds run time configuration information used by SDK components."""
11
11
 
12
12
  """
13
- Used to set pangea domain (and port if needed), it should not include service subdomain
13
+ Used to set Pangea domain (and port if needed), it should not include service subdomain
14
14
  just for particular use cases when environment = "local", domain could be set to an url including:
15
15
  scheme (http:// or https://), subdomain, domain and port.
16
16
 
@@ -19,7 +19,7 @@ class PangeaConfig:
19
19
 
20
20
  """
21
21
  Used to generate service url.
22
- It should be only 'production' or 'local' in case of particular services that can run locally as Redact
22
+ It should be only 'production' or 'local' in cases of particular services that can run locally as Redact.
23
23
 
24
24
  """
25
25
  environment: str = "production"