pangea-sdk 3.7.0__py3-none-any.whl → 3.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +1 -1
- pangea/asyncio/request.py +117 -25
- pangea/asyncio/services/audit.py +97 -13
- pangea/asyncio/services/authn.py +15 -2
- pangea/asyncio/services/base.py +9 -6
- pangea/asyncio/services/file_scan.py +3 -4
- pangea/asyncio/services/intel.py +1 -2
- pangea/asyncio/services/redact.py +1 -2
- pangea/asyncio/services/vault.py +7 -8
- pangea/config.py +10 -18
- pangea/dump_audit.py +1 -0
- pangea/exceptions.py +8 -0
- pangea/request.py +153 -66
- pangea/response.py +39 -4
- pangea/services/audit/audit.py +98 -44
- pangea/services/audit/exceptions.py +1 -2
- pangea/services/audit/models.py +46 -21
- pangea/services/audit/signing.py +1 -0
- pangea/services/audit/util.py +1 -0
- pangea/services/authn/authn.py +18 -0
- pangea/services/authn/models.py +9 -9
- pangea/services/base.py +12 -9
- pangea/services/embargo.py +1 -2
- pangea/services/file_scan.py +3 -4
- pangea/services/intel.py +1 -2
- pangea/services/redact.py +1 -2
- pangea/services/vault/vault.py +6 -6
- pangea/utils.py +0 -1
- {pangea_sdk-3.7.0.dist-info → pangea_sdk-3.7.1.dist-info}/METADATA +24 -2
- pangea_sdk-3.7.1.dist-info/RECORD +44 -0
- pangea_sdk-3.7.0.dist-info/RECORD +0 -44
- {pangea_sdk-3.7.0.dist-info → pangea_sdk-3.7.1.dist-info}/WHEEL +0 -0
pangea/__init__.py
CHANGED
pangea/asyncio/request.py
CHANGED
@@ -7,12 +7,11 @@ import time
|
|
7
7
|
from typing import Dict, List, Optional, Tuple, Type, Union
|
8
8
|
|
9
9
|
import aiohttp
|
10
|
-
import pangea.exceptions as pe
|
11
10
|
from aiohttp import FormData
|
12
11
|
|
13
|
-
|
14
|
-
from pangea.request import PangeaRequestBase
|
15
|
-
from pangea.response import
|
12
|
+
import pangea.exceptions as pe
|
13
|
+
from pangea.request import MultipartResponse, PangeaRequestBase
|
14
|
+
from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
16
15
|
from pangea.utils import default_encoder
|
17
16
|
|
18
17
|
|
@@ -30,7 +29,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
30
29
|
endpoint: str,
|
31
30
|
result_class: Type[PangeaResponseResult],
|
32
31
|
data: Union[str, Dict] = {},
|
33
|
-
files:
|
32
|
+
files: List[Tuple] = [],
|
34
33
|
poll_result: bool = True,
|
35
34
|
url: Optional[str] = None,
|
36
35
|
) -> PangeaResponse:
|
@@ -56,7 +55,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
56
55
|
)
|
57
56
|
transfer_method = data.get("transfer_method", None) # type: ignore[union-attr]
|
58
57
|
|
59
|
-
if files
|
58
|
+
if files and type(data) is dict and (transfer_method == TransferMethod.POST_URL.value):
|
60
59
|
requests_response = await self._full_post_presigned_url(
|
61
60
|
endpoint, result_class=result_class, data=data, files=files
|
62
61
|
)
|
@@ -66,10 +65,26 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
66
65
|
)
|
67
66
|
|
68
67
|
await self._check_http_errors(requests_response)
|
69
|
-
json_resp = await requests_response.json()
|
70
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "post", "url": url, "response": json_resp}))
|
71
68
|
|
72
|
-
|
69
|
+
if "multipart/form-data" in requests_response.headers.get("content-type", ""):
|
70
|
+
multipart_response = await self._process_multipart_response(requests_response)
|
71
|
+
pangea_response: PangeaResponse = PangeaResponse(
|
72
|
+
requests_response,
|
73
|
+
result_class=result_class,
|
74
|
+
json=multipart_response.pangea_json,
|
75
|
+
attached_files=multipart_response.attached_files,
|
76
|
+
)
|
77
|
+
else:
|
78
|
+
try:
|
79
|
+
json_resp = await requests_response.json()
|
80
|
+
self.logger.debug(
|
81
|
+
json.dumps({"service": self.service, "action": "post", "url": url, "response": json_resp})
|
82
|
+
)
|
83
|
+
|
84
|
+
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)
|
85
|
+
except aiohttp.ContentTypeError as e:
|
86
|
+
raise pe.PangeaException(f"Failed to decode json response. {e}. Body: {await requests_response.text()}")
|
87
|
+
|
73
88
|
if poll_result:
|
74
89
|
pangea_response = await self._handle_queued_result(pangea_response)
|
75
90
|
|
@@ -77,7 +92,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
77
92
|
|
78
93
|
async def get(
|
79
94
|
self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
|
80
|
-
) -> PangeaResponse:
|
95
|
+
) -> PangeaResponse[Type[PangeaResponseResult]]:
|
81
96
|
"""Makes the GET call to a Pangea Service endpoint.
|
82
97
|
|
83
98
|
Args:
|
@@ -115,7 +130,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
115
130
|
raise pe.ServiceTemporarilyUnavailable(await resp.json())
|
116
131
|
|
117
132
|
async def poll_result_by_id(
|
118
|
-
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
133
|
+
self, request_id: str, result_class: Union[Type[PangeaResponseResult], Type[dict]], check_response: bool = True
|
119
134
|
):
|
120
135
|
path = self._get_poll_path(request_id)
|
121
136
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
@@ -157,12 +172,84 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
157
172
|
if resp.status < 200 or resp.status >= 300:
|
158
173
|
raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status}", await resp.text())
|
159
174
|
|
175
|
+
async def download_file(self, url: str, filename: Optional[str] = None) -> AttachedFile:
|
176
|
+
self.logger.debug(
|
177
|
+
json.dumps(
|
178
|
+
{
|
179
|
+
"service": self.service,
|
180
|
+
"action": "download_file",
|
181
|
+
"url": url,
|
182
|
+
"status": "start",
|
183
|
+
}
|
184
|
+
)
|
185
|
+
)
|
186
|
+
async with self.session.get(url, headers={}) as response:
|
187
|
+
if response.status == 200:
|
188
|
+
if filename is None:
|
189
|
+
content_disposition = response.headers.get("Content-Disposition", "")
|
190
|
+
filename = self._get_filename_from_content_disposition(content_disposition)
|
191
|
+
if filename is None:
|
192
|
+
filename = self._get_filename_from_url(url)
|
193
|
+
if filename is None:
|
194
|
+
filename = "default_filename"
|
195
|
+
|
196
|
+
content_type = response.headers.get("Content-Type", "")
|
197
|
+
self.logger.debug(
|
198
|
+
json.dumps(
|
199
|
+
{
|
200
|
+
"service": self.service,
|
201
|
+
"action": "download_file",
|
202
|
+
"url": url,
|
203
|
+
"filename": filename,
|
204
|
+
"status": "success",
|
205
|
+
}
|
206
|
+
)
|
207
|
+
)
|
208
|
+
|
209
|
+
return AttachedFile(filename=filename, file=await response.read(), content_type=content_type)
|
210
|
+
else:
|
211
|
+
raise pe.DownloadFileError(f"Failed to download file. Status: {response.status}", await response.text())
|
212
|
+
|
213
|
+
async def _get_pangea_json(self, reader: aiohttp.MultipartReader) -> Optional[Dict]:
|
214
|
+
# Iterate through parts
|
215
|
+
async for part in reader:
|
216
|
+
return await part.json()
|
217
|
+
return None
|
218
|
+
|
219
|
+
async def _get_attached_files(self, reader: aiohttp.MultipartReader) -> List[AttachedFile]:
|
220
|
+
files = []
|
221
|
+
i = 0
|
222
|
+
|
223
|
+
async for part in reader:
|
224
|
+
content_type = part.headers.get("Content-Type", "")
|
225
|
+
content_disposition = part.headers.get("Content-Disposition", "")
|
226
|
+
name = self._get_filename_from_content_disposition(content_disposition)
|
227
|
+
if name is None:
|
228
|
+
name = f"default_file_name_{i}"
|
229
|
+
i += 1
|
230
|
+
files.append(AttachedFile(name, await part.read(), content_type))
|
231
|
+
|
232
|
+
return files
|
233
|
+
|
234
|
+
async def _process_multipart_response(self, resp: aiohttp.ClientResponse) -> MultipartResponse:
|
235
|
+
# Parse the multipart response
|
236
|
+
multipart_reader = aiohttp.MultipartReader.from_response(resp)
|
237
|
+
|
238
|
+
pangea_json = await self._get_pangea_json(multipart_reader) # type: ignore[arg-type]
|
239
|
+
self.logger.debug(
|
240
|
+
json.dumps({"service": self.service, "action": "multipart response", "response": pangea_json})
|
241
|
+
)
|
242
|
+
|
243
|
+
multipart_reader = multipart_reader.__aiter__()
|
244
|
+
attached_files = await self._get_attached_files(multipart_reader) # type: ignore[arg-type]
|
245
|
+
return MultipartResponse(pangea_json, attached_files) # type: ignore[arg-type]
|
246
|
+
|
160
247
|
async def _http_post(
|
161
248
|
self,
|
162
249
|
url: str,
|
163
250
|
headers: Dict = {},
|
164
251
|
data: Union[str, Dict] = {},
|
165
|
-
files:
|
252
|
+
files: List[Tuple] = [],
|
166
253
|
presigned_url_post: bool = False,
|
167
254
|
) -> aiohttp.ClientResponse:
|
168
255
|
if files:
|
@@ -193,26 +280,29 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
193
280
|
self.logger.debug(
|
194
281
|
json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
|
195
282
|
)
|
196
|
-
|
197
|
-
|
198
|
-
form.add_field(name, value[1], filename=value[0], content_type=value[2])
|
199
|
-
return await self.session.put(url, headers=headers, data=form)
|
283
|
+
_, value = files[0]
|
284
|
+
return await self.session.put(url, headers=headers, data=value[1])
|
200
285
|
|
201
286
|
async def _full_post_presigned_url(
|
202
287
|
self,
|
203
288
|
endpoint: str,
|
204
289
|
result_class: Type[PangeaResponseResult],
|
205
290
|
data: Union[str, Dict] = {},
|
206
|
-
files:
|
291
|
+
files: List[Tuple] = [],
|
207
292
|
):
|
208
|
-
if len(files) == 0:
|
293
|
+
if len(files) == 0:
|
209
294
|
raise AttributeError("files attribute should have at least 1 file")
|
210
295
|
|
211
296
|
response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
|
212
|
-
|
213
|
-
|
297
|
+
if response.accepted_result is None:
|
298
|
+
raise pe.PangeaException("No accepted_result field when requesting presigned url")
|
299
|
+
if response.accepted_result.post_url is None:
|
300
|
+
raise pe.PresignedURLException("No presigned url", response)
|
301
|
+
|
302
|
+
data_to_presigned = response.accepted_result.post_form_data
|
303
|
+
presigned_url = response.accepted_result.post_url
|
214
304
|
|
215
|
-
await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
305
|
+
await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
216
306
|
return response.raw_response
|
217
307
|
|
218
308
|
async def request_presigned_url(
|
@@ -232,14 +322,16 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
232
322
|
raise e
|
233
323
|
|
234
324
|
# Receive 202
|
235
|
-
return await self._poll_presigned_url(accepted_exception.response)
|
325
|
+
return await self._poll_presigned_url(accepted_exception.response)
|
236
326
|
|
237
|
-
async def _poll_presigned_url(
|
327
|
+
async def _poll_presigned_url(
|
328
|
+
self, response: PangeaResponse[Type[PangeaResponseResult]]
|
329
|
+
) -> PangeaResponse[Type[PangeaResponseResult]]:
|
238
330
|
if response.http_status != 202:
|
239
331
|
raise AttributeError("Response should be 202")
|
240
332
|
|
241
333
|
if response.accepted_result is not None and response.accepted_result.has_upload_url:
|
242
|
-
return response
|
334
|
+
return response
|
243
335
|
|
244
336
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
245
337
|
retry_count = 1
|
@@ -276,7 +368,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
276
368
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
277
369
|
|
278
370
|
if loop_resp.accepted_result is not None and not loop_resp.accepted_result.has_upload_url:
|
279
|
-
return loop_resp
|
371
|
+
return loop_resp
|
280
372
|
else:
|
281
373
|
raise loop_exc
|
282
374
|
|
pangea/asyncio/services/audit.py
CHANGED
@@ -1,18 +1,24 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import datetime
|
4
|
-
from typing import Any, Dict, List, Optional, Union
|
4
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
5
5
|
|
6
6
|
import pangea.exceptions as pexc
|
7
|
+
from pangea.asyncio.services.base import ServiceBaseAsync
|
7
8
|
from pangea.response import PangeaResponse
|
8
9
|
from pangea.services.audit.audit import AuditBase
|
9
10
|
from pangea.services.audit.exceptions import AuditException
|
10
11
|
from pangea.services.audit.models import (
|
12
|
+
DownloadFormat,
|
13
|
+
DownloadRequest,
|
14
|
+
DownloadResult,
|
11
15
|
Event,
|
12
16
|
LogBulkResult,
|
13
17
|
LogResult,
|
18
|
+
PublishedRoot,
|
14
19
|
RootRequest,
|
15
20
|
RootResult,
|
21
|
+
RootSource,
|
16
22
|
SearchOrder,
|
17
23
|
SearchOrderBy,
|
18
24
|
SearchOutput,
|
@@ -22,8 +28,6 @@ from pangea.services.audit.models import (
|
|
22
28
|
)
|
23
29
|
from pangea.services.audit.util import format_datetime
|
24
30
|
|
25
|
-
from .base import ServiceBaseAsync
|
26
|
-
|
27
31
|
|
28
32
|
class AuditAsync(ServiceBaseAsync, AuditBase):
|
29
33
|
"""Audit service client.
|
@@ -175,8 +179,10 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
175
179
|
"""
|
176
180
|
|
177
181
|
input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
|
178
|
-
response = await self.request.post(
|
179
|
-
|
182
|
+
response: PangeaResponse[LogResult] = await self.request.post(
|
183
|
+
"v1/log", LogResult, data=input.dict(exclude_none=True)
|
184
|
+
)
|
185
|
+
if response.success and response.result is not None:
|
180
186
|
self._process_log_result(response.result, verify=verify)
|
181
187
|
return response
|
182
188
|
|
@@ -209,8 +215,10 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
209
215
|
"""
|
210
216
|
|
211
217
|
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
212
|
-
response = await self.request.post(
|
213
|
-
|
218
|
+
response: PangeaResponse[LogBulkResult] = await self.request.post(
|
219
|
+
"v2/log", LogBulkResult, data=input.dict(exclude_none=True)
|
220
|
+
)
|
221
|
+
if response.success and response.result is not None:
|
214
222
|
for result in response.result.results:
|
215
223
|
self._process_log_result(result, verify=True)
|
216
224
|
return response
|
@@ -245,12 +253,12 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
245
253
|
|
246
254
|
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
247
255
|
try:
|
248
|
-
response = await self.request.post(
|
256
|
+
response: PangeaResponse[LogBulkResult] = await self.request.post(
|
249
257
|
"v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
|
250
258
|
)
|
251
259
|
except pexc.AcceptedRequestException as e:
|
252
260
|
return e.response
|
253
|
-
if response.success:
|
261
|
+
if response.success and response.result:
|
254
262
|
for result in response.result.results:
|
255
263
|
self._process_log_result(result, verify=True)
|
256
264
|
return response
|
@@ -264,7 +272,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
264
272
|
end: Optional[Union[datetime.datetime, str]] = None,
|
265
273
|
limit: Optional[int] = None,
|
266
274
|
max_results: Optional[int] = None,
|
267
|
-
search_restriction: Optional[
|
275
|
+
search_restriction: Optional[Dict[str, Sequence[str]]] = None,
|
268
276
|
verbose: Optional[bool] = None,
|
269
277
|
verify_consistency: bool = False,
|
270
278
|
verify_events: bool = True,
|
@@ -293,7 +301,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
293
301
|
end (datetime, optional): An RFC-3339 formatted timestamp, or relative time adjustment from the current time.
|
294
302
|
limit (int, optional): Optional[int] = None,
|
295
303
|
max_results (int, optional): Maximum number of results to return.
|
296
|
-
search_restriction (
|
304
|
+
search_restriction (Dict[str, Sequence[str]], optional): A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
|
297
305
|
verbose (bool, optional): If true, response include root and membership and consistency proofs.
|
298
306
|
verify_consistency (bool): True to verify logs consistency
|
299
307
|
verify_events (bool): True to verify hash events and signatures
|
@@ -326,7 +334,12 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
326
334
|
verbose=verbose,
|
327
335
|
)
|
328
336
|
|
329
|
-
response = await self.request.post(
|
337
|
+
response: PangeaResponse[SearchOutput] = await self.request.post(
|
338
|
+
"v1/search", SearchOutput, data=input.dict(exclude_none=True)
|
339
|
+
)
|
340
|
+
if verify_consistency:
|
341
|
+
await self.update_published_roots(response.result) # type: ignore[arg-type]
|
342
|
+
|
330
343
|
return self.handle_search_response(response, verify_consistency, verify_events)
|
331
344
|
|
332
345
|
async def results(
|
@@ -334,6 +347,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
334
347
|
id: str,
|
335
348
|
limit: Optional[int] = 20,
|
336
349
|
offset: Optional[int] = 0,
|
350
|
+
assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None,
|
337
351
|
verify_consistency: bool = False,
|
338
352
|
verify_events: bool = True,
|
339
353
|
) -> PangeaResponse[SearchResultOutput]:
|
@@ -348,6 +362,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
348
362
|
id (string): the id of a search action, found in `response.result.id`
|
349
363
|
limit (integer, optional): the maximum number of results to return, default is 20
|
350
364
|
offset (integer, optional): the position of the first result to return, default is 0
|
365
|
+
assert_search_restriction (Dict[str, Sequence[str]], optional): Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
|
351
366
|
verify_consistency (bool): True to verify logs consistency
|
352
367
|
verify_events (bool): True to verify hash events and signatures
|
353
368
|
Raises:
|
@@ -365,7 +380,8 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
365
380
|
result_res: PangeaResponse[SearchResultsOutput] = audit.results(
|
366
381
|
id=search_res.result.id,
|
367
382
|
limit=10,
|
368
|
-
offset=0
|
383
|
+
offset=0,
|
384
|
+
assert_search_restriction={'source': ["monitor"]})
|
369
385
|
"""
|
370
386
|
|
371
387
|
if limit <= 0: # type: ignore[operator]
|
@@ -378,8 +394,12 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
378
394
|
id=id,
|
379
395
|
limit=limit,
|
380
396
|
offset=offset,
|
397
|
+
assert_search_restriction=assert_search_restriction,
|
381
398
|
)
|
382
399
|
response = await self.request.post("v1/results", SearchResultOutput, data=input.dict(exclude_none=True))
|
400
|
+
if verify_consistency and response.result is not None:
|
401
|
+
await self.update_published_roots(response.result)
|
402
|
+
|
383
403
|
return self.handle_results_response(response, verify_consistency, verify_events)
|
384
404
|
|
385
405
|
async def root(self, tree_size: Optional[int] = None) -> PangeaResponse[RootResult]:
|
@@ -405,3 +425,67 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
|
|
405
425
|
"""
|
406
426
|
input = RootRequest(tree_size=tree_size)
|
407
427
|
return await self.request.post("v1/root", RootResult, data=input.dict(exclude_none=True))
|
428
|
+
|
429
|
+
async def download_results(
|
430
|
+
self, result_id: str, format: Optional[DownloadFormat] = None
|
431
|
+
) -> PangeaResponse[DownloadResult]:
|
432
|
+
"""
|
433
|
+
Download search results
|
434
|
+
|
435
|
+
Get all search results as a compressed (gzip) CSV file.
|
436
|
+
|
437
|
+
OperationId: audit_post_v1_download_results
|
438
|
+
|
439
|
+
Args:
|
440
|
+
result_id: ID returned by the search API.
|
441
|
+
format: Format for the records.
|
442
|
+
|
443
|
+
Returns:
|
444
|
+
URL where search results can be downloaded.
|
445
|
+
|
446
|
+
Raises:
|
447
|
+
AuditException: If an Audit-based API exception occurs.
|
448
|
+
PangeaAPIException: If an API exception occurs.
|
449
|
+
|
450
|
+
Examples:
|
451
|
+
response = await audit.download_results(
|
452
|
+
result_id="pas_[...]",
|
453
|
+
format=DownloadFormat.JSON,
|
454
|
+
)
|
455
|
+
"""
|
456
|
+
|
457
|
+
input = DownloadRequest(result_id=result_id, format=format)
|
458
|
+
return await self.request.post("v1/download_results", DownloadResult, data=input.dict(exclude_none=True))
|
459
|
+
|
460
|
+
async def update_published_roots(self, result: SearchResultOutput):
|
461
|
+
"""Fetches series of published root hashes from Arweave
|
462
|
+
|
463
|
+
This is used for subsequent calls to verify_consistency_proof(). Root hashes
|
464
|
+
are published on [Arweave](https://arweave.net).
|
465
|
+
|
466
|
+
Args:
|
467
|
+
result (SearchResultOutput): Result object from previous call to AuditAsync.search() or AuditAsync.results()
|
468
|
+
|
469
|
+
Raises:
|
470
|
+
AuditException: If an audit based api exception happens
|
471
|
+
PangeaAPIException: If an API Error happens
|
472
|
+
"""
|
473
|
+
|
474
|
+
if not result.root:
|
475
|
+
return
|
476
|
+
|
477
|
+
tree_sizes, arweave_roots = self._get_tree_sizes_and_roots(result)
|
478
|
+
|
479
|
+
# fill the missing roots from the server (if allowed)
|
480
|
+
for tree_size in tree_sizes:
|
481
|
+
pub_root = None
|
482
|
+
if tree_size in arweave_roots:
|
483
|
+
pub_root = PublishedRoot(**arweave_roots[tree_size].dict(exclude_none=True))
|
484
|
+
pub_root.source = RootSource.ARWEAVE
|
485
|
+
elif self.allow_server_roots:
|
486
|
+
resp = await self.root(tree_size=tree_size)
|
487
|
+
if resp.success and resp.result is not None:
|
488
|
+
pub_root = PublishedRoot(**resp.result.data.dict(exclude_none=True))
|
489
|
+
pub_root.source = RootSource.PANGEA
|
490
|
+
if pub_root is not None:
|
491
|
+
self.pub_roots[tree_size] = pub_root
|
pangea/asyncio/services/authn.py
CHANGED
@@ -4,10 +4,9 @@
|
|
4
4
|
from typing import Dict, List, Optional, Union
|
5
5
|
|
6
6
|
import pangea.services.authn.models as m
|
7
|
+
from pangea.asyncio.services.base import ServiceBaseAsync
|
7
8
|
from pangea.response import PangeaResponse
|
8
9
|
|
9
|
-
from .base import ServiceBaseAsync
|
10
|
-
|
11
10
|
SERVICE_NAME = "authn"
|
12
11
|
|
13
12
|
|
@@ -114,6 +113,9 @@ class AuthNAsync(ServiceBaseAsync):
|
|
114
113
|
Examples:
|
115
114
|
response = authn.session.list()
|
116
115
|
"""
|
116
|
+
if isinstance(filter, dict):
|
117
|
+
filter = m.SessionListFilter(**filter)
|
118
|
+
|
117
119
|
input = m.SessionListRequest(filter=filter, last=last, order=order, order_by=order_by, size=size)
|
118
120
|
return await self.request.post("v2/session/list", m.SessionListResults, data=input.dict(exclude_none=True))
|
119
121
|
|
@@ -272,6 +274,9 @@ class AuthNAsync(ServiceBaseAsync):
|
|
272
274
|
token="ptu_wuk7tvtpswyjtlsx52b7yyi2l7zotv4a",
|
273
275
|
)
|
274
276
|
"""
|
277
|
+
if isinstance(filter, dict):
|
278
|
+
filter = m.SessionListFilter(**filter)
|
279
|
+
|
275
280
|
input = m.ClientSessionListRequest(
|
276
281
|
token=token, filter=filter, last=last, order=order, order_by=order_by, size=size
|
277
282
|
)
|
@@ -593,6 +598,9 @@ class AuthNAsync(ServiceBaseAsync):
|
|
593
598
|
Examples:
|
594
599
|
response = authn.user.list()
|
595
600
|
"""
|
601
|
+
if isinstance(filter, dict):
|
602
|
+
filter = m.UserListFilter(**filter)
|
603
|
+
|
596
604
|
input = m.UserListRequest(
|
597
605
|
filter=filter,
|
598
606
|
last=last,
|
@@ -642,6 +650,9 @@ class AuthNAsync(ServiceBaseAsync):
|
|
642
650
|
Examples:
|
643
651
|
response = authn.user.invites.list()
|
644
652
|
"""
|
653
|
+
if isinstance(filter, dict):
|
654
|
+
filter = m.UserInviteListFilter(**filter)
|
655
|
+
|
645
656
|
input = m.UserInviteListRequest(filter=filter, last=last, order=order, order_by=order_by, size=size)
|
646
657
|
return await self.request.post(
|
647
658
|
"v2/user/invite/list", m.UserInviteListResult, data=input.dict(exclude_none=True)
|
@@ -1064,6 +1075,8 @@ class AuthNAsync(ServiceBaseAsync):
|
|
1064
1075
|
Examples:
|
1065
1076
|
response = authn.agreements.list()
|
1066
1077
|
"""
|
1078
|
+
if isinstance(filter, dict):
|
1079
|
+
filter = m.AgreementListFilter(**filter)
|
1067
1080
|
|
1068
1081
|
input = m.AgreementListRequest(filter=filter, last=last, order=order, order_by=order_by, size=size)
|
1069
1082
|
return await self.request.post(
|
pangea/asyncio/services/base.py
CHANGED
@@ -1,18 +1,18 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
|
4
|
-
from typing import Optional, Type, Union
|
4
|
+
from typing import Dict, Optional, Type, Union
|
5
5
|
|
6
6
|
from pangea.asyncio.request import PangeaRequestAsync
|
7
7
|
from pangea.exceptions import AcceptedRequestException
|
8
|
-
from pangea.response import PangeaResponse, PangeaResponseResult
|
9
|
-
from pangea.services.base import ServiceBase
|
8
|
+
from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult
|
9
|
+
from pangea.services.base import PangeaRequest, ServiceBase
|
10
10
|
|
11
11
|
|
12
12
|
class ServiceBaseAsync(ServiceBase):
|
13
13
|
@property
|
14
|
-
def request(self):
|
15
|
-
if
|
14
|
+
def request(self) -> PangeaRequestAsync: # type: ignore[override]
|
15
|
+
if self._request is None or isinstance(self._request, PangeaRequest):
|
16
16
|
self._request = PangeaRequestAsync(
|
17
17
|
config=self.config,
|
18
18
|
token=self.token,
|
@@ -28,7 +28,7 @@ class ServiceBaseAsync(ServiceBase):
|
|
28
28
|
exception: Optional[AcceptedRequestException] = None,
|
29
29
|
response: Optional[PangeaResponse] = None,
|
30
30
|
request_id: Optional[str] = None,
|
31
|
-
result_class: Union[Type[PangeaResponseResult],
|
31
|
+
result_class: Union[Type[PangeaResponseResult], Type[Dict]] = dict,
|
32
32
|
) -> PangeaResponse:
|
33
33
|
"""
|
34
34
|
Poll result
|
@@ -58,6 +58,9 @@ class ServiceBaseAsync(ServiceBase):
|
|
58
58
|
else:
|
59
59
|
raise AttributeError("Need to set exception, response or request_id")
|
60
60
|
|
61
|
+
async def download_file(self, url: str, filename: Optional[str] = None) -> AttachedFile: # type: ignore[override]
|
62
|
+
return await self.request.download_file(url=url, filename=filename)
|
63
|
+
|
61
64
|
async def close(self):
|
62
65
|
await self.request.session.close()
|
63
66
|
# Loop over all attributes to check if they are derived from ServiceBaseAsync and close them
|
@@ -2,16 +2,15 @@
|
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import io
|
4
4
|
import logging
|
5
|
-
from typing import Dict, Optional
|
5
|
+
from typing import Dict, List, Optional, Tuple
|
6
6
|
|
7
7
|
import pangea.services.file_scan as m
|
8
8
|
from pangea.asyncio.request import PangeaRequestAsync
|
9
|
+
from pangea.asyncio.services.base import ServiceBaseAsync
|
9
10
|
from pangea.request import PangeaConfig
|
10
11
|
from pangea.response import PangeaResponse, TransferMethod
|
11
12
|
from pangea.utils import FileUploadParams, get_file_upload_params
|
12
13
|
|
13
|
-
from .base import ServiceBaseAsync
|
14
|
-
|
15
14
|
|
16
15
|
class FileScanAsync(ServiceBaseAsync):
|
17
16
|
"""FileScan service client.
|
@@ -96,7 +95,7 @@ class FileScanAsync(ServiceBaseAsync):
|
|
96
95
|
size = params.size
|
97
96
|
else:
|
98
97
|
crc, sha, size = None, None, None
|
99
|
-
files = [("upload", ("filename", file, "application/octet-stream"))]
|
98
|
+
files: List[Tuple] = [("upload", ("filename", file, "application/octet-stream"))]
|
100
99
|
else:
|
101
100
|
raise ValueError("Need to set file_path or file arguments")
|
102
101
|
|
pangea/asyncio/services/intel.py
CHANGED
@@ -4,11 +4,10 @@ import hashlib
|
|
4
4
|
from typing import List, Optional
|
5
5
|
|
6
6
|
import pangea.services.intel as m
|
7
|
+
from pangea.asyncio.services.base import ServiceBaseAsync
|
7
8
|
from pangea.response import PangeaResponse
|
8
9
|
from pangea.utils import hash_256_filepath
|
9
10
|
|
10
|
-
from .base import ServiceBaseAsync
|
11
|
-
|
12
11
|
|
13
12
|
class FileIntelAsync(ServiceBaseAsync):
|
14
13
|
"""File Intel service client
|
@@ -4,10 +4,9 @@
|
|
4
4
|
from typing import Dict, List, Optional, Union
|
5
5
|
|
6
6
|
import pangea.services.redact as m
|
7
|
+
from pangea.asyncio.services.base import ServiceBaseAsync
|
7
8
|
from pangea.response import PangeaResponse
|
8
9
|
|
9
|
-
from .base import ServiceBaseAsync
|
10
|
-
|
11
10
|
|
12
11
|
class RedactAsync(ServiceBaseAsync):
|
13
12
|
"""Redact service client.
|
pangea/asyncio/services/vault.py
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
import datetime
|
4
4
|
from typing import Dict, Optional, Union
|
5
5
|
|
6
|
+
from pangea.asyncio.services.base import ServiceBaseAsync
|
6
7
|
from pangea.response import PangeaResponse
|
7
8
|
from pangea.services.vault.models.asymmetric import (
|
8
9
|
AsymmetricGenerateRequest,
|
@@ -47,8 +48,8 @@ from pangea.services.vault.models.common import (
|
|
47
48
|
StateChangeRequest,
|
48
49
|
StateChangeResult,
|
49
50
|
SymmetricAlgorithm,
|
50
|
-
TDict,
|
51
51
|
Tags,
|
52
|
+
TDict,
|
52
53
|
UpdateRequest,
|
53
54
|
UpdateResult,
|
54
55
|
)
|
@@ -69,8 +70,6 @@ from pangea.services.vault.models.symmetric import (
|
|
69
70
|
SymmetricStoreResult,
|
70
71
|
)
|
71
72
|
|
72
|
-
from .base import ServiceBaseAsync
|
73
|
-
|
74
73
|
|
75
74
|
class VaultAsync(ServiceBaseAsync):
|
76
75
|
"""Vault service client.
|
@@ -92,7 +91,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
92
91
|
vault_config = PangeaConfig(domain="pangea.cloud")
|
93
92
|
|
94
93
|
# Setup Pangea Vault service
|
95
|
-
vault = Vault(token=PANGEA_VAULT_TOKEN, config=
|
94
|
+
vault = Vault(token=PANGEA_VAULT_TOKEN, config=vault_config)
|
96
95
|
"""
|
97
96
|
|
98
97
|
service_name = "vault"
|
@@ -859,8 +858,8 @@ class VaultAsync(ServiceBaseAsync):
|
|
859
858
|
|
860
859
|
Default is `deactivated`.
|
861
860
|
public_key (EncodedPublicKey, optional): The public key (in PEM format)
|
862
|
-
private_key
|
863
|
-
key
|
861
|
+
private_key (EncodedPrivateKey, optional): The private key (in PEM format)
|
862
|
+
key (EncodedSymmetricKey, optional): The key material (in base64)
|
864
863
|
|
865
864
|
Raises:
|
866
865
|
PangeaAPIException: If an API Error happens
|
@@ -1222,7 +1221,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1222
1221
|
)
|
1223
1222
|
"""
|
1224
1223
|
|
1225
|
-
input = EncryptStructuredRequest(
|
1224
|
+
input: EncryptStructuredRequest[TDict] = EncryptStructuredRequest(
|
1226
1225
|
id=id, structured_data=structured_data, filter=filter, version=version, additional_data=additional_data
|
1227
1226
|
)
|
1228
1227
|
return await self.request.post(
|
@@ -1271,7 +1270,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1271
1270
|
)
|
1272
1271
|
"""
|
1273
1272
|
|
1274
|
-
input = EncryptStructuredRequest(
|
1273
|
+
input: EncryptStructuredRequest[TDict] = EncryptStructuredRequest(
|
1275
1274
|
id=id, structured_data=structured_data, filter=filter, version=version, additional_data=additional_data
|
1276
1275
|
)
|
1277
1276
|
return await self.request.post(
|