pangea-sdk 3.2.0__py3-none-any.whl → 3.4.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- pangea/__init__.py +1 -1
- pangea/asyncio/request.py +127 -96
- pangea/asyncio/services/base.py +20 -3
- pangea/asyncio/services/file_scan.py +75 -5
- pangea/asyncio/services/redact.py +2 -7
- pangea/config.py +2 -2
- pangea/request.py +135 -96
- pangea/response.py +13 -2
- pangea/services/audit/audit.py +7 -7
- pangea/services/authn/authn.py +4 -1
- pangea/services/authn/models.py +2 -1
- pangea/services/base.py +17 -4
- pangea/services/file_scan.py +68 -11
- pangea/services/redact.py +2 -7
- pangea/services/vault/vault.py +2 -2
- pangea/utils.py +9 -2
- {pangea_sdk-3.2.0.dist-info → pangea_sdk-3.4.0.dist-info}/METADATA +13 -13
- {pangea_sdk-3.2.0.dist-info → pangea_sdk-3.4.0.dist-info}/RECORD +19 -19
- {pangea_sdk-3.2.0.dist-info → pangea_sdk-3.4.0.dist-info}/WHEEL +0 -0
pangea/__init__.py
CHANGED
pangea/asyncio/request.py
CHANGED
@@ -51,12 +51,10 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
51
51
|
if self.config_id and data.get("config_id", None) is None:
|
52
52
|
data["config_id"] = self.config_id
|
53
53
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
):
|
59
|
-
requests_response = await self._post_presigned_url(
|
54
|
+
transfer_method = data.get("transfer_method", None)
|
55
|
+
|
56
|
+
if files is not None and type(data) is dict and (transfer_method == TransferMethod.POST_URL.value):
|
57
|
+
requests_response = await self._full_post_presigned_url(
|
60
58
|
endpoint, result_class=result_class, data=data, files=files
|
61
59
|
)
|
62
60
|
else:
|
@@ -64,14 +62,92 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
64
62
|
url, headers=self._headers(), data=data, files=files, presigned_url_post=False
|
65
63
|
)
|
66
64
|
|
67
|
-
|
68
|
-
|
69
|
-
|
65
|
+
json_resp = await requests_response.json()
|
66
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "post", "url": url, "response": json_resp}))
|
67
|
+
|
68
|
+
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)
|
70
69
|
if poll_result:
|
71
70
|
pangea_response = await self._handle_queued_result(pangea_response)
|
72
71
|
|
73
72
|
return self._check_response(pangea_response)
|
74
73
|
|
74
|
+
async def get(
|
75
|
+
self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
|
76
|
+
) -> PangeaResponse:
|
77
|
+
"""Makes the GET call to a Pangea Service endpoint.
|
78
|
+
|
79
|
+
Args:
|
80
|
+
endpoint(str): The Pangea Service API endpoint.
|
81
|
+
path(str): Additional URL path
|
82
|
+
|
83
|
+
Returns:
|
84
|
+
PangeaResponse which contains the response in its entirety and
|
85
|
+
various properties to retrieve individual fields
|
86
|
+
"""
|
87
|
+
|
88
|
+
url = self._url(path)
|
89
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
90
|
+
|
91
|
+
async with self.session.get(url, headers=self._headers()) as requests_response:
|
92
|
+
pangea_response = PangeaResponse(
|
93
|
+
requests_response, result_class=result_class, json=await requests_response.json()
|
94
|
+
)
|
95
|
+
|
96
|
+
self.logger.debug(
|
97
|
+
json.dumps(
|
98
|
+
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
99
|
+
default=default_encoder,
|
100
|
+
)
|
101
|
+
)
|
102
|
+
|
103
|
+
if check_response is False:
|
104
|
+
return pangea_response
|
105
|
+
|
106
|
+
return self._check_response(pangea_response)
|
107
|
+
|
108
|
+
async def poll_result_by_id(
|
109
|
+
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
110
|
+
):
|
111
|
+
path = self._get_poll_path(request_id)
|
112
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
113
|
+
return await self.get(path, result_class, check_response=check_response)
|
114
|
+
|
115
|
+
async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
116
|
+
request_id = response.request_id
|
117
|
+
if not request_id:
|
118
|
+
raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
|
119
|
+
|
120
|
+
if response.status != ResponseStatus.ACCEPTED.value:
|
121
|
+
raise pe.PangeaException("Response already proccesed")
|
122
|
+
|
123
|
+
return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
124
|
+
|
125
|
+
async def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
|
126
|
+
# Send form request with file and upload_details as body
|
127
|
+
resp = await self._http_post(url=url, data=data, files=files, presigned_url_post=True)
|
128
|
+
self.logger.debug(
|
129
|
+
json.dumps(
|
130
|
+
{"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
|
131
|
+
default=default_encoder,
|
132
|
+
)
|
133
|
+
)
|
134
|
+
|
135
|
+
if resp.status < 200 or resp.status >= 300:
|
136
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", resp.text)
|
137
|
+
|
138
|
+
async def put_presigned_url(self, url: str, files: List[Tuple]):
|
139
|
+
# Send put request with file as body
|
140
|
+
resp = await self._http_put(url=url, files=files)
|
141
|
+
self.logger.debug(
|
142
|
+
json.dumps(
|
143
|
+
{"service": self.service, "action": "put presigned", "url": url, "response": await resp.text()},
|
144
|
+
default=default_encoder,
|
145
|
+
)
|
146
|
+
)
|
147
|
+
|
148
|
+
if resp.status < 200 or resp.status >= 300:
|
149
|
+
raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status}", await resp.text())
|
150
|
+
|
75
151
|
async def _http_post(
|
76
152
|
self,
|
77
153
|
url: str,
|
@@ -105,7 +181,21 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
105
181
|
|
106
182
|
return await self.session.post(url, headers=headers, data=data_send)
|
107
183
|
|
108
|
-
async def
|
184
|
+
async def _http_put(
|
185
|
+
self,
|
186
|
+
url: str,
|
187
|
+
files: List[Tuple],
|
188
|
+
headers: Dict = {},
|
189
|
+
) -> aiohttp.ClientResponse:
|
190
|
+
self.logger.debug(
|
191
|
+
json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
|
192
|
+
)
|
193
|
+
form = FormData()
|
194
|
+
name, value = files[0]
|
195
|
+
form.add_field(name, value[1], filename=value[0], content_type=value[2])
|
196
|
+
return await self.session.put(url, headers=headers, data=form)
|
197
|
+
|
198
|
+
async def _full_post_presigned_url(
|
109
199
|
self,
|
110
200
|
endpoint: str,
|
111
201
|
result_class: Type[PangeaResponseResult],
|
@@ -115,61 +205,52 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
115
205
|
if len(files) == 0:
|
116
206
|
raise AttributeError("files attribute should have at least 1 file")
|
117
207
|
|
208
|
+
response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
|
209
|
+
data_to_presigned = response.accepted_result.post_form_data
|
210
|
+
presigned_url = response.accepted_result.post_url
|
211
|
+
|
212
|
+
await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
213
|
+
return response.raw_response
|
214
|
+
|
215
|
+
async def request_presigned_url(
|
216
|
+
self,
|
217
|
+
endpoint: str,
|
218
|
+
result_class: Type[PangeaResponseResult],
|
219
|
+
data: Union[str, Dict] = {},
|
220
|
+
) -> PangeaResponse:
|
118
221
|
# Send request
|
119
222
|
try:
|
120
223
|
# This should return 202 (AcceptedRequestException)
|
121
224
|
resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
122
225
|
raise pe.PresignedURLException("Should return 202", resp)
|
123
|
-
|
124
226
|
except pe.AcceptedRequestException as e:
|
125
227
|
accepted_exception = e
|
126
228
|
except Exception as e:
|
127
229
|
raise e
|
128
230
|
|
129
|
-
# Receive 202
|
130
|
-
|
131
|
-
data_to_presigned = result.accepted_status.upload_details
|
132
|
-
presigned_url = result.accepted_status.upload_url
|
231
|
+
# Receive 202
|
232
|
+
return await self._poll_presigned_url(accepted_exception.response)
|
133
233
|
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
json.dumps(
|
138
|
-
{
|
139
|
-
"service": self.service,
|
140
|
-
"action": "post presigned",
|
141
|
-
"url": presigned_url,
|
142
|
-
"response": await resp.text(),
|
143
|
-
},
|
144
|
-
default=default_encoder,
|
145
|
-
)
|
146
|
-
)
|
234
|
+
async def _poll_presigned_url(self, response: PangeaResponse) -> AcceptedResult:
|
235
|
+
if response.http_status != 202:
|
236
|
+
raise AttributeError("Response should be 202")
|
147
237
|
|
148
|
-
if
|
149
|
-
|
150
|
-
|
151
|
-
return accepted_exception.response.raw_response
|
152
|
-
|
153
|
-
async def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
|
154
|
-
if type(initial_exc) is not pe.AcceptedRequestException:
|
155
|
-
raise AttributeError("Exception should be of type AcceptedRequestException")
|
156
|
-
|
157
|
-
if initial_exc.accepted_result.accepted_status.upload_url:
|
158
|
-
return initial_exc.accepted_result
|
238
|
+
if response.accepted_result is not None and response.accepted_result.has_upload_url:
|
239
|
+
return response
|
159
240
|
|
160
241
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
161
242
|
retry_count = 1
|
162
243
|
start = time.time()
|
163
|
-
|
244
|
+
loop_resp = response
|
164
245
|
|
165
246
|
while (
|
166
|
-
|
167
|
-
and not
|
247
|
+
loop_resp.accepted_result is not None
|
248
|
+
and not loop_resp.accepted_result.has_upload_url
|
168
249
|
and not self._reach_timeout(start)
|
169
250
|
):
|
170
251
|
await asyncio.sleep(self._get_delay(retry_count, start))
|
171
252
|
try:
|
172
|
-
await self.poll_result_once(
|
253
|
+
await self.poll_result_once(response, check_response=False)
|
173
254
|
msg = "Polling presigned url return 200 instead of 202"
|
174
255
|
self.logger.debug(
|
175
256
|
json.dumps(
|
@@ -179,6 +260,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
179
260
|
raise pe.PangeaException(msg)
|
180
261
|
except pe.AcceptedRequestException as e:
|
181
262
|
retry_count += 1
|
263
|
+
loop_resp = e.response
|
182
264
|
loop_exc = e
|
183
265
|
except Exception as e:
|
184
266
|
self.logger.debug(
|
@@ -190,8 +272,8 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
190
272
|
|
191
273
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
192
274
|
|
193
|
-
if
|
194
|
-
return
|
275
|
+
if loop_resp.accepted_result is not None and not loop_resp.accepted_result.has_upload_url:
|
276
|
+
return loop_resp
|
195
277
|
else:
|
196
278
|
raise loop_exc
|
197
279
|
|
@@ -207,57 +289,6 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
207
289
|
|
208
290
|
return response
|
209
291
|
|
210
|
-
async def get(
|
211
|
-
self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
|
212
|
-
) -> PangeaResponse:
|
213
|
-
"""Makes the GET call to a Pangea Service endpoint.
|
214
|
-
|
215
|
-
Args:
|
216
|
-
endpoint(str): The Pangea Service API endpoint.
|
217
|
-
path(str): Additional URL path
|
218
|
-
|
219
|
-
Returns:
|
220
|
-
PangeaResponse which contains the response in its entirety and
|
221
|
-
various properties to retrieve individual fields
|
222
|
-
"""
|
223
|
-
|
224
|
-
url = self._url(path)
|
225
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
226
|
-
|
227
|
-
async with self.session.get(url, headers=self._headers()) as requests_response:
|
228
|
-
pangea_response = PangeaResponse(
|
229
|
-
requests_response, result_class=result_class, json=await requests_response.json()
|
230
|
-
)
|
231
|
-
|
232
|
-
self.logger.debug(
|
233
|
-
json.dumps(
|
234
|
-
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
235
|
-
default=default_encoder,
|
236
|
-
)
|
237
|
-
)
|
238
|
-
|
239
|
-
if check_response is False:
|
240
|
-
return pangea_response
|
241
|
-
|
242
|
-
return self._check_response(pangea_response)
|
243
|
-
|
244
|
-
async def poll_result_by_id(
|
245
|
-
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
246
|
-
):
|
247
|
-
path = self._get_poll_path(request_id)
|
248
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
249
|
-
return await self.get(path, result_class, check_response=check_response)
|
250
|
-
|
251
|
-
async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
252
|
-
request_id = response.request_id
|
253
|
-
if not request_id:
|
254
|
-
raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
|
255
|
-
|
256
|
-
if response.status != ResponseStatus.ACCEPTED.value:
|
257
|
-
raise pe.PangeaException("Response already proccesed")
|
258
|
-
|
259
|
-
return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
260
|
-
|
261
292
|
async def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
|
262
293
|
retry_count = 1
|
263
294
|
start = time.time()
|
pangea/asyncio/services/base.py
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
|
4
|
+
from typing import Optional, Type, Union
|
5
|
+
|
4
6
|
from pangea.asyncio.request import PangeaRequestAsync
|
5
7
|
from pangea.exceptions import AcceptedRequestException
|
6
|
-
from pangea.response import PangeaResponse
|
8
|
+
from pangea.response import PangeaResponse, PangeaResponseResult
|
7
9
|
from pangea.services.base import ServiceBase
|
8
10
|
|
9
11
|
|
@@ -21,7 +23,13 @@ class ServiceBaseAsync(ServiceBase):
|
|
21
23
|
|
22
24
|
return self._request
|
23
25
|
|
24
|
-
async def poll_result(
|
26
|
+
async def poll_result(
|
27
|
+
self,
|
28
|
+
exception: Optional[AcceptedRequestException] = None,
|
29
|
+
response: Optional[PangeaResponse] = None,
|
30
|
+
request_id: Optional[str] = None,
|
31
|
+
result_class: Union[Type[PangeaResponseResult], dict] = dict,
|
32
|
+
) -> PangeaResponse:
|
25
33
|
"""
|
26
34
|
Poll result
|
27
35
|
|
@@ -39,7 +47,16 @@ class ServiceBaseAsync(ServiceBase):
|
|
39
47
|
Examples:
|
40
48
|
response = service.poll_result(exception)
|
41
49
|
"""
|
42
|
-
|
50
|
+
if exception is not None:
|
51
|
+
return await self.request.poll_result_once(exception.response, check_response=True)
|
52
|
+
elif response is not None:
|
53
|
+
return await self.request.poll_result_once(response, check_response=True)
|
54
|
+
elif request_id is not None:
|
55
|
+
return await self.request.poll_result_by_id(
|
56
|
+
request_id=request_id, result_class=result_class, check_response=True
|
57
|
+
)
|
58
|
+
else:
|
59
|
+
raise AttributeError("Need to set exception, response or request_id")
|
43
60
|
|
44
61
|
async def close(self):
|
45
62
|
await self.request.session.close()
|
@@ -1,11 +1,14 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import io
|
4
|
-
|
4
|
+
import logging
|
5
|
+
from typing import Dict, Optional
|
5
6
|
|
6
7
|
import pangea.services.file_scan as m
|
7
|
-
from pangea.
|
8
|
-
from pangea.
|
8
|
+
from pangea.asyncio.request import PangeaRequestAsync
|
9
|
+
from pangea.request import PangeaConfig
|
10
|
+
from pangea.response import PangeaResponse, TransferMethod
|
11
|
+
from pangea.utils import FileUploadParams, get_file_upload_params
|
9
12
|
|
10
13
|
from .base import ServiceBaseAsync
|
11
14
|
|
@@ -46,6 +49,8 @@ class FileScanAsync(ServiceBaseAsync):
|
|
46
49
|
raw: Optional[bool] = None,
|
47
50
|
provider: Optional[str] = None,
|
48
51
|
sync_call: bool = True,
|
52
|
+
transfer_method: TransferMethod = TransferMethod.POST_URL,
|
53
|
+
source_url: Optional[str] = None,
|
49
54
|
) -> PangeaResponse[m.FileScanResult]:
|
50
55
|
"""
|
51
56
|
Scan
|
@@ -84,13 +89,78 @@ class FileScanAsync(ServiceBaseAsync):
|
|
84
89
|
if file or file_path:
|
85
90
|
if file_path:
|
86
91
|
file = open(file_path, "rb")
|
87
|
-
|
92
|
+
if transfer_method == TransferMethod.POST_URL:
|
93
|
+
params = get_file_upload_params(file)
|
94
|
+
crc = params.crc_hex
|
95
|
+
sha = params.sha256_hex
|
96
|
+
size = params.size
|
97
|
+
else:
|
98
|
+
crc, sha, size = None, None, None
|
88
99
|
files = [("upload", ("filename", file, "application/octet-stream"))]
|
89
100
|
else:
|
90
101
|
raise ValueError("Need to set file_path or file arguments")
|
91
102
|
|
92
103
|
input = m.FileScanRequest(
|
93
|
-
verbose=verbose,
|
104
|
+
verbose=verbose,
|
105
|
+
raw=raw,
|
106
|
+
provider=provider,
|
107
|
+
crc32c=crc,
|
108
|
+
sha256=sha,
|
109
|
+
size=size,
|
110
|
+
transfer_method=transfer_method,
|
111
|
+
source_url=source_url,
|
94
112
|
)
|
95
113
|
data = input.dict(exclude_none=True)
|
96
114
|
return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
|
115
|
+
|
116
|
+
async def request_upload_url(
|
117
|
+
self,
|
118
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
119
|
+
params: Optional[FileUploadParams] = None,
|
120
|
+
verbose: Optional[bool] = None,
|
121
|
+
raw: Optional[bool] = None,
|
122
|
+
provider: Optional[str] = None,
|
123
|
+
) -> PangeaResponse[m.FileScanResult]:
|
124
|
+
input = m.FileScanRequest(
|
125
|
+
verbose=verbose,
|
126
|
+
raw=raw,
|
127
|
+
provider=provider,
|
128
|
+
transfer_method=transfer_method,
|
129
|
+
)
|
130
|
+
if params is not None and (transfer_method == TransferMethod.POST_URL):
|
131
|
+
input.crc32c = params.crc_hex
|
132
|
+
input.sha256 = params.sha256_hex
|
133
|
+
input.size = params.size
|
134
|
+
|
135
|
+
data = input.dict(exclude_none=True)
|
136
|
+
return await self.request.request_presigned_url("v1/scan", m.FileScanResult, data=data)
|
137
|
+
|
138
|
+
|
139
|
+
class FileUploaderAsync:
|
140
|
+
def __init__(self):
|
141
|
+
self.logger = logging.getLogger("pangea")
|
142
|
+
self._request: PangeaRequestAsync = PangeaRequestAsync(
|
143
|
+
config=PangeaConfig(),
|
144
|
+
token="",
|
145
|
+
service="FileScanUploader",
|
146
|
+
logger=self.logger,
|
147
|
+
)
|
148
|
+
|
149
|
+
async def upload_file(
|
150
|
+
self,
|
151
|
+
url: str,
|
152
|
+
file: io.BufferedReader,
|
153
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
154
|
+
file_details: Optional[Dict] = None,
|
155
|
+
):
|
156
|
+
if transfer_method == TransferMethod.PUT_URL:
|
157
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
158
|
+
await self._request.put_presigned_url(url=url, files=files)
|
159
|
+
elif transfer_method == TransferMethod.POST_URL:
|
160
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
161
|
+
await self._request.post_presigned_url(url=url, data=file_details, files=files)
|
162
|
+
else:
|
163
|
+
raise ValueError(f"Transfer method not supported: {transfer_method}")
|
164
|
+
|
165
|
+
async def close(self):
|
166
|
+
await self._request.session.close()
|
@@ -36,13 +36,8 @@ class RedactAsync(ServiceBaseAsync):
|
|
36
36
|
|
37
37
|
service_name = "redact"
|
38
38
|
|
39
|
-
def __init__(
|
40
|
-
|
41
|
-
token,
|
42
|
-
config=None,
|
43
|
-
logger_name="pangea",
|
44
|
-
):
|
45
|
-
super().__init__(token, config, logger_name)
|
39
|
+
def __init__(self, token, config=None, logger_name="pangea", config_id: Optional[str] = None):
|
40
|
+
super().__init__(token, config, logger_name, config_id=config_id)
|
46
41
|
|
47
42
|
async def redact(
|
48
43
|
self,
|
pangea/config.py
CHANGED
@@ -10,7 +10,7 @@ class PangeaConfig:
|
|
10
10
|
"""Holds run time configuration information used by SDK components."""
|
11
11
|
|
12
12
|
"""
|
13
|
-
Used to set
|
13
|
+
Used to set Pangea domain (and port if needed), it should not include service subdomain
|
14
14
|
just for particular use cases when environment = "local", domain could be set to an url including:
|
15
15
|
scheme (http:// or https://), subdomain, domain and port.
|
16
16
|
|
@@ -19,7 +19,7 @@ class PangeaConfig:
|
|
19
19
|
|
20
20
|
"""
|
21
21
|
Used to generate service url.
|
22
|
-
It should be only 'production' or 'local' in
|
22
|
+
It should be only 'production' or 'local' in cases of particular services that can run locally as Redact.
|
23
23
|
|
24
24
|
"""
|
25
25
|
environment: str = "production"
|
pangea/request.py
CHANGED
@@ -12,7 +12,7 @@ import pangea
|
|
12
12
|
import pangea.exceptions as pe
|
13
13
|
import requests
|
14
14
|
from pangea.config import PangeaConfig
|
15
|
-
from pangea.response import
|
15
|
+
from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
16
16
|
from pangea.utils import default_encoder
|
17
17
|
from requests.adapters import HTTPAdapter, Retry
|
18
18
|
|
@@ -203,30 +203,126 @@ class PangeaRequest(PangeaRequestBase):
|
|
203
203
|
if self.config_id and data.get("config_id", None) is None:
|
204
204
|
data["config_id"] = self.config_id
|
205
205
|
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
206
|
+
transfer_method = data.get("transfer_method", None)
|
207
|
+
|
208
|
+
if files is not None and type(data) is dict and (transfer_method == TransferMethod.POST_URL.value):
|
209
|
+
requests_response = self._full_post_presigned_url(
|
210
|
+
endpoint, result_class=result_class, data=data, files=files
|
211
|
+
)
|
212
212
|
else:
|
213
213
|
requests_response = self._http_post(
|
214
214
|
url, headers=self._headers(), data=data, files=files, multipart_post=True
|
215
215
|
)
|
216
216
|
|
217
|
-
|
217
|
+
json_resp = requests_response.json()
|
218
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "post", "url": url, "response": json_resp}))
|
219
|
+
|
220
|
+
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)
|
218
221
|
if poll_result:
|
219
222
|
pangea_response = self._handle_queued_result(pangea_response)
|
220
223
|
|
221
224
|
return self._check_response(pangea_response)
|
222
225
|
|
226
|
+
def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
|
227
|
+
"""Makes the GET call to a Pangea Service endpoint.
|
228
|
+
|
229
|
+
Args:
|
230
|
+
endpoint(str): The Pangea Service API endpoint.
|
231
|
+
path(str): Additional URL path
|
232
|
+
|
233
|
+
Returns:
|
234
|
+
PangeaResponse which contains the response in its entirety and
|
235
|
+
various properties to retrieve individual fields
|
236
|
+
"""
|
237
|
+
|
238
|
+
url = self._url(path)
|
239
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
240
|
+
requests_response = self.session.get(url, headers=self._headers())
|
241
|
+
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
|
242
|
+
|
243
|
+
self.logger.debug(
|
244
|
+
json.dumps(
|
245
|
+
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
246
|
+
default=default_encoder,
|
247
|
+
)
|
248
|
+
)
|
249
|
+
|
250
|
+
if check_response is False:
|
251
|
+
return pangea_response
|
252
|
+
|
253
|
+
return self._check_response(pangea_response)
|
254
|
+
|
255
|
+
def poll_result_by_id(
|
256
|
+
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
257
|
+
):
|
258
|
+
path = self._get_poll_path(request_id)
|
259
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
260
|
+
return self.get(path, result_class, check_response=check_response)
|
261
|
+
|
262
|
+
def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
263
|
+
request_id = response.request_id
|
264
|
+
if not request_id:
|
265
|
+
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
266
|
+
|
267
|
+
if response.status != ResponseStatus.ACCEPTED.value:
|
268
|
+
raise pe.PangeaException("Response already proccesed")
|
269
|
+
|
270
|
+
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
271
|
+
|
272
|
+
def request_presigned_url(
|
273
|
+
self,
|
274
|
+
endpoint: str,
|
275
|
+
result_class: Type[PangeaResponseResult],
|
276
|
+
data: Union[str, Dict] = {},
|
277
|
+
) -> PangeaResponse:
|
278
|
+
# Send request
|
279
|
+
try:
|
280
|
+
# This should return 202 (AcceptedRequestException)
|
281
|
+
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
282
|
+
raise pe.PresignedURLException("Should return 202", resp)
|
283
|
+
|
284
|
+
except pe.AcceptedRequestException as e:
|
285
|
+
accepted_exception = e
|
286
|
+
except Exception as e:
|
287
|
+
raise e
|
288
|
+
|
289
|
+
# Receive 202
|
290
|
+
return self._poll_presigned_url(accepted_exception.response)
|
291
|
+
|
292
|
+
def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
|
293
|
+
# Send form request with file and upload_details as body
|
294
|
+
resp = self._http_post(url=url, data=data, files=files, multipart_post=False)
|
295
|
+
self.logger.debug(
|
296
|
+
json.dumps(
|
297
|
+
{"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
|
298
|
+
default=default_encoder,
|
299
|
+
)
|
300
|
+
)
|
301
|
+
|
302
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
303
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
|
304
|
+
|
305
|
+
def put_presigned_url(self, url: str, files: List[Tuple]):
|
306
|
+
# Send put request with file as body
|
307
|
+
resp = self._http_put(url=url, files=files)
|
308
|
+
self.logger.debug(
|
309
|
+
json.dumps(
|
310
|
+
{"service": self.service, "action": "put presigned", "url": url, "response": resp.text},
|
311
|
+
default=default_encoder,
|
312
|
+
)
|
313
|
+
)
|
314
|
+
|
315
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
316
|
+
raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status_code}", resp.text)
|
317
|
+
|
318
|
+
# Start internal methods
|
223
319
|
def _http_post(
|
224
320
|
self,
|
225
321
|
url: str,
|
226
322
|
headers: Dict = {},
|
227
323
|
data: Union[str, Dict] = {},
|
228
324
|
files: Optional[List[Tuple]] = None,
|
229
|
-
multipart_post: bool = True,
|
325
|
+
multipart_post: bool = True, # Multipart or form post
|
230
326
|
) -> requests.Response:
|
231
327
|
self.logger.debug(
|
232
328
|
json.dumps(
|
@@ -237,6 +333,17 @@ class PangeaRequest(PangeaRequestBase):
|
|
237
333
|
data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
|
238
334
|
return self.session.post(url, headers=headers, data=data_send, files=files)
|
239
335
|
|
336
|
+
def _http_put(
|
337
|
+
self,
|
338
|
+
url: str,
|
339
|
+
files: List[Tuple],
|
340
|
+
headers: Dict = {},
|
341
|
+
) -> requests.Response:
|
342
|
+
self.logger.debug(
|
343
|
+
json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
|
344
|
+
)
|
345
|
+
return self.session.put(url, headers=headers, files=files)
|
346
|
+
|
240
347
|
def _http_post_process(
|
241
348
|
self, data: Union[str, Dict] = {}, files: Optional[List[Tuple]] = None, multipart_post: bool = True
|
242
349
|
):
|
@@ -263,7 +370,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
263
370
|
|
264
371
|
return data, files
|
265
372
|
|
266
|
-
def
|
373
|
+
def _full_post_presigned_url(
|
267
374
|
self,
|
268
375
|
endpoint: str,
|
269
376
|
result_class: Type[PangeaResponseResult],
|
@@ -273,35 +380,12 @@ class PangeaRequest(PangeaRequestBase):
|
|
273
380
|
if len(files) == 0:
|
274
381
|
raise AttributeError("files attribute should have at least 1 file")
|
275
382
|
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
280
|
-
raise pe.PresignedURLException("Should return 202", resp)
|
281
|
-
|
282
|
-
except pe.AcceptedRequestException as e:
|
283
|
-
accepted_exception = e
|
284
|
-
except Exception as e:
|
285
|
-
raise e
|
286
|
-
|
287
|
-
# Receive 202 with accepted_status
|
288
|
-
result = self._poll_presigned_url(accepted_exception)
|
289
|
-
data_to_presigned = result.accepted_status.upload_details
|
290
|
-
presigned_url = result.accepted_status.upload_url
|
291
|
-
|
292
|
-
# Send multipart request with file and upload_details as body
|
293
|
-
resp = self._http_post(url=presigned_url, data=data_to_presigned, files=files, multipart_post=False)
|
294
|
-
self.logger.debug(
|
295
|
-
json.dumps(
|
296
|
-
{"service": self.service, "action": "post presigned", "url": presigned_url, "response": resp.text},
|
297
|
-
default=default_encoder,
|
298
|
-
)
|
299
|
-
)
|
300
|
-
|
301
|
-
if resp.status_code < 200 or resp.status_code >= 300:
|
302
|
-
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
|
383
|
+
response = self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
|
384
|
+
data_to_presigned = response.accepted_result.post_form_data
|
385
|
+
presigned_url = response.accepted_result.post_url
|
303
386
|
|
304
|
-
|
387
|
+
self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
388
|
+
return response.raw_response
|
305
389
|
|
306
390
|
def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
|
307
391
|
if self._queued_retry_enabled and response.raw_response.status_code == 202:
|
@@ -315,52 +399,6 @@ class PangeaRequest(PangeaRequestBase):
|
|
315
399
|
|
316
400
|
return response
|
317
401
|
|
318
|
-
def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
|
319
|
-
"""Makes the GET call to a Pangea Service endpoint.
|
320
|
-
|
321
|
-
Args:
|
322
|
-
endpoint(str): The Pangea Service API endpoint.
|
323
|
-
path(str): Additional URL path
|
324
|
-
|
325
|
-
Returns:
|
326
|
-
PangeaResponse which contains the response in its entirety and
|
327
|
-
various properties to retrieve individual fields
|
328
|
-
"""
|
329
|
-
|
330
|
-
url = self._url(path)
|
331
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
332
|
-
requests_response = self.session.get(url, headers=self._headers())
|
333
|
-
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
|
334
|
-
|
335
|
-
self.logger.debug(
|
336
|
-
json.dumps(
|
337
|
-
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
338
|
-
default=default_encoder,
|
339
|
-
)
|
340
|
-
)
|
341
|
-
|
342
|
-
if check_response is False:
|
343
|
-
return pangea_response
|
344
|
-
|
345
|
-
return self._check_response(pangea_response)
|
346
|
-
|
347
|
-
def poll_result_by_id(
|
348
|
-
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
349
|
-
):
|
350
|
-
path = self._get_poll_path(request_id)
|
351
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
352
|
-
return self.get(path, result_class, check_response=check_response)
|
353
|
-
|
354
|
-
def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
355
|
-
request_id = response.request_id
|
356
|
-
if not request_id:
|
357
|
-
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
358
|
-
|
359
|
-
if response.status != ResponseStatus.ACCEPTED.value:
|
360
|
-
raise pe.PangeaException("Response already proccesed")
|
361
|
-
|
362
|
-
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
363
|
-
|
364
402
|
def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
|
365
403
|
retry_count = 1
|
366
404
|
start = time.time()
|
@@ -373,26 +411,26 @@ class PangeaRequest(PangeaRequestBase):
|
|
373
411
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
|
374
412
|
return self._check_response(response)
|
375
413
|
|
376
|
-
def _poll_presigned_url(self,
|
377
|
-
if
|
378
|
-
raise AttributeError("
|
414
|
+
def _poll_presigned_url(self, response: PangeaResponse) -> PangeaResponse:
|
415
|
+
if response.http_status != 202:
|
416
|
+
raise AttributeError("Response should be 202")
|
379
417
|
|
380
|
-
if
|
381
|
-
return
|
418
|
+
if response.accepted_result is not None and response.accepted_result.has_upload_url:
|
419
|
+
return response
|
382
420
|
|
383
421
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
384
422
|
retry_count = 1
|
385
423
|
start = time.time()
|
386
|
-
|
424
|
+
loop_resp = response
|
387
425
|
|
388
426
|
while (
|
389
|
-
|
390
|
-
and not
|
427
|
+
loop_resp.accepted_result is not None
|
428
|
+
and not loop_resp.accepted_result.has_upload_url
|
391
429
|
and not self._reach_timeout(start)
|
392
430
|
):
|
393
431
|
time.sleep(self._get_delay(retry_count, start))
|
394
432
|
try:
|
395
|
-
self.poll_result_once(
|
433
|
+
self.poll_result_once(loop_resp, check_response=False)
|
396
434
|
msg = "Polling presigned url return 200 instead of 202"
|
397
435
|
self.logger.debug(
|
398
436
|
json.dumps(
|
@@ -402,6 +440,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
402
440
|
raise pe.PangeaException(msg)
|
403
441
|
except pe.AcceptedRequestException as e:
|
404
442
|
retry_count += 1
|
443
|
+
loop_resp = e.response
|
405
444
|
loop_exc = e
|
406
445
|
except Exception as e:
|
407
446
|
self.logger.debug(
|
@@ -409,12 +448,12 @@ class PangeaRequest(PangeaRequestBase):
|
|
409
448
|
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
|
410
449
|
)
|
411
450
|
)
|
412
|
-
raise pe.PresignedURLException("Failed to pull Presigned URL",
|
451
|
+
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_resp, e)
|
413
452
|
|
414
453
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
415
454
|
|
416
|
-
if
|
417
|
-
return
|
455
|
+
if loop_resp.accepted_result is not None and not loop_resp.accepted_result.has_upload_url:
|
456
|
+
return loop_resp
|
418
457
|
else:
|
419
458
|
raise loop_exc
|
420
459
|
|
pangea/response.py
CHANGED
@@ -13,8 +13,10 @@ T = TypeVar("T")
|
|
13
13
|
|
14
14
|
|
15
15
|
class TransferMethod(str, enum.Enum):
|
16
|
-
DIRECT = "direct"
|
17
16
|
MULTIPART = "multipart"
|
17
|
+
POST_URL = "post-url"
|
18
|
+
PUT_URL = "put-url"
|
19
|
+
SOURCE_URL = "source-url"
|
18
20
|
|
19
21
|
def __str__(self):
|
20
22
|
return str(self.value)
|
@@ -76,7 +78,16 @@ class AcceptedStatus(APIResponseModel):
|
|
76
78
|
|
77
79
|
|
78
80
|
class AcceptedResult(PangeaResponseResult):
|
79
|
-
|
81
|
+
ttl_mins: int
|
82
|
+
retry_counter: int
|
83
|
+
location: str
|
84
|
+
post_url: Optional[str] = None
|
85
|
+
post_form_data: Dict[str, Any] = {}
|
86
|
+
put_url: Optional[str] = None
|
87
|
+
|
88
|
+
@property
|
89
|
+
def has_upload_url(self) -> bool:
|
90
|
+
return self.post_url is not None or self.put_url is not None
|
80
91
|
|
81
92
|
|
82
93
|
class PangeaError(PangeaResponseResult):
|
pangea/services/audit/audit.py
CHANGED
@@ -462,7 +462,7 @@ class Audit(ServiceBase, AuditBase):
|
|
462
462
|
|
463
463
|
Examples:
|
464
464
|
log_response = audit.log(
|
465
|
-
message="hello world",
|
465
|
+
message="hello world",
|
466
466
|
verbose=True,
|
467
467
|
)
|
468
468
|
"""
|
@@ -551,7 +551,7 @@ class Audit(ServiceBase, AuditBase):
|
|
551
551
|
|
552
552
|
Examples:
|
553
553
|
log_response = audit.log_bulk(
|
554
|
-
events=[{"message": "hello world"}],
|
554
|
+
events=[{"message": "hello world"}],
|
555
555
|
verbose=True,
|
556
556
|
)
|
557
557
|
"""
|
@@ -590,7 +590,7 @@ class Audit(ServiceBase, AuditBase):
|
|
590
590
|
|
591
591
|
Examples:
|
592
592
|
log_response = audit.log_bulk_async(
|
593
|
-
events=[{"message": "hello world"}],
|
593
|
+
events=[{"message": "hello world"}],
|
594
594
|
verbose=True,
|
595
595
|
)
|
596
596
|
"""
|
@@ -664,10 +664,10 @@ class Audit(ServiceBase, AuditBase):
|
|
664
664
|
|
665
665
|
Examples:
|
666
666
|
response = audit.search(
|
667
|
-
query="message:test",
|
668
|
-
search_restriction={'source': ["monitor"]},
|
669
|
-
limit=1,
|
670
|
-
verify_consistency=True,
|
667
|
+
query="message:test",
|
668
|
+
search_restriction={'source': ["monitor"]},
|
669
|
+
limit=1,
|
670
|
+
verify_consistency=True,
|
671
671
|
verify_events=True,
|
672
672
|
)
|
673
673
|
"""
|
pangea/services/authn/authn.py
CHANGED
@@ -520,9 +520,10 @@ class AuthN(ServiceBase):
|
|
520
520
|
|
521
521
|
def update(
|
522
522
|
self,
|
523
|
-
disabled: bool,
|
523
|
+
disabled: Optional[bool] = None,
|
524
524
|
id: Optional[str] = None,
|
525
525
|
email: Optional[str] = None,
|
526
|
+
unlock: Optional[bool] = None,
|
526
527
|
) -> PangeaResponse[m.UserUpdateResult]:
|
527
528
|
"""
|
528
529
|
Update user's settings
|
@@ -534,6 +535,7 @@ class AuthN(ServiceBase):
|
|
534
535
|
Args:
|
535
536
|
disabled (bool): New disabled value.
|
536
537
|
Disabling a user account will prevent them from logging in.
|
538
|
+
unlock (bool): Unlock a user account if it has been locked out due to failed Authentication attempts.
|
537
539
|
id (str, optional): The identity of a user or a service
|
538
540
|
email (str, optional): An email address
|
539
541
|
|
@@ -552,6 +554,7 @@ class AuthN(ServiceBase):
|
|
552
554
|
id=id,
|
553
555
|
email=email,
|
554
556
|
disabled=disabled,
|
557
|
+
unlock=unlock,
|
555
558
|
)
|
556
559
|
|
557
560
|
return self.request.post("v2/user/update", m.UserUpdateResult, data=input.dict(exclude_none=True))
|
pangea/services/authn/models.py
CHANGED
@@ -363,7 +363,8 @@ class UserProfileUpdateResult(User):
|
|
363
363
|
class UserUpdateRequest(APIRequestModel):
|
364
364
|
id: Optional[str] = None
|
365
365
|
email: Optional[str] = None
|
366
|
-
disabled: bool
|
366
|
+
disabled: Optional[bool] = None
|
367
|
+
unlock: Optional[bool] = None
|
367
368
|
|
368
369
|
|
369
370
|
class UserUpdateResult(User):
|
pangea/services/base.py
CHANGED
@@ -3,13 +3,13 @@
|
|
3
3
|
|
4
4
|
import copy
|
5
5
|
import logging
|
6
|
-
from typing import Optional, Union
|
6
|
+
from typing import Optional, Type, Union
|
7
7
|
|
8
8
|
from pangea.asyncio.request import PangeaRequestAsync
|
9
9
|
from pangea.config import PangeaConfig
|
10
10
|
from pangea.exceptions import AcceptedRequestException
|
11
11
|
from pangea.request import PangeaRequest
|
12
|
-
from pangea.response import PangeaResponse
|
12
|
+
from pangea.response import PangeaResponse, PangeaResponseResult
|
13
13
|
|
14
14
|
|
15
15
|
class ServiceBase(object):
|
@@ -50,7 +50,13 @@ class ServiceBase(object):
|
|
50
50
|
|
51
51
|
return self._request
|
52
52
|
|
53
|
-
def poll_result(
|
53
|
+
def poll_result(
|
54
|
+
self,
|
55
|
+
exception: Optional[AcceptedRequestException] = None,
|
56
|
+
response: Optional[PangeaResponse] = None,
|
57
|
+
request_id: Optional[str] = None,
|
58
|
+
result_class: Union[Type[PangeaResponseResult], dict] = dict,
|
59
|
+
) -> PangeaResponse:
|
54
60
|
"""
|
55
61
|
Poll result
|
56
62
|
|
@@ -68,4 +74,11 @@ class ServiceBase(object):
|
|
68
74
|
Examples:
|
69
75
|
response = service.poll_result(exception)
|
70
76
|
"""
|
71
|
-
|
77
|
+
if exception is not None:
|
78
|
+
return self.request.poll_result_once(exception.response, check_response=True)
|
79
|
+
elif response is not None:
|
80
|
+
return self.request.poll_result_once(response, check_response=True)
|
81
|
+
elif request_id is not None:
|
82
|
+
return self.request.poll_result_by_id(request_id=request_id, result_class=result_class, check_response=True)
|
83
|
+
else:
|
84
|
+
raise AttributeError("Need to set exception, response or request_id")
|
pangea/services/file_scan.py
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import io
|
4
|
+
import logging
|
4
5
|
from typing import Dict, List, Optional
|
5
6
|
|
7
|
+
from pangea.request import PangeaConfig, PangeaRequest
|
6
8
|
from pangea.response import APIRequestModel, PangeaResponse, PangeaResponseResult, TransferMethod
|
7
|
-
from pangea.utils import
|
9
|
+
from pangea.utils import FileUploadParams, get_file_upload_params
|
8
10
|
|
9
11
|
from .base import ServiceBase
|
10
12
|
|
@@ -21,10 +23,11 @@ class FileScanRequest(APIRequestModel):
|
|
21
23
|
verbose: Optional[bool] = None
|
22
24
|
raw: Optional[bool] = None
|
23
25
|
provider: Optional[str] = None
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
26
|
+
size: Optional[int] = None
|
27
|
+
crc32c: Optional[str] = None
|
28
|
+
sha256: Optional[str] = None
|
29
|
+
source_url: Optional[str] = None
|
30
|
+
transfer_method: TransferMethod = TransferMethod.POST_URL
|
28
31
|
|
29
32
|
|
30
33
|
class FileScanData(PangeaResponseResult):
|
@@ -79,7 +82,8 @@ class FileScan(ServiceBase):
|
|
79
82
|
raw: Optional[bool] = None,
|
80
83
|
provider: Optional[str] = None,
|
81
84
|
sync_call: bool = True,
|
82
|
-
transfer_method: TransferMethod = TransferMethod.
|
85
|
+
transfer_method: TransferMethod = TransferMethod.POST_URL,
|
86
|
+
source_url: Optional[str] = None,
|
83
87
|
) -> PangeaResponse[FileScanResult]:
|
84
88
|
"""
|
85
89
|
Scan
|
@@ -118,8 +122,11 @@ class FileScan(ServiceBase):
|
|
118
122
|
if file or file_path:
|
119
123
|
if file_path:
|
120
124
|
file = open(file_path, "rb")
|
121
|
-
if transfer_method == TransferMethod.
|
122
|
-
|
125
|
+
if transfer_method == TransferMethod.POST_URL:
|
126
|
+
params = get_file_upload_params(file)
|
127
|
+
crc = params.crc_hex
|
128
|
+
sha = params.sha256_hex
|
129
|
+
size = params.size
|
123
130
|
else:
|
124
131
|
crc, sha, size = None, None, None
|
125
132
|
files = [("upload", ("filename", file, "application/octet-stream"))]
|
@@ -130,10 +137,60 @@ class FileScan(ServiceBase):
|
|
130
137
|
verbose=verbose,
|
131
138
|
raw=raw,
|
132
139
|
provider=provider,
|
133
|
-
|
134
|
-
|
135
|
-
|
140
|
+
crc32c=crc,
|
141
|
+
sha256=sha,
|
142
|
+
size=size,
|
136
143
|
transfer_method=transfer_method,
|
144
|
+
source_url=source_url,
|
137
145
|
)
|
138
146
|
data = input.dict(exclude_none=True)
|
139
147
|
return self.request.post("v1/scan", FileScanResult, data=data, files=files, poll_result=sync_call)
|
148
|
+
|
149
|
+
def request_upload_url(
|
150
|
+
self,
|
151
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
152
|
+
params: Optional[FileUploadParams] = None,
|
153
|
+
verbose: Optional[bool] = None,
|
154
|
+
raw: Optional[bool] = None,
|
155
|
+
provider: Optional[str] = None,
|
156
|
+
) -> PangeaResponse[FileScanResult]:
|
157
|
+
input = FileScanRequest(
|
158
|
+
verbose=verbose,
|
159
|
+
raw=raw,
|
160
|
+
provider=provider,
|
161
|
+
transfer_method=transfer_method,
|
162
|
+
)
|
163
|
+
if params is not None and (transfer_method == TransferMethod.POST_URL):
|
164
|
+
input.crc32c = params.crc_hex
|
165
|
+
input.sha256 = params.sha256_hex
|
166
|
+
input.size = params.size
|
167
|
+
|
168
|
+
data = input.dict(exclude_none=True)
|
169
|
+
return self.request.request_presigned_url("v1/scan", FileScanResult, data=data)
|
170
|
+
|
171
|
+
|
172
|
+
class FileUploader:
|
173
|
+
def __init__(self):
|
174
|
+
self.logger = logging.getLogger("pangea")
|
175
|
+
self._request = PangeaRequest(
|
176
|
+
config=PangeaConfig(),
|
177
|
+
token="",
|
178
|
+
service="FileScanUploader",
|
179
|
+
logger=self.logger,
|
180
|
+
)
|
181
|
+
|
182
|
+
def upload_file(
|
183
|
+
self,
|
184
|
+
url: str,
|
185
|
+
file: io.BufferedReader,
|
186
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
187
|
+
file_details: Optional[Dict] = None,
|
188
|
+
):
|
189
|
+
if transfer_method == TransferMethod.PUT_URL:
|
190
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
191
|
+
self._request.put_presigned_url(url=url, files=files)
|
192
|
+
elif transfer_method == TransferMethod.POST_URL:
|
193
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
194
|
+
self._request.post_presigned_url(url=url, data=file_details, files=files)
|
195
|
+
else:
|
196
|
+
raise ValueError(f"Transfer method not supported: {transfer_method}")
|
pangea/services/redact.py
CHANGED
@@ -134,13 +134,8 @@ class Redact(ServiceBase):
|
|
134
134
|
|
135
135
|
service_name = "redact"
|
136
136
|
|
137
|
-
def __init__(
|
138
|
-
|
139
|
-
token,
|
140
|
-
config=None,
|
141
|
-
logger_name="pangea",
|
142
|
-
):
|
143
|
-
super().__init__(token, config, logger_name)
|
137
|
+
def __init__(self, token, config=None, logger_name="pangea", config_id: Optional[str] = None):
|
138
|
+
super().__init__(token, config, logger_name, config_id=config_id)
|
144
139
|
|
145
140
|
def redact(
|
146
141
|
self,
|
pangea/services/vault/vault.py
CHANGED
@@ -537,7 +537,7 @@ class Vault(ServiceBase):
|
|
537
537
|
|
538
538
|
Generate a symmetric key
|
539
539
|
|
540
|
-
OperationId: vault_post_v1_key_generate
|
540
|
+
OperationId: vault_post_v1_key_generate 2
|
541
541
|
|
542
542
|
Args:
|
543
543
|
algorithm (SymmetricAlgorithm): The algorithm of the key
|
@@ -611,7 +611,7 @@ class Vault(ServiceBase):
|
|
611
611
|
|
612
612
|
Generate an asymmetric key
|
613
613
|
|
614
|
-
OperationId: vault_post_v1_key_generate
|
614
|
+
OperationId: vault_post_v1_key_generate 1
|
615
615
|
|
616
616
|
Args:
|
617
617
|
algorithm (AsymmetricAlgorithm): The algorithm of the key
|
pangea/utils.py
CHANGED
@@ -8,6 +8,7 @@ from collections import OrderedDict
|
|
8
8
|
from hashlib import new, sha1, sha256, sha512
|
9
9
|
|
10
10
|
from google_crc32c import Checksum as CRC32C
|
11
|
+
from pydantic import BaseModel
|
11
12
|
|
12
13
|
|
13
14
|
def format_datetime(dt: datetime.datetime) -> str:
|
@@ -100,7 +101,13 @@ def get_prefix(hash: str, len: int = 5):
|
|
100
101
|
return hash[0:len]
|
101
102
|
|
102
103
|
|
103
|
-
|
104
|
+
class FileUploadParams(BaseModel):
|
105
|
+
crc_hex: str
|
106
|
+
sha256_hex: str
|
107
|
+
size: int
|
108
|
+
|
109
|
+
|
110
|
+
def get_file_upload_params(file: io.BufferedReader) -> FileUploadParams:
|
104
111
|
if "b" not in file.mode:
|
105
112
|
raise AttributeError("File need to be open in binary mode")
|
106
113
|
|
@@ -118,4 +125,4 @@ def get_presigned_url_upload_params(file: io.BufferedReader):
|
|
118
125
|
size += len(chunk)
|
119
126
|
|
120
127
|
file.seek(0) # restart reading
|
121
|
-
return crc.hexdigest().decode("utf-8"), sha.hexdigest(), size
|
128
|
+
return FileUploadParams(crc_hex=crc.hexdigest().decode("utf-8"), sha256_hex=sha.hexdigest(), size=size)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: pangea-sdk
|
3
|
-
Version: 3.
|
3
|
+
Version: 3.4.0
|
4
4
|
Summary: Pangea API SDK
|
5
5
|
License: MIT
|
6
6
|
Keywords: Pangea,SDK,Audit
|
@@ -15,10 +15,10 @@ Classifier: Programming Language :: Python :: 3.10
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
16
16
|
Classifier: Topic :: Software Development
|
17
17
|
Classifier: Topic :: Software Development :: Libraries
|
18
|
-
Requires-Dist: aiohttp (>=3.8.
|
18
|
+
Requires-Dist: aiohttp (>=3.8.6,<4.0.0)
|
19
19
|
Requires-Dist: alive-progress (>=2.4.1,<3.0.0)
|
20
20
|
Requires-Dist: asyncio (>=3.4.3,<4.0.0)
|
21
|
-
Requires-Dist: cryptography (==41.0.
|
21
|
+
Requires-Dist: cryptography (==41.0.5)
|
22
22
|
Requires-Dist: deprecated (>=1.2.13,<2.0.0)
|
23
23
|
Requires-Dist: google-crc32c (>=1.5.0,<2.0.0)
|
24
24
|
Requires-Dist: pydantic (>=1.10.2,<2.0.0)
|
@@ -59,18 +59,18 @@ poetry add pangea-sdk
|
|
59
59
|
|
60
60
|
## Usage
|
61
61
|
|
62
|
-
For
|
62
|
+
For sample apps, look at the [/examples](https://github.com/pangeacyber/pangea-python/tree/main/examples) folder in this repository. There you will find basic sample apps for each of the services supported on this SDK. Each service folder has a README.md with instructions to install, setup, and run the sample app.
|
63
63
|
|
64
64
|
|
65
65
|
## Asyncio support
|
66
66
|
|
67
|
-
We have added support to asyncio library using aiohttp in order to support async/await calls to all our services.
|
67
|
+
We have added support to the asyncio library using aiohttp in order to support async/await calls to all our services.
|
68
68
|
Async services classes are inside [pangea/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk/pangea/asyncio) folder, and examples about how to use them are in [/examples/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/examples/asyncio).
|
69
69
|
|
70
70
|
|
71
71
|
### Secure Audit Service - Integrity Tools
|
72
72
|
|
73
|
-
Python Pangea SDK
|
73
|
+
The Python Pangea SDK also includes some extra features to validate Audit Service log's integrity. Here we explain how to run them.
|
74
74
|
|
75
75
|
#### Verify audit data
|
76
76
|
|
@@ -101,7 +101,7 @@ curl -H "Authorization: Bearer ${PANGEA_TOKEN}" -X POST -H 'Content-Type: applic
|
|
101
101
|
|
102
102
|
Download all audit logs for a given time range. Start and end date should be provided,
|
103
103
|
a variety of formats is supported, including ISO-8601. The result is stored in a
|
104
|
-
|
104
|
+
json file (one json per line).
|
105
105
|
|
106
106
|
```
|
107
107
|
usage: python -m pangea.dump_audit [-h] [--token TOKEN] [--domain DOMAIN] [--output OUTPUT] start end
|
@@ -126,8 +126,8 @@ options:
|
|
126
126
|
|
127
127
|
#### Perform Exhaustive Verification of Audit Data
|
128
128
|
|
129
|
-
This script performs extensive verification on a range of events of the log stream.
|
130
|
-
and the membership proof, it checks that there
|
129
|
+
This script performs extensive verification on a range of events of the log stream. Apart from verifying the hash
|
130
|
+
and the membership proof, it checks that there are no omissions in the stream, i.e. all the events are present and properly located.
|
131
131
|
|
132
132
|
```
|
133
133
|
usage: python -m pangea.deep_verify [-h] [--token TOKEN] [--domain DOMAIN] --file FILE
|
@@ -153,9 +153,9 @@ It accepts multiple file formats:
|
|
153
153
|
|
154
154
|
## Reporting issues and new features
|
155
155
|
|
156
|
-
If
|
157
|
-
We would need you to provide some basic information
|
158
|
-
Also feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/)
|
156
|
+
If you run into an issue using or testing this SDK or if you have a new feature request, feel free to open an issue by [clicking here](https://github.com/pangeacyber/pangea-python/issues).
|
157
|
+
We would need you to provide some basic information, such as what SDK version you are using, the stack trace if you got it, the framework used, and steps to reproduce the issue.
|
158
|
+
Also, feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/).
|
159
159
|
|
160
160
|
|
161
161
|
## Contributing
|
@@ -168,5 +168,5 @@ These linters will run on every `git commit` operation.
|
|
168
168
|
|
169
169
|
### Send a PR
|
170
170
|
|
171
|
-
If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in code or an error in documents we
|
171
|
+
If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in the code or an error in documents, we really appreciate it and after review and approval, you will be included in our [contributors list](https://github.com/pangeacyber/pangea-python/blob/main/packages/pangea-sdk/CONTRIBUTING.md).
|
172
172
|
|
@@ -1,43 +1,43 @@
|
|
1
|
-
pangea/__init__.py,sha256=
|
2
|
-
pangea/asyncio/request.py,sha256=
|
1
|
+
pangea/__init__.py,sha256=IdKL3uBCgPwFlLeZHTLAAY32MvYJBkmSYEBKIOoYXzI,200
|
2
|
+
pangea/asyncio/request.py,sha256=HXD0LscN0lfGoN9Nu0C8fMMkAZT7LLg2UotbD-A57q4,12628
|
3
3
|
pangea/asyncio/services/__init__.py,sha256=_CEDza6E9VmEs6d_vubWetfeqTogH7UxT7XrTVRw4Mo,290
|
4
4
|
pangea/asyncio/services/audit.py,sha256=xHvz5ReJpmk_StK3mr_uDqCI80J6Aj7-EglEBWS0Qqc,16480
|
5
5
|
pangea/asyncio/services/authn.py,sha256=HssvrZ3fr2oP5sgYBj1Rr5Yf-fKiHhrWTue5LOAeyOk,43288
|
6
|
-
pangea/asyncio/services/base.py,sha256=
|
6
|
+
pangea/asyncio/services/base.py,sha256=hmdY5-IS_H2WGMeVfzotn11Z1c2B2iJTHi4USBbmptA,2304
|
7
7
|
pangea/asyncio/services/embargo.py,sha256=8WguyWZUaGVwGpNzic5h8QzLueirA9WpBBik4mpCTeA,3056
|
8
|
-
pangea/asyncio/services/file_scan.py,sha256=
|
8
|
+
pangea/asyncio/services/file_scan.py,sha256=o_6Cddk8bXXlh4Aj_DUQJYlE9Q6U0XGPlZxML88PTfw,6151
|
9
9
|
pangea/asyncio/services/intel.py,sha256=LdJSxCohXsI4Vk8bG74jXTBYAL9N45lb5LIX7nBXqLI,22248
|
10
|
-
pangea/asyncio/services/redact.py,sha256=
|
10
|
+
pangea/asyncio/services/redact.py,sha256=W9eKMw1XyY6tgRudRrtuCpyQYhgttvmsYngoiMld0C4,5152
|
11
11
|
pangea/asyncio/services/vault.py,sha256=JhxUdmPNYZzp3eiAQFR6vsfyS9L3ddKWes-Cv7rj2ow,43425
|
12
12
|
pangea/audit_logger.py,sha256=zCSsq0kvw4Pb_aIBb7APfaYsTqd9dmohYLFld2ir5lc,3778
|
13
|
-
pangea/config.py,sha256=
|
13
|
+
pangea/config.py,sha256=kFu9mLMkFpkM7wdT8Ymvskx6DGLfRZKeKfQRVZTgbyA,1670
|
14
14
|
pangea/deep_verify.py,sha256=WiA_2gqtrSCQYoTMKX9ILlNgnknsH4UxZpJXLVG2uyc,8343
|
15
15
|
pangea/deprecated.py,sha256=IjFYEVvY1E0ld0SMkEYC1o62MAleX3nnT1If2dFVbHo,608
|
16
16
|
pangea/dump_audit.py,sha256=Ws0KuZyHoaySsQ2lq9EKK2iw65O8x4zL1Mii0ChDh0k,6511
|
17
17
|
pangea/exceptions.py,sha256=JMx_Dym7W2cgcPSHN4bz9Rlrm-3BZGNpIYu3VBXZLBU,5277
|
18
|
-
pangea/request.py,sha256=
|
19
|
-
pangea/response.py,sha256=
|
18
|
+
pangea/request.py,sha256=GShc12dltLSapfayHChnHEmL9l4feyYCA-O-PQ4k_VY,18641
|
19
|
+
pangea/response.py,sha256=8cA08NjP_2jIq73lybBaBK0N7XnYLd0f9rOHNcnESg4,5552
|
20
20
|
pangea/services/__init__.py,sha256=auqKaEAOLiazHCzOQVwrUwd2ABFw_bF-ptoOq1bpa68,253
|
21
|
-
pangea/services/audit/audit.py,sha256=
|
21
|
+
pangea/services/audit/audit.py,sha256=50zkQE2HQ0cQtFroFSVjUxX1M4RQsf6pBUeMidSkEHI,30295
|
22
22
|
pangea/services/audit/exceptions.py,sha256=CVdaQZCvQKx1n-iIjWz5wnStUGU6cXDwKqe7MoijAXk,451
|
23
23
|
pangea/services/audit/models.py,sha256=ztLcB7XNRudn3GquyW8gp_TlCrH3va32Tw0nMpO7g6k,12244
|
24
24
|
pangea/services/audit/signing.py,sha256=JWYutDNV5XZZzLASwdUJ-9gMHlGd8-h4IjIbKz7JMJM,5261
|
25
25
|
pangea/services/audit/util.py,sha256=2EM3K9R-T9FBKegklmLLoWa1IDm84uGjmJZVO3mYEhk,7568
|
26
|
-
pangea/services/authn/authn.py,sha256=
|
27
|
-
pangea/services/authn/models.py,sha256=
|
28
|
-
pangea/services/base.py,sha256=
|
26
|
+
pangea/services/authn/authn.py,sha256=sGQUPQ8VLEMtu8AkEhWPyY6Q6AxCsfrX-XNUzkC-o1M,42774
|
27
|
+
pangea/services/authn/models.py,sha256=V-hj1KfbSuaWJMVgd-Q3fWTsnbj3EdPumnzXtjHOR8g,18150
|
28
|
+
pangea/services/base.py,sha256=bRSuuhLv6iQrPVjhasOjTBqOgxTRRhCD204v3tNjVOE,2731
|
29
29
|
pangea/services/embargo.py,sha256=F3jx7SbElnjhbDEUR3oHfWsQ8G8zf_kfp6_q1_vmOIc,3871
|
30
|
-
pangea/services/file_scan.py,sha256=
|
30
|
+
pangea/services/file_scan.py,sha256=iwqvJcaNE1nJN4VpLcYKtgWFfgnbLLxnbUylhIdEPTk,6882
|
31
31
|
pangea/services/intel.py,sha256=oTvTxCtdnoaZRlt0anUKnkRT2FVfc9LG8ftDIX09jHk,30399
|
32
|
-
pangea/services/redact.py,sha256=
|
32
|
+
pangea/services/redact.py,sha256=vH4sg_t8MnoSjSY-F42JZhe8EfRmVCclslDN7U1hNP4,7756
|
33
33
|
pangea/services/vault/models/asymmetric.py,sha256=ac2Exc66elXxO-HxBqtvLPQWNI7y_00kb6SVqBPKecA,1450
|
34
34
|
pangea/services/vault/models/common.py,sha256=D_xZlqjEc0XO9G_aoed3YuPNTNYB5S2DoZJRIJVoliQ,9095
|
35
35
|
pangea/services/vault/models/secret.py,sha256=cLgEj-_BeGkB4-pmSeTkWVyasFbaJwcEltIEcOyf1U8,481
|
36
36
|
pangea/services/vault/models/symmetric.py,sha256=5N2n6FDStB1CLPfpd4p-6Ig__Nt-EyurhjCWfEyws2k,1330
|
37
|
-
pangea/services/vault/vault.py,sha256=
|
37
|
+
pangea/services/vault/vault.py,sha256=t3KU59FjUfVawbMZt7A51cDQoyapftChdihTqllVrTw,43160
|
38
38
|
pangea/tools.py,sha256=A9gkHpAkECwqnUKA-O09vG-n6Gmq2g_LYr61BUs-n54,6403
|
39
|
-
pangea/utils.py,sha256=
|
39
|
+
pangea/utils.py,sha256=09dHX9CKmyKNWccSF2z7LyCYYYduZ9AWP6D5Ylnfty8,3360
|
40
40
|
pangea/verify_audit.py,sha256=gWhde7gETKSWfBaMm5gEckAO2xmYB_vmgcZ_4FvvyfU,10616
|
41
|
-
pangea_sdk-3.
|
42
|
-
pangea_sdk-3.
|
43
|
-
pangea_sdk-3.
|
41
|
+
pangea_sdk-3.4.0.dist-info/METADATA,sha256=HkC7uL-xyRL0Vlcqoe16pHLZmo2JdNW4z0_iywI9Z3c,6934
|
42
|
+
pangea_sdk-3.4.0.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
|
43
|
+
pangea_sdk-3.4.0.dist-info/RECORD,,
|
File without changes
|