pangea-sdk 3.2.0__py3-none-any.whl → 3.3.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- pangea/__init__.py +1 -1
- pangea/asyncio/request.py +122 -88
- pangea/asyncio/services/base.py +20 -3
- pangea/asyncio/services/file_scan.py +75 -5
- pangea/asyncio/services/redact.py +2 -7
- pangea/config.py +2 -2
- pangea/request.py +131 -91
- pangea/response.py +4 -1
- pangea/services/audit/audit.py +7 -7
- pangea/services/authn/authn.py +4 -1
- pangea/services/authn/models.py +2 -1
- pangea/services/base.py +17 -4
- pangea/services/file_scan.py +59 -3
- pangea/services/redact.py +2 -7
- pangea/services/vault/vault.py +2 -2
- pangea/utils.py +9 -2
- {pangea_sdk-3.2.0.dist-info → pangea_sdk-3.3.0.dist-info}/METADATA +13 -13
- {pangea_sdk-3.2.0.dist-info → pangea_sdk-3.3.0.dist-info}/RECORD +19 -19
- {pangea_sdk-3.2.0.dist-info → pangea_sdk-3.3.0.dist-info}/WHEEL +0 -0
pangea/__init__.py
CHANGED
pangea/asyncio/request.py
CHANGED
@@ -51,12 +51,14 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
51
51
|
if self.config_id and data.get("config_id", None) is None:
|
52
52
|
data["config_id"] = self.config_id
|
53
53
|
|
54
|
+
transfer_method = data.get("transfer_method", None)
|
55
|
+
|
54
56
|
if (
|
55
57
|
files is not None
|
56
58
|
and type(data) is dict
|
57
|
-
and
|
59
|
+
and (transfer_method == TransferMethod.DIRECT.value or transfer_method == TransferMethod.POST_URL.value)
|
58
60
|
):
|
59
|
-
requests_response = await self.
|
61
|
+
requests_response = await self._full_post_presigned_url(
|
60
62
|
endpoint, result_class=result_class, data=data, files=files
|
61
63
|
)
|
62
64
|
else:
|
@@ -72,6 +74,83 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
72
74
|
|
73
75
|
return self._check_response(pangea_response)
|
74
76
|
|
77
|
+
async def get(
|
78
|
+
self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
|
79
|
+
) -> PangeaResponse:
|
80
|
+
"""Makes the GET call to a Pangea Service endpoint.
|
81
|
+
|
82
|
+
Args:
|
83
|
+
endpoint(str): The Pangea Service API endpoint.
|
84
|
+
path(str): Additional URL path
|
85
|
+
|
86
|
+
Returns:
|
87
|
+
PangeaResponse which contains the response in its entirety and
|
88
|
+
various properties to retrieve individual fields
|
89
|
+
"""
|
90
|
+
|
91
|
+
url = self._url(path)
|
92
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
93
|
+
|
94
|
+
async with self.session.get(url, headers=self._headers()) as requests_response:
|
95
|
+
pangea_response = PangeaResponse(
|
96
|
+
requests_response, result_class=result_class, json=await requests_response.json()
|
97
|
+
)
|
98
|
+
|
99
|
+
self.logger.debug(
|
100
|
+
json.dumps(
|
101
|
+
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
102
|
+
default=default_encoder,
|
103
|
+
)
|
104
|
+
)
|
105
|
+
|
106
|
+
if check_response is False:
|
107
|
+
return pangea_response
|
108
|
+
|
109
|
+
return self._check_response(pangea_response)
|
110
|
+
|
111
|
+
async def poll_result_by_id(
|
112
|
+
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
113
|
+
):
|
114
|
+
path = self._get_poll_path(request_id)
|
115
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
116
|
+
return await self.get(path, result_class, check_response=check_response)
|
117
|
+
|
118
|
+
async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
119
|
+
request_id = response.request_id
|
120
|
+
if not request_id:
|
121
|
+
raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
|
122
|
+
|
123
|
+
if response.status != ResponseStatus.ACCEPTED.value:
|
124
|
+
raise pe.PangeaException("Response already proccesed")
|
125
|
+
|
126
|
+
return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
127
|
+
|
128
|
+
async def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
|
129
|
+
# Send form request with file and upload_details as body
|
130
|
+
resp = await self._http_post(url=url, data=data, files=files, presigned_url_post=True)
|
131
|
+
self.logger.debug(
|
132
|
+
json.dumps(
|
133
|
+
{"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
|
134
|
+
default=default_encoder,
|
135
|
+
)
|
136
|
+
)
|
137
|
+
|
138
|
+
if resp.status < 200 or resp.status >= 300:
|
139
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", resp.text)
|
140
|
+
|
141
|
+
async def put_presigned_url(self, url: str, files: List[Tuple]):
|
142
|
+
# Send put request with file as body
|
143
|
+
resp = await self._http_put(url=url, files=files)
|
144
|
+
self.logger.debug(
|
145
|
+
json.dumps(
|
146
|
+
{"service": self.service, "action": "put presigned", "url": url, "response": resp.text},
|
147
|
+
default=default_encoder,
|
148
|
+
)
|
149
|
+
)
|
150
|
+
|
151
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
152
|
+
raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status_code}", resp.text)
|
153
|
+
|
75
154
|
async def _http_post(
|
76
155
|
self,
|
77
156
|
url: str,
|
@@ -105,7 +184,21 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
105
184
|
|
106
185
|
return await self.session.post(url, headers=headers, data=data_send)
|
107
186
|
|
108
|
-
async def
|
187
|
+
async def _http_put(
|
188
|
+
self,
|
189
|
+
url: str,
|
190
|
+
files: List[Tuple],
|
191
|
+
headers: Dict = {},
|
192
|
+
) -> aiohttp.ClientResponse:
|
193
|
+
self.logger.debug(
|
194
|
+
json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
|
195
|
+
)
|
196
|
+
form = FormData()
|
197
|
+
name, value = files[0]
|
198
|
+
form.add_field(name, value[1], filename=value[0], content_type=value[2])
|
199
|
+
return self.session.put(url, headers=headers, data=form)
|
200
|
+
|
201
|
+
async def _full_post_presigned_url(
|
109
202
|
self,
|
110
203
|
endpoint: str,
|
111
204
|
result_class: Type[PangeaResponseResult],
|
@@ -115,61 +208,52 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
115
208
|
if len(files) == 0:
|
116
209
|
raise AttributeError("files attribute should have at least 1 file")
|
117
210
|
|
211
|
+
response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
|
212
|
+
data_to_presigned = response.accepted_result.accepted_status.upload_details
|
213
|
+
presigned_url = response.accepted_result.accepted_status.upload_url
|
214
|
+
|
215
|
+
await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
216
|
+
return response.raw_response
|
217
|
+
|
218
|
+
async def request_presigned_url(
|
219
|
+
self,
|
220
|
+
endpoint: str,
|
221
|
+
result_class: Type[PangeaResponseResult],
|
222
|
+
data: Union[str, Dict] = {},
|
223
|
+
) -> PangeaResponse:
|
118
224
|
# Send request
|
119
225
|
try:
|
120
226
|
# This should return 202 (AcceptedRequestException)
|
121
227
|
resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
122
228
|
raise pe.PresignedURLException("Should return 202", resp)
|
123
|
-
|
124
229
|
except pe.AcceptedRequestException as e:
|
125
230
|
accepted_exception = e
|
126
231
|
except Exception as e:
|
127
232
|
raise e
|
128
233
|
|
129
234
|
# Receive 202 with accepted_status
|
130
|
-
|
131
|
-
data_to_presigned = result.accepted_status.upload_details
|
132
|
-
presigned_url = result.accepted_status.upload_url
|
133
|
-
|
134
|
-
# Send multipart request with file and upload_details as body
|
135
|
-
resp = await self._http_post(url=presigned_url, data=data_to_presigned, files=files, presigned_url_post=True)
|
136
|
-
self.logger.debug(
|
137
|
-
json.dumps(
|
138
|
-
{
|
139
|
-
"service": self.service,
|
140
|
-
"action": "post presigned",
|
141
|
-
"url": presigned_url,
|
142
|
-
"response": await resp.text(),
|
143
|
-
},
|
144
|
-
default=default_encoder,
|
145
|
-
)
|
146
|
-
)
|
147
|
-
|
148
|
-
if resp.status < 200 or resp.status >= 300:
|
149
|
-
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
|
150
|
-
|
151
|
-
return accepted_exception.response.raw_response
|
235
|
+
return await self._poll_presigned_url(accepted_exception.response)
|
152
236
|
|
153
|
-
async def _poll_presigned_url(self,
|
154
|
-
if
|
155
|
-
raise AttributeError("
|
237
|
+
async def _poll_presigned_url(self, response: PangeaResponse) -> AcceptedResult:
|
238
|
+
if response.http_status != 202:
|
239
|
+
raise AttributeError("Response should be 202")
|
156
240
|
|
157
|
-
if
|
158
|
-
return
|
241
|
+
if response.accepted_result.accepted_status.upload_url:
|
242
|
+
return response
|
159
243
|
|
160
244
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
161
245
|
retry_count = 1
|
162
246
|
start = time.time()
|
163
|
-
|
247
|
+
loop_resp = response
|
164
248
|
|
165
249
|
while (
|
166
|
-
|
167
|
-
and not
|
250
|
+
loop_resp.accepted_result is not None
|
251
|
+
and not loop_resp.accepted_result.accepted_status.upload_url
|
168
252
|
and not self._reach_timeout(start)
|
169
253
|
):
|
170
254
|
await asyncio.sleep(self._get_delay(retry_count, start))
|
171
255
|
try:
|
172
|
-
await self.poll_result_once(
|
256
|
+
await self.poll_result_once(response, check_response=False)
|
173
257
|
msg = "Polling presigned url return 200 instead of 202"
|
174
258
|
self.logger.debug(
|
175
259
|
json.dumps(
|
@@ -179,6 +263,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
179
263
|
raise pe.PangeaException(msg)
|
180
264
|
except pe.AcceptedRequestException as e:
|
181
265
|
retry_count += 1
|
266
|
+
loop_resp = e.response
|
182
267
|
loop_exc = e
|
183
268
|
except Exception as e:
|
184
269
|
self.logger.debug(
|
@@ -190,8 +275,8 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
190
275
|
|
191
276
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
192
277
|
|
193
|
-
if
|
194
|
-
return
|
278
|
+
if loop_resp.accepted_result is not None and not loop_resp.accepted_result.accepted_status.upload_url:
|
279
|
+
return loop_resp
|
195
280
|
else:
|
196
281
|
raise loop_exc
|
197
282
|
|
@@ -207,57 +292,6 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
207
292
|
|
208
293
|
return response
|
209
294
|
|
210
|
-
async def get(
|
211
|
-
self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
|
212
|
-
) -> PangeaResponse:
|
213
|
-
"""Makes the GET call to a Pangea Service endpoint.
|
214
|
-
|
215
|
-
Args:
|
216
|
-
endpoint(str): The Pangea Service API endpoint.
|
217
|
-
path(str): Additional URL path
|
218
|
-
|
219
|
-
Returns:
|
220
|
-
PangeaResponse which contains the response in its entirety and
|
221
|
-
various properties to retrieve individual fields
|
222
|
-
"""
|
223
|
-
|
224
|
-
url = self._url(path)
|
225
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
226
|
-
|
227
|
-
async with self.session.get(url, headers=self._headers()) as requests_response:
|
228
|
-
pangea_response = PangeaResponse(
|
229
|
-
requests_response, result_class=result_class, json=await requests_response.json()
|
230
|
-
)
|
231
|
-
|
232
|
-
self.logger.debug(
|
233
|
-
json.dumps(
|
234
|
-
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
235
|
-
default=default_encoder,
|
236
|
-
)
|
237
|
-
)
|
238
|
-
|
239
|
-
if check_response is False:
|
240
|
-
return pangea_response
|
241
|
-
|
242
|
-
return self._check_response(pangea_response)
|
243
|
-
|
244
|
-
async def poll_result_by_id(
|
245
|
-
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
246
|
-
):
|
247
|
-
path = self._get_poll_path(request_id)
|
248
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
249
|
-
return await self.get(path, result_class, check_response=check_response)
|
250
|
-
|
251
|
-
async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
252
|
-
request_id = response.request_id
|
253
|
-
if not request_id:
|
254
|
-
raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
|
255
|
-
|
256
|
-
if response.status != ResponseStatus.ACCEPTED.value:
|
257
|
-
raise pe.PangeaException("Response already proccesed")
|
258
|
-
|
259
|
-
return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
260
|
-
|
261
295
|
async def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
|
262
296
|
retry_count = 1
|
263
297
|
start = time.time()
|
pangea/asyncio/services/base.py
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
|
4
|
+
from typing import Optional, Type, Union
|
5
|
+
|
4
6
|
from pangea.asyncio.request import PangeaRequestAsync
|
5
7
|
from pangea.exceptions import AcceptedRequestException
|
6
|
-
from pangea.response import PangeaResponse
|
8
|
+
from pangea.response import PangeaResponse, PangeaResponseResult
|
7
9
|
from pangea.services.base import ServiceBase
|
8
10
|
|
9
11
|
|
@@ -21,7 +23,13 @@ class ServiceBaseAsync(ServiceBase):
|
|
21
23
|
|
22
24
|
return self._request
|
23
25
|
|
24
|
-
async def poll_result(
|
26
|
+
async def poll_result(
|
27
|
+
self,
|
28
|
+
exception: Optional[AcceptedRequestException] = None,
|
29
|
+
response: Optional[PangeaResponse] = None,
|
30
|
+
request_id: Optional[str] = None,
|
31
|
+
result_class: Union[Type[PangeaResponseResult], dict] = dict,
|
32
|
+
) -> PangeaResponse:
|
25
33
|
"""
|
26
34
|
Poll result
|
27
35
|
|
@@ -39,7 +47,16 @@ class ServiceBaseAsync(ServiceBase):
|
|
39
47
|
Examples:
|
40
48
|
response = service.poll_result(exception)
|
41
49
|
"""
|
42
|
-
|
50
|
+
if exception is not None:
|
51
|
+
return await self.request.poll_result_once(exception.response, check_response=True)
|
52
|
+
elif response is not None:
|
53
|
+
return await self.request.poll_result_once(response, check_response=True)
|
54
|
+
elif request_id is not None:
|
55
|
+
return await self.request.poll_result_by_id(
|
56
|
+
request_id=request_id, result_class=result_class, check_response=True
|
57
|
+
)
|
58
|
+
else:
|
59
|
+
raise AttributeError("Need to set exception, response or request_id")
|
43
60
|
|
44
61
|
async def close(self):
|
45
62
|
await self.request.session.close()
|
@@ -1,11 +1,14 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import io
|
4
|
-
|
4
|
+
import logging
|
5
|
+
from typing import Dict, Optional
|
5
6
|
|
6
7
|
import pangea.services.file_scan as m
|
7
|
-
from pangea.
|
8
|
-
from pangea.
|
8
|
+
from pangea.asyncio.request import PangeaRequestAsync
|
9
|
+
from pangea.request import PangeaConfig
|
10
|
+
from pangea.response import PangeaResponse, TransferMethod
|
11
|
+
from pangea.utils import FileUploadParams, get_file_upload_params
|
9
12
|
|
10
13
|
from .base import ServiceBaseAsync
|
11
14
|
|
@@ -46,6 +49,7 @@ class FileScanAsync(ServiceBaseAsync):
|
|
46
49
|
raw: Optional[bool] = None,
|
47
50
|
provider: Optional[str] = None,
|
48
51
|
sync_call: bool = True,
|
52
|
+
transfer_method: TransferMethod = TransferMethod.DIRECT,
|
49
53
|
) -> PangeaResponse[m.FileScanResult]:
|
50
54
|
"""
|
51
55
|
Scan
|
@@ -84,13 +88,79 @@ class FileScanAsync(ServiceBaseAsync):
|
|
84
88
|
if file or file_path:
|
85
89
|
if file_path:
|
86
90
|
file = open(file_path, "rb")
|
87
|
-
|
91
|
+
if transfer_method == TransferMethod.DIRECT or transfer_method == TransferMethod.POST_URL:
|
92
|
+
params = get_file_upload_params(file)
|
93
|
+
crc = params.crc_hex
|
94
|
+
sha = params.sha256_hex
|
95
|
+
size = params.size
|
96
|
+
else:
|
97
|
+
crc, sha, size = None, None, None
|
88
98
|
files = [("upload", ("filename", file, "application/octet-stream"))]
|
89
99
|
else:
|
90
100
|
raise ValueError("Need to set file_path or file arguments")
|
91
101
|
|
92
102
|
input = m.FileScanRequest(
|
93
|
-
verbose=verbose,
|
103
|
+
verbose=verbose,
|
104
|
+
raw=raw,
|
105
|
+
provider=provider,
|
106
|
+
transfer_crc32c=crc,
|
107
|
+
transfer_sha256=sha,
|
108
|
+
transfer_size=size,
|
109
|
+
transfer_method=transfer_method,
|
94
110
|
)
|
95
111
|
data = input.dict(exclude_none=True)
|
96
112
|
return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
|
113
|
+
|
114
|
+
async def request_upload_url(
|
115
|
+
self,
|
116
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
117
|
+
params: Optional[FileUploadParams] = None,
|
118
|
+
verbose: Optional[bool] = None,
|
119
|
+
raw: Optional[bool] = None,
|
120
|
+
provider: Optional[str] = None,
|
121
|
+
) -> PangeaResponse[m.FileScanResult]:
|
122
|
+
input = m.FileScanRequest(
|
123
|
+
verbose=verbose,
|
124
|
+
raw=raw,
|
125
|
+
provider=provider,
|
126
|
+
transfer_method=transfer_method,
|
127
|
+
)
|
128
|
+
if params is not None and (
|
129
|
+
transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT
|
130
|
+
):
|
131
|
+
input.transfer_crc32c = params.crc_hex
|
132
|
+
input.transfer_sha256 = params.sha256_hex
|
133
|
+
input.transfer_size = params.size
|
134
|
+
|
135
|
+
data = input.dict(exclude_none=True)
|
136
|
+
return await self.request.request_presigned_url("v1/scan", m.FileScanResult, data=data)
|
137
|
+
|
138
|
+
|
139
|
+
class FileUploaderAsync:
|
140
|
+
def __init__(self):
|
141
|
+
self.logger = logging.getLogger("pangea")
|
142
|
+
self._request: PangeaRequestAsync = PangeaRequestAsync(
|
143
|
+
config=PangeaConfig(),
|
144
|
+
token="",
|
145
|
+
service="FileScanUploader",
|
146
|
+
logger=self.logger,
|
147
|
+
)
|
148
|
+
|
149
|
+
async def upload_file(
|
150
|
+
self,
|
151
|
+
url: str,
|
152
|
+
file: io.BufferedReader,
|
153
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
154
|
+
file_details: Optional[Dict] = None,
|
155
|
+
):
|
156
|
+
if transfer_method == TransferMethod.PUT_URL:
|
157
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
158
|
+
await self._request.put_presigned_url(url=url, files=files)
|
159
|
+
elif transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT:
|
160
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
161
|
+
await self._request.post_presigned_url(url=url, data=file_details, files=files)
|
162
|
+
else:
|
163
|
+
raise ValueError(f"Transfer method not supported: {transfer_method}")
|
164
|
+
|
165
|
+
async def close(self):
|
166
|
+
await self._request.session.close()
|
@@ -36,13 +36,8 @@ class RedactAsync(ServiceBaseAsync):
|
|
36
36
|
|
37
37
|
service_name = "redact"
|
38
38
|
|
39
|
-
def __init__(
|
40
|
-
|
41
|
-
token,
|
42
|
-
config=None,
|
43
|
-
logger_name="pangea",
|
44
|
-
):
|
45
|
-
super().__init__(token, config, logger_name)
|
39
|
+
def __init__(self, token, config=None, logger_name="pangea", config_id: Optional[str] = None):
|
40
|
+
super().__init__(token, config, logger_name, config_id=config_id)
|
46
41
|
|
47
42
|
async def redact(
|
48
43
|
self,
|
pangea/config.py
CHANGED
@@ -10,7 +10,7 @@ class PangeaConfig:
|
|
10
10
|
"""Holds run time configuration information used by SDK components."""
|
11
11
|
|
12
12
|
"""
|
13
|
-
Used to set
|
13
|
+
Used to set Pangea domain (and port if needed), it should not include service subdomain
|
14
14
|
just for particular use cases when environment = "local", domain could be set to an url including:
|
15
15
|
scheme (http:// or https://), subdomain, domain and port.
|
16
16
|
|
@@ -19,7 +19,7 @@ class PangeaConfig:
|
|
19
19
|
|
20
20
|
"""
|
21
21
|
Used to generate service url.
|
22
|
-
It should be only 'production' or 'local' in
|
22
|
+
It should be only 'production' or 'local' in cases of particular services that can run locally as Redact.
|
23
23
|
|
24
24
|
"""
|
25
25
|
environment: str = "production"
|
pangea/request.py
CHANGED
@@ -12,7 +12,7 @@ import pangea
|
|
12
12
|
import pangea.exceptions as pe
|
13
13
|
import requests
|
14
14
|
from pangea.config import PangeaConfig
|
15
|
-
from pangea.response import
|
15
|
+
from pangea.response import PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
16
16
|
from pangea.utils import default_encoder
|
17
17
|
from requests.adapters import HTTPAdapter, Retry
|
18
18
|
|
@@ -203,12 +203,16 @@ class PangeaRequest(PangeaRequestBase):
|
|
203
203
|
if self.config_id and data.get("config_id", None) is None:
|
204
204
|
data["config_id"] = self.config_id
|
205
205
|
|
206
|
+
transfer_method = data.get("transfer_method", None)
|
207
|
+
|
206
208
|
if (
|
207
209
|
files is not None
|
208
210
|
and type(data) is dict
|
209
|
-
and
|
211
|
+
and (transfer_method == TransferMethod.DIRECT.value or transfer_method == TransferMethod.POST_URL.value)
|
210
212
|
):
|
211
|
-
requests_response = self.
|
213
|
+
requests_response = self._full_post_presigned_url(
|
214
|
+
endpoint, result_class=result_class, data=data, files=files
|
215
|
+
)
|
212
216
|
else:
|
213
217
|
requests_response = self._http_post(
|
214
218
|
url, headers=self._headers(), data=data, files=files, multipart_post=True
|
@@ -220,13 +224,106 @@ class PangeaRequest(PangeaRequestBase):
|
|
220
224
|
|
221
225
|
return self._check_response(pangea_response)
|
222
226
|
|
227
|
+
def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
|
228
|
+
"""Makes the GET call to a Pangea Service endpoint.
|
229
|
+
|
230
|
+
Args:
|
231
|
+
endpoint(str): The Pangea Service API endpoint.
|
232
|
+
path(str): Additional URL path
|
233
|
+
|
234
|
+
Returns:
|
235
|
+
PangeaResponse which contains the response in its entirety and
|
236
|
+
various properties to retrieve individual fields
|
237
|
+
"""
|
238
|
+
|
239
|
+
url = self._url(path)
|
240
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
241
|
+
requests_response = self.session.get(url, headers=self._headers())
|
242
|
+
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
|
243
|
+
|
244
|
+
self.logger.debug(
|
245
|
+
json.dumps(
|
246
|
+
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
247
|
+
default=default_encoder,
|
248
|
+
)
|
249
|
+
)
|
250
|
+
|
251
|
+
if check_response is False:
|
252
|
+
return pangea_response
|
253
|
+
|
254
|
+
return self._check_response(pangea_response)
|
255
|
+
|
256
|
+
def poll_result_by_id(
|
257
|
+
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
258
|
+
):
|
259
|
+
path = self._get_poll_path(request_id)
|
260
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
261
|
+
return self.get(path, result_class, check_response=check_response)
|
262
|
+
|
263
|
+
def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
264
|
+
request_id = response.request_id
|
265
|
+
if not request_id:
|
266
|
+
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
267
|
+
|
268
|
+
if response.status != ResponseStatus.ACCEPTED.value:
|
269
|
+
raise pe.PangeaException("Response already proccesed")
|
270
|
+
|
271
|
+
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
272
|
+
|
273
|
+
def request_presigned_url(
|
274
|
+
self,
|
275
|
+
endpoint: str,
|
276
|
+
result_class: Type[PangeaResponseResult],
|
277
|
+
data: Union[str, Dict] = {},
|
278
|
+
) -> PangeaResponse:
|
279
|
+
# Send request
|
280
|
+
try:
|
281
|
+
# This should return 202 (AcceptedRequestException)
|
282
|
+
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
283
|
+
raise pe.PresignedURLException("Should return 202", resp)
|
284
|
+
|
285
|
+
except pe.AcceptedRequestException as e:
|
286
|
+
accepted_exception = e
|
287
|
+
except Exception as e:
|
288
|
+
raise e
|
289
|
+
|
290
|
+
# Receive 202 with accepted_status
|
291
|
+
return self._poll_presigned_url(accepted_exception.response)
|
292
|
+
|
293
|
+
def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
|
294
|
+
# Send form request with file and upload_details as body
|
295
|
+
resp = self._http_post(url=url, data=data, files=files, multipart_post=False)
|
296
|
+
self.logger.debug(
|
297
|
+
json.dumps(
|
298
|
+
{"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
|
299
|
+
default=default_encoder,
|
300
|
+
)
|
301
|
+
)
|
302
|
+
|
303
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
304
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
|
305
|
+
|
306
|
+
def put_presigned_url(self, url: str, files: List[Tuple]):
|
307
|
+
# Send put request with file as body
|
308
|
+
resp = self._http_put(url=url, files=files)
|
309
|
+
self.logger.debug(
|
310
|
+
json.dumps(
|
311
|
+
{"service": self.service, "action": "put presigned", "url": url, "response": resp.text},
|
312
|
+
default=default_encoder,
|
313
|
+
)
|
314
|
+
)
|
315
|
+
|
316
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
317
|
+
raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status_code}", resp.text)
|
318
|
+
|
319
|
+
# Start internal methods
|
223
320
|
def _http_post(
|
224
321
|
self,
|
225
322
|
url: str,
|
226
323
|
headers: Dict = {},
|
227
324
|
data: Union[str, Dict] = {},
|
228
325
|
files: Optional[List[Tuple]] = None,
|
229
|
-
multipart_post: bool = True,
|
326
|
+
multipart_post: bool = True, # Multipart or form post
|
230
327
|
) -> requests.Response:
|
231
328
|
self.logger.debug(
|
232
329
|
json.dumps(
|
@@ -237,6 +334,17 @@ class PangeaRequest(PangeaRequestBase):
|
|
237
334
|
data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
|
238
335
|
return self.session.post(url, headers=headers, data=data_send, files=files)
|
239
336
|
|
337
|
+
def _http_put(
|
338
|
+
self,
|
339
|
+
url: str,
|
340
|
+
files: List[Tuple],
|
341
|
+
headers: Dict = {},
|
342
|
+
) -> requests.Response:
|
343
|
+
self.logger.debug(
|
344
|
+
json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
|
345
|
+
)
|
346
|
+
return self.session.put(url, headers=headers, files=files)
|
347
|
+
|
240
348
|
def _http_post_process(
|
241
349
|
self, data: Union[str, Dict] = {}, files: Optional[List[Tuple]] = None, multipart_post: bool = True
|
242
350
|
):
|
@@ -263,7 +371,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
263
371
|
|
264
372
|
return data, files
|
265
373
|
|
266
|
-
def
|
374
|
+
def _full_post_presigned_url(
|
267
375
|
self,
|
268
376
|
endpoint: str,
|
269
377
|
result_class: Type[PangeaResponseResult],
|
@@ -273,35 +381,12 @@ class PangeaRequest(PangeaRequestBase):
|
|
273
381
|
if len(files) == 0:
|
274
382
|
raise AttributeError("files attribute should have at least 1 file")
|
275
383
|
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
280
|
-
raise pe.PresignedURLException("Should return 202", resp)
|
281
|
-
|
282
|
-
except pe.AcceptedRequestException as e:
|
283
|
-
accepted_exception = e
|
284
|
-
except Exception as e:
|
285
|
-
raise e
|
286
|
-
|
287
|
-
# Receive 202 with accepted_status
|
288
|
-
result = self._poll_presigned_url(accepted_exception)
|
289
|
-
data_to_presigned = result.accepted_status.upload_details
|
290
|
-
presigned_url = result.accepted_status.upload_url
|
291
|
-
|
292
|
-
# Send multipart request with file and upload_details as body
|
293
|
-
resp = self._http_post(url=presigned_url, data=data_to_presigned, files=files, multipart_post=False)
|
294
|
-
self.logger.debug(
|
295
|
-
json.dumps(
|
296
|
-
{"service": self.service, "action": "post presigned", "url": presigned_url, "response": resp.text},
|
297
|
-
default=default_encoder,
|
298
|
-
)
|
299
|
-
)
|
300
|
-
|
301
|
-
if resp.status_code < 200 or resp.status_code >= 300:
|
302
|
-
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status_code}", resp.text)
|
384
|
+
response = self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
|
385
|
+
data_to_presigned = response.accepted_result.accepted_status.upload_details
|
386
|
+
presigned_url = response.accepted_result.accepted_status.upload_url
|
303
387
|
|
304
|
-
|
388
|
+
self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
389
|
+
return response.raw_response
|
305
390
|
|
306
391
|
def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
|
307
392
|
if self._queued_retry_enabled and response.raw_response.status_code == 202:
|
@@ -315,52 +400,6 @@ class PangeaRequest(PangeaRequestBase):
|
|
315
400
|
|
316
401
|
return response
|
317
402
|
|
318
|
-
def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
|
319
|
-
"""Makes the GET call to a Pangea Service endpoint.
|
320
|
-
|
321
|
-
Args:
|
322
|
-
endpoint(str): The Pangea Service API endpoint.
|
323
|
-
path(str): Additional URL path
|
324
|
-
|
325
|
-
Returns:
|
326
|
-
PangeaResponse which contains the response in its entirety and
|
327
|
-
various properties to retrieve individual fields
|
328
|
-
"""
|
329
|
-
|
330
|
-
url = self._url(path)
|
331
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
332
|
-
requests_response = self.session.get(url, headers=self._headers())
|
333
|
-
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=requests_response.json())
|
334
|
-
|
335
|
-
self.logger.debug(
|
336
|
-
json.dumps(
|
337
|
-
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
338
|
-
default=default_encoder,
|
339
|
-
)
|
340
|
-
)
|
341
|
-
|
342
|
-
if check_response is False:
|
343
|
-
return pangea_response
|
344
|
-
|
345
|
-
return self._check_response(pangea_response)
|
346
|
-
|
347
|
-
def poll_result_by_id(
|
348
|
-
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
349
|
-
):
|
350
|
-
path = self._get_poll_path(request_id)
|
351
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
352
|
-
return self.get(path, result_class, check_response=check_response)
|
353
|
-
|
354
|
-
def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
355
|
-
request_id = response.request_id
|
356
|
-
if not request_id:
|
357
|
-
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
358
|
-
|
359
|
-
if response.status != ResponseStatus.ACCEPTED.value:
|
360
|
-
raise pe.PangeaException("Response already proccesed")
|
361
|
-
|
362
|
-
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
363
|
-
|
364
403
|
def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
|
365
404
|
retry_count = 1
|
366
405
|
start = time.time()
|
@@ -373,26 +412,26 @@ class PangeaRequest(PangeaRequestBase):
|
|
373
412
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
|
374
413
|
return self._check_response(response)
|
375
414
|
|
376
|
-
def _poll_presigned_url(self,
|
377
|
-
if
|
378
|
-
raise AttributeError("
|
415
|
+
def _poll_presigned_url(self, response: PangeaResponse) -> PangeaResponse:
|
416
|
+
if response.http_status != 202:
|
417
|
+
raise AttributeError("Response should be 202")
|
379
418
|
|
380
|
-
if
|
381
|
-
return
|
419
|
+
if response.accepted_result.accepted_status.upload_url:
|
420
|
+
return response
|
382
421
|
|
383
422
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
384
423
|
retry_count = 1
|
385
424
|
start = time.time()
|
386
|
-
|
425
|
+
loop_resp = response
|
387
426
|
|
388
427
|
while (
|
389
|
-
|
390
|
-
and not
|
428
|
+
loop_resp.accepted_result is not None
|
429
|
+
and not loop_resp.accepted_result.accepted_status.upload_url
|
391
430
|
and not self._reach_timeout(start)
|
392
431
|
):
|
393
432
|
time.sleep(self._get_delay(retry_count, start))
|
394
433
|
try:
|
395
|
-
self.poll_result_once(
|
434
|
+
self.poll_result_once(loop_resp, check_response=False)
|
396
435
|
msg = "Polling presigned url return 200 instead of 202"
|
397
436
|
self.logger.debug(
|
398
437
|
json.dumps(
|
@@ -402,6 +441,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
402
441
|
raise pe.PangeaException(msg)
|
403
442
|
except pe.AcceptedRequestException as e:
|
404
443
|
retry_count += 1
|
444
|
+
loop_resp = e.response
|
405
445
|
loop_exc = e
|
406
446
|
except Exception as e:
|
407
447
|
self.logger.debug(
|
@@ -409,12 +449,12 @@ class PangeaRequest(PangeaRequestBase):
|
|
409
449
|
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
|
410
450
|
)
|
411
451
|
)
|
412
|
-
raise pe.PresignedURLException("Failed to pull Presigned URL",
|
452
|
+
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_resp, e)
|
413
453
|
|
414
454
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
415
455
|
|
416
|
-
if
|
417
|
-
return
|
456
|
+
if loop_resp.accepted_result is not None and not loop_resp.accepted_result.accepted_status.upload_url:
|
457
|
+
return loop_resp
|
418
458
|
else:
|
419
459
|
raise loop_exc
|
420
460
|
|
pangea/response.py
CHANGED
@@ -13,8 +13,11 @@ T = TypeVar("T")
|
|
13
13
|
|
14
14
|
|
15
15
|
class TransferMethod(str, enum.Enum):
|
16
|
-
DIRECT = "
|
16
|
+
DIRECT = "post-url" # deprecated, use POST_URL instead
|
17
17
|
MULTIPART = "multipart"
|
18
|
+
POST_URL = "post-url"
|
19
|
+
PUT_URL = "put-url"
|
20
|
+
# URL = "url"
|
18
21
|
|
19
22
|
def __str__(self):
|
20
23
|
return str(self.value)
|
pangea/services/audit/audit.py
CHANGED
@@ -462,7 +462,7 @@ class Audit(ServiceBase, AuditBase):
|
|
462
462
|
|
463
463
|
Examples:
|
464
464
|
log_response = audit.log(
|
465
|
-
message="hello world",
|
465
|
+
message="hello world",
|
466
466
|
verbose=True,
|
467
467
|
)
|
468
468
|
"""
|
@@ -551,7 +551,7 @@ class Audit(ServiceBase, AuditBase):
|
|
551
551
|
|
552
552
|
Examples:
|
553
553
|
log_response = audit.log_bulk(
|
554
|
-
events=[{"message": "hello world"}],
|
554
|
+
events=[{"message": "hello world"}],
|
555
555
|
verbose=True,
|
556
556
|
)
|
557
557
|
"""
|
@@ -590,7 +590,7 @@ class Audit(ServiceBase, AuditBase):
|
|
590
590
|
|
591
591
|
Examples:
|
592
592
|
log_response = audit.log_bulk_async(
|
593
|
-
events=[{"message": "hello world"}],
|
593
|
+
events=[{"message": "hello world"}],
|
594
594
|
verbose=True,
|
595
595
|
)
|
596
596
|
"""
|
@@ -664,10 +664,10 @@ class Audit(ServiceBase, AuditBase):
|
|
664
664
|
|
665
665
|
Examples:
|
666
666
|
response = audit.search(
|
667
|
-
query="message:test",
|
668
|
-
search_restriction={'source': ["monitor"]},
|
669
|
-
limit=1,
|
670
|
-
verify_consistency=True,
|
667
|
+
query="message:test",
|
668
|
+
search_restriction={'source': ["monitor"]},
|
669
|
+
limit=1,
|
670
|
+
verify_consistency=True,
|
671
671
|
verify_events=True,
|
672
672
|
)
|
673
673
|
"""
|
pangea/services/authn/authn.py
CHANGED
@@ -520,9 +520,10 @@ class AuthN(ServiceBase):
|
|
520
520
|
|
521
521
|
def update(
|
522
522
|
self,
|
523
|
-
disabled: bool,
|
523
|
+
disabled: Optional[bool] = None,
|
524
524
|
id: Optional[str] = None,
|
525
525
|
email: Optional[str] = None,
|
526
|
+
unlock: Optional[bool] = None,
|
526
527
|
) -> PangeaResponse[m.UserUpdateResult]:
|
527
528
|
"""
|
528
529
|
Update user's settings
|
@@ -534,6 +535,7 @@ class AuthN(ServiceBase):
|
|
534
535
|
Args:
|
535
536
|
disabled (bool): New disabled value.
|
536
537
|
Disabling a user account will prevent them from logging in.
|
538
|
+
unlock (bool): Unlock a user account if it has been locked out due to failed Authentication attempts.
|
537
539
|
id (str, optional): The identity of a user or a service
|
538
540
|
email (str, optional): An email address
|
539
541
|
|
@@ -552,6 +554,7 @@ class AuthN(ServiceBase):
|
|
552
554
|
id=id,
|
553
555
|
email=email,
|
554
556
|
disabled=disabled,
|
557
|
+
unlock=unlock,
|
555
558
|
)
|
556
559
|
|
557
560
|
return self.request.post("v2/user/update", m.UserUpdateResult, data=input.dict(exclude_none=True))
|
pangea/services/authn/models.py
CHANGED
@@ -363,7 +363,8 @@ class UserProfileUpdateResult(User):
|
|
363
363
|
class UserUpdateRequest(APIRequestModel):
|
364
364
|
id: Optional[str] = None
|
365
365
|
email: Optional[str] = None
|
366
|
-
disabled: bool
|
366
|
+
disabled: Optional[bool] = None
|
367
|
+
unlock: Optional[bool] = None
|
367
368
|
|
368
369
|
|
369
370
|
class UserUpdateResult(User):
|
pangea/services/base.py
CHANGED
@@ -3,13 +3,13 @@
|
|
3
3
|
|
4
4
|
import copy
|
5
5
|
import logging
|
6
|
-
from typing import Optional, Union
|
6
|
+
from typing import Optional, Type, Union
|
7
7
|
|
8
8
|
from pangea.asyncio.request import PangeaRequestAsync
|
9
9
|
from pangea.config import PangeaConfig
|
10
10
|
from pangea.exceptions import AcceptedRequestException
|
11
11
|
from pangea.request import PangeaRequest
|
12
|
-
from pangea.response import PangeaResponse
|
12
|
+
from pangea.response import PangeaResponse, PangeaResponseResult
|
13
13
|
|
14
14
|
|
15
15
|
class ServiceBase(object):
|
@@ -50,7 +50,13 @@ class ServiceBase(object):
|
|
50
50
|
|
51
51
|
return self._request
|
52
52
|
|
53
|
-
def poll_result(
|
53
|
+
def poll_result(
|
54
|
+
self,
|
55
|
+
exception: Optional[AcceptedRequestException] = None,
|
56
|
+
response: Optional[PangeaResponse] = None,
|
57
|
+
request_id: Optional[str] = None,
|
58
|
+
result_class: Union[Type[PangeaResponseResult], dict] = dict,
|
59
|
+
) -> PangeaResponse:
|
54
60
|
"""
|
55
61
|
Poll result
|
56
62
|
|
@@ -68,4 +74,11 @@ class ServiceBase(object):
|
|
68
74
|
Examples:
|
69
75
|
response = service.poll_result(exception)
|
70
76
|
"""
|
71
|
-
|
77
|
+
if exception is not None:
|
78
|
+
return self.request.poll_result_once(exception.response, check_response=True)
|
79
|
+
elif response is not None:
|
80
|
+
return self.request.poll_result_once(response, check_response=True)
|
81
|
+
elif request_id is not None:
|
82
|
+
return self.request.poll_result_by_id(request_id=request_id, result_class=result_class, check_response=True)
|
83
|
+
else:
|
84
|
+
raise AttributeError("Need to set exception, response or request_id")
|
pangea/services/file_scan.py
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import io
|
4
|
+
import logging
|
4
5
|
from typing import Dict, List, Optional
|
5
6
|
|
7
|
+
from pangea.request import PangeaConfig, PangeaRequest
|
6
8
|
from pangea.response import APIRequestModel, PangeaResponse, PangeaResponseResult, TransferMethod
|
7
|
-
from pangea.utils import
|
9
|
+
from pangea.utils import FileUploadParams, get_file_upload_params
|
8
10
|
|
9
11
|
from .base import ServiceBase
|
10
12
|
|
@@ -118,8 +120,11 @@ class FileScan(ServiceBase):
|
|
118
120
|
if file or file_path:
|
119
121
|
if file_path:
|
120
122
|
file = open(file_path, "rb")
|
121
|
-
if transfer_method == TransferMethod.DIRECT:
|
122
|
-
|
123
|
+
if transfer_method == TransferMethod.DIRECT or transfer_method == TransferMethod.POST_URL:
|
124
|
+
params = get_file_upload_params(file)
|
125
|
+
crc = params.crc_hex
|
126
|
+
sha = params.sha256_hex
|
127
|
+
size = params.size
|
123
128
|
else:
|
124
129
|
crc, sha, size = None, None, None
|
125
130
|
files = [("upload", ("filename", file, "application/octet-stream"))]
|
@@ -137,3 +142,54 @@ class FileScan(ServiceBase):
|
|
137
142
|
)
|
138
143
|
data = input.dict(exclude_none=True)
|
139
144
|
return self.request.post("v1/scan", FileScanResult, data=data, files=files, poll_result=sync_call)
|
145
|
+
|
146
|
+
def request_upload_url(
|
147
|
+
self,
|
148
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
149
|
+
params: Optional[FileUploadParams] = None,
|
150
|
+
verbose: Optional[bool] = None,
|
151
|
+
raw: Optional[bool] = None,
|
152
|
+
provider: Optional[str] = None,
|
153
|
+
) -> PangeaResponse[FileScanResult]:
|
154
|
+
input = FileScanRequest(
|
155
|
+
verbose=verbose,
|
156
|
+
raw=raw,
|
157
|
+
provider=provider,
|
158
|
+
transfer_method=transfer_method,
|
159
|
+
)
|
160
|
+
if params is not None and (
|
161
|
+
transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT
|
162
|
+
):
|
163
|
+
input.transfer_crc32c = params.crc_hex
|
164
|
+
input.transfer_sha256 = params.sha256_hex
|
165
|
+
input.transfer_size = params.size
|
166
|
+
|
167
|
+
data = input.dict(exclude_none=True)
|
168
|
+
return self.request.request_presigned_url("v1/scan", FileScanResult, data=data)
|
169
|
+
|
170
|
+
|
171
|
+
class FileUploader:
|
172
|
+
def __init__(self):
|
173
|
+
self.logger = logging.getLogger("pangea")
|
174
|
+
self._request = PangeaRequest(
|
175
|
+
config=PangeaConfig(),
|
176
|
+
token="",
|
177
|
+
service="FileScanUploader",
|
178
|
+
logger=self.logger,
|
179
|
+
)
|
180
|
+
|
181
|
+
def upload_file(
|
182
|
+
self,
|
183
|
+
url: str,
|
184
|
+
file: io.BufferedReader,
|
185
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
186
|
+
file_details: Optional[Dict] = None,
|
187
|
+
):
|
188
|
+
if transfer_method == TransferMethod.PUT_URL:
|
189
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
190
|
+
self._request.put_presigned_url(url=url, files=files)
|
191
|
+
elif transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT:
|
192
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
193
|
+
self._request.post_presigned_url(url=url, data=file_details, files=files)
|
194
|
+
else:
|
195
|
+
raise ValueError(f"Transfer method not supported: {transfer_method}")
|
pangea/services/redact.py
CHANGED
@@ -134,13 +134,8 @@ class Redact(ServiceBase):
|
|
134
134
|
|
135
135
|
service_name = "redact"
|
136
136
|
|
137
|
-
def __init__(
|
138
|
-
|
139
|
-
token,
|
140
|
-
config=None,
|
141
|
-
logger_name="pangea",
|
142
|
-
):
|
143
|
-
super().__init__(token, config, logger_name)
|
137
|
+
def __init__(self, token, config=None, logger_name="pangea", config_id: Optional[str] = None):
|
138
|
+
super().__init__(token, config, logger_name, config_id=config_id)
|
144
139
|
|
145
140
|
def redact(
|
146
141
|
self,
|
pangea/services/vault/vault.py
CHANGED
@@ -537,7 +537,7 @@ class Vault(ServiceBase):
|
|
537
537
|
|
538
538
|
Generate a symmetric key
|
539
539
|
|
540
|
-
OperationId: vault_post_v1_key_generate
|
540
|
+
OperationId: vault_post_v1_key_generate 2
|
541
541
|
|
542
542
|
Args:
|
543
543
|
algorithm (SymmetricAlgorithm): The algorithm of the key
|
@@ -611,7 +611,7 @@ class Vault(ServiceBase):
|
|
611
611
|
|
612
612
|
Generate an asymmetric key
|
613
613
|
|
614
|
-
OperationId: vault_post_v1_key_generate
|
614
|
+
OperationId: vault_post_v1_key_generate 1
|
615
615
|
|
616
616
|
Args:
|
617
617
|
algorithm (AsymmetricAlgorithm): The algorithm of the key
|
pangea/utils.py
CHANGED
@@ -8,6 +8,7 @@ from collections import OrderedDict
|
|
8
8
|
from hashlib import new, sha1, sha256, sha512
|
9
9
|
|
10
10
|
from google_crc32c import Checksum as CRC32C
|
11
|
+
from pydantic import BaseModel
|
11
12
|
|
12
13
|
|
13
14
|
def format_datetime(dt: datetime.datetime) -> str:
|
@@ -100,7 +101,13 @@ def get_prefix(hash: str, len: int = 5):
|
|
100
101
|
return hash[0:len]
|
101
102
|
|
102
103
|
|
103
|
-
|
104
|
+
class FileUploadParams(BaseModel):
|
105
|
+
crc_hex: str
|
106
|
+
sha256_hex: str
|
107
|
+
size: int
|
108
|
+
|
109
|
+
|
110
|
+
def get_file_upload_params(file: io.BufferedReader) -> FileUploadParams:
|
104
111
|
if "b" not in file.mode:
|
105
112
|
raise AttributeError("File need to be open in binary mode")
|
106
113
|
|
@@ -118,4 +125,4 @@ def get_presigned_url_upload_params(file: io.BufferedReader):
|
|
118
125
|
size += len(chunk)
|
119
126
|
|
120
127
|
file.seek(0) # restart reading
|
121
|
-
return crc.hexdigest().decode("utf-8"), sha.hexdigest(), size
|
128
|
+
return FileUploadParams(crc_hex=crc.hexdigest().decode("utf-8"), sha256_hex=sha.hexdigest(), size=size)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: pangea-sdk
|
3
|
-
Version: 3.
|
3
|
+
Version: 3.3.0
|
4
4
|
Summary: Pangea API SDK
|
5
5
|
License: MIT
|
6
6
|
Keywords: Pangea,SDK,Audit
|
@@ -15,10 +15,10 @@ Classifier: Programming Language :: Python :: 3.10
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
16
16
|
Classifier: Topic :: Software Development
|
17
17
|
Classifier: Topic :: Software Development :: Libraries
|
18
|
-
Requires-Dist: aiohttp (>=3.8.
|
18
|
+
Requires-Dist: aiohttp (>=3.8.6,<4.0.0)
|
19
19
|
Requires-Dist: alive-progress (>=2.4.1,<3.0.0)
|
20
20
|
Requires-Dist: asyncio (>=3.4.3,<4.0.0)
|
21
|
-
Requires-Dist: cryptography (==41.0.
|
21
|
+
Requires-Dist: cryptography (==41.0.5)
|
22
22
|
Requires-Dist: deprecated (>=1.2.13,<2.0.0)
|
23
23
|
Requires-Dist: google-crc32c (>=1.5.0,<2.0.0)
|
24
24
|
Requires-Dist: pydantic (>=1.10.2,<2.0.0)
|
@@ -59,18 +59,18 @@ poetry add pangea-sdk
|
|
59
59
|
|
60
60
|
## Usage
|
61
61
|
|
62
|
-
For
|
62
|
+
For sample apps, look at the [/examples](https://github.com/pangeacyber/pangea-python/tree/main/examples) folder in this repository. There you will find basic sample apps for each of the services supported on this SDK. Each service folder has a README.md with instructions to install, setup, and run the sample app.
|
63
63
|
|
64
64
|
|
65
65
|
## Asyncio support
|
66
66
|
|
67
|
-
We have added support to asyncio library using aiohttp in order to support async/await calls to all our services.
|
67
|
+
We have added support to the asyncio library using aiohttp in order to support async/await calls to all our services.
|
68
68
|
Async services classes are inside [pangea/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk/pangea/asyncio) folder, and examples about how to use them are in [/examples/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/examples/asyncio).
|
69
69
|
|
70
70
|
|
71
71
|
### Secure Audit Service - Integrity Tools
|
72
72
|
|
73
|
-
Python Pangea SDK
|
73
|
+
The Python Pangea SDK also includes some extra features to validate Audit Service log's integrity. Here we explain how to run them.
|
74
74
|
|
75
75
|
#### Verify audit data
|
76
76
|
|
@@ -101,7 +101,7 @@ curl -H "Authorization: Bearer ${PANGEA_TOKEN}" -X POST -H 'Content-Type: applic
|
|
101
101
|
|
102
102
|
Download all audit logs for a given time range. Start and end date should be provided,
|
103
103
|
a variety of formats is supported, including ISO-8601. The result is stored in a
|
104
|
-
|
104
|
+
json file (one json per line).
|
105
105
|
|
106
106
|
```
|
107
107
|
usage: python -m pangea.dump_audit [-h] [--token TOKEN] [--domain DOMAIN] [--output OUTPUT] start end
|
@@ -126,8 +126,8 @@ options:
|
|
126
126
|
|
127
127
|
#### Perform Exhaustive Verification of Audit Data
|
128
128
|
|
129
|
-
This script performs extensive verification on a range of events of the log stream.
|
130
|
-
and the membership proof, it checks that there
|
129
|
+
This script performs extensive verification on a range of events of the log stream. Apart from verifying the hash
|
130
|
+
and the membership proof, it checks that there are no omissions in the stream, i.e. all the events are present and properly located.
|
131
131
|
|
132
132
|
```
|
133
133
|
usage: python -m pangea.deep_verify [-h] [--token TOKEN] [--domain DOMAIN] --file FILE
|
@@ -153,9 +153,9 @@ It accepts multiple file formats:
|
|
153
153
|
|
154
154
|
## Reporting issues and new features
|
155
155
|
|
156
|
-
If
|
157
|
-
We would need you to provide some basic information
|
158
|
-
Also feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/)
|
156
|
+
If you run into an issue using or testing this SDK or if you have a new feature request, feel free to open an issue by [clicking here](https://github.com/pangeacyber/pangea-python/issues).
|
157
|
+
We would need you to provide some basic information, such as what SDK version you are using, the stack trace if you got it, the framework used, and steps to reproduce the issue.
|
158
|
+
Also, feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/).
|
159
159
|
|
160
160
|
|
161
161
|
## Contributing
|
@@ -168,5 +168,5 @@ These linters will run on every `git commit` operation.
|
|
168
168
|
|
169
169
|
### Send a PR
|
170
170
|
|
171
|
-
If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in code or an error in documents we
|
171
|
+
If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in the code or an error in documents, we really appreciate it and after review and approval, you will be included in our [contributors list](https://github.com/pangeacyber/pangea-python/blob/main/packages/pangea-sdk/CONTRIBUTING.md).
|
172
172
|
|
@@ -1,43 +1,43 @@
|
|
1
|
-
pangea/__init__.py,sha256=
|
2
|
-
pangea/asyncio/request.py,sha256=
|
1
|
+
pangea/__init__.py,sha256=GgLmx-eZWNCnGMva8AGtHJ4JesBsSPfw8YdDVyuKXOo,200
|
2
|
+
pangea/asyncio/request.py,sha256=akSpGTjC8j8Jr6uK40Zu8v9HQXX_QOf7dKgweq5K6DM,12642
|
3
3
|
pangea/asyncio/services/__init__.py,sha256=_CEDza6E9VmEs6d_vubWetfeqTogH7UxT7XrTVRw4Mo,290
|
4
4
|
pangea/asyncio/services/audit.py,sha256=xHvz5ReJpmk_StK3mr_uDqCI80J6Aj7-EglEBWS0Qqc,16480
|
5
5
|
pangea/asyncio/services/authn.py,sha256=HssvrZ3fr2oP5sgYBj1Rr5Yf-fKiHhrWTue5LOAeyOk,43288
|
6
|
-
pangea/asyncio/services/base.py,sha256=
|
6
|
+
pangea/asyncio/services/base.py,sha256=hmdY5-IS_H2WGMeVfzotn11Z1c2B2iJTHi4USBbmptA,2304
|
7
7
|
pangea/asyncio/services/embargo.py,sha256=8WguyWZUaGVwGpNzic5h8QzLueirA9WpBBik4mpCTeA,3056
|
8
|
-
pangea/asyncio/services/file_scan.py,sha256=
|
8
|
+
pangea/asyncio/services/file_scan.py,sha256=DnysLA0oEsxkHPdt4Pal6oWKiYdl7TZz8eX20Bs_Zl8,6280
|
9
9
|
pangea/asyncio/services/intel.py,sha256=LdJSxCohXsI4Vk8bG74jXTBYAL9N45lb5LIX7nBXqLI,22248
|
10
|
-
pangea/asyncio/services/redact.py,sha256=
|
10
|
+
pangea/asyncio/services/redact.py,sha256=W9eKMw1XyY6tgRudRrtuCpyQYhgttvmsYngoiMld0C4,5152
|
11
11
|
pangea/asyncio/services/vault.py,sha256=JhxUdmPNYZzp3eiAQFR6vsfyS9L3ddKWes-Cv7rj2ow,43425
|
12
12
|
pangea/audit_logger.py,sha256=zCSsq0kvw4Pb_aIBb7APfaYsTqd9dmohYLFld2ir5lc,3778
|
13
|
-
pangea/config.py,sha256=
|
13
|
+
pangea/config.py,sha256=kFu9mLMkFpkM7wdT8Ymvskx6DGLfRZKeKfQRVZTgbyA,1670
|
14
14
|
pangea/deep_verify.py,sha256=WiA_2gqtrSCQYoTMKX9ILlNgnknsH4UxZpJXLVG2uyc,8343
|
15
15
|
pangea/deprecated.py,sha256=IjFYEVvY1E0ld0SMkEYC1o62MAleX3nnT1If2dFVbHo,608
|
16
16
|
pangea/dump_audit.py,sha256=Ws0KuZyHoaySsQ2lq9EKK2iw65O8x4zL1Mii0ChDh0k,6511
|
17
17
|
pangea/exceptions.py,sha256=JMx_Dym7W2cgcPSHN4bz9Rlrm-3BZGNpIYu3VBXZLBU,5277
|
18
|
-
pangea/request.py,sha256=
|
19
|
-
pangea/response.py,sha256=
|
18
|
+
pangea/request.py,sha256=5BP-wFI6x7lfJ41-6bTcpcg3pyviMgrX_4nX_M9yyXg,18640
|
19
|
+
pangea/response.py,sha256=mOgeFqafEH_MdDILSo-ju5cWicXlvSd1CXzbsHObiJo,5363
|
20
20
|
pangea/services/__init__.py,sha256=auqKaEAOLiazHCzOQVwrUwd2ABFw_bF-ptoOq1bpa68,253
|
21
|
-
pangea/services/audit/audit.py,sha256=
|
21
|
+
pangea/services/audit/audit.py,sha256=50zkQE2HQ0cQtFroFSVjUxX1M4RQsf6pBUeMidSkEHI,30295
|
22
22
|
pangea/services/audit/exceptions.py,sha256=CVdaQZCvQKx1n-iIjWz5wnStUGU6cXDwKqe7MoijAXk,451
|
23
23
|
pangea/services/audit/models.py,sha256=ztLcB7XNRudn3GquyW8gp_TlCrH3va32Tw0nMpO7g6k,12244
|
24
24
|
pangea/services/audit/signing.py,sha256=JWYutDNV5XZZzLASwdUJ-9gMHlGd8-h4IjIbKz7JMJM,5261
|
25
25
|
pangea/services/audit/util.py,sha256=2EM3K9R-T9FBKegklmLLoWa1IDm84uGjmJZVO3mYEhk,7568
|
26
|
-
pangea/services/authn/authn.py,sha256=
|
27
|
-
pangea/services/authn/models.py,sha256=
|
28
|
-
pangea/services/base.py,sha256=
|
26
|
+
pangea/services/authn/authn.py,sha256=sGQUPQ8VLEMtu8AkEhWPyY6Q6AxCsfrX-XNUzkC-o1M,42774
|
27
|
+
pangea/services/authn/models.py,sha256=V-hj1KfbSuaWJMVgd-Q3fWTsnbj3EdPumnzXtjHOR8g,18150
|
28
|
+
pangea/services/base.py,sha256=bRSuuhLv6iQrPVjhasOjTBqOgxTRRhCD204v3tNjVOE,2731
|
29
29
|
pangea/services/embargo.py,sha256=F3jx7SbElnjhbDEUR3oHfWsQ8G8zf_kfp6_q1_vmOIc,3871
|
30
|
-
pangea/services/file_scan.py,sha256=
|
30
|
+
pangea/services/file_scan.py,sha256=K4zDhZtY-Co622QgSOueGfkCaAJVy04mO6sqQ22TBS4,6999
|
31
31
|
pangea/services/intel.py,sha256=oTvTxCtdnoaZRlt0anUKnkRT2FVfc9LG8ftDIX09jHk,30399
|
32
|
-
pangea/services/redact.py,sha256=
|
32
|
+
pangea/services/redact.py,sha256=vH4sg_t8MnoSjSY-F42JZhe8EfRmVCclslDN7U1hNP4,7756
|
33
33
|
pangea/services/vault/models/asymmetric.py,sha256=ac2Exc66elXxO-HxBqtvLPQWNI7y_00kb6SVqBPKecA,1450
|
34
34
|
pangea/services/vault/models/common.py,sha256=D_xZlqjEc0XO9G_aoed3YuPNTNYB5S2DoZJRIJVoliQ,9095
|
35
35
|
pangea/services/vault/models/secret.py,sha256=cLgEj-_BeGkB4-pmSeTkWVyasFbaJwcEltIEcOyf1U8,481
|
36
36
|
pangea/services/vault/models/symmetric.py,sha256=5N2n6FDStB1CLPfpd4p-6Ig__Nt-EyurhjCWfEyws2k,1330
|
37
|
-
pangea/services/vault/vault.py,sha256=
|
37
|
+
pangea/services/vault/vault.py,sha256=t3KU59FjUfVawbMZt7A51cDQoyapftChdihTqllVrTw,43160
|
38
38
|
pangea/tools.py,sha256=A9gkHpAkECwqnUKA-O09vG-n6Gmq2g_LYr61BUs-n54,6403
|
39
|
-
pangea/utils.py,sha256=
|
39
|
+
pangea/utils.py,sha256=09dHX9CKmyKNWccSF2z7LyCYYYduZ9AWP6D5Ylnfty8,3360
|
40
40
|
pangea/verify_audit.py,sha256=gWhde7gETKSWfBaMm5gEckAO2xmYB_vmgcZ_4FvvyfU,10616
|
41
|
-
pangea_sdk-3.
|
42
|
-
pangea_sdk-3.
|
43
|
-
pangea_sdk-3.
|
41
|
+
pangea_sdk-3.3.0.dist-info/METADATA,sha256=7xobhjV_WaLEGoK4-jsG9YZG_PqDEhYlXD3c6CnnxS0,6934
|
42
|
+
pangea_sdk-3.3.0.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
|
43
|
+
pangea_sdk-3.3.0.dist-info/RECORD,,
|
File without changes
|