pangea-sdk 3.8.0b1__py3-none-any.whl → 5.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +1 -1
- pangea/asyncio/file_uploader.py +1 -1
- pangea/asyncio/request.py +49 -31
- pangea/asyncio/services/__init__.py +2 -0
- pangea/asyncio/services/audit.py +192 -31
- pangea/asyncio/services/authn.py +187 -109
- pangea/asyncio/services/authz.py +285 -0
- pangea/asyncio/services/base.py +21 -2
- pangea/asyncio/services/embargo.py +2 -2
- pangea/asyncio/services/file_scan.py +24 -9
- pangea/asyncio/services/intel.py +108 -34
- pangea/asyncio/services/redact.py +72 -4
- pangea/asyncio/services/sanitize.py +217 -0
- pangea/asyncio/services/share.py +246 -73
- pangea/asyncio/services/vault.py +1710 -750
- pangea/crypto/rsa.py +135 -0
- pangea/deep_verify.py +7 -1
- pangea/dump_audit.py +9 -8
- pangea/request.py +83 -59
- pangea/response.py +49 -31
- pangea/services/__init__.py +2 -0
- pangea/services/audit/audit.py +205 -42
- pangea/services/audit/models.py +56 -8
- pangea/services/audit/signing.py +6 -5
- pangea/services/audit/util.py +3 -3
- pangea/services/authn/authn.py +140 -70
- pangea/services/authn/models.py +167 -11
- pangea/services/authz.py +400 -0
- pangea/services/base.py +39 -8
- pangea/services/embargo.py +2 -2
- pangea/services/file_scan.py +32 -15
- pangea/services/intel.py +157 -32
- pangea/services/redact.py +152 -4
- pangea/services/sanitize.py +388 -0
- pangea/services/share/share.py +683 -107
- pangea/services/vault/models/asymmetric.py +120 -18
- pangea/services/vault/models/common.py +439 -141
- pangea/services/vault/models/keys.py +94 -0
- pangea/services/vault/models/secret.py +27 -3
- pangea/services/vault/models/symmetric.py +68 -22
- pangea/services/vault/vault.py +1690 -749
- pangea/tools.py +6 -7
- pangea/utils.py +16 -27
- pangea/verify_audit.py +270 -83
- {pangea_sdk-3.8.0b1.dist-info → pangea_sdk-5.3.0.dist-info}/METADATA +43 -35
- pangea_sdk-5.3.0.dist-info/RECORD +56 -0
- {pangea_sdk-3.8.0b1.dist-info → pangea_sdk-5.3.0.dist-info}/WHEEL +1 -1
- pangea_sdk-3.8.0b1.dist-info/RECORD +0 -50
pangea/crypto/rsa.py
ADDED
@@ -0,0 +1,135 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import base64
|
4
|
+
from typing import TYPE_CHECKING
|
5
|
+
|
6
|
+
from cryptography.hazmat.backends import default_backend
|
7
|
+
from cryptography.hazmat.primitives import hashes, serialization
|
8
|
+
from cryptography.hazmat.primitives.asymmetric import padding, rsa
|
9
|
+
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
10
|
+
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
|
11
|
+
|
12
|
+
from pangea.services.vault.models.common import ExportEncryptionAlgorithm
|
13
|
+
from pangea.services.vault.models.symmetric import SymmetricKeyEncryptionAlgorithm
|
14
|
+
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
from pangea.services.vault.models.common import ExportResult
|
17
|
+
|
18
|
+
|
19
|
+
def generate_key_pair() -> tuple[rsa.RSAPrivateKey, rsa.RSAPublicKey]:
|
20
|
+
# Generate a 4096-bit RSA key pair
|
21
|
+
private_key = rsa.generate_private_key(
|
22
|
+
public_exponent=65537,
|
23
|
+
key_size=4096,
|
24
|
+
)
|
25
|
+
|
26
|
+
# Extract the public key from the private key
|
27
|
+
public_key = private_key.public_key()
|
28
|
+
return private_key, public_key
|
29
|
+
|
30
|
+
|
31
|
+
def decrypt_sha512(private_key: rsa.RSAPrivateKey, encrypted_message: bytes) -> bytes:
|
32
|
+
# Decrypt the message using the private key and OAEP padding
|
33
|
+
return private_key.decrypt(
|
34
|
+
encrypted_message,
|
35
|
+
padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA512()), algorithm=hashes.SHA512(), label=None),
|
36
|
+
)
|
37
|
+
|
38
|
+
|
39
|
+
def encrypt_sha512(public_key: rsa.RSAPublicKey, message: bytes) -> bytes:
|
40
|
+
# Encrypt the message using the public key and OAEP padding
|
41
|
+
return public_key.encrypt(
|
42
|
+
message, padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA512()), algorithm=hashes.SHA512(), label=None)
|
43
|
+
)
|
44
|
+
|
45
|
+
|
46
|
+
def private_key_to_pem(private_key: rsa.RSAPrivateKey) -> bytes:
|
47
|
+
# Serialize private key to PEM format
|
48
|
+
return private_key.private_bytes(
|
49
|
+
encoding=serialization.Encoding.PEM,
|
50
|
+
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
51
|
+
encryption_algorithm=serialization.NoEncryption(),
|
52
|
+
)
|
53
|
+
|
54
|
+
|
55
|
+
def public_key_to_pem(public_key: rsa.RSAPublicKey) -> bytes:
|
56
|
+
# Serialize public key to PEM format
|
57
|
+
return public_key.public_bytes(
|
58
|
+
encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo
|
59
|
+
)
|
60
|
+
|
61
|
+
|
62
|
+
_AES_GCM_IV_SIZE = 12
|
63
|
+
"""Standard nonce size for GCM."""
|
64
|
+
|
65
|
+
_KEY_LENGTH = 32
|
66
|
+
"""AES-256 key length in bytes."""
|
67
|
+
|
68
|
+
|
69
|
+
def kem_decrypt(
|
70
|
+
private_key: rsa.RSAPrivateKey,
|
71
|
+
iv: bytes,
|
72
|
+
ciphertext: bytes,
|
73
|
+
symmetric_algorithm: str,
|
74
|
+
asymmetric_algorithm: str,
|
75
|
+
encrypted_salt: bytes,
|
76
|
+
password: str,
|
77
|
+
iteration_count: int,
|
78
|
+
hash_algorithm: str,
|
79
|
+
) -> str:
|
80
|
+
if symmetric_algorithm.casefold() != SymmetricKeyEncryptionAlgorithm.AES_GCM_256.value.casefold():
|
81
|
+
raise NotImplementedError(f"Unsupported symmetric algorithm: {symmetric_algorithm}")
|
82
|
+
|
83
|
+
if asymmetric_algorithm != ExportEncryptionAlgorithm.RSA_NO_PADDING_4096_KEM:
|
84
|
+
raise NotImplementedError(f"Unsupported asymmetric algorithm: {asymmetric_algorithm}")
|
85
|
+
|
86
|
+
if hash_algorithm.casefold() != "SHA512".casefold():
|
87
|
+
raise NotImplementedError(f"Unsupported hash algorithm: {hash_algorithm}")
|
88
|
+
|
89
|
+
# No-padding RSA decryption.
|
90
|
+
n = private_key.private_numbers().public_numbers.n
|
91
|
+
salt = pow(
|
92
|
+
int.from_bytes(encrypted_salt, byteorder="big"),
|
93
|
+
private_key.private_numbers().d,
|
94
|
+
n,
|
95
|
+
).to_bytes(n.bit_length() // 8, byteorder="big")
|
96
|
+
|
97
|
+
kdf = PBKDF2HMAC(
|
98
|
+
algorithm=hashes.SHA512(), length=_KEY_LENGTH, salt=salt, iterations=iteration_count, backend=default_backend()
|
99
|
+
)
|
100
|
+
symmetric_key = kdf.derive(password.encode("utf-8"))
|
101
|
+
|
102
|
+
decrypted = AESGCM(symmetric_key).decrypt(nonce=iv, data=ciphertext, associated_data=None)
|
103
|
+
|
104
|
+
return decrypted.decode("ascii")
|
105
|
+
|
106
|
+
|
107
|
+
def kem_decrypt_export_result(*, result: ExportResult, password: str, private_key: rsa.RSAPrivateKey) -> str:
|
108
|
+
"""Decrypt the exported result of a KEM operation."""
|
109
|
+
cipher_encoded = result.private_key or result.key
|
110
|
+
if not cipher_encoded:
|
111
|
+
raise TypeError("`private_key` or `key` should be set.")
|
112
|
+
|
113
|
+
assert result.encrypted_salt
|
114
|
+
assert result.symmetric_algorithm
|
115
|
+
assert result.asymmetric_algorithm
|
116
|
+
assert result.iteration_count
|
117
|
+
assert result.hash_algorithm
|
118
|
+
|
119
|
+
cipher_with_iv = base64.b64decode(cipher_encoded)
|
120
|
+
encrypted_salt = base64.b64decode(result.encrypted_salt)
|
121
|
+
|
122
|
+
iv = cipher_with_iv[:_AES_GCM_IV_SIZE]
|
123
|
+
cipher = cipher_with_iv[_AES_GCM_IV_SIZE:]
|
124
|
+
|
125
|
+
return kem_decrypt(
|
126
|
+
private_key=private_key,
|
127
|
+
iv=iv,
|
128
|
+
ciphertext=cipher,
|
129
|
+
password=password,
|
130
|
+
encrypted_salt=encrypted_salt,
|
131
|
+
symmetric_algorithm=result.symmetric_algorithm,
|
132
|
+
asymmetric_algorithm=result.asymmetric_algorithm,
|
133
|
+
iteration_count=result.iteration_count,
|
134
|
+
hash_algorithm=result.hash_algorithm,
|
135
|
+
)
|
pangea/deep_verify.py
CHANGED
@@ -263,8 +263,14 @@ def main():
|
|
263
263
|
audit = init_audit(args.token, args.domain)
|
264
264
|
errors = deep_verify(audit, args.file)
|
265
265
|
|
266
|
-
print("\n\
|
266
|
+
print("\n\nWarnings:")
|
267
|
+
val = errors["not_persisted"]
|
268
|
+
print(f"\tnot_persisted: {val}")
|
269
|
+
|
270
|
+
print("\nTotal errors:")
|
267
271
|
for key, val in errors.items():
|
272
|
+
if key == "not_persisted":
|
273
|
+
continue
|
268
274
|
print(f"\t{key.title()}: {val}")
|
269
275
|
print()
|
270
276
|
|
pangea/dump_audit.py
CHANGED
@@ -19,7 +19,7 @@ from pangea.utils import default_encoder
|
|
19
19
|
|
20
20
|
|
21
21
|
def dump_event(output: io.TextIOWrapper, row: SearchEvent, resp: PangeaResponse[SearchOutput]):
|
22
|
-
row_data = filter_deep_none(row.
|
22
|
+
row_data = filter_deep_none(row.model_dump())
|
23
23
|
if resp.result and resp.result.root:
|
24
24
|
row_data["tree_size"] = resp.result.root.size
|
25
25
|
output.write(json.dumps(row_data, default=default_encoder) + "\n")
|
@@ -63,11 +63,12 @@ def dump_before(audit: Audit, output: io.TextIOWrapper, start: datetime) -> int:
|
|
63
63
|
cnt = 0
|
64
64
|
if search_res.result and search_res.result.count > 0:
|
65
65
|
leaf_index = search_res.result.events[0].leaf_index
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
66
|
+
if leaf_index is not None:
|
67
|
+
for row in reversed(search_res.result.events):
|
68
|
+
if row.leaf_index != leaf_index:
|
69
|
+
break
|
70
|
+
dump_event(output, row, search_res)
|
71
|
+
cnt += 1
|
71
72
|
print(f"Dumping before... {cnt} events")
|
72
73
|
return cnt
|
73
74
|
|
@@ -89,7 +90,7 @@ def dump_after(audit: Audit, output: io.TextIOWrapper, start: datetime, last_eve
|
|
89
90
|
cnt = 0
|
90
91
|
if search_res.result and search_res.result.count > 0:
|
91
92
|
leaf_index = search_res.result.events[0].leaf_index
|
92
|
-
if leaf_index == last_leaf_index:
|
93
|
+
if leaf_index is not None and leaf_index == last_leaf_index:
|
93
94
|
start_idx: int = 1 if last_event_hash == search_res.result.events[0].hash else 0
|
94
95
|
for row in search_res.result.events[start_idx:]:
|
95
96
|
if row.leaf_index != leaf_index:
|
@@ -124,7 +125,7 @@ def dump_page(
|
|
124
125
|
msg = f"Dumping... {search_res.result.count} events"
|
125
126
|
|
126
127
|
if search_res.result.count <= 1:
|
127
|
-
return end, 0
|
128
|
+
return end, 0, True, "", 0
|
128
129
|
|
129
130
|
offset = 0
|
130
131
|
result_id = search_res.result.id
|
pangea/request.py
CHANGED
@@ -1,16 +1,18 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
3
4
|
|
4
5
|
import copy
|
5
6
|
import json
|
6
7
|
import logging
|
7
8
|
import time
|
8
|
-
from typing import Dict, List, Optional, Tuple, Type, Union
|
9
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Type, Union
|
9
10
|
|
10
|
-
import aiohttp
|
11
11
|
import requests
|
12
|
+
from pydantic import BaseModel
|
12
13
|
from requests.adapters import HTTPAdapter, Retry
|
13
|
-
from requests_toolbelt import MultipartDecoder # type: ignore
|
14
|
+
from requests_toolbelt import MultipartDecoder # type: ignore[import-untyped]
|
15
|
+
from typing_extensions import TypeVar
|
14
16
|
|
15
17
|
import pangea
|
16
18
|
import pangea.exceptions as pe
|
@@ -18,8 +20,11 @@ from pangea.config import PangeaConfig
|
|
18
20
|
from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
19
21
|
from pangea.utils import default_encoder
|
20
22
|
|
23
|
+
if TYPE_CHECKING:
|
24
|
+
import aiohttp
|
21
25
|
|
22
|
-
|
26
|
+
|
27
|
+
class MultipartResponse:
|
23
28
|
pangea_json: Dict[str, str]
|
24
29
|
attached_files: List = []
|
25
30
|
|
@@ -28,7 +33,7 @@ class MultipartResponse(object):
|
|
28
33
|
self.attached_files = attached_files
|
29
34
|
|
30
35
|
|
31
|
-
class PangeaRequestBase
|
36
|
+
class PangeaRequestBase:
|
32
37
|
def __init__(
|
33
38
|
self, config: PangeaConfig, token: str, service: str, logger: logging.Logger, config_id: Optional[str] = None
|
34
39
|
):
|
@@ -126,8 +131,7 @@ class PangeaRequestBase(object):
|
|
126
131
|
filename_parts = content_disposition.split("name=")
|
127
132
|
if len(filename_parts) > 1:
|
128
133
|
return filename_parts[1].split(";")[0].strip('"')
|
129
|
-
|
130
|
-
return None
|
134
|
+
return None
|
131
135
|
|
132
136
|
def _get_filename_from_url(self, url: str) -> Optional[str]:
|
133
137
|
return url.split("/")[-1].split("?")[0]
|
@@ -154,39 +158,42 @@ class PangeaRequestBase(object):
|
|
154
158
|
|
155
159
|
if status == ResponseStatus.VALIDATION_ERR.value:
|
156
160
|
raise pe.ValidationException(summary, response)
|
157
|
-
|
161
|
+
if status == ResponseStatus.TOO_MANY_REQUESTS.value:
|
158
162
|
raise pe.RateLimitException(summary, response)
|
159
|
-
|
163
|
+
if status == ResponseStatus.NO_CREDIT.value:
|
160
164
|
raise pe.NoCreditException(summary, response)
|
161
|
-
|
165
|
+
if status == ResponseStatus.UNAUTHORIZED.value:
|
162
166
|
raise pe.UnauthorizedException(self.service, response)
|
163
|
-
|
167
|
+
if status == ResponseStatus.SERVICE_NOT_ENABLED.value:
|
164
168
|
raise pe.ServiceNotEnabledException(self.service, response)
|
165
|
-
|
169
|
+
if status == ResponseStatus.PROVIDER_ERR.value:
|
166
170
|
raise pe.ProviderErrorException(summary, response)
|
167
|
-
|
171
|
+
if status in (ResponseStatus.MISSING_CONFIG_ID_SCOPE.value, ResponseStatus.MISSING_CONFIG_ID.value):
|
168
172
|
raise pe.MissingConfigID(self.service, response)
|
169
|
-
|
173
|
+
if status == ResponseStatus.SERVICE_NOT_AVAILABLE.value:
|
170
174
|
raise pe.ServiceNotAvailableException(summary, response)
|
171
|
-
|
175
|
+
if status == ResponseStatus.TREE_NOT_FOUND.value:
|
172
176
|
raise pe.TreeNotFoundException(summary, response)
|
173
|
-
|
177
|
+
if status == ResponseStatus.IP_NOT_FOUND.value:
|
174
178
|
raise pe.IPNotFoundException(summary, response)
|
175
|
-
|
179
|
+
if status == ResponseStatus.BAD_OFFSET.value:
|
176
180
|
raise pe.BadOffsetException(summary, response)
|
177
|
-
|
181
|
+
if status == ResponseStatus.FORBIDDEN_VAULT_OPERATION.value:
|
178
182
|
raise pe.ForbiddenVaultOperation(summary, response)
|
179
|
-
|
183
|
+
if status == ResponseStatus.VAULT_ITEM_NOT_FOUND.value:
|
180
184
|
raise pe.VaultItemNotFound(summary, response)
|
181
|
-
|
182
|
-
raise pe.NotFound(str(response.raw_response.url) if response.raw_response is not None else "", response)
|
183
|
-
|
185
|
+
if status == ResponseStatus.NOT_FOUND.value:
|
186
|
+
raise pe.NotFound(str(response.raw_response.url) if response.raw_response is not None else "", response)
|
187
|
+
if status == ResponseStatus.INTERNAL_SERVER_ERROR.value:
|
184
188
|
raise pe.InternalServerError(response)
|
185
|
-
|
189
|
+
if status == ResponseStatus.ACCEPTED.value:
|
186
190
|
raise pe.AcceptedRequestException(response)
|
187
191
|
raise pe.PangeaAPIException(f"{summary} ", response)
|
188
192
|
|
189
193
|
|
194
|
+
TResult = TypeVar("TResult", bound=PangeaResponseResult)
|
195
|
+
|
196
|
+
|
190
197
|
class PangeaRequest(PangeaRequestBase):
|
191
198
|
"""An object that makes direct calls to Pangea Service APIs.
|
192
199
|
|
@@ -202,12 +209,12 @@ class PangeaRequest(PangeaRequestBase):
|
|
202
209
|
def post(
|
203
210
|
self,
|
204
211
|
endpoint: str,
|
205
|
-
result_class: Type[
|
206
|
-
data:
|
212
|
+
result_class: Type[TResult],
|
213
|
+
data: str | BaseModel | dict[str, Any] | None = None,
|
207
214
|
files: Optional[List[Tuple]] = None,
|
208
215
|
poll_result: bool = True,
|
209
216
|
url: Optional[str] = None,
|
210
|
-
) -> PangeaResponse:
|
217
|
+
) -> PangeaResponse[TResult]:
|
211
218
|
"""Makes the POST call to a Pangea Service endpoint.
|
212
219
|
|
213
220
|
Args:
|
@@ -218,6 +225,13 @@ class PangeaRequest(PangeaRequestBase):
|
|
218
225
|
PangeaResponse which contains the response in its entirety and
|
219
226
|
various properties to retrieve individual fields
|
220
227
|
"""
|
228
|
+
|
229
|
+
if isinstance(data, BaseModel):
|
230
|
+
data = data.model_dump(exclude_none=True)
|
231
|
+
|
232
|
+
if data is None:
|
233
|
+
data = {}
|
234
|
+
|
221
235
|
if url is None:
|
222
236
|
url = self._url(endpoint)
|
223
237
|
|
@@ -318,32 +332,33 @@ class PangeaRequest(PangeaRequestBase):
|
|
318
332
|
return self.session.post(url, headers=headers, data=data_send, files=files)
|
319
333
|
|
320
334
|
def _http_post_process(
|
321
|
-
self,
|
335
|
+
self,
|
336
|
+
data: Union[str, Dict] = {},
|
337
|
+
files: Optional[Sequence[Tuple[str, Tuple[Any, str, str]]]] = None,
|
338
|
+
multipart_post: bool = True,
|
322
339
|
):
|
323
340
|
if files:
|
324
341
|
if multipart_post is True:
|
325
342
|
data_send: str = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
326
343
|
multi = [("request", (None, data_send, "application/json"))]
|
327
|
-
multi.extend(files)
|
328
|
-
files = multi
|
344
|
+
multi.extend(files)
|
345
|
+
files = multi
|
329
346
|
return None, files
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
342
|
-
return data_send, None
|
347
|
+
# Post to presigned url as form
|
348
|
+
data_send: list = [] # type: ignore[no-redef]
|
349
|
+
for k, v in data.items(): # type: ignore[union-attr]
|
350
|
+
data_send.append((k, v)) # type: ignore[attr-defined]
|
351
|
+
# When posting to presigned url, file key should be 'file'
|
352
|
+
files = { # type: ignore[assignment]
|
353
|
+
"file": files[0][1],
|
354
|
+
}
|
355
|
+
return data_send, files
|
356
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
357
|
+
return data_send, None
|
343
358
|
|
344
359
|
return data, files
|
345
360
|
|
346
|
-
def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse[
|
361
|
+
def _handle_queued_result(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
|
347
362
|
if self._queued_retry_enabled and response.http_status == 202:
|
348
363
|
self.logger.debug(
|
349
364
|
json.dumps(
|
@@ -355,7 +370,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
355
370
|
|
356
371
|
return response
|
357
372
|
|
358
|
-
def get(self, path: str, result_class: Type[
|
373
|
+
def get(self, path: str, result_class: Type[TResult], check_response: bool = True) -> PangeaResponse[TResult]:
|
359
374
|
"""Makes the GET call to a Pangea Service endpoint.
|
360
375
|
|
361
376
|
Args:
|
@@ -387,7 +402,20 @@ class PangeaRequest(PangeaRequestBase):
|
|
387
402
|
|
388
403
|
return self._check_response(pangea_response)
|
389
404
|
|
390
|
-
def download_file(self, url: str, filename:
|
405
|
+
def download_file(self, url: str, filename: str | None = None) -> AttachedFile:
|
406
|
+
"""
|
407
|
+
Download file
|
408
|
+
|
409
|
+
Download a file from the specified URL and save it with the given
|
410
|
+
filename.
|
411
|
+
|
412
|
+
Args:
|
413
|
+
url: URL of the file to download
|
414
|
+
filename: Name to save the downloaded file as. If not provided, the
|
415
|
+
filename will be determined from the Content-Disposition header or
|
416
|
+
the URL.
|
417
|
+
"""
|
418
|
+
|
391
419
|
self.logger.debug(
|
392
420
|
json.dumps(
|
393
421
|
{
|
@@ -423,25 +451,24 @@ class PangeaRequest(PangeaRequestBase):
|
|
423
451
|
)
|
424
452
|
)
|
425
453
|
return AttachedFile(filename=filename, file=response.content, content_type=content_type)
|
426
|
-
|
427
|
-
raise pe.DownloadFileError(f"Failed to download file. Status: {response.status_code}", response.text)
|
454
|
+
raise pe.DownloadFileError(f"Failed to download file. Status: {response.status_code}", response.text)
|
428
455
|
|
429
456
|
def poll_result_by_id(
|
430
|
-
self, request_id: str, result_class:
|
431
|
-
):
|
457
|
+
self, request_id: str, result_class: Type[TResult], check_response: bool = True
|
458
|
+
) -> PangeaResponse[TResult]:
|
432
459
|
path = self._get_poll_path(request_id)
|
433
460
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
434
|
-
return self.get(path, result_class, check_response=check_response)
|
461
|
+
return self.get(path, result_class, check_response=check_response)
|
435
462
|
|
436
463
|
def poll_result_once(
|
437
|
-
self, response: PangeaResponse, check_response: bool = True
|
438
|
-
) -> PangeaResponse[
|
464
|
+
self, response: PangeaResponse[TResult], check_response: bool = True
|
465
|
+
) -> PangeaResponse[TResult]:
|
439
466
|
request_id = response.request_id
|
440
467
|
if not request_id:
|
441
468
|
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
442
469
|
|
443
470
|
if response.status != ResponseStatus.ACCEPTED.value:
|
444
|
-
raise pe.PangeaException("Response already
|
471
|
+
raise pe.PangeaException("Response already processed")
|
445
472
|
|
446
473
|
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
447
474
|
|
@@ -526,7 +553,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
526
553
|
self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
527
554
|
return response.raw_response
|
528
555
|
|
529
|
-
def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse[
|
556
|
+
def _poll_result_retry(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
|
530
557
|
retry_count = 1
|
531
558
|
start = time.time()
|
532
559
|
|
@@ -538,9 +565,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
538
565
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
|
539
566
|
return self._check_response(response)
|
540
567
|
|
541
|
-
def _poll_presigned_url(
|
542
|
-
self, response: PangeaResponse[Type[PangeaResponseResult]]
|
543
|
-
) -> PangeaResponse[Type[PangeaResponseResult]]:
|
568
|
+
def _poll_presigned_url(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
|
544
569
|
if response.http_status != 202:
|
545
570
|
raise AttributeError("Response should be 202")
|
546
571
|
|
@@ -583,8 +608,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
583
608
|
|
584
609
|
if loop_resp.accepted_result is not None and not loop_resp.accepted_result.has_upload_url:
|
585
610
|
return loop_resp
|
586
|
-
|
587
|
-
raise loop_exc
|
611
|
+
raise loop_exc
|
588
612
|
|
589
613
|
def _init_session(self) -> requests.Session:
|
590
614
|
retry_config = Retry(
|
pangea/response.py
CHANGED
@@ -3,16 +3,15 @@
|
|
3
3
|
import datetime
|
4
4
|
import enum
|
5
5
|
import os
|
6
|
-
from typing import Any, Dict, Generic, List, Optional, Type,
|
6
|
+
from typing import Any, Dict, Generic, List, Optional, Type, Union
|
7
7
|
|
8
8
|
import aiohttp
|
9
9
|
import requests
|
10
|
-
from pydantic import BaseModel
|
10
|
+
from pydantic import BaseModel, ConfigDict, PlainSerializer
|
11
|
+
from typing_extensions import Annotated, TypeVar
|
11
12
|
|
12
13
|
from pangea.utils import format_datetime
|
13
14
|
|
14
|
-
T = TypeVar("T")
|
15
|
-
|
16
15
|
|
17
16
|
class AttachedFile(object):
|
18
17
|
filename: str
|
@@ -50,10 +49,24 @@ class AttachedFile(object):
|
|
50
49
|
|
51
50
|
|
52
51
|
class TransferMethod(str, enum.Enum):
|
52
|
+
"""Transfer methods for uploading file data."""
|
53
|
+
|
53
54
|
MULTIPART = "multipart"
|
54
55
|
POST_URL = "post-url"
|
55
56
|
PUT_URL = "put-url"
|
56
57
|
SOURCE_URL = "source-url"
|
58
|
+
"""
|
59
|
+
A `source-url` is a caller-specified URL where the Pangea APIs can fetch the
|
60
|
+
contents of the input file. When calling a Pangea API with a
|
61
|
+
`transfer_method` of `source-url`, you must also specify a `source_url`
|
62
|
+
input parameter that provides a URL to the input file. The source URL can be
|
63
|
+
a presigned URL created by the caller, and it will be used to download the
|
64
|
+
content of the input file. The `source-url` transfer method is useful when
|
65
|
+
you already have a file in your storage and can provide a URL from which
|
66
|
+
Pangea API can fetch the input file—there is no need to transfer it to
|
67
|
+
Pangea with a separate POST or PUT request.
|
68
|
+
"""
|
69
|
+
|
57
70
|
DEST_URL = "dest-url"
|
58
71
|
|
59
72
|
def __str__(self):
|
@@ -63,24 +76,17 @@ class TransferMethod(str, enum.Enum):
|
|
63
76
|
return str(self.value)
|
64
77
|
|
65
78
|
|
79
|
+
PangeaDateTime = Annotated[datetime.datetime, PlainSerializer(format_datetime)]
|
80
|
+
|
81
|
+
|
66
82
|
# API response should accept arbitrary fields to make them accept possible new parameters
|
67
83
|
class APIResponseModel(BaseModel):
|
68
|
-
|
69
|
-
arbitrary_types_allowed = True
|
70
|
-
# allow parameters despite they are not declared in model. Make SDK accept server new parameters
|
71
|
-
extra = "allow"
|
84
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
|
72
85
|
|
73
86
|
|
74
87
|
# API request models doesn't not allow arbitrary fields
|
75
88
|
class APIRequestModel(BaseModel):
|
76
|
-
|
77
|
-
arbitrary_types_allowed = True
|
78
|
-
extra = (
|
79
|
-
"allow" # allow parameters despite they are not declared in model. Make SDK accept server new parameters
|
80
|
-
)
|
81
|
-
json_encoders = {
|
82
|
-
datetime.datetime: format_datetime,
|
83
|
-
}
|
89
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
|
84
90
|
|
85
91
|
|
86
92
|
class PangeaResponseResult(APIResponseModel):
|
@@ -155,38 +161,50 @@ class ResponseStatus(str, enum.Enum):
|
|
155
161
|
|
156
162
|
|
157
163
|
class ResponseHeader(APIResponseModel):
|
158
|
-
"""
|
159
|
-
Pangea response API header.
|
160
|
-
|
161
|
-
Arguments:
|
162
|
-
request_id -- The request ID.
|
163
|
-
request_time -- The time the request was issued, ISO8601.
|
164
|
-
response_time -- The time the response was issued, ISO8601.
|
165
|
-
status -- Pangea response status
|
166
|
-
summary -- The summary of the response.
|
167
|
-
"""
|
164
|
+
"""Pangea response API header."""
|
168
165
|
|
169
166
|
request_id: str
|
167
|
+
"""A unique identifier assigned to each request made to the API."""
|
168
|
+
|
170
169
|
request_time: str
|
170
|
+
"""
|
171
|
+
Timestamp indicating the exact moment when a request is made to the API.
|
172
|
+
"""
|
173
|
+
|
171
174
|
response_time: str
|
175
|
+
"""
|
176
|
+
Duration it takes for the API to process a request and generate a response.
|
177
|
+
"""
|
178
|
+
|
172
179
|
status: str
|
180
|
+
"""
|
181
|
+
Represents the status or outcome of the API request.
|
182
|
+
"""
|
183
|
+
|
173
184
|
summary: str
|
185
|
+
"""
|
186
|
+
Provides a concise and brief overview of the purpose or primary objective of
|
187
|
+
the API endpoint.
|
188
|
+
"""
|
189
|
+
|
190
|
+
|
191
|
+
T = TypeVar("T", bound=PangeaResponseResult)
|
174
192
|
|
175
193
|
|
176
|
-
class PangeaResponse(Generic[T]
|
194
|
+
class PangeaResponse(ResponseHeader, Generic[T]):
|
177
195
|
raw_result: Optional[Dict[str, Any]] = None
|
178
196
|
raw_response: Optional[Union[requests.Response, aiohttp.ClientResponse]] = None
|
179
197
|
result: Optional[T] = None
|
180
198
|
pangea_error: Optional[PangeaError] = None
|
181
199
|
accepted_result: Optional[AcceptedResult] = None
|
182
|
-
result_class:
|
200
|
+
result_class: Type[T] = PangeaResponseResult # type: ignore[assignment]
|
183
201
|
_json: Any
|
184
202
|
attached_files: List[AttachedFile] = []
|
185
203
|
|
186
204
|
def __init__(
|
187
205
|
self,
|
188
206
|
response: requests.Response,
|
189
|
-
result_class:
|
207
|
+
result_class: Type[T],
|
190
208
|
json: dict,
|
191
209
|
attached_files: List[AttachedFile] = [],
|
192
210
|
):
|
@@ -198,7 +216,7 @@ class PangeaResponse(Generic[T], ResponseHeader):
|
|
198
216
|
self.attached_files = attached_files
|
199
217
|
|
200
218
|
self.result = (
|
201
|
-
self.result_class(**self.raw_result)
|
219
|
+
self.result_class(**self.raw_result)
|
202
220
|
if self.raw_result is not None and issubclass(self.result_class, PangeaResponseResult) and self.success
|
203
221
|
else None
|
204
222
|
)
|
@@ -230,4 +248,4 @@ class PangeaResponse(Generic[T], ResponseHeader):
|
|
230
248
|
|
231
249
|
@property
|
232
250
|
def url(self) -> str:
|
233
|
-
return str(self.raw_response.url) # type: ignore[
|
251
|
+
return str(self.raw_response.url) # type: ignore[union-attr]
|
pangea/services/__init__.py
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
from .audit.audit import Audit
|
2
2
|
from .authn.authn import AuthN
|
3
|
+
from .authz import AuthZ
|
3
4
|
from .embargo import Embargo
|
4
5
|
from .file_scan import FileScan
|
5
6
|
from .intel import DomainIntel, FileIntel, IpIntel, UrlIntel, UserIntel
|
6
7
|
from .redact import Redact
|
8
|
+
from .sanitize import Sanitize
|
7
9
|
from .share.share import Share
|
8
10
|
from .vault.vault import Vault
|