pangea-sdk 3.8.0b4__py3-none-any.whl → 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +1 -2
- pangea/asyncio/request.py +17 -22
- pangea/asyncio/services/__init__.py +0 -2
- pangea/asyncio/services/audit.py +188 -23
- pangea/asyncio/services/authn.py +167 -108
- pangea/asyncio/services/authz.py +36 -45
- pangea/asyncio/services/embargo.py +2 -2
- pangea/asyncio/services/file_scan.py +3 -3
- pangea/asyncio/services/intel.py +44 -26
- pangea/asyncio/services/redact.py +60 -4
- pangea/asyncio/services/vault.py +145 -30
- pangea/dump_audit.py +1 -1
- pangea/request.py +30 -24
- pangea/response.py +34 -42
- pangea/services/__init__.py +0 -2
- pangea/services/audit/audit.py +202 -34
- pangea/services/audit/models.py +56 -8
- pangea/services/audit/util.py +3 -3
- pangea/services/authn/authn.py +116 -65
- pangea/services/authn/models.py +88 -4
- pangea/services/authz.py +51 -56
- pangea/services/base.py +23 -6
- pangea/services/embargo.py +2 -2
- pangea/services/file_scan.py +3 -2
- pangea/services/intel.py +25 -23
- pangea/services/redact.py +124 -4
- pangea/services/vault/models/common.py +121 -6
- pangea/services/vault/models/symmetric.py +2 -2
- pangea/services/vault/vault.py +143 -32
- pangea/utils.py +20 -109
- pangea/verify_audit.py +267 -83
- {pangea_sdk-3.8.0b4.dist-info → pangea_sdk-4.0.0.dist-info}/METADATA +12 -20
- pangea_sdk-4.0.0.dist-info/RECORD +46 -0
- {pangea_sdk-3.8.0b4.dist-info → pangea_sdk-4.0.0.dist-info}/WHEEL +1 -1
- pangea/asyncio/__init__.py +0 -1
- pangea/asyncio/file_uploader.py +0 -39
- pangea/asyncio/services/sanitize.py +0 -185
- pangea/asyncio/services/share.py +0 -573
- pangea/file_uploader.py +0 -35
- pangea/services/sanitize.py +0 -275
- pangea/services/share/file_format.py +0 -170
- pangea/services/share/share.py +0 -877
- pangea_sdk-3.8.0b4.dist-info/RECORD +0 -54
pangea/asyncio/services/vault.py
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import datetime
|
4
6
|
from typing import Dict, Optional, Union
|
5
7
|
|
6
8
|
from pangea.asyncio.services.base import ServiceBaseAsync
|
9
|
+
from pangea.config import PangeaConfig
|
7
10
|
from pangea.response import PangeaResponse
|
8
11
|
from pangea.services.vault.models.asymmetric import (
|
9
12
|
AsymmetricGenerateRequest,
|
@@ -17,6 +20,8 @@ from pangea.services.vault.models.asymmetric import (
|
|
17
20
|
)
|
18
21
|
from pangea.services.vault.models.common import (
|
19
22
|
AsymmetricAlgorithm,
|
23
|
+
DecryptTransformRequest,
|
24
|
+
DecryptTransformResult,
|
20
25
|
DeleteRequest,
|
21
26
|
DeleteResult,
|
22
27
|
EncodedPrivateKey,
|
@@ -24,6 +29,8 @@ from pangea.services.vault.models.common import (
|
|
24
29
|
EncodedSymmetricKey,
|
25
30
|
EncryptStructuredRequest,
|
26
31
|
EncryptStructuredResult,
|
32
|
+
EncryptTransformRequest,
|
33
|
+
EncryptTransformResult,
|
27
34
|
FolderCreateRequest,
|
28
35
|
FolderCreateResult,
|
29
36
|
GetRequest,
|
@@ -50,6 +57,7 @@ from pangea.services.vault.models.common import (
|
|
50
57
|
SymmetricAlgorithm,
|
51
58
|
Tags,
|
52
59
|
TDict,
|
60
|
+
TransformAlphabet,
|
53
61
|
UpdateRequest,
|
54
62
|
UpdateResult,
|
55
63
|
)
|
@@ -98,10 +106,24 @@ class VaultAsync(ServiceBaseAsync):
|
|
98
106
|
|
99
107
|
def __init__(
|
100
108
|
self,
|
101
|
-
token,
|
102
|
-
config=None,
|
103
|
-
logger_name="pangea",
|
104
|
-
):
|
109
|
+
token: str,
|
110
|
+
config: PangeaConfig | None = None,
|
111
|
+
logger_name: str = "pangea",
|
112
|
+
) -> None:
|
113
|
+
"""
|
114
|
+
Vault client
|
115
|
+
|
116
|
+
Initializes a new Vault client.
|
117
|
+
|
118
|
+
Args:
|
119
|
+
token: Pangea API token.
|
120
|
+
config: Configuration.
|
121
|
+
logger_name: Logger name.
|
122
|
+
|
123
|
+
Examples:
|
124
|
+
config = PangeaConfig(domain="pangea_domain")
|
125
|
+
vault = VaultAsync(token="pangea_token", config=config)
|
126
|
+
"""
|
105
127
|
super().__init__(token, config, logger_name)
|
106
128
|
|
107
129
|
# Delete endpoint
|
@@ -129,7 +151,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
129
151
|
input = DeleteRequest(
|
130
152
|
id=id,
|
131
153
|
)
|
132
|
-
return await self.request.post("v1/delete", DeleteResult, data=input.
|
154
|
+
return await self.request.post("v1/delete", DeleteResult, data=input.model_dump(exclude_none=True))
|
133
155
|
|
134
156
|
# Get endpoint
|
135
157
|
async def get(
|
@@ -176,7 +198,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
176
198
|
verbose=verbose,
|
177
199
|
version_state=version_state,
|
178
200
|
)
|
179
|
-
return await self.request.post("v1/get", GetResult, data=input.
|
201
|
+
return await self.request.post("v1/get", GetResult, data=input.model_dump(exclude_none=True))
|
180
202
|
|
181
203
|
# List endpoint
|
182
204
|
async def list(
|
@@ -232,7 +254,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
232
254
|
)
|
233
255
|
"""
|
234
256
|
input = ListRequest(filter=filter, last=last, order=order, order_by=order_by, size=size)
|
235
|
-
return await self.request.post("v1/list", ListResult, data=input.
|
257
|
+
return await self.request.post("v1/list", ListResult, data=input.model_dump(exclude_none=True))
|
236
258
|
|
237
259
|
# Update endpoint
|
238
260
|
async def update(
|
@@ -313,7 +335,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
313
335
|
expiration=expiration,
|
314
336
|
item_state=item_state,
|
315
337
|
)
|
316
|
-
return await self.request.post("v1/update", UpdateResult, data=input.
|
338
|
+
return await self.request.post("v1/update", UpdateResult, data=input.model_dump(exclude_none=True))
|
317
339
|
|
318
340
|
async def secret_store(
|
319
341
|
self,
|
@@ -383,7 +405,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
383
405
|
rotation_state=rotation_state,
|
384
406
|
expiration=expiration,
|
385
407
|
)
|
386
|
-
return await self.request.post("v1/secret/store", SecretStoreResult, data=input.
|
408
|
+
return await self.request.post("v1/secret/store", SecretStoreResult, data=input.model_dump(exclude_none=True))
|
387
409
|
|
388
410
|
async def pangea_token_store(
|
389
411
|
self,
|
@@ -453,7 +475,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
453
475
|
rotation_state=rotation_state,
|
454
476
|
expiration=expiration,
|
455
477
|
)
|
456
|
-
return await self.request.post("v1/secret/store", SecretStoreResult, data=input.
|
478
|
+
return await self.request.post("v1/secret/store", SecretStoreResult, data=input.model_dump(exclude_none=True))
|
457
479
|
|
458
480
|
# Rotate endpoint
|
459
481
|
async def secret_rotate(
|
@@ -493,7 +515,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
493
515
|
)
|
494
516
|
"""
|
495
517
|
input = SecretRotateRequest(id=id, secret=secret, rotation_state=rotation_state)
|
496
|
-
return await self.request.post("v1/secret/rotate", SecretRotateResult, data=input.
|
518
|
+
return await self.request.post("v1/secret/rotate", SecretRotateResult, data=input.model_dump(exclude_none=True))
|
497
519
|
|
498
520
|
# Rotate endpoint
|
499
521
|
async def pangea_token_rotate(self, id: str) -> PangeaResponse[SecretRotateResult]:
|
@@ -521,7 +543,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
521
543
|
)
|
522
544
|
"""
|
523
545
|
input = SecretRotateRequest(id=id) # type: ignore[call-arg]
|
524
|
-
return await self.request.post("v1/secret/rotate", SecretRotateResult, data=input.
|
546
|
+
return await self.request.post("v1/secret/rotate", SecretRotateResult, data=input.model_dump(exclude_none=True))
|
525
547
|
|
526
548
|
async def symmetric_generate(
|
527
549
|
self,
|
@@ -595,7 +617,9 @@ class VaultAsync(ServiceBaseAsync):
|
|
595
617
|
rotation_state=rotation_state,
|
596
618
|
expiration=expiration,
|
597
619
|
)
|
598
|
-
return await self.request.post(
|
620
|
+
return await self.request.post(
|
621
|
+
"v1/key/generate", SymmetricGenerateResult, data=input.model_dump(exclude_none=True)
|
622
|
+
)
|
599
623
|
|
600
624
|
async def asymmetric_generate(
|
601
625
|
self,
|
@@ -669,7 +693,9 @@ class VaultAsync(ServiceBaseAsync):
|
|
669
693
|
rotation_state=rotation_state,
|
670
694
|
expiration=expiration,
|
671
695
|
)
|
672
|
-
return await self.request.post(
|
696
|
+
return await self.request.post(
|
697
|
+
"v1/key/generate", AsymmetricGenerateResult, data=input.model_dump(exclude_none=True)
|
698
|
+
)
|
673
699
|
|
674
700
|
# Store endpoints
|
675
701
|
async def asymmetric_store(
|
@@ -752,7 +778,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
752
778
|
rotation_state=rotation_state,
|
753
779
|
expiration=expiration,
|
754
780
|
)
|
755
|
-
return await self.request.post("v1/key/store", AsymmetricStoreResult, data=input.
|
781
|
+
return await self.request.post("v1/key/store", AsymmetricStoreResult, data=input.model_dump(exclude_none=True))
|
756
782
|
|
757
783
|
async def symmetric_store(
|
758
784
|
self,
|
@@ -830,7 +856,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
830
856
|
rotation_state=rotation_state,
|
831
857
|
expiration=expiration,
|
832
858
|
)
|
833
|
-
return await self.request.post("v1/key/store", SymmetricStoreResult, data=input.
|
859
|
+
return await self.request.post("v1/key/store", SymmetricStoreResult, data=input.model_dump(exclude_none=True))
|
834
860
|
|
835
861
|
# Rotate endpoint
|
836
862
|
async def key_rotate(
|
@@ -879,7 +905,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
879
905
|
input = KeyRotateRequest(
|
880
906
|
id=id, public_key=public_key, private_key=private_key, key=key, rotation_state=rotation_state
|
881
907
|
)
|
882
|
-
return await self.request.post("v1/key/rotate", KeyRotateResult, data=input.
|
908
|
+
return await self.request.post("v1/key/rotate", KeyRotateResult, data=input.model_dump(exclude_none=True))
|
883
909
|
|
884
910
|
# Encrypt
|
885
911
|
async def encrypt(self, id: str, plain_text: str, version: Optional[int] = None) -> PangeaResponse[EncryptResult]:
|
@@ -910,8 +936,8 @@ class VaultAsync(ServiceBaseAsync):
|
|
910
936
|
version=1,
|
911
937
|
)
|
912
938
|
"""
|
913
|
-
input = EncryptRequest(id=id, plain_text=plain_text, version=version)
|
914
|
-
return await self.request.post("v1/key/encrypt", EncryptResult, data=input.
|
939
|
+
input = EncryptRequest(id=id, plain_text=plain_text, version=version)
|
940
|
+
return await self.request.post("v1/key/encrypt", EncryptResult, data=input.model_dump(exclude_none=True))
|
915
941
|
|
916
942
|
# Decrypt
|
917
943
|
async def decrypt(self, id: str, cipher_text: str, version: Optional[int] = None) -> PangeaResponse[DecryptResult]:
|
@@ -942,8 +968,8 @@ class VaultAsync(ServiceBaseAsync):
|
|
942
968
|
version=1,
|
943
969
|
)
|
944
970
|
"""
|
945
|
-
input = DecryptRequest(id=id, cipher_text=cipher_text, version=version)
|
946
|
-
return await self.request.post("v1/key/decrypt", DecryptResult, data=input.
|
971
|
+
input = DecryptRequest(id=id, cipher_text=cipher_text, version=version)
|
972
|
+
return await self.request.post("v1/key/decrypt", DecryptResult, data=input.model_dump(exclude_none=True))
|
947
973
|
|
948
974
|
# Sign
|
949
975
|
async def sign(self, id: str, message: str, version: Optional[int] = None) -> PangeaResponse[SignResult]:
|
@@ -975,7 +1001,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
975
1001
|
)
|
976
1002
|
"""
|
977
1003
|
input = SignRequest(id=id, message=message, version=version)
|
978
|
-
return await self.request.post("v1/key/sign", SignResult, data=input.
|
1004
|
+
return await self.request.post("v1/key/sign", SignResult, data=input.model_dump(exclude_none=True))
|
979
1005
|
|
980
1006
|
# Verify
|
981
1007
|
async def verify(
|
@@ -1016,7 +1042,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1016
1042
|
signature=signature,
|
1017
1043
|
version=version,
|
1018
1044
|
)
|
1019
|
-
return await self.request.post("v1/key/verify", VerifyResult, data=input.
|
1045
|
+
return await self.request.post("v1/key/verify", VerifyResult, data=input.model_dump(exclude_none=True))
|
1020
1046
|
|
1021
1047
|
async def jwt_verify(self, jws: str) -> PangeaResponse[JWTVerifyResult]:
|
1022
1048
|
"""
|
@@ -1043,7 +1069,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1043
1069
|
)
|
1044
1070
|
"""
|
1045
1071
|
input = JWTVerifyRequest(jws=jws)
|
1046
|
-
return await self.request.post("v1/key/verify/jwt", JWTVerifyResult, data=input.
|
1072
|
+
return await self.request.post("v1/key/verify/jwt", JWTVerifyResult, data=input.model_dump(exclude_none=True))
|
1047
1073
|
|
1048
1074
|
async def jwt_sign(self, id: str, payload: str) -> PangeaResponse[JWTSignResult]:
|
1049
1075
|
"""
|
@@ -1072,7 +1098,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1072
1098
|
)
|
1073
1099
|
"""
|
1074
1100
|
input = JWTSignRequest(id=id, payload=payload)
|
1075
|
-
return await self.request.post("v1/key/sign/jwt", JWTSignResult, data=input.
|
1101
|
+
return await self.request.post("v1/key/sign/jwt", JWTSignResult, data=input.model_dump(exclude_none=True))
|
1076
1102
|
|
1077
1103
|
# Get endpoint
|
1078
1104
|
async def jwk_get(self, id: str, version: Optional[str] = None) -> PangeaResponse[JWKGetResult]:
|
@@ -1103,7 +1129,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1103
1129
|
)
|
1104
1130
|
"""
|
1105
1131
|
input = JWKGetRequest(id=id, version=version)
|
1106
|
-
return await self.request.post("v1/get/jwk", JWKGetResult, data=input.
|
1132
|
+
return await self.request.post("v1/get/jwk", JWKGetResult, data=input.model_dump(exclude_none=True))
|
1107
1133
|
|
1108
1134
|
# State change
|
1109
1135
|
async def state_change(
|
@@ -1142,7 +1168,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1142
1168
|
)
|
1143
1169
|
"""
|
1144
1170
|
input = StateChangeRequest(id=id, state=state, version=version, destroy_period=destroy_period)
|
1145
|
-
return await self.request.post("v1/state/change", StateChangeResult, data=input.
|
1171
|
+
return await self.request.post("v1/state/change", StateChangeResult, data=input.model_dump(exclude_none=True))
|
1146
1172
|
|
1147
1173
|
# Folder create
|
1148
1174
|
async def folder_create(
|
@@ -1179,7 +1205,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1179
1205
|
)
|
1180
1206
|
"""
|
1181
1207
|
input = FolderCreateRequest(name=name, folder=folder, metadata=metadata, tags=tags)
|
1182
|
-
return await self.request.post("v1/folder/create", FolderCreateResult, data=input.
|
1208
|
+
return await self.request.post("v1/folder/create", FolderCreateResult, data=input.model_dump(exclude_none=True))
|
1183
1209
|
|
1184
1210
|
# Encrypt structured
|
1185
1211
|
async def encrypt_structured(
|
@@ -1227,7 +1253,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1227
1253
|
return await self.request.post(
|
1228
1254
|
"v1/key/encrypt/structured",
|
1229
1255
|
EncryptStructuredResult,
|
1230
|
-
data=input.
|
1256
|
+
data=input.model_dump(exclude_none=True),
|
1231
1257
|
)
|
1232
1258
|
|
1233
1259
|
# Decrypt structured
|
@@ -1276,5 +1302,94 @@ class VaultAsync(ServiceBaseAsync):
|
|
1276
1302
|
return await self.request.post(
|
1277
1303
|
"v1/key/decrypt/structured",
|
1278
1304
|
EncryptStructuredResult,
|
1279
|
-
data=input.
|
1305
|
+
data=input.model_dump(exclude_none=True),
|
1306
|
+
)
|
1307
|
+
|
1308
|
+
async def encrypt_transform(
|
1309
|
+
self,
|
1310
|
+
id: str,
|
1311
|
+
plain_text: str,
|
1312
|
+
alphabet: TransformAlphabet,
|
1313
|
+
tweak: str | None = None,
|
1314
|
+
version: int | None = None,
|
1315
|
+
) -> PangeaResponse[EncryptTransformResult]:
|
1316
|
+
"""
|
1317
|
+
Encrypt transform
|
1318
|
+
|
1319
|
+
Encrypt using a format-preserving algorithm (FPE).
|
1320
|
+
|
1321
|
+
OperationId: vault_post_v1_key_encrypt_transform
|
1322
|
+
|
1323
|
+
Args:
|
1324
|
+
id: The item ID.
|
1325
|
+
plain_text: A message to be encrypted.
|
1326
|
+
alphabet: Set of characters to use for format-preserving encryption (FPE).
|
1327
|
+
tweak: User provided tweak string. If not provided, a random string will be generated and returned.
|
1328
|
+
version: The item version. Defaults to the current version.
|
1329
|
+
|
1330
|
+
Raises:
|
1331
|
+
PangeaAPIException: If an API error happens.
|
1332
|
+
|
1333
|
+
Returns:
|
1334
|
+
A `PangeaResponse` containing the encrypted message.
|
1335
|
+
|
1336
|
+
Examples:
|
1337
|
+
await vault.encrypt_transform(
|
1338
|
+
id="pvi_[...]",
|
1339
|
+
plain_text="message to encrypt",
|
1340
|
+
alphabet=TransformAlphabet.ALPHANUMERIC,
|
1341
|
+
tweak="MTIzMTIzMT==",
|
1342
|
+
)
|
1343
|
+
"""
|
1344
|
+
|
1345
|
+
input = EncryptTransformRequest(
|
1346
|
+
id=id,
|
1347
|
+
plain_text=plain_text,
|
1348
|
+
alphabet=alphabet,
|
1349
|
+
tweak=tweak,
|
1350
|
+
version=version,
|
1351
|
+
)
|
1352
|
+
return await self.request.post(
|
1353
|
+
"v1/key/encrypt/transform",
|
1354
|
+
EncryptTransformResult,
|
1355
|
+
data=input.model_dump(exclude_none=True),
|
1356
|
+
)
|
1357
|
+
|
1358
|
+
async def decrypt_transform(
|
1359
|
+
self, id: str, cipher_text: str, tweak: str, alphabet: TransformAlphabet, version: int | None = None
|
1360
|
+
) -> PangeaResponse[DecryptTransformResult]:
|
1361
|
+
"""
|
1362
|
+
Decrypt transform
|
1363
|
+
|
1364
|
+
Decrypt using a format-preserving algorithm (FPE).
|
1365
|
+
|
1366
|
+
OperationId: vault_post_v1_key_decrypt_transform
|
1367
|
+
|
1368
|
+
Args:
|
1369
|
+
id: The item ID.
|
1370
|
+
cipher_text: A message encrypted by Vault.
|
1371
|
+
tweak: User provided tweak string.
|
1372
|
+
alphabet: Set of characters to use for format-preserving encryption (FPE).
|
1373
|
+
version: The item version. Defaults to the current version.
|
1374
|
+
|
1375
|
+
Raises:
|
1376
|
+
PangeaAPIException: If an API error happens.
|
1377
|
+
|
1378
|
+
Returns:
|
1379
|
+
A `PangeaResponse` containing the decrypted message.
|
1380
|
+
|
1381
|
+
Examples:
|
1382
|
+
await vault.decrypt_transform(
|
1383
|
+
id="pvi_[...]",
|
1384
|
+
cipher_text="encrypted message",
|
1385
|
+
tweak="MTIzMTIzMT==",
|
1386
|
+
alphabet=TransformAlphabet.ALPHANUMERIC,
|
1387
|
+
)
|
1388
|
+
"""
|
1389
|
+
|
1390
|
+
input = DecryptTransformRequest(id=id, cipher_text=cipher_text, tweak=tweak, alphabet=alphabet, version=version)
|
1391
|
+
return await self.request.post(
|
1392
|
+
"v1/key/decrypt/transform",
|
1393
|
+
DecryptTransformResult,
|
1394
|
+
data=input.model_dump(exclude_none=True),
|
1280
1395
|
)
|
pangea/dump_audit.py
CHANGED
@@ -19,7 +19,7 @@ from pangea.utils import default_encoder
|
|
19
19
|
|
20
20
|
|
21
21
|
def dump_event(output: io.TextIOWrapper, row: SearchEvent, resp: PangeaResponse[SearchOutput]):
|
22
|
-
row_data = filter_deep_none(row.
|
22
|
+
row_data = filter_deep_none(row.model_dump())
|
23
23
|
if resp.result and resp.result.root:
|
24
24
|
row_data["tree_size"] = resp.result.root.size
|
25
25
|
output.write(json.dumps(row_data, default=default_encoder) + "\n")
|
pangea/request.py
CHANGED
@@ -5,12 +5,12 @@ import copy
|
|
5
5
|
import json
|
6
6
|
import logging
|
7
7
|
import time
|
8
|
-
from typing import Dict, List, Optional, Tuple, Type, Union
|
8
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Type, Union
|
9
9
|
|
10
|
-
import aiohttp
|
11
10
|
import requests
|
12
11
|
from requests.adapters import HTTPAdapter, Retry
|
13
12
|
from requests_toolbelt import MultipartDecoder # type: ignore
|
13
|
+
from typing_extensions import TypeVar
|
14
14
|
|
15
15
|
import pangea
|
16
16
|
import pangea.exceptions as pe
|
@@ -18,6 +18,9 @@ from pangea.config import PangeaConfig
|
|
18
18
|
from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
|
19
19
|
from pangea.utils import default_encoder
|
20
20
|
|
21
|
+
if TYPE_CHECKING:
|
22
|
+
import aiohttp
|
23
|
+
|
21
24
|
|
22
25
|
class MultipartResponse(object):
|
23
26
|
pangea_json: Dict[str, str]
|
@@ -187,6 +190,9 @@ class PangeaRequestBase(object):
|
|
187
190
|
raise pe.PangeaAPIException(f"{summary} ", response)
|
188
191
|
|
189
192
|
|
193
|
+
TResult = TypeVar("TResult", bound=PangeaResponseResult)
|
194
|
+
|
195
|
+
|
190
196
|
class PangeaRequest(PangeaRequestBase):
|
191
197
|
"""An object that makes direct calls to Pangea Service APIs.
|
192
198
|
|
@@ -202,12 +208,12 @@ class PangeaRequest(PangeaRequestBase):
|
|
202
208
|
def post(
|
203
209
|
self,
|
204
210
|
endpoint: str,
|
205
|
-
result_class: Type[
|
211
|
+
result_class: Type[TResult],
|
206
212
|
data: Union[str, Dict] = {},
|
207
213
|
files: Optional[List[Tuple]] = None,
|
208
214
|
poll_result: bool = True,
|
209
215
|
url: Optional[str] = None,
|
210
|
-
) -> PangeaResponse:
|
216
|
+
) -> PangeaResponse[TResult]:
|
211
217
|
"""Makes the POST call to a Pangea Service endpoint.
|
212
218
|
|
213
219
|
Args:
|
@@ -318,14 +324,17 @@ class PangeaRequest(PangeaRequestBase):
|
|
318
324
|
return self.session.post(url, headers=headers, data=data_send, files=files)
|
319
325
|
|
320
326
|
def _http_post_process(
|
321
|
-
self,
|
327
|
+
self,
|
328
|
+
data: Union[str, Dict] = {},
|
329
|
+
files: Optional[Sequence[Tuple[str, Tuple[Any, str, str]]]] = None,
|
330
|
+
multipart_post: bool = True,
|
322
331
|
):
|
323
332
|
if files:
|
324
333
|
if multipart_post is True:
|
325
334
|
data_send: str = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
326
335
|
multi = [("request", (None, data_send, "application/json"))]
|
327
|
-
multi.extend(files)
|
328
|
-
files = multi
|
336
|
+
multi.extend(files)
|
337
|
+
files = multi
|
329
338
|
return None, files
|
330
339
|
else:
|
331
340
|
# Post to presigned url as form
|
@@ -343,7 +352,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
343
352
|
|
344
353
|
return data, files
|
345
354
|
|
346
|
-
def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse[
|
355
|
+
def _handle_queued_result(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
|
347
356
|
if self._queued_retry_enabled and response.http_status == 202:
|
348
357
|
self.logger.debug(
|
349
358
|
json.dumps(
|
@@ -355,7 +364,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
355
364
|
|
356
365
|
return response
|
357
366
|
|
358
|
-
def get(self, path: str, result_class: Type[
|
367
|
+
def get(self, path: str, result_class: Type[TResult], check_response: bool = True) -> PangeaResponse[TResult]:
|
359
368
|
"""Makes the GET call to a Pangea Service endpoint.
|
360
369
|
|
361
370
|
Args:
|
@@ -427,21 +436,21 @@ class PangeaRequest(PangeaRequestBase):
|
|
427
436
|
raise pe.DownloadFileError(f"Failed to download file. Status: {response.status_code}", response.text)
|
428
437
|
|
429
438
|
def poll_result_by_id(
|
430
|
-
self, request_id: str, result_class:
|
431
|
-
):
|
439
|
+
self, request_id: str, result_class: Type[TResult], check_response: bool = True
|
440
|
+
) -> PangeaResponse[TResult]:
|
432
441
|
path = self._get_poll_path(request_id)
|
433
442
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
434
|
-
return self.get(path, result_class, check_response=check_response)
|
443
|
+
return self.get(path, result_class, check_response=check_response)
|
435
444
|
|
436
445
|
def poll_result_once(
|
437
|
-
self, response: PangeaResponse, check_response: bool = True
|
438
|
-
) -> PangeaResponse[
|
446
|
+
self, response: PangeaResponse[TResult], check_response: bool = True
|
447
|
+
) -> PangeaResponse[TResult]:
|
439
448
|
request_id = response.request_id
|
440
449
|
if not request_id:
|
441
450
|
raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
|
442
451
|
|
443
452
|
if response.status != ResponseStatus.ACCEPTED.value:
|
444
|
-
raise pe.PangeaException("Response already
|
453
|
+
raise pe.PangeaException("Response already processed")
|
445
454
|
|
446
455
|
return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
447
456
|
|
@@ -453,8 +462,10 @@ class PangeaRequest(PangeaRequestBase):
|
|
453
462
|
) -> PangeaResponse:
|
454
463
|
# Send request
|
455
464
|
try:
|
456
|
-
# This should return 202 (AcceptedRequestException)
|
457
|
-
|
465
|
+
# This should return 202 (AcceptedRequestException)
|
466
|
+
resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
467
|
+
raise pe.PresignedURLException("Should return 202", resp)
|
468
|
+
|
458
469
|
except pe.AcceptedRequestException as e:
|
459
470
|
accepted_exception = e
|
460
471
|
except Exception as e:
|
@@ -512,9 +523,6 @@ class PangeaRequest(PangeaRequestBase):
|
|
512
523
|
raise AttributeError("files attribute should have at least 1 file")
|
513
524
|
|
514
525
|
response = self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
|
515
|
-
|
516
|
-
if response.success: # This should only happen when uploading a zero bytes file
|
517
|
-
return response.raw_response
|
518
526
|
if response.accepted_result is None:
|
519
527
|
raise pe.PangeaException("No accepted_result field when requesting presigned url")
|
520
528
|
if response.accepted_result.post_url is None:
|
@@ -526,7 +534,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
526
534
|
self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
527
535
|
return response.raw_response
|
528
536
|
|
529
|
-
def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse[
|
537
|
+
def _poll_result_retry(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
|
530
538
|
retry_count = 1
|
531
539
|
start = time.time()
|
532
540
|
|
@@ -538,9 +546,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
538
546
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
|
539
547
|
return self._check_response(response)
|
540
548
|
|
541
|
-
def _poll_presigned_url(
|
542
|
-
self, response: PangeaResponse[Type[PangeaResponseResult]]
|
543
|
-
) -> PangeaResponse[Type[PangeaResponseResult]]:
|
549
|
+
def _poll_presigned_url(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
|
544
550
|
if response.http_status != 202:
|
545
551
|
raise AttributeError("Response should be 202")
|
546
552
|
|
pangea/response.py
CHANGED
@@ -3,16 +3,15 @@
|
|
3
3
|
import datetime
|
4
4
|
import enum
|
5
5
|
import os
|
6
|
-
from typing import Any, Dict, Generic, List, Optional, Type,
|
6
|
+
from typing import Any, Dict, Generic, List, Optional, Type, Union
|
7
7
|
|
8
8
|
import aiohttp
|
9
9
|
import requests
|
10
|
-
from pydantic import BaseModel
|
10
|
+
from pydantic import BaseModel, ConfigDict, PlainSerializer
|
11
|
+
from typing_extensions import Annotated, TypeVar
|
11
12
|
|
12
13
|
from pangea.utils import format_datetime
|
13
14
|
|
14
|
-
T = TypeVar("T")
|
15
|
-
|
16
15
|
|
17
16
|
class AttachedFile(object):
|
18
17
|
filename: str
|
@@ -29,7 +28,6 @@ class AttachedFile(object):
|
|
29
28
|
filename = self.filename if self.filename else "default_save_filename"
|
30
29
|
|
31
30
|
filepath = os.path.join(dest_folder, filename)
|
32
|
-
filepath = self._find_available_file(filepath)
|
33
31
|
directory = os.path.dirname(filepath)
|
34
32
|
if not os.path.exists(directory):
|
35
33
|
os.makedirs(directory)
|
@@ -37,17 +35,6 @@ class AttachedFile(object):
|
|
37
35
|
with open(filepath, "wb") as file:
|
38
36
|
file.write(self.file)
|
39
37
|
|
40
|
-
def _find_available_file(self, file_path):
|
41
|
-
base_name, ext = os.path.splitext(file_path)
|
42
|
-
counter = 1
|
43
|
-
while os.path.exists(file_path):
|
44
|
-
if ext:
|
45
|
-
file_path = f"{base_name}_{counter}{ext}"
|
46
|
-
else:
|
47
|
-
file_path = f"{base_name}_{counter}"
|
48
|
-
counter += 1
|
49
|
-
return file_path
|
50
|
-
|
51
38
|
|
52
39
|
class TransferMethod(str, enum.Enum):
|
53
40
|
MULTIPART = "multipart"
|
@@ -63,24 +50,17 @@ class TransferMethod(str, enum.Enum):
|
|
63
50
|
return str(self.value)
|
64
51
|
|
65
52
|
|
53
|
+
PangeaDateTime = Annotated[datetime.datetime, PlainSerializer(format_datetime)]
|
54
|
+
|
55
|
+
|
66
56
|
# API response should accept arbitrary fields to make them accept possible new parameters
|
67
57
|
class APIResponseModel(BaseModel):
|
68
|
-
|
69
|
-
arbitrary_types_allowed = True
|
70
|
-
# allow parameters despite they are not declared in model. Make SDK accept server new parameters
|
71
|
-
extra = "allow"
|
58
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
|
72
59
|
|
73
60
|
|
74
61
|
# API request models doesn't not allow arbitrary fields
|
75
62
|
class APIRequestModel(BaseModel):
|
76
|
-
|
77
|
-
arbitrary_types_allowed = True
|
78
|
-
extra = (
|
79
|
-
"allow" # allow parameters despite they are not declared in model. Make SDK accept server new parameters
|
80
|
-
)
|
81
|
-
json_encoders = {
|
82
|
-
datetime.datetime: format_datetime,
|
83
|
-
}
|
63
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
|
84
64
|
|
85
65
|
|
86
66
|
class PangeaResponseResult(APIResponseModel):
|
@@ -155,38 +135,50 @@ class ResponseStatus(str, enum.Enum):
|
|
155
135
|
|
156
136
|
|
157
137
|
class ResponseHeader(APIResponseModel):
|
158
|
-
"""
|
159
|
-
Pangea response API header.
|
160
|
-
|
161
|
-
Arguments:
|
162
|
-
request_id -- The request ID.
|
163
|
-
request_time -- The time the request was issued, ISO8601.
|
164
|
-
response_time -- The time the response was issued, ISO8601.
|
165
|
-
status -- Pangea response status
|
166
|
-
summary -- The summary of the response.
|
167
|
-
"""
|
138
|
+
"""Pangea response API header."""
|
168
139
|
|
169
140
|
request_id: str
|
141
|
+
"""A unique identifier assigned to each request made to the API."""
|
142
|
+
|
170
143
|
request_time: str
|
144
|
+
"""
|
145
|
+
Timestamp indicating the exact moment when a request is made to the API.
|
146
|
+
"""
|
147
|
+
|
171
148
|
response_time: str
|
149
|
+
"""
|
150
|
+
Duration it takes for the API to process a request and generate a response.
|
151
|
+
"""
|
152
|
+
|
172
153
|
status: str
|
154
|
+
"""
|
155
|
+
Represents the status or outcome of the API request.
|
156
|
+
"""
|
157
|
+
|
173
158
|
summary: str
|
159
|
+
"""
|
160
|
+
Provides a concise and brief overview of the purpose or primary objective of
|
161
|
+
the API endpoint.
|
162
|
+
"""
|
163
|
+
|
164
|
+
|
165
|
+
T = TypeVar("T", bound=PangeaResponseResult)
|
174
166
|
|
175
167
|
|
176
|
-
class PangeaResponse(Generic[T]
|
168
|
+
class PangeaResponse(ResponseHeader, Generic[T]):
|
177
169
|
raw_result: Optional[Dict[str, Any]] = None
|
178
170
|
raw_response: Optional[Union[requests.Response, aiohttp.ClientResponse]] = None
|
179
171
|
result: Optional[T] = None
|
180
172
|
pangea_error: Optional[PangeaError] = None
|
181
173
|
accepted_result: Optional[AcceptedResult] = None
|
182
|
-
result_class:
|
174
|
+
result_class: Type[T] = PangeaResponseResult # type: ignore[assignment]
|
183
175
|
_json: Any
|
184
176
|
attached_files: List[AttachedFile] = []
|
185
177
|
|
186
178
|
def __init__(
|
187
179
|
self,
|
188
180
|
response: requests.Response,
|
189
|
-
result_class:
|
181
|
+
result_class: Type[T],
|
190
182
|
json: dict,
|
191
183
|
attached_files: List[AttachedFile] = [],
|
192
184
|
):
|
@@ -198,7 +190,7 @@ class PangeaResponse(Generic[T], ResponseHeader):
|
|
198
190
|
self.attached_files = attached_files
|
199
191
|
|
200
192
|
self.result = (
|
201
|
-
self.result_class(**self.raw_result)
|
193
|
+
self.result_class(**self.raw_result)
|
202
194
|
if self.raw_result is not None and issubclass(self.result_class, PangeaResponseResult) and self.success
|
203
195
|
else None
|
204
196
|
)
|