pangea-sdk 3.1.0__py3-none-any.whl → 3.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +1 -1
- pangea/asyncio/request.py +122 -88
- pangea/asyncio/services/audit.py +82 -3
- pangea/asyncio/services/base.py +20 -3
- pangea/asyncio/services/file_scan.py +75 -5
- pangea/asyncio/services/redact.py +2 -7
- pangea/config.py +2 -2
- pangea/exceptions.py +1 -1
- pangea/request.py +131 -91
- pangea/response.py +10 -4
- pangea/services/audit/audit.py +163 -62
- pangea/services/audit/models.py +32 -0
- pangea/services/authn/authn.py +4 -1
- pangea/services/authn/models.py +2 -1
- pangea/services/base.py +17 -4
- pangea/services/file_scan.py +59 -3
- pangea/services/redact.py +2 -7
- pangea/services/vault/vault.py +2 -2
- pangea/utils.py +9 -2
- {pangea_sdk-3.1.0.dist-info → pangea_sdk-3.3.0.dist-info}/METADATA +13 -13
- {pangea_sdk-3.1.0.dist-info → pangea_sdk-3.3.0.dist-info}/RECORD +22 -22
- {pangea_sdk-3.1.0.dist-info → pangea_sdk-3.3.0.dist-info}/WHEEL +0 -0
pangea/services/audit/audit.py
CHANGED
@@ -2,14 +2,18 @@
|
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import datetime
|
4
4
|
import json
|
5
|
-
from typing import Any, Dict, Optional, Union
|
5
|
+
from typing import Any, Dict, List, Optional, Union
|
6
6
|
|
7
|
+
import pangea.exceptions as pexc
|
7
8
|
from pangea.response import PangeaResponse
|
8
9
|
from pangea.services.audit.exceptions import AuditException, EventCorruption
|
9
10
|
from pangea.services.audit.models import (
|
10
11
|
Event,
|
11
12
|
EventEnvelope,
|
12
13
|
EventVerification,
|
14
|
+
LogBulkRequest,
|
15
|
+
LogBulkResult,
|
16
|
+
LogEvent,
|
13
17
|
LogRequest,
|
14
18
|
LogResult,
|
15
19
|
PublishedRoot,
|
@@ -56,7 +60,29 @@ class AuditBase:
|
|
56
60
|
self.prev_unpublished_root_hash: Optional[str] = None
|
57
61
|
self.tenant_id = tenant_id
|
58
62
|
|
59
|
-
def
|
63
|
+
def _get_log_request(
|
64
|
+
self, events: Union[dict, List[dict]], sign_local: bool, verify: bool, verbose: Optional[bool]
|
65
|
+
) -> Union[LogRequest, LogBulkResult]:
|
66
|
+
if isinstance(events, list):
|
67
|
+
request_events: List[LogEvent] = []
|
68
|
+
for e in events:
|
69
|
+
request_events.append(self._process_log(e, sign_local=sign_local))
|
70
|
+
|
71
|
+
input = LogBulkRequest(events=request_events, verbose=verbose)
|
72
|
+
|
73
|
+
elif isinstance(events, dict):
|
74
|
+
log = self._process_log(events, sign_local=sign_local)
|
75
|
+
input = LogRequest(event=log.event, signature=log.signature, public_key=log.public_key)
|
76
|
+
input.verbose = True if verify else verbose
|
77
|
+
if verify and self.prev_unpublished_root_hash:
|
78
|
+
input.prev_root = self.prev_unpublished_root_hash
|
79
|
+
|
80
|
+
else:
|
81
|
+
raise AttributeError(f"events should be a dict or a list[dict] and it is {type(events)}")
|
82
|
+
|
83
|
+
return input
|
84
|
+
|
85
|
+
def _process_log(self, event: dict, sign_local: bool) -> LogEvent:
|
60
86
|
if event.get("tenant_id", None) is None and self.tenant_id:
|
61
87
|
event["tenant_id"] = self.tenant_id
|
62
88
|
|
@@ -66,70 +92,60 @@ class AuditBase:
|
|
66
92
|
if sign_local is True and self.signer is None:
|
67
93
|
raise AuditException("Error: the `signing` parameter set, but `signer` is not configured")
|
68
94
|
|
69
|
-
|
70
|
-
|
95
|
+
signature = None
|
96
|
+
pki = None
|
71
97
|
if sign_local is True:
|
72
98
|
data2sign = canonicalize_event(event)
|
73
99
|
signature = self.signer.sign(data2sign)
|
74
|
-
if signature is
|
75
|
-
input.signature = signature
|
76
|
-
else:
|
100
|
+
if signature is None:
|
77
101
|
raise AuditException("Error: failure signing message")
|
78
102
|
|
79
103
|
# Add public key value to public key info and serialize
|
80
|
-
self.
|
81
|
-
|
82
|
-
if verify:
|
83
|
-
input.verbose = True
|
84
|
-
if self.prev_unpublished_root_hash:
|
85
|
-
input.prev_root = self.prev_unpublished_root_hash
|
104
|
+
pki = self._get_public_key_info(self.signer, self.public_key_info)
|
86
105
|
|
87
|
-
return
|
88
|
-
|
89
|
-
def handle_log_response(self, response: PangeaResponse, verify: bool) -> PangeaResponse[LogResult]:
|
90
|
-
if not response.success:
|
91
|
-
return response
|
106
|
+
return LogEvent(event=event, signature=signature, public_key=pki)
|
92
107
|
|
93
|
-
|
108
|
+
def _process_log_result(self, result: LogResult, verify: bool):
|
109
|
+
new_unpublished_root_hash = result.unpublished_root
|
94
110
|
|
95
111
|
if verify:
|
96
|
-
if
|
112
|
+
if result.envelope:
|
97
113
|
# verify event hash
|
98
|
-
if
|
114
|
+
if result.hash and not verify_envelope_hash(result.envelope, result.hash):
|
99
115
|
# it's a extreme case, it's OK to raise an exception
|
100
|
-
raise EventCorruption("Error: Event hash failed.",
|
116
|
+
raise EventCorruption("Error: Event hash failed.", result.envelope)
|
101
117
|
|
102
|
-
|
118
|
+
result.signature_verification = self.verify_signature(result.envelope)
|
103
119
|
|
104
120
|
if new_unpublished_root_hash:
|
105
|
-
if
|
121
|
+
if result.membership_proof is not None:
|
106
122
|
# verify membership proofs
|
107
|
-
membership_proof = decode_membership_proof(
|
123
|
+
membership_proof = decode_membership_proof(result.membership_proof)
|
108
124
|
if verify_membership_proof(
|
109
|
-
node_hash=decode_hash(
|
125
|
+
node_hash=decode_hash(result.hash),
|
110
126
|
root_hash=decode_hash(new_unpublished_root_hash),
|
111
127
|
proof=membership_proof,
|
112
128
|
):
|
113
|
-
|
129
|
+
result.membership_verification = EventVerification.PASS
|
114
130
|
else:
|
115
|
-
|
131
|
+
result.membership_verification = EventVerification.FAIL
|
116
132
|
|
117
133
|
# verify consistency proofs (following events)
|
118
|
-
if
|
119
|
-
consistency_proof = decode_consistency_proof(
|
134
|
+
if result.consistency_proof is not None and self.prev_unpublished_root_hash:
|
135
|
+
consistency_proof = decode_consistency_proof(result.consistency_proof)
|
120
136
|
if verify_consistency_proof(
|
121
137
|
new_root=decode_hash(new_unpublished_root_hash),
|
122
138
|
prev_root=decode_hash(self.prev_unpublished_root_hash),
|
123
139
|
proof=consistency_proof,
|
124
140
|
):
|
125
|
-
|
141
|
+
result.consistency_verification = EventVerification.PASS
|
126
142
|
else:
|
127
|
-
|
143
|
+
result.consistency_verification = EventVerification.FAIL
|
128
144
|
|
129
145
|
# Update prev unpublished root
|
130
146
|
if new_unpublished_root_hash:
|
131
147
|
self.prev_unpublished_root_hash = new_unpublished_root_hash
|
132
|
-
return
|
148
|
+
return
|
133
149
|
|
134
150
|
def handle_results_response(
|
135
151
|
self, response: PangeaResponse[SearchResultOutput], verify_consistency: bool = False, verify_events: bool = True
|
@@ -348,12 +364,10 @@ class AuditBase:
|
|
348
364
|
else:
|
349
365
|
return EventVerification.NONE
|
350
366
|
|
351
|
-
def
|
367
|
+
def _get_public_key_info(self, signer: Signer, public_key_info: Dict[str, str]):
|
352
368
|
public_key_info["key"] = signer.get_public_key_PEM()
|
353
369
|
public_key_info["algorithm"] = signer.get_algorithm()
|
354
|
-
|
355
|
-
public_key_info, ensure_ascii=False, allow_nan=False, separators=(",", ":"), sort_keys=True
|
356
|
-
)
|
370
|
+
return json.dumps(public_key_info, ensure_ascii=False, allow_nan=False, separators=(",", ":"), sort_keys=True)
|
357
371
|
|
358
372
|
|
359
373
|
class Audit(ServiceBase, AuditBase):
|
@@ -444,16 +458,13 @@ class Audit(ServiceBase, AuditBase):
|
|
444
458
|
A PangeaResponse where the hash of event data and optional verbose
|
445
459
|
results are returned in the response.result field.
|
446
460
|
Available response fields can be found in our
|
447
|
-
[API documentation](https://pangea.cloud/docs/api/audit
|
461
|
+
[API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
|
448
462
|
|
449
463
|
Examples:
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
print(f"Request Error: {e.response.summary}")
|
455
|
-
for err in e.errors:
|
456
|
-
print(f"\\t{err.detail} \\n")
|
464
|
+
log_response = audit.log(
|
465
|
+
message="hello world",
|
466
|
+
verbose=True,
|
467
|
+
)
|
457
468
|
"""
|
458
469
|
|
459
470
|
event = Event(
|
@@ -481,6 +492,7 @@ class Audit(ServiceBase, AuditBase):
|
|
481
492
|
Log an entry
|
482
493
|
|
483
494
|
Create a log entry in the Secure Audit Log.
|
495
|
+
|
484
496
|
Args:
|
485
497
|
event (dict[str, Any]): event to be logged
|
486
498
|
verify (bool, optional): True to verify logs consistency after response.
|
@@ -493,11 +505,11 @@ class Audit(ServiceBase, AuditBase):
|
|
493
505
|
Returns:
|
494
506
|
A PangeaResponse where the hash of event data and optional verbose
|
495
507
|
results are returned in the response.result field.
|
496
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit
|
508
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
|
497
509
|
|
498
510
|
Examples:
|
499
511
|
try:
|
500
|
-
log_response = audit.log({"message"
|
512
|
+
log_response = audit.log({"message": "hello world"}, verbose=True)
|
501
513
|
print(f"Response. Hash: {log_response.result.hash}")
|
502
514
|
except pe.PangeaAPIException as e:
|
503
515
|
print(f"Request Error: {e.response.summary}")
|
@@ -505,9 +517,98 @@ class Audit(ServiceBase, AuditBase):
|
|
505
517
|
print(f"\\t{err.detail} \\n")
|
506
518
|
"""
|
507
519
|
|
508
|
-
input = self.
|
520
|
+
input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
|
509
521
|
response = self.request.post("v1/log", LogResult, data=input.dict(exclude_none=True))
|
510
|
-
|
522
|
+
if response.success:
|
523
|
+
self._process_log_result(response.result, verify=verify)
|
524
|
+
return response
|
525
|
+
|
526
|
+
def log_bulk(
|
527
|
+
self,
|
528
|
+
events: List[Dict[str, Any]],
|
529
|
+
sign_local: bool = False,
|
530
|
+
verbose: Optional[bool] = None,
|
531
|
+
) -> PangeaResponse[LogBulkResult]:
|
532
|
+
"""
|
533
|
+
Log multiple entries
|
534
|
+
|
535
|
+
Create multiple log entries in the Secure Audit Log.
|
536
|
+
|
537
|
+
OperationId: audit_post_v2_log
|
538
|
+
|
539
|
+
Args:
|
540
|
+
events (List[dict[str, Any]]): events to be logged
|
541
|
+
sign_local (bool, optional): True to sign event with local key.
|
542
|
+
verbose (bool, optional): True to get a more verbose response.
|
543
|
+
Raises:
|
544
|
+
AuditException: If an audit based api exception happens
|
545
|
+
PangeaAPIException: If an API Error happens
|
546
|
+
|
547
|
+
Returns:
|
548
|
+
A PangeaResponse where the hash of event data and optional verbose
|
549
|
+
results are returned in the response.result field.
|
550
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v2/log).
|
551
|
+
|
552
|
+
Examples:
|
553
|
+
log_response = audit.log_bulk(
|
554
|
+
events=[{"message": "hello world"}],
|
555
|
+
verbose=True,
|
556
|
+
)
|
557
|
+
"""
|
558
|
+
|
559
|
+
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
560
|
+
response = self.request.post("v2/log", LogBulkResult, data=input.dict(exclude_none=True))
|
561
|
+
|
562
|
+
if response.success:
|
563
|
+
for result in response.result.results:
|
564
|
+
self._process_log_result(result, verify=True)
|
565
|
+
return response
|
566
|
+
|
567
|
+
def log_bulk_async(
|
568
|
+
self,
|
569
|
+
events: List[Dict[str, Any]],
|
570
|
+
sign_local: bool = False,
|
571
|
+
verbose: Optional[bool] = None,
|
572
|
+
) -> PangeaResponse[LogBulkResult]:
|
573
|
+
"""
|
574
|
+
Log multiple entries asynchronously
|
575
|
+
|
576
|
+
Asynchronously create multiple log entries in the Secure Audit Log.
|
577
|
+
|
578
|
+
Args:
|
579
|
+
events (List[dict[str, Any]]): events to be logged
|
580
|
+
sign_local (bool, optional): True to sign event with local key.
|
581
|
+
verbose (bool, optional): True to get a more verbose response.
|
582
|
+
Raises:
|
583
|
+
AuditException: If an audit based api exception happens
|
584
|
+
PangeaAPIException: If an API Error happens
|
585
|
+
|
586
|
+
Returns:
|
587
|
+
A PangeaResponse where the hash of event data and optional verbose
|
588
|
+
results are returned in the response.result field.
|
589
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v2/log_async).
|
590
|
+
|
591
|
+
Examples:
|
592
|
+
log_response = audit.log_bulk_async(
|
593
|
+
events=[{"message": "hello world"}],
|
594
|
+
verbose=True,
|
595
|
+
)
|
596
|
+
"""
|
597
|
+
|
598
|
+
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
599
|
+
|
600
|
+
try:
|
601
|
+
# Calling to v2 methods will return always a 202.
|
602
|
+
response = self.request.post(
|
603
|
+
"v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
|
604
|
+
)
|
605
|
+
except pexc.AcceptedRequestException as e:
|
606
|
+
return e.response
|
607
|
+
|
608
|
+
if response.success:
|
609
|
+
for result in response.result.results:
|
610
|
+
self._process_log_result(result, verify=True)
|
611
|
+
return response
|
511
612
|
|
512
613
|
def search(
|
513
614
|
self,
|
@@ -558,11 +659,17 @@ class Audit(ServiceBase, AuditBase):
|
|
558
659
|
|
559
660
|
Returns:
|
560
661
|
A PangeaResponse[SearchOutput] where the first page of matched events is returned in the
|
561
|
-
response.result field. Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit
|
562
|
-
Pagination can be found in the [search results endpoint](https://pangea.cloud/docs/api/audit
|
662
|
+
response.result field. Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/search).
|
663
|
+
Pagination can be found in the [search results endpoint](https://pangea.cloud/docs/api/audit#/v1/results).
|
563
664
|
|
564
665
|
Examples:
|
565
|
-
response
|
666
|
+
response = audit.search(
|
667
|
+
query="message:test",
|
668
|
+
search_restriction={'source': ["monitor"]},
|
669
|
+
limit=1,
|
670
|
+
verify_consistency=True,
|
671
|
+
verify_events=True,
|
672
|
+
)
|
566
673
|
"""
|
567
674
|
|
568
675
|
if verify_consistency:
|
@@ -609,17 +716,11 @@ class Audit(ServiceBase, AuditBase):
|
|
609
716
|
PangeaAPIException: If an API Error happens
|
610
717
|
|
611
718
|
Examples:
|
612
|
-
|
613
|
-
|
614
|
-
search_restriction={'source': ["monitor"]},
|
615
|
-
limit=100,
|
616
|
-
verify_consistency=True,
|
617
|
-
verify_events=True)
|
618
|
-
|
619
|
-
result_res: PangeaResponse[SearchResultsOutput] = audit.results(
|
620
|
-
id=search_res.result.id,
|
719
|
+
response = audit.results(
|
720
|
+
id="pas_sqilrhruwu54uggihqj3aie24wrctakr",
|
621
721
|
limit=10,
|
622
|
-
offset=0
|
722
|
+
offset=0,
|
723
|
+
)
|
623
724
|
"""
|
624
725
|
|
625
726
|
if limit <= 0:
|
pangea/services/audit/models.py
CHANGED
@@ -146,6 +146,34 @@ class LogRequest(APIRequestModel):
|
|
146
146
|
prev_root: Optional[str] = None
|
147
147
|
|
148
148
|
|
149
|
+
class LogEvent(APIRequestModel):
|
150
|
+
"""
|
151
|
+
Event to perform a log action
|
152
|
+
|
153
|
+
Arguments:
|
154
|
+
event -- A structured event describing an auditable activity.
|
155
|
+
signature -- An optional client-side signature for forgery protection.
|
156
|
+
public_key -- The base64-encoded ed25519 public key used for the signature, if one is provided.
|
157
|
+
"""
|
158
|
+
|
159
|
+
event: Dict[str, Any]
|
160
|
+
signature: Optional[str] = None
|
161
|
+
public_key: Optional[str] = None
|
162
|
+
|
163
|
+
|
164
|
+
class LogBulkRequest(APIRequestModel):
|
165
|
+
"""
|
166
|
+
Request to perform a bulk log action
|
167
|
+
|
168
|
+
Arguments:
|
169
|
+
events -- A list structured events describing an auditable activity.
|
170
|
+
verbose -- If true, be verbose in the response; include membership proof, unpublished root and consistency proof, etc.
|
171
|
+
"""
|
172
|
+
|
173
|
+
events: List[LogEvent]
|
174
|
+
verbose: Optional[bool] = None
|
175
|
+
|
176
|
+
|
149
177
|
class LogResult(PangeaResponseResult):
|
150
178
|
"""
|
151
179
|
Result class after an audit log action
|
@@ -167,6 +195,10 @@ class LogResult(PangeaResponseResult):
|
|
167
195
|
signature_verification: EventVerification = EventVerification.NONE
|
168
196
|
|
169
197
|
|
198
|
+
class LogBulkResult(PangeaResponseResult):
|
199
|
+
results: List[LogResult] = []
|
200
|
+
|
201
|
+
|
170
202
|
class SearchRestriction(APIResponseModel):
|
171
203
|
"""
|
172
204
|
Set of restrictions when perform an audit search action
|
pangea/services/authn/authn.py
CHANGED
@@ -520,9 +520,10 @@ class AuthN(ServiceBase):
|
|
520
520
|
|
521
521
|
def update(
|
522
522
|
self,
|
523
|
-
disabled: bool,
|
523
|
+
disabled: Optional[bool] = None,
|
524
524
|
id: Optional[str] = None,
|
525
525
|
email: Optional[str] = None,
|
526
|
+
unlock: Optional[bool] = None,
|
526
527
|
) -> PangeaResponse[m.UserUpdateResult]:
|
527
528
|
"""
|
528
529
|
Update user's settings
|
@@ -534,6 +535,7 @@ class AuthN(ServiceBase):
|
|
534
535
|
Args:
|
535
536
|
disabled (bool): New disabled value.
|
536
537
|
Disabling a user account will prevent them from logging in.
|
538
|
+
unlock (bool): Unlock a user account if it has been locked out due to failed Authentication attempts.
|
537
539
|
id (str, optional): The identity of a user or a service
|
538
540
|
email (str, optional): An email address
|
539
541
|
|
@@ -552,6 +554,7 @@ class AuthN(ServiceBase):
|
|
552
554
|
id=id,
|
553
555
|
email=email,
|
554
556
|
disabled=disabled,
|
557
|
+
unlock=unlock,
|
555
558
|
)
|
556
559
|
|
557
560
|
return self.request.post("v2/user/update", m.UserUpdateResult, data=input.dict(exclude_none=True))
|
pangea/services/authn/models.py
CHANGED
@@ -363,7 +363,8 @@ class UserProfileUpdateResult(User):
|
|
363
363
|
class UserUpdateRequest(APIRequestModel):
|
364
364
|
id: Optional[str] = None
|
365
365
|
email: Optional[str] = None
|
366
|
-
disabled: bool
|
366
|
+
disabled: Optional[bool] = None
|
367
|
+
unlock: Optional[bool] = None
|
367
368
|
|
368
369
|
|
369
370
|
class UserUpdateResult(User):
|
pangea/services/base.py
CHANGED
@@ -3,13 +3,13 @@
|
|
3
3
|
|
4
4
|
import copy
|
5
5
|
import logging
|
6
|
-
from typing import Optional, Union
|
6
|
+
from typing import Optional, Type, Union
|
7
7
|
|
8
8
|
from pangea.asyncio.request import PangeaRequestAsync
|
9
9
|
from pangea.config import PangeaConfig
|
10
10
|
from pangea.exceptions import AcceptedRequestException
|
11
11
|
from pangea.request import PangeaRequest
|
12
|
-
from pangea.response import PangeaResponse
|
12
|
+
from pangea.response import PangeaResponse, PangeaResponseResult
|
13
13
|
|
14
14
|
|
15
15
|
class ServiceBase(object):
|
@@ -50,7 +50,13 @@ class ServiceBase(object):
|
|
50
50
|
|
51
51
|
return self._request
|
52
52
|
|
53
|
-
def poll_result(
|
53
|
+
def poll_result(
|
54
|
+
self,
|
55
|
+
exception: Optional[AcceptedRequestException] = None,
|
56
|
+
response: Optional[PangeaResponse] = None,
|
57
|
+
request_id: Optional[str] = None,
|
58
|
+
result_class: Union[Type[PangeaResponseResult], dict] = dict,
|
59
|
+
) -> PangeaResponse:
|
54
60
|
"""
|
55
61
|
Poll result
|
56
62
|
|
@@ -68,4 +74,11 @@ class ServiceBase(object):
|
|
68
74
|
Examples:
|
69
75
|
response = service.poll_result(exception)
|
70
76
|
"""
|
71
|
-
|
77
|
+
if exception is not None:
|
78
|
+
return self.request.poll_result_once(exception.response, check_response=True)
|
79
|
+
elif response is not None:
|
80
|
+
return self.request.poll_result_once(response, check_response=True)
|
81
|
+
elif request_id is not None:
|
82
|
+
return self.request.poll_result_by_id(request_id=request_id, result_class=result_class, check_response=True)
|
83
|
+
else:
|
84
|
+
raise AttributeError("Need to set exception, response or request_id")
|
pangea/services/file_scan.py
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import io
|
4
|
+
import logging
|
4
5
|
from typing import Dict, List, Optional
|
5
6
|
|
7
|
+
from pangea.request import PangeaConfig, PangeaRequest
|
6
8
|
from pangea.response import APIRequestModel, PangeaResponse, PangeaResponseResult, TransferMethod
|
7
|
-
from pangea.utils import
|
9
|
+
from pangea.utils import FileUploadParams, get_file_upload_params
|
8
10
|
|
9
11
|
from .base import ServiceBase
|
10
12
|
|
@@ -118,8 +120,11 @@ class FileScan(ServiceBase):
|
|
118
120
|
if file or file_path:
|
119
121
|
if file_path:
|
120
122
|
file = open(file_path, "rb")
|
121
|
-
if transfer_method == TransferMethod.DIRECT:
|
122
|
-
|
123
|
+
if transfer_method == TransferMethod.DIRECT or transfer_method == TransferMethod.POST_URL:
|
124
|
+
params = get_file_upload_params(file)
|
125
|
+
crc = params.crc_hex
|
126
|
+
sha = params.sha256_hex
|
127
|
+
size = params.size
|
123
128
|
else:
|
124
129
|
crc, sha, size = None, None, None
|
125
130
|
files = [("upload", ("filename", file, "application/octet-stream"))]
|
@@ -137,3 +142,54 @@ class FileScan(ServiceBase):
|
|
137
142
|
)
|
138
143
|
data = input.dict(exclude_none=True)
|
139
144
|
return self.request.post("v1/scan", FileScanResult, data=data, files=files, poll_result=sync_call)
|
145
|
+
|
146
|
+
def request_upload_url(
|
147
|
+
self,
|
148
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
149
|
+
params: Optional[FileUploadParams] = None,
|
150
|
+
verbose: Optional[bool] = None,
|
151
|
+
raw: Optional[bool] = None,
|
152
|
+
provider: Optional[str] = None,
|
153
|
+
) -> PangeaResponse[FileScanResult]:
|
154
|
+
input = FileScanRequest(
|
155
|
+
verbose=verbose,
|
156
|
+
raw=raw,
|
157
|
+
provider=provider,
|
158
|
+
transfer_method=transfer_method,
|
159
|
+
)
|
160
|
+
if params is not None and (
|
161
|
+
transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT
|
162
|
+
):
|
163
|
+
input.transfer_crc32c = params.crc_hex
|
164
|
+
input.transfer_sha256 = params.sha256_hex
|
165
|
+
input.transfer_size = params.size
|
166
|
+
|
167
|
+
data = input.dict(exclude_none=True)
|
168
|
+
return self.request.request_presigned_url("v1/scan", FileScanResult, data=data)
|
169
|
+
|
170
|
+
|
171
|
+
class FileUploader:
|
172
|
+
def __init__(self):
|
173
|
+
self.logger = logging.getLogger("pangea")
|
174
|
+
self._request = PangeaRequest(
|
175
|
+
config=PangeaConfig(),
|
176
|
+
token="",
|
177
|
+
service="FileScanUploader",
|
178
|
+
logger=self.logger,
|
179
|
+
)
|
180
|
+
|
181
|
+
def upload_file(
|
182
|
+
self,
|
183
|
+
url: str,
|
184
|
+
file: io.BufferedReader,
|
185
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
186
|
+
file_details: Optional[Dict] = None,
|
187
|
+
):
|
188
|
+
if transfer_method == TransferMethod.PUT_URL:
|
189
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
190
|
+
self._request.put_presigned_url(url=url, files=files)
|
191
|
+
elif transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT:
|
192
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
193
|
+
self._request.post_presigned_url(url=url, data=file_details, files=files)
|
194
|
+
else:
|
195
|
+
raise ValueError(f"Transfer method not supported: {transfer_method}")
|
pangea/services/redact.py
CHANGED
@@ -134,13 +134,8 @@ class Redact(ServiceBase):
|
|
134
134
|
|
135
135
|
service_name = "redact"
|
136
136
|
|
137
|
-
def __init__(
|
138
|
-
|
139
|
-
token,
|
140
|
-
config=None,
|
141
|
-
logger_name="pangea",
|
142
|
-
):
|
143
|
-
super().__init__(token, config, logger_name)
|
137
|
+
def __init__(self, token, config=None, logger_name="pangea", config_id: Optional[str] = None):
|
138
|
+
super().__init__(token, config, logger_name, config_id=config_id)
|
144
139
|
|
145
140
|
def redact(
|
146
141
|
self,
|
pangea/services/vault/vault.py
CHANGED
@@ -537,7 +537,7 @@ class Vault(ServiceBase):
|
|
537
537
|
|
538
538
|
Generate a symmetric key
|
539
539
|
|
540
|
-
OperationId: vault_post_v1_key_generate
|
540
|
+
OperationId: vault_post_v1_key_generate 2
|
541
541
|
|
542
542
|
Args:
|
543
543
|
algorithm (SymmetricAlgorithm): The algorithm of the key
|
@@ -611,7 +611,7 @@ class Vault(ServiceBase):
|
|
611
611
|
|
612
612
|
Generate an asymmetric key
|
613
613
|
|
614
|
-
OperationId: vault_post_v1_key_generate
|
614
|
+
OperationId: vault_post_v1_key_generate 1
|
615
615
|
|
616
616
|
Args:
|
617
617
|
algorithm (AsymmetricAlgorithm): The algorithm of the key
|
pangea/utils.py
CHANGED
@@ -8,6 +8,7 @@ from collections import OrderedDict
|
|
8
8
|
from hashlib import new, sha1, sha256, sha512
|
9
9
|
|
10
10
|
from google_crc32c import Checksum as CRC32C
|
11
|
+
from pydantic import BaseModel
|
11
12
|
|
12
13
|
|
13
14
|
def format_datetime(dt: datetime.datetime) -> str:
|
@@ -100,7 +101,13 @@ def get_prefix(hash: str, len: int = 5):
|
|
100
101
|
return hash[0:len]
|
101
102
|
|
102
103
|
|
103
|
-
|
104
|
+
class FileUploadParams(BaseModel):
|
105
|
+
crc_hex: str
|
106
|
+
sha256_hex: str
|
107
|
+
size: int
|
108
|
+
|
109
|
+
|
110
|
+
def get_file_upload_params(file: io.BufferedReader) -> FileUploadParams:
|
104
111
|
if "b" not in file.mode:
|
105
112
|
raise AttributeError("File need to be open in binary mode")
|
106
113
|
|
@@ -118,4 +125,4 @@ def get_presigned_url_upload_params(file: io.BufferedReader):
|
|
118
125
|
size += len(chunk)
|
119
126
|
|
120
127
|
file.seek(0) # restart reading
|
121
|
-
return crc.hexdigest().decode("utf-8"), sha.hexdigest(), size
|
128
|
+
return FileUploadParams(crc_hex=crc.hexdigest().decode("utf-8"), sha256_hex=sha.hexdigest(), size=size)
|