pangea-sdk 3.8.0b1__py3-none-any.whl → 5.3.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- pangea/__init__.py +1 -1
- pangea/asyncio/file_uploader.py +1 -1
- pangea/asyncio/request.py +49 -31
- pangea/asyncio/services/__init__.py +2 -0
- pangea/asyncio/services/audit.py +192 -31
- pangea/asyncio/services/authn.py +187 -109
- pangea/asyncio/services/authz.py +285 -0
- pangea/asyncio/services/base.py +21 -2
- pangea/asyncio/services/embargo.py +2 -2
- pangea/asyncio/services/file_scan.py +24 -9
- pangea/asyncio/services/intel.py +108 -34
- pangea/asyncio/services/redact.py +72 -4
- pangea/asyncio/services/sanitize.py +217 -0
- pangea/asyncio/services/share.py +246 -73
- pangea/asyncio/services/vault.py +1710 -750
- pangea/crypto/rsa.py +135 -0
- pangea/deep_verify.py +7 -1
- pangea/dump_audit.py +9 -8
- pangea/request.py +83 -59
- pangea/response.py +49 -31
- pangea/services/__init__.py +2 -0
- pangea/services/audit/audit.py +205 -42
- pangea/services/audit/models.py +56 -8
- pangea/services/audit/signing.py +6 -5
- pangea/services/audit/util.py +3 -3
- pangea/services/authn/authn.py +140 -70
- pangea/services/authn/models.py +167 -11
- pangea/services/authz.py +400 -0
- pangea/services/base.py +39 -8
- pangea/services/embargo.py +2 -2
- pangea/services/file_scan.py +32 -15
- pangea/services/intel.py +157 -32
- pangea/services/redact.py +152 -4
- pangea/services/sanitize.py +388 -0
- pangea/services/share/share.py +683 -107
- pangea/services/vault/models/asymmetric.py +120 -18
- pangea/services/vault/models/common.py +439 -141
- pangea/services/vault/models/keys.py +94 -0
- pangea/services/vault/models/secret.py +27 -3
- pangea/services/vault/models/symmetric.py +68 -22
- pangea/services/vault/vault.py +1690 -749
- pangea/tools.py +6 -7
- pangea/utils.py +16 -27
- pangea/verify_audit.py +270 -83
- {pangea_sdk-3.8.0b1.dist-info → pangea_sdk-5.3.0.dist-info}/METADATA +43 -35
- pangea_sdk-5.3.0.dist-info/RECORD +56 -0
- {pangea_sdk-3.8.0b1.dist-info → pangea_sdk-5.3.0.dist-info}/WHEEL +1 -1
- pangea_sdk-3.8.0b1.dist-info/RECORD +0 -50
pangea/services/audit/audit.py
CHANGED
@@ -1,11 +1,14 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import datetime
|
4
6
|
import json
|
5
|
-
from typing import Any, Dict, List, Optional, Set, Tuple, Union
|
7
|
+
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
|
6
8
|
|
7
9
|
import pangea.exceptions as pexc
|
8
|
-
from pangea.
|
10
|
+
from pangea.config import PangeaConfig
|
11
|
+
from pangea.response import PangeaResponse, PangeaResponseResult
|
9
12
|
from pangea.services.audit.exceptions import AuditException, EventCorruption
|
10
13
|
from pangea.services.audit.models import (
|
11
14
|
DownloadFormat,
|
@@ -14,6 +17,7 @@ from pangea.services.audit.models import (
|
|
14
17
|
Event,
|
15
18
|
EventEnvelope,
|
16
19
|
EventVerification,
|
20
|
+
ExportRequest,
|
17
21
|
LogBulkRequest,
|
18
22
|
LogBulkResult,
|
19
23
|
LogEvent,
|
@@ -51,8 +55,8 @@ from pangea.utils import canonicalize_nested_json
|
|
51
55
|
|
52
56
|
class AuditBase:
|
53
57
|
def __init__(
|
54
|
-
self, private_key_file: str = "", public_key_info:
|
55
|
-
):
|
58
|
+
self, private_key_file: str = "", public_key_info: dict[str, str] = {}, tenant_id: str | None = None
|
59
|
+
) -> None:
|
56
60
|
self.pub_roots: Dict[int, PublishedRoot] = {}
|
57
61
|
self.buffer_data: Optional[str] = None
|
58
62
|
self.signer: Optional[Signer] = Signer(private_key_file) if private_key_file else None
|
@@ -195,7 +199,7 @@ class AuditBase:
|
|
195
199
|
|
196
200
|
# verify consistency proofs
|
197
201
|
if self.can_verify_consistency_proof(search_event):
|
198
|
-
if self.verify_consistency_proof(
|
202
|
+
if search_event.leaf_index is not None and self.verify_consistency_proof(search_event.leaf_index):
|
199
203
|
search_event.consistency_verification = EventVerification.PASS
|
200
204
|
else:
|
201
205
|
search_event.consistency_verification = EventVerification.FAIL
|
@@ -218,7 +222,7 @@ class AuditBase:
|
|
218
222
|
tree_sizes.difference_update(self.pub_roots.keys())
|
219
223
|
|
220
224
|
if tree_sizes:
|
221
|
-
arweave_roots = get_arweave_published_roots(result.root.tree_name,
|
225
|
+
arweave_roots = get_arweave_published_roots(result.root.tree_name, tree_sizes)
|
222
226
|
else:
|
223
227
|
arweave_roots = {}
|
224
228
|
|
@@ -280,7 +284,7 @@ class AuditBase:
|
|
280
284
|
"""
|
281
285
|
return event.published and event.leaf_index is not None and event.leaf_index >= 0 # type: ignore[return-value]
|
282
286
|
|
283
|
-
def verify_consistency_proof(self,
|
287
|
+
def verify_consistency_proof(self, tree_size: int) -> bool:
|
284
288
|
"""
|
285
289
|
Verify consistency proof
|
286
290
|
|
@@ -289,18 +293,17 @@ class AuditBase:
|
|
289
293
|
Read more at: [What is a consistency proof?](https://pangea.cloud/docs/audit/merkle-trees#what-is-a-consistency-proof)
|
290
294
|
|
291
295
|
Args:
|
292
|
-
|
293
|
-
event (SearchEvent): Audit event to be verified.
|
296
|
+
leaf_index (int): The tree size of the root to be verified.
|
294
297
|
|
295
298
|
Returns:
|
296
299
|
bool: True if consistency proof is verified, False otherwise.
|
297
300
|
"""
|
298
301
|
|
299
|
-
if
|
302
|
+
if tree_size == 0:
|
300
303
|
return True
|
301
304
|
|
302
|
-
curr_root = pub_roots.get(
|
303
|
-
prev_root = pub_roots.get(
|
305
|
+
curr_root = self.pub_roots.get(tree_size + 1)
|
306
|
+
prev_root = self.pub_roots.get(tree_size)
|
304
307
|
|
305
308
|
if not curr_root or not prev_root:
|
306
309
|
return False
|
@@ -310,9 +313,12 @@ class AuditBase:
|
|
310
313
|
):
|
311
314
|
return False
|
312
315
|
|
316
|
+
if curr_root.consistency_proof is None:
|
317
|
+
return False
|
318
|
+
|
313
319
|
curr_root_hash = decode_hash(curr_root.root_hash)
|
314
320
|
prev_root_hash = decode_hash(prev_root.root_hash)
|
315
|
-
proof = decode_consistency_proof(curr_root.consistency_proof)
|
321
|
+
proof = decode_consistency_proof(curr_root.consistency_proof)
|
316
322
|
|
317
323
|
return verify_consistency_proof(curr_root_hash, prev_root_hash, proof)
|
318
324
|
|
@@ -332,7 +338,9 @@ class AuditBase:
|
|
332
338
|
if audit_envelope and audit_envelope.signature and public_key:
|
333
339
|
v = Verifier()
|
334
340
|
verification = v.verify_signature(
|
335
|
-
audit_envelope.signature,
|
341
|
+
audit_envelope.signature,
|
342
|
+
canonicalize_event(Event(**audit_envelope.event)),
|
343
|
+
public_key,
|
336
344
|
)
|
337
345
|
if verification is not None:
|
338
346
|
return EventVerification.PASS if verification else EventVerification.FAIL
|
@@ -374,14 +382,32 @@ class Audit(ServiceBase, AuditBase):
|
|
374
382
|
|
375
383
|
def __init__(
|
376
384
|
self,
|
377
|
-
token,
|
378
|
-
config=None,
|
385
|
+
token: str,
|
386
|
+
config: PangeaConfig | None = None,
|
379
387
|
private_key_file: str = "",
|
380
|
-
public_key_info:
|
381
|
-
tenant_id:
|
382
|
-
logger_name="pangea",
|
383
|
-
config_id:
|
384
|
-
):
|
388
|
+
public_key_info: dict[str, str] = {},
|
389
|
+
tenant_id: str | None = None,
|
390
|
+
logger_name: str = "pangea",
|
391
|
+
config_id: str | None = None,
|
392
|
+
) -> None:
|
393
|
+
"""
|
394
|
+
Audit client
|
395
|
+
|
396
|
+
Initializes a new Audit client.
|
397
|
+
|
398
|
+
Args:
|
399
|
+
token: Pangea API token.
|
400
|
+
config: Configuration.
|
401
|
+
private_key_file: Private key filepath.
|
402
|
+
public_key_info: Public key information.
|
403
|
+
tenant_id: Tenant ID.
|
404
|
+
logger_name: Logger name.
|
405
|
+
config_id: Configuration ID.
|
406
|
+
|
407
|
+
Examples:
|
408
|
+
config = PangeaConfig(domain="pangea_domain")
|
409
|
+
audit = Audit(token="pangea_token", config=config)
|
410
|
+
"""
|
385
411
|
# FIXME: Temporary check to deprecate config_id from PangeaConfig.
|
386
412
|
# Delete it when deprecate PangeaConfig.config_id
|
387
413
|
if config_id and config is not None and config.config_id is not None:
|
@@ -466,7 +492,7 @@ class Audit(ServiceBase, AuditBase):
|
|
466
492
|
verbose: Optional[bool] = None,
|
467
493
|
) -> PangeaResponse[LogResult]:
|
468
494
|
"""
|
469
|
-
Log an
|
495
|
+
Log an event
|
470
496
|
|
471
497
|
Create a log entry in the Secure Audit Log.
|
472
498
|
|
@@ -475,6 +501,7 @@ class Audit(ServiceBase, AuditBase):
|
|
475
501
|
verify (bool, optional): True to verify logs consistency after response.
|
476
502
|
sign_local (bool, optional): True to sign event with local key.
|
477
503
|
verbose (bool, optional): True to get a more verbose response.
|
504
|
+
|
478
505
|
Raises:
|
479
506
|
AuditException: If an audit based api exception happens
|
480
507
|
PangeaAPIException: If an API Error happens
|
@@ -485,17 +512,13 @@ class Audit(ServiceBase, AuditBase):
|
|
485
512
|
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
|
486
513
|
|
487
514
|
Examples:
|
488
|
-
|
489
|
-
log_response = audit.log({"message": "hello world"}, verbose=True)
|
490
|
-
print(f"Response. Hash: {log_response.result.hash}")
|
491
|
-
except pe.PangeaAPIException as e:
|
492
|
-
print(f"Request Error: {e.response.summary}")
|
493
|
-
for err in e.errors:
|
494
|
-
print(f"\\t{err.detail} \\n")
|
515
|
+
response = audit.log_event({"message": "hello world"}, verbose=True)
|
495
516
|
"""
|
496
517
|
|
497
518
|
input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
|
498
|
-
response: PangeaResponse[LogResult] = self.request.post(
|
519
|
+
response: PangeaResponse[LogResult] = self.request.post(
|
520
|
+
"v1/log", LogResult, data=input.model_dump(exclude_none=True)
|
521
|
+
)
|
499
522
|
if response.success and response.result is not None:
|
500
523
|
self._process_log_result(response.result, verify=verify)
|
501
524
|
return response
|
@@ -535,7 +558,7 @@ class Audit(ServiceBase, AuditBase):
|
|
535
558
|
|
536
559
|
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
537
560
|
response: PangeaResponse[LogBulkResult] = self.request.post(
|
538
|
-
"v2/log", LogBulkResult, data=input.
|
561
|
+
"v2/log", LogBulkResult, data=input.model_dump(exclude_none=True)
|
539
562
|
)
|
540
563
|
|
541
564
|
if response.success and response.result is not None:
|
@@ -579,7 +602,7 @@ class Audit(ServiceBase, AuditBase):
|
|
579
602
|
try:
|
580
603
|
# Calling to v2 methods will return always a 202.
|
581
604
|
response: PangeaResponse[LogBulkResult] = self.request.post(
|
582
|
-
"v2/log_async", LogBulkResult, data=input.
|
605
|
+
"v2/log_async", LogBulkResult, data=input.model_dump(exclude_none=True), poll_result=False
|
583
606
|
)
|
584
607
|
except pexc.AcceptedRequestException as e:
|
585
608
|
return e.response
|
@@ -598,10 +621,11 @@ class Audit(ServiceBase, AuditBase):
|
|
598
621
|
end: Optional[Union[datetime.datetime, str]] = None,
|
599
622
|
limit: Optional[int] = None,
|
600
623
|
max_results: Optional[int] = None,
|
601
|
-
search_restriction: Optional[
|
624
|
+
search_restriction: Optional[Dict[str, Sequence[str]]] = None,
|
602
625
|
verbose: Optional[bool] = None,
|
603
626
|
verify_consistency: bool = False,
|
604
627
|
verify_events: bool = True,
|
628
|
+
return_context: Optional[bool] = None,
|
605
629
|
) -> PangeaResponse[SearchOutput]:
|
606
630
|
"""
|
607
631
|
Search the log
|
@@ -627,10 +651,11 @@ class Audit(ServiceBase, AuditBase):
|
|
627
651
|
end (datetime, optional): An RFC-3339 formatted timestamp, or relative time adjustment from the current time.
|
628
652
|
limit (int, optional): Optional[int] = None,
|
629
653
|
max_results (int, optional): Maximum number of results to return.
|
630
|
-
search_restriction (
|
654
|
+
search_restriction (Dict[str, Sequence[str]], optional): A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
|
631
655
|
verbose (bool, optional): If true, response include root and membership and consistency proofs.
|
632
656
|
verify_consistency (bool): True to verify logs consistency
|
633
657
|
verify_events (bool): True to verify hash events and signatures
|
658
|
+
return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
634
659
|
|
635
660
|
Raises:
|
636
661
|
AuditException: If an audit based api exception happens
|
@@ -664,10 +689,11 @@ class Audit(ServiceBase, AuditBase):
|
|
664
689
|
max_results=max_results,
|
665
690
|
search_restriction=search_restriction,
|
666
691
|
verbose=verbose,
|
692
|
+
return_context=return_context,
|
667
693
|
)
|
668
694
|
|
669
695
|
response: PangeaResponse[SearchOutput] = self.request.post(
|
670
|
-
"v1/search", SearchOutput, data=input.
|
696
|
+
"v1/search", SearchOutput, data=input.model_dump(exclude_none=True)
|
671
697
|
)
|
672
698
|
if verify_consistency and response.result is not None:
|
673
699
|
self.update_published_roots(response.result)
|
@@ -678,8 +704,10 @@ class Audit(ServiceBase, AuditBase):
|
|
678
704
|
id: str,
|
679
705
|
limit: Optional[int] = 20,
|
680
706
|
offset: Optional[int] = 0,
|
707
|
+
assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None,
|
681
708
|
verify_consistency: bool = False,
|
682
709
|
verify_events: bool = True,
|
710
|
+
return_context: Optional[bool] = None,
|
683
711
|
) -> PangeaResponse[SearchResultOutput]:
|
684
712
|
"""
|
685
713
|
Results of a search
|
@@ -692,8 +720,10 @@ class Audit(ServiceBase, AuditBase):
|
|
692
720
|
id (string): the id of a search action, found in `response.result.id`
|
693
721
|
limit (integer, optional): the maximum number of results to return, default is 20
|
694
722
|
offset (integer, optional): the position of the first result to return, default is 0
|
723
|
+
assert_search_restriction (Dict[str, Sequence[str]], optional): Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
|
695
724
|
verify_consistency (bool): True to verify logs consistency
|
696
725
|
verify_events (bool): True to verify hash events and signatures
|
726
|
+
return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
697
727
|
Raises:
|
698
728
|
AuditException: If an audit based api exception happens
|
699
729
|
PangeaAPIException: If an API Error happens
|
@@ -716,14 +746,117 @@ class Audit(ServiceBase, AuditBase):
|
|
716
746
|
id=id,
|
717
747
|
limit=limit,
|
718
748
|
offset=offset,
|
749
|
+
assert_search_restriction=assert_search_restriction,
|
750
|
+
return_context=return_context,
|
719
751
|
)
|
720
752
|
response: PangeaResponse[SearchResultOutput] = self.request.post(
|
721
|
-
"v1/results", SearchResultOutput, data=input.
|
753
|
+
"v1/results", SearchResultOutput, data=input.model_dump(exclude_none=True)
|
722
754
|
)
|
723
755
|
if verify_consistency and response.result is not None:
|
724
756
|
self.update_published_roots(response.result)
|
725
757
|
return self.handle_results_response(response, verify_consistency, verify_events)
|
726
758
|
|
759
|
+
def export(
|
760
|
+
self,
|
761
|
+
*,
|
762
|
+
format: DownloadFormat = DownloadFormat.CSV,
|
763
|
+
start: Optional[datetime.datetime] = None,
|
764
|
+
end: Optional[datetime.datetime] = None,
|
765
|
+
order: Optional[SearchOrder] = None,
|
766
|
+
order_by: Optional[str] = None,
|
767
|
+
verbose: bool = True,
|
768
|
+
) -> PangeaResponse[PangeaResponseResult]:
|
769
|
+
"""
|
770
|
+
Export from the audit log
|
771
|
+
|
772
|
+
Bulk export of data from the Secure Audit Log, with optional filtering.
|
773
|
+
|
774
|
+
OperationId: audit_post_v1_export
|
775
|
+
|
776
|
+
Args:
|
777
|
+
format: Format for the records.
|
778
|
+
start: The start of the time range to perform the search on.
|
779
|
+
end: The end of the time range to perform the search on. If omitted,
|
780
|
+
then all records up to the latest will be searched.
|
781
|
+
order: Specify the sort order of the response.
|
782
|
+
order_by: Name of column to sort the results by.
|
783
|
+
verbose: Whether or not to include the root hash of the tree and the
|
784
|
+
membership proof for each record.
|
785
|
+
|
786
|
+
Raises:
|
787
|
+
AuditException: If an audit based api exception happens
|
788
|
+
PangeaAPIException: If an API Error happens
|
789
|
+
|
790
|
+
Examples:
|
791
|
+
export_res = audit.export(verbose=False)
|
792
|
+
|
793
|
+
# Export may take several dozens of minutes, so polling for the result
|
794
|
+
# should be done in a loop. That is omitted here for brevity's sake.
|
795
|
+
try:
|
796
|
+
audit.poll_result(request_id=export_res.request_id)
|
797
|
+
except AcceptedRequestException:
|
798
|
+
# Retry later.
|
799
|
+
|
800
|
+
# Download the result when it's ready.
|
801
|
+
download_res = audit.download_results(request_id=export_res.request_id)
|
802
|
+
download_res.result.dest_url
|
803
|
+
# => https://pangea-runtime.s3.amazonaws.com/audit/xxxxx/search_results_[...]
|
804
|
+
"""
|
805
|
+
input = ExportRequest(
|
806
|
+
format=format,
|
807
|
+
start=start,
|
808
|
+
end=end,
|
809
|
+
order=order,
|
810
|
+
order_by=order_by,
|
811
|
+
verbose=verbose,
|
812
|
+
)
|
813
|
+
try:
|
814
|
+
return self.request.post(
|
815
|
+
"v1/export", PangeaResponseResult, data=input.model_dump(exclude_none=True), poll_result=False
|
816
|
+
)
|
817
|
+
except pexc.AcceptedRequestException as e:
|
818
|
+
return e.response
|
819
|
+
|
820
|
+
def log_stream(self, data: dict) -> PangeaResponse[PangeaResponseResult]:
|
821
|
+
"""
|
822
|
+
Log streaming endpoint
|
823
|
+
|
824
|
+
This API allows 3rd party vendors (like Auth0) to stream events to this
|
825
|
+
endpoint where the structure of the payload varies across different
|
826
|
+
vendors.
|
827
|
+
|
828
|
+
OperationId: audit_post_v1_log_stream
|
829
|
+
|
830
|
+
Args:
|
831
|
+
data: Event data. The exact schema of this will vary by vendor.
|
832
|
+
|
833
|
+
Raises:
|
834
|
+
AuditException: If an audit based api exception happens
|
835
|
+
PangeaAPIException: If an API Error happens
|
836
|
+
|
837
|
+
Examples:
|
838
|
+
data = {
|
839
|
+
"logs": [
|
840
|
+
{
|
841
|
+
"log_id": "some log ID",
|
842
|
+
"data": {
|
843
|
+
"date": "2024-03-29T17:26:50.193Z",
|
844
|
+
"type": "sapi",
|
845
|
+
"description": "Create a log stream",
|
846
|
+
"client_id": "some client ID",
|
847
|
+
"ip": "127.0.0.1",
|
848
|
+
"user_agent": "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0",
|
849
|
+
"user_id": "some user ID",
|
850
|
+
},
|
851
|
+
}
|
852
|
+
# ...
|
853
|
+
]
|
854
|
+
}
|
855
|
+
|
856
|
+
response = audit.log_stream(data)
|
857
|
+
"""
|
858
|
+
return self.request.post("v1/log_stream", PangeaResponseResult, data=data)
|
859
|
+
|
727
860
|
def root(self, tree_size: Optional[int] = None) -> PangeaResponse[RootResult]:
|
728
861
|
"""
|
729
862
|
Tamperproof verification
|
@@ -746,10 +879,14 @@ class Audit(ServiceBase, AuditBase):
|
|
746
879
|
response = audit.root(tree_size=7)
|
747
880
|
"""
|
748
881
|
input = RootRequest(tree_size=tree_size)
|
749
|
-
return self.request.post("v1/root", RootResult, data=input.
|
882
|
+
return self.request.post("v1/root", RootResult, data=input.model_dump(exclude_none=True))
|
750
883
|
|
751
884
|
def download_results(
|
752
|
-
self,
|
885
|
+
self,
|
886
|
+
result_id: Optional[str] = None,
|
887
|
+
format: DownloadFormat = DownloadFormat.CSV,
|
888
|
+
request_id: Optional[str] = None,
|
889
|
+
return_context: Optional[bool] = None,
|
753
890
|
) -> PangeaResponse[DownloadResult]:
|
754
891
|
"""
|
755
892
|
Download search results
|
@@ -761,6 +898,8 @@ class Audit(ServiceBase, AuditBase):
|
|
761
898
|
Args:
|
762
899
|
result_id: ID returned by the search API.
|
763
900
|
format: Format for the records.
|
901
|
+
request_id: ID returned by the export API.
|
902
|
+
return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
764
903
|
|
765
904
|
Returns:
|
766
905
|
URL where search results can be downloaded.
|
@@ -776,8 +915,13 @@ class Audit(ServiceBase, AuditBase):
|
|
776
915
|
)
|
777
916
|
"""
|
778
917
|
|
779
|
-
|
780
|
-
|
918
|
+
if request_id is None and result_id is None:
|
919
|
+
raise ValueError("must pass one of `request_id` or `result_id`")
|
920
|
+
|
921
|
+
input = DownloadRequest(
|
922
|
+
request_id=request_id, result_id=result_id, format=format, return_context=return_context
|
923
|
+
)
|
924
|
+
return self.request.post("v1/download_results", DownloadResult, data=input.model_dump(exclude_none=True))
|
781
925
|
|
782
926
|
def update_published_roots(self, result: SearchResultOutput):
|
783
927
|
"""Fetches series of published root hashes from Arweave
|
@@ -802,12 +946,31 @@ class Audit(ServiceBase, AuditBase):
|
|
802
946
|
for tree_size in tree_sizes:
|
803
947
|
pub_root = None
|
804
948
|
if tree_size in arweave_roots:
|
805
|
-
pub_root = PublishedRoot(**arweave_roots[tree_size].
|
949
|
+
pub_root = PublishedRoot(**arweave_roots[tree_size].model_dump(exclude_none=True))
|
806
950
|
pub_root.source = RootSource.ARWEAVE
|
807
951
|
elif self.allow_server_roots:
|
808
952
|
resp = self.root(tree_size=tree_size)
|
809
953
|
if resp.success and resp.result is not None:
|
810
|
-
pub_root = PublishedRoot(**resp.result.data.
|
954
|
+
pub_root = PublishedRoot(**resp.result.data.model_dump(exclude_none=True))
|
811
955
|
pub_root.source = RootSource.PANGEA
|
812
956
|
if pub_root is not None:
|
813
957
|
self.pub_roots[tree_size] = pub_root
|
958
|
+
|
959
|
+
self._fix_consistency_proofs(tree_sizes)
|
960
|
+
|
961
|
+
def _fix_consistency_proofs(self, tree_sizes: Iterable[int]) -> None:
|
962
|
+
# on very rare occasions, the consistency proof in Arweave may be wrong
|
963
|
+
# override it with the proof from pangea (not the root hash, just the proof)
|
964
|
+
for tree_size in tree_sizes:
|
965
|
+
if tree_size not in self.pub_roots or tree_size - 1 not in self.pub_roots:
|
966
|
+
continue
|
967
|
+
|
968
|
+
if self.pub_roots[tree_size].source == RootSource.PANGEA:
|
969
|
+
continue
|
970
|
+
|
971
|
+
if self.verify_consistency_proof(tree_size):
|
972
|
+
continue
|
973
|
+
|
974
|
+
resp = self.root(tree_size=tree_size)
|
975
|
+
if resp.success and resp.result is not None and resp.result.data is not None:
|
976
|
+
self.pub_roots[tree_size].consistency_proof = resp.result.data.consistency_proof
|
pangea/services/audit/models.py
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import datetime
|
4
6
|
import enum
|
5
|
-
from typing import Any, Dict, List, Optional, Union
|
7
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
6
8
|
|
7
|
-
from pangea.response import APIRequestModel, APIResponseModel, PangeaResponseResult
|
9
|
+
from pangea.response import APIRequestModel, APIResponseModel, PangeaDateTime, PangeaResponseResult
|
8
10
|
|
9
11
|
|
10
12
|
class EventVerification(str, enum.Enum):
|
@@ -19,14 +21,14 @@ class EventVerification(str, enum.Enum):
|
|
19
21
|
return str(self.value)
|
20
22
|
|
21
23
|
|
22
|
-
class Event(
|
24
|
+
class Event(Dict[str, Any]):
|
23
25
|
"""
|
24
26
|
Event to perform an auditable activity
|
25
27
|
|
26
28
|
Auxiliary class to be compatible with older SDKs
|
27
29
|
"""
|
28
30
|
|
29
|
-
def __init__(self, **data):
|
31
|
+
def __init__(self, **data) -> None:
|
30
32
|
super().__init__(**data)
|
31
33
|
|
32
34
|
@property
|
@@ -124,7 +126,7 @@ class EventEnvelope(APIResponseModel):
|
|
124
126
|
event: Dict[str, Any]
|
125
127
|
signature: Optional[str] = None
|
126
128
|
public_key: Optional[str] = None
|
127
|
-
received_at:
|
129
|
+
received_at: PangeaDateTime
|
128
130
|
|
129
131
|
|
130
132
|
class LogRequest(APIRequestModel):
|
@@ -269,6 +271,7 @@ class SearchRequest(APIRequestModel):
|
|
269
271
|
max_results -- Maximum number of results to return.
|
270
272
|
search_restriction -- A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
|
271
273
|
verbose -- If true, include root, membership and consistency proofs in response.
|
274
|
+
return_context -- Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
272
275
|
"""
|
273
276
|
|
274
277
|
query: str
|
@@ -279,8 +282,9 @@ class SearchRequest(APIRequestModel):
|
|
279
282
|
end: Optional[str] = None
|
280
283
|
limit: Optional[int] = None
|
281
284
|
max_results: Optional[int] = None
|
282
|
-
search_restriction: Optional[
|
285
|
+
search_restriction: Optional[Dict[str, Sequence[str]]] = None
|
283
286
|
verbose: Optional[bool] = None
|
287
|
+
return_context: Optional[bool] = None
|
284
288
|
|
285
289
|
|
286
290
|
class RootRequest(APIRequestModel):
|
@@ -361,6 +365,7 @@ class SearchEvent(APIResponseModel):
|
|
361
365
|
consistency_verification -- Consistency verification calculated if required.
|
362
366
|
membership_verification -- Membership verification calculated if required.
|
363
367
|
signature_verification -- Signature verification calculated if required.
|
368
|
+
fpe_context -- The context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
364
369
|
"""
|
365
370
|
|
366
371
|
envelope: EventEnvelope
|
@@ -371,6 +376,7 @@ class SearchEvent(APIResponseModel):
|
|
371
376
|
consistency_verification: EventVerification = EventVerification.NONE
|
372
377
|
membership_verification: EventVerification = EventVerification.NONE
|
373
378
|
signature_verification: EventVerification = EventVerification.NONE
|
379
|
+
fpe_context: Optional[str] = None
|
374
380
|
|
375
381
|
|
376
382
|
class SearchResultOutput(PangeaResponseResult):
|
@@ -404,7 +410,7 @@ class SearchOutput(SearchResultOutput):
|
|
404
410
|
"""
|
405
411
|
|
406
412
|
id: str
|
407
|
-
expires_at:
|
413
|
+
expires_at: PangeaDateTime
|
408
414
|
|
409
415
|
|
410
416
|
class SearchResultRequest(APIRequestModel):
|
@@ -415,11 +421,15 @@ class SearchResultRequest(APIRequestModel):
|
|
415
421
|
id -- A search results identifier returned by the search call.
|
416
422
|
limit -- Number of audit records to include from the first page of the results.
|
417
423
|
offset -- Offset from the start of the result set to start returning results from.
|
424
|
+
assert_search_restriction -- Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
|
425
|
+
return_context -- Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
418
426
|
"""
|
419
427
|
|
420
428
|
id: str
|
421
429
|
limit: Optional[int] = 20
|
422
430
|
offset: Optional[int] = 0
|
431
|
+
assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None
|
432
|
+
return_context: Optional[bool] = None
|
423
433
|
|
424
434
|
|
425
435
|
class DownloadFormat(str, enum.Enum):
|
@@ -437,13 +447,51 @@ class DownloadFormat(str, enum.Enum):
|
|
437
447
|
|
438
448
|
|
439
449
|
class DownloadRequest(APIRequestModel):
|
440
|
-
|
450
|
+
request_id: Optional[str] = None
|
451
|
+
"""ID returned by the export API."""
|
452
|
+
|
453
|
+
result_id: Optional[str] = None
|
441
454
|
"""ID returned by the search API."""
|
442
455
|
|
443
456
|
format: Optional[str] = None
|
444
457
|
"""Format for the records."""
|
445
458
|
|
459
|
+
return_context: Optional[bool] = None
|
460
|
+
"""Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption."""
|
461
|
+
|
446
462
|
|
447
463
|
class DownloadResult(PangeaResponseResult):
|
448
464
|
dest_url: str
|
449
465
|
"""URL where search results can be downloaded."""
|
466
|
+
|
467
|
+
expires_at: str
|
468
|
+
"""
|
469
|
+
The time when the results will no longer be available to page through via
|
470
|
+
the results API.
|
471
|
+
"""
|
472
|
+
|
473
|
+
|
474
|
+
class ExportRequest(APIRequestModel):
|
475
|
+
format: DownloadFormat = DownloadFormat.CSV
|
476
|
+
"""Format for the records."""
|
477
|
+
|
478
|
+
start: Optional[datetime.datetime] = None
|
479
|
+
"""The start of the time range to perform the search on."""
|
480
|
+
|
481
|
+
end: Optional[datetime.datetime] = None
|
482
|
+
"""
|
483
|
+
The end of the time range to perform the search on. If omitted, then all
|
484
|
+
records up to the latest will be searched.
|
485
|
+
"""
|
486
|
+
|
487
|
+
order_by: Optional[str] = None
|
488
|
+
"""Name of column to sort the results by."""
|
489
|
+
|
490
|
+
order: Optional[SearchOrder] = None
|
491
|
+
"""Specify the sort order of the response."""
|
492
|
+
|
493
|
+
verbose: bool = True
|
494
|
+
"""
|
495
|
+
Whether or not to include the root hash of the tree and the membership proof
|
496
|
+
for each record.
|
497
|
+
"""
|
pangea/services/audit/signing.py
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
from abc import ABC, abstractmethod
|
4
6
|
from typing import Optional
|
5
7
|
|
@@ -10,7 +12,7 @@ from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes, Pub
|
|
10
12
|
|
11
13
|
from pangea.exceptions import PangeaException
|
12
14
|
from pangea.services.audit.util import b64decode, b64decode_ascii, b64encode_ascii
|
13
|
-
from pangea.services.vault.models.
|
15
|
+
from pangea.services.vault.models.asymmetric import AsymmetricKeySigningAlgorithm
|
14
16
|
|
15
17
|
|
16
18
|
class AlgorithmSigner(ABC):
|
@@ -43,7 +45,7 @@ class ED25519Signer(AlgorithmSigner):
|
|
43
45
|
)
|
44
46
|
|
45
47
|
def get_algorithm(self) -> str:
|
46
|
-
return
|
48
|
+
return AsymmetricKeySigningAlgorithm.ED25519.value
|
47
49
|
|
48
50
|
|
49
51
|
signers = {
|
@@ -96,7 +98,7 @@ class Signer:
|
|
96
98
|
|
97
99
|
for func in (serialization.load_pem_private_key, serialization.load_ssh_private_key):
|
98
100
|
try:
|
99
|
-
return func(private_key, None)
|
101
|
+
return func(private_key, None)
|
100
102
|
except exceptions.UnsupportedAlgorithm as e:
|
101
103
|
raise e
|
102
104
|
except ValueError:
|
@@ -144,8 +146,7 @@ class Verifier:
|
|
144
146
|
for cls, verifier in verifiers.items():
|
145
147
|
if isinstance(pubkey, cls):
|
146
148
|
return verifier(pubkey).verify(message_bytes, signature_bytes)
|
147
|
-
|
148
|
-
raise PangeaException(f"Not supported public key type: {type(pubkey)}")
|
149
|
+
raise PangeaException(f"Not supported public key type: {type(pubkey)}")
|
149
150
|
|
150
151
|
def _decode_public_key(self, public_key: bytes):
|
151
152
|
"""Parse a public key in PEM or ssh format"""
|
pangea/services/audit/util.py
CHANGED
@@ -7,7 +7,7 @@ from binascii import hexlify, unhexlify
|
|
7
7
|
from dataclasses import dataclass
|
8
8
|
from datetime import datetime
|
9
9
|
from hashlib import sha256
|
10
|
-
from typing import Dict, List, Optional
|
10
|
+
from typing import Collection, Dict, List, Optional
|
11
11
|
|
12
12
|
import requests
|
13
13
|
|
@@ -61,7 +61,7 @@ def verify_hash(hash1: str, hash2: str) -> bool:
|
|
61
61
|
|
62
62
|
|
63
63
|
def verify_envelope_hash(envelope: EventEnvelope, hash: str):
|
64
|
-
return verify_hash(hash_dict(normalize_log(envelope.
|
64
|
+
return verify_hash(hash_dict(normalize_log(envelope.model_dump(exclude_none=True))), hash)
|
65
65
|
|
66
66
|
|
67
67
|
def canonicalize_event(event: Event) -> bytes:
|
@@ -192,7 +192,7 @@ def arweave_graphql_url():
|
|
192
192
|
return f"{ARWEAVE_BASE_URL}/graphql"
|
193
193
|
|
194
194
|
|
195
|
-
def get_arweave_published_roots(tree_name: str, tree_sizes:
|
195
|
+
def get_arweave_published_roots(tree_name: str, tree_sizes: Collection[int]) -> Dict[int, PublishedRoot]:
|
196
196
|
if len(tree_sizes) == 0:
|
197
197
|
return {}
|
198
198
|
|