pangea-sdk 3.8.0b1__py3-none-any.whl → 5.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +1 -1
- pangea/asyncio/file_uploader.py +1 -1
- pangea/asyncio/request.py +49 -31
- pangea/asyncio/services/__init__.py +2 -0
- pangea/asyncio/services/audit.py +192 -31
- pangea/asyncio/services/authn.py +187 -109
- pangea/asyncio/services/authz.py +285 -0
- pangea/asyncio/services/base.py +21 -2
- pangea/asyncio/services/embargo.py +2 -2
- pangea/asyncio/services/file_scan.py +24 -9
- pangea/asyncio/services/intel.py +108 -34
- pangea/asyncio/services/redact.py +72 -4
- pangea/asyncio/services/sanitize.py +217 -0
- pangea/asyncio/services/share.py +246 -73
- pangea/asyncio/services/vault.py +1710 -750
- pangea/crypto/rsa.py +135 -0
- pangea/deep_verify.py +7 -1
- pangea/dump_audit.py +9 -8
- pangea/request.py +83 -59
- pangea/response.py +49 -31
- pangea/services/__init__.py +2 -0
- pangea/services/audit/audit.py +205 -42
- pangea/services/audit/models.py +56 -8
- pangea/services/audit/signing.py +6 -5
- pangea/services/audit/util.py +3 -3
- pangea/services/authn/authn.py +140 -70
- pangea/services/authn/models.py +167 -11
- pangea/services/authz.py +400 -0
- pangea/services/base.py +39 -8
- pangea/services/embargo.py +2 -2
- pangea/services/file_scan.py +32 -15
- pangea/services/intel.py +157 -32
- pangea/services/redact.py +152 -4
- pangea/services/sanitize.py +388 -0
- pangea/services/share/share.py +683 -107
- pangea/services/vault/models/asymmetric.py +120 -18
- pangea/services/vault/models/common.py +439 -141
- pangea/services/vault/models/keys.py +94 -0
- pangea/services/vault/models/secret.py +27 -3
- pangea/services/vault/models/symmetric.py +68 -22
- pangea/services/vault/vault.py +1690 -749
- pangea/tools.py +6 -7
- pangea/utils.py +16 -27
- pangea/verify_audit.py +270 -83
- {pangea_sdk-3.8.0b1.dist-info → pangea_sdk-5.3.0.dist-info}/METADATA +43 -35
- pangea_sdk-5.3.0.dist-info/RECORD +56 -0
- {pangea_sdk-3.8.0b1.dist-info → pangea_sdk-5.3.0.dist-info}/WHEEL +1 -1
- pangea_sdk-3.8.0b1.dist-info/RECORD +0 -50
pangea/services/audit/audit.py
CHANGED
@@ -1,11 +1,14 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import datetime
|
4
6
|
import json
|
5
|
-
from typing import Any, Dict, List, Optional, Set, Tuple, Union
|
7
|
+
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
|
6
8
|
|
7
9
|
import pangea.exceptions as pexc
|
8
|
-
from pangea.
|
10
|
+
from pangea.config import PangeaConfig
|
11
|
+
from pangea.response import PangeaResponse, PangeaResponseResult
|
9
12
|
from pangea.services.audit.exceptions import AuditException, EventCorruption
|
10
13
|
from pangea.services.audit.models import (
|
11
14
|
DownloadFormat,
|
@@ -14,6 +17,7 @@ from pangea.services.audit.models import (
|
|
14
17
|
Event,
|
15
18
|
EventEnvelope,
|
16
19
|
EventVerification,
|
20
|
+
ExportRequest,
|
17
21
|
LogBulkRequest,
|
18
22
|
LogBulkResult,
|
19
23
|
LogEvent,
|
@@ -51,8 +55,8 @@ from pangea.utils import canonicalize_nested_json
|
|
51
55
|
|
52
56
|
class AuditBase:
|
53
57
|
def __init__(
|
54
|
-
self, private_key_file: str = "", public_key_info:
|
55
|
-
):
|
58
|
+
self, private_key_file: str = "", public_key_info: dict[str, str] = {}, tenant_id: str | None = None
|
59
|
+
) -> None:
|
56
60
|
self.pub_roots: Dict[int, PublishedRoot] = {}
|
57
61
|
self.buffer_data: Optional[str] = None
|
58
62
|
self.signer: Optional[Signer] = Signer(private_key_file) if private_key_file else None
|
@@ -195,7 +199,7 @@ class AuditBase:
|
|
195
199
|
|
196
200
|
# verify consistency proofs
|
197
201
|
if self.can_verify_consistency_proof(search_event):
|
198
|
-
if self.verify_consistency_proof(
|
202
|
+
if search_event.leaf_index is not None and self.verify_consistency_proof(search_event.leaf_index):
|
199
203
|
search_event.consistency_verification = EventVerification.PASS
|
200
204
|
else:
|
201
205
|
search_event.consistency_verification = EventVerification.FAIL
|
@@ -218,7 +222,7 @@ class AuditBase:
|
|
218
222
|
tree_sizes.difference_update(self.pub_roots.keys())
|
219
223
|
|
220
224
|
if tree_sizes:
|
221
|
-
arweave_roots = get_arweave_published_roots(result.root.tree_name,
|
225
|
+
arweave_roots = get_arweave_published_roots(result.root.tree_name, tree_sizes)
|
222
226
|
else:
|
223
227
|
arweave_roots = {}
|
224
228
|
|
@@ -280,7 +284,7 @@ class AuditBase:
|
|
280
284
|
"""
|
281
285
|
return event.published and event.leaf_index is not None and event.leaf_index >= 0 # type: ignore[return-value]
|
282
286
|
|
283
|
-
def verify_consistency_proof(self,
|
287
|
+
def verify_consistency_proof(self, tree_size: int) -> bool:
|
284
288
|
"""
|
285
289
|
Verify consistency proof
|
286
290
|
|
@@ -289,18 +293,17 @@ class AuditBase:
|
|
289
293
|
Read more at: [What is a consistency proof?](https://pangea.cloud/docs/audit/merkle-trees#what-is-a-consistency-proof)
|
290
294
|
|
291
295
|
Args:
|
292
|
-
|
293
|
-
event (SearchEvent): Audit event to be verified.
|
296
|
+
leaf_index (int): The tree size of the root to be verified.
|
294
297
|
|
295
298
|
Returns:
|
296
299
|
bool: True if consistency proof is verified, False otherwise.
|
297
300
|
"""
|
298
301
|
|
299
|
-
if
|
302
|
+
if tree_size == 0:
|
300
303
|
return True
|
301
304
|
|
302
|
-
curr_root = pub_roots.get(
|
303
|
-
prev_root = pub_roots.get(
|
305
|
+
curr_root = self.pub_roots.get(tree_size + 1)
|
306
|
+
prev_root = self.pub_roots.get(tree_size)
|
304
307
|
|
305
308
|
if not curr_root or not prev_root:
|
306
309
|
return False
|
@@ -310,9 +313,12 @@ class AuditBase:
|
|
310
313
|
):
|
311
314
|
return False
|
312
315
|
|
316
|
+
if curr_root.consistency_proof is None:
|
317
|
+
return False
|
318
|
+
|
313
319
|
curr_root_hash = decode_hash(curr_root.root_hash)
|
314
320
|
prev_root_hash = decode_hash(prev_root.root_hash)
|
315
|
-
proof = decode_consistency_proof(curr_root.consistency_proof)
|
321
|
+
proof = decode_consistency_proof(curr_root.consistency_proof)
|
316
322
|
|
317
323
|
return verify_consistency_proof(curr_root_hash, prev_root_hash, proof)
|
318
324
|
|
@@ -332,7 +338,9 @@ class AuditBase:
|
|
332
338
|
if audit_envelope and audit_envelope.signature and public_key:
|
333
339
|
v = Verifier()
|
334
340
|
verification = v.verify_signature(
|
335
|
-
audit_envelope.signature,
|
341
|
+
audit_envelope.signature,
|
342
|
+
canonicalize_event(Event(**audit_envelope.event)),
|
343
|
+
public_key,
|
336
344
|
)
|
337
345
|
if verification is not None:
|
338
346
|
return EventVerification.PASS if verification else EventVerification.FAIL
|
@@ -374,14 +382,32 @@ class Audit(ServiceBase, AuditBase):
|
|
374
382
|
|
375
383
|
def __init__(
|
376
384
|
self,
|
377
|
-
token,
|
378
|
-
config=None,
|
385
|
+
token: str,
|
386
|
+
config: PangeaConfig | None = None,
|
379
387
|
private_key_file: str = "",
|
380
|
-
public_key_info:
|
381
|
-
tenant_id:
|
382
|
-
logger_name="pangea",
|
383
|
-
config_id:
|
384
|
-
):
|
388
|
+
public_key_info: dict[str, str] = {},
|
389
|
+
tenant_id: str | None = None,
|
390
|
+
logger_name: str = "pangea",
|
391
|
+
config_id: str | None = None,
|
392
|
+
) -> None:
|
393
|
+
"""
|
394
|
+
Audit client
|
395
|
+
|
396
|
+
Initializes a new Audit client.
|
397
|
+
|
398
|
+
Args:
|
399
|
+
token: Pangea API token.
|
400
|
+
config: Configuration.
|
401
|
+
private_key_file: Private key filepath.
|
402
|
+
public_key_info: Public key information.
|
403
|
+
tenant_id: Tenant ID.
|
404
|
+
logger_name: Logger name.
|
405
|
+
config_id: Configuration ID.
|
406
|
+
|
407
|
+
Examples:
|
408
|
+
config = PangeaConfig(domain="pangea_domain")
|
409
|
+
audit = Audit(token="pangea_token", config=config)
|
410
|
+
"""
|
385
411
|
# FIXME: Temporary check to deprecate config_id from PangeaConfig.
|
386
412
|
# Delete it when deprecate PangeaConfig.config_id
|
387
413
|
if config_id and config is not None and config.config_id is not None:
|
@@ -466,7 +492,7 @@ class Audit(ServiceBase, AuditBase):
|
|
466
492
|
verbose: Optional[bool] = None,
|
467
493
|
) -> PangeaResponse[LogResult]:
|
468
494
|
"""
|
469
|
-
Log an
|
495
|
+
Log an event
|
470
496
|
|
471
497
|
Create a log entry in the Secure Audit Log.
|
472
498
|
|
@@ -475,6 +501,7 @@ class Audit(ServiceBase, AuditBase):
|
|
475
501
|
verify (bool, optional): True to verify logs consistency after response.
|
476
502
|
sign_local (bool, optional): True to sign event with local key.
|
477
503
|
verbose (bool, optional): True to get a more verbose response.
|
504
|
+
|
478
505
|
Raises:
|
479
506
|
AuditException: If an audit based api exception happens
|
480
507
|
PangeaAPIException: If an API Error happens
|
@@ -485,17 +512,13 @@ class Audit(ServiceBase, AuditBase):
|
|
485
512
|
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
|
486
513
|
|
487
514
|
Examples:
|
488
|
-
|
489
|
-
log_response = audit.log({"message": "hello world"}, verbose=True)
|
490
|
-
print(f"Response. Hash: {log_response.result.hash}")
|
491
|
-
except pe.PangeaAPIException as e:
|
492
|
-
print(f"Request Error: {e.response.summary}")
|
493
|
-
for err in e.errors:
|
494
|
-
print(f"\\t{err.detail} \\n")
|
515
|
+
response = audit.log_event({"message": "hello world"}, verbose=True)
|
495
516
|
"""
|
496
517
|
|
497
518
|
input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
|
498
|
-
response: PangeaResponse[LogResult] = self.request.post(
|
519
|
+
response: PangeaResponse[LogResult] = self.request.post(
|
520
|
+
"v1/log", LogResult, data=input.model_dump(exclude_none=True)
|
521
|
+
)
|
499
522
|
if response.success and response.result is not None:
|
500
523
|
self._process_log_result(response.result, verify=verify)
|
501
524
|
return response
|
@@ -535,7 +558,7 @@ class Audit(ServiceBase, AuditBase):
|
|
535
558
|
|
536
559
|
input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
|
537
560
|
response: PangeaResponse[LogBulkResult] = self.request.post(
|
538
|
-
"v2/log", LogBulkResult, data=input.
|
561
|
+
"v2/log", LogBulkResult, data=input.model_dump(exclude_none=True)
|
539
562
|
)
|
540
563
|
|
541
564
|
if response.success and response.result is not None:
|
@@ -579,7 +602,7 @@ class Audit(ServiceBase, AuditBase):
|
|
579
602
|
try:
|
580
603
|
# Calling to v2 methods will return always a 202.
|
581
604
|
response: PangeaResponse[LogBulkResult] = self.request.post(
|
582
|
-
"v2/log_async", LogBulkResult, data=input.
|
605
|
+
"v2/log_async", LogBulkResult, data=input.model_dump(exclude_none=True), poll_result=False
|
583
606
|
)
|
584
607
|
except pexc.AcceptedRequestException as e:
|
585
608
|
return e.response
|
@@ -598,10 +621,11 @@ class Audit(ServiceBase, AuditBase):
|
|
598
621
|
end: Optional[Union[datetime.datetime, str]] = None,
|
599
622
|
limit: Optional[int] = None,
|
600
623
|
max_results: Optional[int] = None,
|
601
|
-
search_restriction: Optional[
|
624
|
+
search_restriction: Optional[Dict[str, Sequence[str]]] = None,
|
602
625
|
verbose: Optional[bool] = None,
|
603
626
|
verify_consistency: bool = False,
|
604
627
|
verify_events: bool = True,
|
628
|
+
return_context: Optional[bool] = None,
|
605
629
|
) -> PangeaResponse[SearchOutput]:
|
606
630
|
"""
|
607
631
|
Search the log
|
@@ -627,10 +651,11 @@ class Audit(ServiceBase, AuditBase):
|
|
627
651
|
end (datetime, optional): An RFC-3339 formatted timestamp, or relative time adjustment from the current time.
|
628
652
|
limit (int, optional): Optional[int] = None,
|
629
653
|
max_results (int, optional): Maximum number of results to return.
|
630
|
-
search_restriction (
|
654
|
+
search_restriction (Dict[str, Sequence[str]], optional): A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
|
631
655
|
verbose (bool, optional): If true, response include root and membership and consistency proofs.
|
632
656
|
verify_consistency (bool): True to verify logs consistency
|
633
657
|
verify_events (bool): True to verify hash events and signatures
|
658
|
+
return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
634
659
|
|
635
660
|
Raises:
|
636
661
|
AuditException: If an audit based api exception happens
|
@@ -664,10 +689,11 @@ class Audit(ServiceBase, AuditBase):
|
|
664
689
|
max_results=max_results,
|
665
690
|
search_restriction=search_restriction,
|
666
691
|
verbose=verbose,
|
692
|
+
return_context=return_context,
|
667
693
|
)
|
668
694
|
|
669
695
|
response: PangeaResponse[SearchOutput] = self.request.post(
|
670
|
-
"v1/search", SearchOutput, data=input.
|
696
|
+
"v1/search", SearchOutput, data=input.model_dump(exclude_none=True)
|
671
697
|
)
|
672
698
|
if verify_consistency and response.result is not None:
|
673
699
|
self.update_published_roots(response.result)
|
@@ -678,8 +704,10 @@ class Audit(ServiceBase, AuditBase):
|
|
678
704
|
id: str,
|
679
705
|
limit: Optional[int] = 20,
|
680
706
|
offset: Optional[int] = 0,
|
707
|
+
assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None,
|
681
708
|
verify_consistency: bool = False,
|
682
709
|
verify_events: bool = True,
|
710
|
+
return_context: Optional[bool] = None,
|
683
711
|
) -> PangeaResponse[SearchResultOutput]:
|
684
712
|
"""
|
685
713
|
Results of a search
|
@@ -692,8 +720,10 @@ class Audit(ServiceBase, AuditBase):
|
|
692
720
|
id (string): the id of a search action, found in `response.result.id`
|
693
721
|
limit (integer, optional): the maximum number of results to return, default is 20
|
694
722
|
offset (integer, optional): the position of the first result to return, default is 0
|
723
|
+
assert_search_restriction (Dict[str, Sequence[str]], optional): Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
|
695
724
|
verify_consistency (bool): True to verify logs consistency
|
696
725
|
verify_events (bool): True to verify hash events and signatures
|
726
|
+
return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
697
727
|
Raises:
|
698
728
|
AuditException: If an audit based api exception happens
|
699
729
|
PangeaAPIException: If an API Error happens
|
@@ -716,14 +746,117 @@ class Audit(ServiceBase, AuditBase):
|
|
716
746
|
id=id,
|
717
747
|
limit=limit,
|
718
748
|
offset=offset,
|
749
|
+
assert_search_restriction=assert_search_restriction,
|
750
|
+
return_context=return_context,
|
719
751
|
)
|
720
752
|
response: PangeaResponse[SearchResultOutput] = self.request.post(
|
721
|
-
"v1/results", SearchResultOutput, data=input.
|
753
|
+
"v1/results", SearchResultOutput, data=input.model_dump(exclude_none=True)
|
722
754
|
)
|
723
755
|
if verify_consistency and response.result is not None:
|
724
756
|
self.update_published_roots(response.result)
|
725
757
|
return self.handle_results_response(response, verify_consistency, verify_events)
|
726
758
|
|
759
|
+
def export(
|
760
|
+
self,
|
761
|
+
*,
|
762
|
+
format: DownloadFormat = DownloadFormat.CSV,
|
763
|
+
start: Optional[datetime.datetime] = None,
|
764
|
+
end: Optional[datetime.datetime] = None,
|
765
|
+
order: Optional[SearchOrder] = None,
|
766
|
+
order_by: Optional[str] = None,
|
767
|
+
verbose: bool = True,
|
768
|
+
) -> PangeaResponse[PangeaResponseResult]:
|
769
|
+
"""
|
770
|
+
Export from the audit log
|
771
|
+
|
772
|
+
Bulk export of data from the Secure Audit Log, with optional filtering.
|
773
|
+
|
774
|
+
OperationId: audit_post_v1_export
|
775
|
+
|
776
|
+
Args:
|
777
|
+
format: Format for the records.
|
778
|
+
start: The start of the time range to perform the search on.
|
779
|
+
end: The end of the time range to perform the search on. If omitted,
|
780
|
+
then all records up to the latest will be searched.
|
781
|
+
order: Specify the sort order of the response.
|
782
|
+
order_by: Name of column to sort the results by.
|
783
|
+
verbose: Whether or not to include the root hash of the tree and the
|
784
|
+
membership proof for each record.
|
785
|
+
|
786
|
+
Raises:
|
787
|
+
AuditException: If an audit based api exception happens
|
788
|
+
PangeaAPIException: If an API Error happens
|
789
|
+
|
790
|
+
Examples:
|
791
|
+
export_res = audit.export(verbose=False)
|
792
|
+
|
793
|
+
# Export may take several dozens of minutes, so polling for the result
|
794
|
+
# should be done in a loop. That is omitted here for brevity's sake.
|
795
|
+
try:
|
796
|
+
audit.poll_result(request_id=export_res.request_id)
|
797
|
+
except AcceptedRequestException:
|
798
|
+
# Retry later.
|
799
|
+
|
800
|
+
# Download the result when it's ready.
|
801
|
+
download_res = audit.download_results(request_id=export_res.request_id)
|
802
|
+
download_res.result.dest_url
|
803
|
+
# => https://pangea-runtime.s3.amazonaws.com/audit/xxxxx/search_results_[...]
|
804
|
+
"""
|
805
|
+
input = ExportRequest(
|
806
|
+
format=format,
|
807
|
+
start=start,
|
808
|
+
end=end,
|
809
|
+
order=order,
|
810
|
+
order_by=order_by,
|
811
|
+
verbose=verbose,
|
812
|
+
)
|
813
|
+
try:
|
814
|
+
return self.request.post(
|
815
|
+
"v1/export", PangeaResponseResult, data=input.model_dump(exclude_none=True), poll_result=False
|
816
|
+
)
|
817
|
+
except pexc.AcceptedRequestException as e:
|
818
|
+
return e.response
|
819
|
+
|
820
|
+
def log_stream(self, data: dict) -> PangeaResponse[PangeaResponseResult]:
|
821
|
+
"""
|
822
|
+
Log streaming endpoint
|
823
|
+
|
824
|
+
This API allows 3rd party vendors (like Auth0) to stream events to this
|
825
|
+
endpoint where the structure of the payload varies across different
|
826
|
+
vendors.
|
827
|
+
|
828
|
+
OperationId: audit_post_v1_log_stream
|
829
|
+
|
830
|
+
Args:
|
831
|
+
data: Event data. The exact schema of this will vary by vendor.
|
832
|
+
|
833
|
+
Raises:
|
834
|
+
AuditException: If an audit based api exception happens
|
835
|
+
PangeaAPIException: If an API Error happens
|
836
|
+
|
837
|
+
Examples:
|
838
|
+
data = {
|
839
|
+
"logs": [
|
840
|
+
{
|
841
|
+
"log_id": "some log ID",
|
842
|
+
"data": {
|
843
|
+
"date": "2024-03-29T17:26:50.193Z",
|
844
|
+
"type": "sapi",
|
845
|
+
"description": "Create a log stream",
|
846
|
+
"client_id": "some client ID",
|
847
|
+
"ip": "127.0.0.1",
|
848
|
+
"user_agent": "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0",
|
849
|
+
"user_id": "some user ID",
|
850
|
+
},
|
851
|
+
}
|
852
|
+
# ...
|
853
|
+
]
|
854
|
+
}
|
855
|
+
|
856
|
+
response = audit.log_stream(data)
|
857
|
+
"""
|
858
|
+
return self.request.post("v1/log_stream", PangeaResponseResult, data=data)
|
859
|
+
|
727
860
|
def root(self, tree_size: Optional[int] = None) -> PangeaResponse[RootResult]:
|
728
861
|
"""
|
729
862
|
Tamperproof verification
|
@@ -746,10 +879,14 @@ class Audit(ServiceBase, AuditBase):
|
|
746
879
|
response = audit.root(tree_size=7)
|
747
880
|
"""
|
748
881
|
input = RootRequest(tree_size=tree_size)
|
749
|
-
return self.request.post("v1/root", RootResult, data=input.
|
882
|
+
return self.request.post("v1/root", RootResult, data=input.model_dump(exclude_none=True))
|
750
883
|
|
751
884
|
def download_results(
|
752
|
-
self,
|
885
|
+
self,
|
886
|
+
result_id: Optional[str] = None,
|
887
|
+
format: DownloadFormat = DownloadFormat.CSV,
|
888
|
+
request_id: Optional[str] = None,
|
889
|
+
return_context: Optional[bool] = None,
|
753
890
|
) -> PangeaResponse[DownloadResult]:
|
754
891
|
"""
|
755
892
|
Download search results
|
@@ -761,6 +898,8 @@ class Audit(ServiceBase, AuditBase):
|
|
761
898
|
Args:
|
762
899
|
result_id: ID returned by the search API.
|
763
900
|
format: Format for the records.
|
901
|
+
request_id: ID returned by the export API.
|
902
|
+
return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
764
903
|
|
765
904
|
Returns:
|
766
905
|
URL where search results can be downloaded.
|
@@ -776,8 +915,13 @@ class Audit(ServiceBase, AuditBase):
|
|
776
915
|
)
|
777
916
|
"""
|
778
917
|
|
779
|
-
|
780
|
-
|
918
|
+
if request_id is None and result_id is None:
|
919
|
+
raise ValueError("must pass one of `request_id` or `result_id`")
|
920
|
+
|
921
|
+
input = DownloadRequest(
|
922
|
+
request_id=request_id, result_id=result_id, format=format, return_context=return_context
|
923
|
+
)
|
924
|
+
return self.request.post("v1/download_results", DownloadResult, data=input.model_dump(exclude_none=True))
|
781
925
|
|
782
926
|
def update_published_roots(self, result: SearchResultOutput):
|
783
927
|
"""Fetches series of published root hashes from Arweave
|
@@ -802,12 +946,31 @@ class Audit(ServiceBase, AuditBase):
|
|
802
946
|
for tree_size in tree_sizes:
|
803
947
|
pub_root = None
|
804
948
|
if tree_size in arweave_roots:
|
805
|
-
pub_root = PublishedRoot(**arweave_roots[tree_size].
|
949
|
+
pub_root = PublishedRoot(**arweave_roots[tree_size].model_dump(exclude_none=True))
|
806
950
|
pub_root.source = RootSource.ARWEAVE
|
807
951
|
elif self.allow_server_roots:
|
808
952
|
resp = self.root(tree_size=tree_size)
|
809
953
|
if resp.success and resp.result is not None:
|
810
|
-
pub_root = PublishedRoot(**resp.result.data.
|
954
|
+
pub_root = PublishedRoot(**resp.result.data.model_dump(exclude_none=True))
|
811
955
|
pub_root.source = RootSource.PANGEA
|
812
956
|
if pub_root is not None:
|
813
957
|
self.pub_roots[tree_size] = pub_root
|
958
|
+
|
959
|
+
self._fix_consistency_proofs(tree_sizes)
|
960
|
+
|
961
|
+
def _fix_consistency_proofs(self, tree_sizes: Iterable[int]) -> None:
|
962
|
+
# on very rare occasions, the consistency proof in Arweave may be wrong
|
963
|
+
# override it with the proof from pangea (not the root hash, just the proof)
|
964
|
+
for tree_size in tree_sizes:
|
965
|
+
if tree_size not in self.pub_roots or tree_size - 1 not in self.pub_roots:
|
966
|
+
continue
|
967
|
+
|
968
|
+
if self.pub_roots[tree_size].source == RootSource.PANGEA:
|
969
|
+
continue
|
970
|
+
|
971
|
+
if self.verify_consistency_proof(tree_size):
|
972
|
+
continue
|
973
|
+
|
974
|
+
resp = self.root(tree_size=tree_size)
|
975
|
+
if resp.success and resp.result is not None and resp.result.data is not None:
|
976
|
+
self.pub_roots[tree_size].consistency_proof = resp.result.data.consistency_proof
|
pangea/services/audit/models.py
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import datetime
|
4
6
|
import enum
|
5
|
-
from typing import Any, Dict, List, Optional, Union
|
7
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
6
8
|
|
7
|
-
from pangea.response import APIRequestModel, APIResponseModel, PangeaResponseResult
|
9
|
+
from pangea.response import APIRequestModel, APIResponseModel, PangeaDateTime, PangeaResponseResult
|
8
10
|
|
9
11
|
|
10
12
|
class EventVerification(str, enum.Enum):
|
@@ -19,14 +21,14 @@ class EventVerification(str, enum.Enum):
|
|
19
21
|
return str(self.value)
|
20
22
|
|
21
23
|
|
22
|
-
class Event(
|
24
|
+
class Event(Dict[str, Any]):
|
23
25
|
"""
|
24
26
|
Event to perform an auditable activity
|
25
27
|
|
26
28
|
Auxiliary class to be compatible with older SDKs
|
27
29
|
"""
|
28
30
|
|
29
|
-
def __init__(self, **data):
|
31
|
+
def __init__(self, **data) -> None:
|
30
32
|
super().__init__(**data)
|
31
33
|
|
32
34
|
@property
|
@@ -124,7 +126,7 @@ class EventEnvelope(APIResponseModel):
|
|
124
126
|
event: Dict[str, Any]
|
125
127
|
signature: Optional[str] = None
|
126
128
|
public_key: Optional[str] = None
|
127
|
-
received_at:
|
129
|
+
received_at: PangeaDateTime
|
128
130
|
|
129
131
|
|
130
132
|
class LogRequest(APIRequestModel):
|
@@ -269,6 +271,7 @@ class SearchRequest(APIRequestModel):
|
|
269
271
|
max_results -- Maximum number of results to return.
|
270
272
|
search_restriction -- A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
|
271
273
|
verbose -- If true, include root, membership and consistency proofs in response.
|
274
|
+
return_context -- Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
272
275
|
"""
|
273
276
|
|
274
277
|
query: str
|
@@ -279,8 +282,9 @@ class SearchRequest(APIRequestModel):
|
|
279
282
|
end: Optional[str] = None
|
280
283
|
limit: Optional[int] = None
|
281
284
|
max_results: Optional[int] = None
|
282
|
-
search_restriction: Optional[
|
285
|
+
search_restriction: Optional[Dict[str, Sequence[str]]] = None
|
283
286
|
verbose: Optional[bool] = None
|
287
|
+
return_context: Optional[bool] = None
|
284
288
|
|
285
289
|
|
286
290
|
class RootRequest(APIRequestModel):
|
@@ -361,6 +365,7 @@ class SearchEvent(APIResponseModel):
|
|
361
365
|
consistency_verification -- Consistency verification calculated if required.
|
362
366
|
membership_verification -- Membership verification calculated if required.
|
363
367
|
signature_verification -- Signature verification calculated if required.
|
368
|
+
fpe_context -- The context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
364
369
|
"""
|
365
370
|
|
366
371
|
envelope: EventEnvelope
|
@@ -371,6 +376,7 @@ class SearchEvent(APIResponseModel):
|
|
371
376
|
consistency_verification: EventVerification = EventVerification.NONE
|
372
377
|
membership_verification: EventVerification = EventVerification.NONE
|
373
378
|
signature_verification: EventVerification = EventVerification.NONE
|
379
|
+
fpe_context: Optional[str] = None
|
374
380
|
|
375
381
|
|
376
382
|
class SearchResultOutput(PangeaResponseResult):
|
@@ -404,7 +410,7 @@ class SearchOutput(SearchResultOutput):
|
|
404
410
|
"""
|
405
411
|
|
406
412
|
id: str
|
407
|
-
expires_at:
|
413
|
+
expires_at: PangeaDateTime
|
408
414
|
|
409
415
|
|
410
416
|
class SearchResultRequest(APIRequestModel):
|
@@ -415,11 +421,15 @@ class SearchResultRequest(APIRequestModel):
|
|
415
421
|
id -- A search results identifier returned by the search call.
|
416
422
|
limit -- Number of audit records to include from the first page of the results.
|
417
423
|
offset -- Offset from the start of the result set to start returning results from.
|
424
|
+
assert_search_restriction -- Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
|
425
|
+
return_context -- Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
|
418
426
|
"""
|
419
427
|
|
420
428
|
id: str
|
421
429
|
limit: Optional[int] = 20
|
422
430
|
offset: Optional[int] = 0
|
431
|
+
assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None
|
432
|
+
return_context: Optional[bool] = None
|
423
433
|
|
424
434
|
|
425
435
|
class DownloadFormat(str, enum.Enum):
|
@@ -437,13 +447,51 @@ class DownloadFormat(str, enum.Enum):
|
|
437
447
|
|
438
448
|
|
439
449
|
class DownloadRequest(APIRequestModel):
|
440
|
-
|
450
|
+
request_id: Optional[str] = None
|
451
|
+
"""ID returned by the export API."""
|
452
|
+
|
453
|
+
result_id: Optional[str] = None
|
441
454
|
"""ID returned by the search API."""
|
442
455
|
|
443
456
|
format: Optional[str] = None
|
444
457
|
"""Format for the records."""
|
445
458
|
|
459
|
+
return_context: Optional[bool] = None
|
460
|
+
"""Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption."""
|
461
|
+
|
446
462
|
|
447
463
|
class DownloadResult(PangeaResponseResult):
|
448
464
|
dest_url: str
|
449
465
|
"""URL where search results can be downloaded."""
|
466
|
+
|
467
|
+
expires_at: str
|
468
|
+
"""
|
469
|
+
The time when the results will no longer be available to page through via
|
470
|
+
the results API.
|
471
|
+
"""
|
472
|
+
|
473
|
+
|
474
|
+
class ExportRequest(APIRequestModel):
|
475
|
+
format: DownloadFormat = DownloadFormat.CSV
|
476
|
+
"""Format for the records."""
|
477
|
+
|
478
|
+
start: Optional[datetime.datetime] = None
|
479
|
+
"""The start of the time range to perform the search on."""
|
480
|
+
|
481
|
+
end: Optional[datetime.datetime] = None
|
482
|
+
"""
|
483
|
+
The end of the time range to perform the search on. If omitted, then all
|
484
|
+
records up to the latest will be searched.
|
485
|
+
"""
|
486
|
+
|
487
|
+
order_by: Optional[str] = None
|
488
|
+
"""Name of column to sort the results by."""
|
489
|
+
|
490
|
+
order: Optional[SearchOrder] = None
|
491
|
+
"""Specify the sort order of the response."""
|
492
|
+
|
493
|
+
verbose: bool = True
|
494
|
+
"""
|
495
|
+
Whether or not to include the root hash of the tree and the membership proof
|
496
|
+
for each record.
|
497
|
+
"""
|
pangea/services/audit/signing.py
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
from abc import ABC, abstractmethod
|
4
6
|
from typing import Optional
|
5
7
|
|
@@ -10,7 +12,7 @@ from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes, Pub
|
|
10
12
|
|
11
13
|
from pangea.exceptions import PangeaException
|
12
14
|
from pangea.services.audit.util import b64decode, b64decode_ascii, b64encode_ascii
|
13
|
-
from pangea.services.vault.models.
|
15
|
+
from pangea.services.vault.models.asymmetric import AsymmetricKeySigningAlgorithm
|
14
16
|
|
15
17
|
|
16
18
|
class AlgorithmSigner(ABC):
|
@@ -43,7 +45,7 @@ class ED25519Signer(AlgorithmSigner):
|
|
43
45
|
)
|
44
46
|
|
45
47
|
def get_algorithm(self) -> str:
|
46
|
-
return
|
48
|
+
return AsymmetricKeySigningAlgorithm.ED25519.value
|
47
49
|
|
48
50
|
|
49
51
|
signers = {
|
@@ -96,7 +98,7 @@ class Signer:
|
|
96
98
|
|
97
99
|
for func in (serialization.load_pem_private_key, serialization.load_ssh_private_key):
|
98
100
|
try:
|
99
|
-
return func(private_key, None)
|
101
|
+
return func(private_key, None)
|
100
102
|
except exceptions.UnsupportedAlgorithm as e:
|
101
103
|
raise e
|
102
104
|
except ValueError:
|
@@ -144,8 +146,7 @@ class Verifier:
|
|
144
146
|
for cls, verifier in verifiers.items():
|
145
147
|
if isinstance(pubkey, cls):
|
146
148
|
return verifier(pubkey).verify(message_bytes, signature_bytes)
|
147
|
-
|
148
|
-
raise PangeaException(f"Not supported public key type: {type(pubkey)}")
|
149
|
+
raise PangeaException(f"Not supported public key type: {type(pubkey)}")
|
149
150
|
|
150
151
|
def _decode_public_key(self, public_key: bytes):
|
151
152
|
"""Parse a public key in PEM or ssh format"""
|
pangea/services/audit/util.py
CHANGED
@@ -7,7 +7,7 @@ from binascii import hexlify, unhexlify
|
|
7
7
|
from dataclasses import dataclass
|
8
8
|
from datetime import datetime
|
9
9
|
from hashlib import sha256
|
10
|
-
from typing import Dict, List, Optional
|
10
|
+
from typing import Collection, Dict, List, Optional
|
11
11
|
|
12
12
|
import requests
|
13
13
|
|
@@ -61,7 +61,7 @@ def verify_hash(hash1: str, hash2: str) -> bool:
|
|
61
61
|
|
62
62
|
|
63
63
|
def verify_envelope_hash(envelope: EventEnvelope, hash: str):
|
64
|
-
return verify_hash(hash_dict(normalize_log(envelope.
|
64
|
+
return verify_hash(hash_dict(normalize_log(envelope.model_dump(exclude_none=True))), hash)
|
65
65
|
|
66
66
|
|
67
67
|
def canonicalize_event(event: Event) -> bytes:
|
@@ -192,7 +192,7 @@ def arweave_graphql_url():
|
|
192
192
|
return f"{ARWEAVE_BASE_URL}/graphql"
|
193
193
|
|
194
194
|
|
195
|
-
def get_arweave_published_roots(tree_name: str, tree_sizes:
|
195
|
+
def get_arweave_published_roots(tree_name: str, tree_sizes: Collection[int]) -> Dict[int, PublishedRoot]:
|
196
196
|
if len(tree_sizes) == 0:
|
197
197
|
return {}
|
198
198
|
|