pangea-sdk 3.8.0b1__py3-none-any.whl → 5.3.0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. pangea/__init__.py +1 -1
  2. pangea/asyncio/file_uploader.py +1 -1
  3. pangea/asyncio/request.py +49 -31
  4. pangea/asyncio/services/__init__.py +2 -0
  5. pangea/asyncio/services/audit.py +192 -31
  6. pangea/asyncio/services/authn.py +187 -109
  7. pangea/asyncio/services/authz.py +285 -0
  8. pangea/asyncio/services/base.py +21 -2
  9. pangea/asyncio/services/embargo.py +2 -2
  10. pangea/asyncio/services/file_scan.py +24 -9
  11. pangea/asyncio/services/intel.py +108 -34
  12. pangea/asyncio/services/redact.py +72 -4
  13. pangea/asyncio/services/sanitize.py +217 -0
  14. pangea/asyncio/services/share.py +246 -73
  15. pangea/asyncio/services/vault.py +1710 -750
  16. pangea/crypto/rsa.py +135 -0
  17. pangea/deep_verify.py +7 -1
  18. pangea/dump_audit.py +9 -8
  19. pangea/request.py +83 -59
  20. pangea/response.py +49 -31
  21. pangea/services/__init__.py +2 -0
  22. pangea/services/audit/audit.py +205 -42
  23. pangea/services/audit/models.py +56 -8
  24. pangea/services/audit/signing.py +6 -5
  25. pangea/services/audit/util.py +3 -3
  26. pangea/services/authn/authn.py +140 -70
  27. pangea/services/authn/models.py +167 -11
  28. pangea/services/authz.py +400 -0
  29. pangea/services/base.py +39 -8
  30. pangea/services/embargo.py +2 -2
  31. pangea/services/file_scan.py +32 -15
  32. pangea/services/intel.py +157 -32
  33. pangea/services/redact.py +152 -4
  34. pangea/services/sanitize.py +388 -0
  35. pangea/services/share/share.py +683 -107
  36. pangea/services/vault/models/asymmetric.py +120 -18
  37. pangea/services/vault/models/common.py +439 -141
  38. pangea/services/vault/models/keys.py +94 -0
  39. pangea/services/vault/models/secret.py +27 -3
  40. pangea/services/vault/models/symmetric.py +68 -22
  41. pangea/services/vault/vault.py +1690 -749
  42. pangea/tools.py +6 -7
  43. pangea/utils.py +16 -27
  44. pangea/verify_audit.py +270 -83
  45. {pangea_sdk-3.8.0b1.dist-info → pangea_sdk-5.3.0.dist-info}/METADATA +43 -35
  46. pangea_sdk-5.3.0.dist-info/RECORD +56 -0
  47. {pangea_sdk-3.8.0b1.dist-info → pangea_sdk-5.3.0.dist-info}/WHEEL +1 -1
  48. pangea_sdk-3.8.0b1.dist-info/RECORD +0 -50
@@ -1,11 +1,14 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
+ from __future__ import annotations
4
+
3
5
  import datetime
4
6
  import json
5
- from typing import Any, Dict, List, Optional, Set, Tuple, Union
7
+ from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
6
8
 
7
9
  import pangea.exceptions as pexc
8
- from pangea.response import PangeaResponse
10
+ from pangea.config import PangeaConfig
11
+ from pangea.response import PangeaResponse, PangeaResponseResult
9
12
  from pangea.services.audit.exceptions import AuditException, EventCorruption
10
13
  from pangea.services.audit.models import (
11
14
  DownloadFormat,
@@ -14,6 +17,7 @@ from pangea.services.audit.models import (
14
17
  Event,
15
18
  EventEnvelope,
16
19
  EventVerification,
20
+ ExportRequest,
17
21
  LogBulkRequest,
18
22
  LogBulkResult,
19
23
  LogEvent,
@@ -51,8 +55,8 @@ from pangea.utils import canonicalize_nested_json
51
55
 
52
56
  class AuditBase:
53
57
  def __init__(
54
- self, private_key_file: str = "", public_key_info: Dict[str, str] = {}, tenant_id: Optional[str] = None
55
- ):
58
+ self, private_key_file: str = "", public_key_info: dict[str, str] = {}, tenant_id: str | None = None
59
+ ) -> None:
56
60
  self.pub_roots: Dict[int, PublishedRoot] = {}
57
61
  self.buffer_data: Optional[str] = None
58
62
  self.signer: Optional[Signer] = Signer(private_key_file) if private_key_file else None
@@ -195,7 +199,7 @@ class AuditBase:
195
199
 
196
200
  # verify consistency proofs
197
201
  if self.can_verify_consistency_proof(search_event):
198
- if self.verify_consistency_proof(self.pub_roots, search_event):
202
+ if search_event.leaf_index is not None and self.verify_consistency_proof(search_event.leaf_index):
199
203
  search_event.consistency_verification = EventVerification.PASS
200
204
  else:
201
205
  search_event.consistency_verification = EventVerification.FAIL
@@ -218,7 +222,7 @@ class AuditBase:
218
222
  tree_sizes.difference_update(self.pub_roots.keys())
219
223
 
220
224
  if tree_sizes:
221
- arweave_roots = get_arweave_published_roots(result.root.tree_name, list(tree_sizes))
225
+ arweave_roots = get_arweave_published_roots(result.root.tree_name, tree_sizes)
222
226
  else:
223
227
  arweave_roots = {}
224
228
 
@@ -280,7 +284,7 @@ class AuditBase:
280
284
  """
281
285
  return event.published and event.leaf_index is not None and event.leaf_index >= 0 # type: ignore[return-value]
282
286
 
283
- def verify_consistency_proof(self, pub_roots: Dict[int, PublishedRoot], event: SearchEvent) -> bool:
287
+ def verify_consistency_proof(self, tree_size: int) -> bool:
284
288
  """
285
289
  Verify consistency proof
286
290
 
@@ -289,18 +293,17 @@ class AuditBase:
289
293
  Read more at: [What is a consistency proof?](https://pangea.cloud/docs/audit/merkle-trees#what-is-a-consistency-proof)
290
294
 
291
295
  Args:
292
- pub_roots (dict[int, Root]): list of published root hashes across time
293
- event (SearchEvent): Audit event to be verified.
296
+ leaf_index (int): The tree size of the root to be verified.
294
297
 
295
298
  Returns:
296
299
  bool: True if consistency proof is verified, False otherwise.
297
300
  """
298
301
 
299
- if event.leaf_index == 0:
302
+ if tree_size == 0:
300
303
  return True
301
304
 
302
- curr_root = pub_roots.get(event.leaf_index + 1) # type: ignore[operator]
303
- prev_root = pub_roots.get(event.leaf_index) # type: ignore[arg-type]
305
+ curr_root = self.pub_roots.get(tree_size + 1)
306
+ prev_root = self.pub_roots.get(tree_size)
304
307
 
305
308
  if not curr_root or not prev_root:
306
309
  return False
@@ -310,9 +313,12 @@ class AuditBase:
310
313
  ):
311
314
  return False
312
315
 
316
+ if curr_root.consistency_proof is None:
317
+ return False
318
+
313
319
  curr_root_hash = decode_hash(curr_root.root_hash)
314
320
  prev_root_hash = decode_hash(prev_root.root_hash)
315
- proof = decode_consistency_proof(curr_root.consistency_proof) # type: ignore[arg-type]
321
+ proof = decode_consistency_proof(curr_root.consistency_proof)
316
322
 
317
323
  return verify_consistency_proof(curr_root_hash, prev_root_hash, proof)
318
324
 
@@ -332,7 +338,9 @@ class AuditBase:
332
338
  if audit_envelope and audit_envelope.signature and public_key:
333
339
  v = Verifier()
334
340
  verification = v.verify_signature(
335
- audit_envelope.signature, canonicalize_event(audit_envelope.event), public_key # type: ignore[arg-type]
341
+ audit_envelope.signature,
342
+ canonicalize_event(Event(**audit_envelope.event)),
343
+ public_key,
336
344
  )
337
345
  if verification is not None:
338
346
  return EventVerification.PASS if verification else EventVerification.FAIL
@@ -374,14 +382,32 @@ class Audit(ServiceBase, AuditBase):
374
382
 
375
383
  def __init__(
376
384
  self,
377
- token,
378
- config=None,
385
+ token: str,
386
+ config: PangeaConfig | None = None,
379
387
  private_key_file: str = "",
380
- public_key_info: Dict[str, str] = {},
381
- tenant_id: Optional[str] = None,
382
- logger_name="pangea",
383
- config_id: Optional[str] = None,
384
- ):
388
+ public_key_info: dict[str, str] = {},
389
+ tenant_id: str | None = None,
390
+ logger_name: str = "pangea",
391
+ config_id: str | None = None,
392
+ ) -> None:
393
+ """
394
+ Audit client
395
+
396
+ Initializes a new Audit client.
397
+
398
+ Args:
399
+ token: Pangea API token.
400
+ config: Configuration.
401
+ private_key_file: Private key filepath.
402
+ public_key_info: Public key information.
403
+ tenant_id: Tenant ID.
404
+ logger_name: Logger name.
405
+ config_id: Configuration ID.
406
+
407
+ Examples:
408
+ config = PangeaConfig(domain="pangea_domain")
409
+ audit = Audit(token="pangea_token", config=config)
410
+ """
385
411
  # FIXME: Temporary check to deprecate config_id from PangeaConfig.
386
412
  # Delete it when deprecate PangeaConfig.config_id
387
413
  if config_id and config is not None and config.config_id is not None:
@@ -466,7 +492,7 @@ class Audit(ServiceBase, AuditBase):
466
492
  verbose: Optional[bool] = None,
467
493
  ) -> PangeaResponse[LogResult]:
468
494
  """
469
- Log an entry
495
+ Log an event
470
496
 
471
497
  Create a log entry in the Secure Audit Log.
472
498
 
@@ -475,6 +501,7 @@ class Audit(ServiceBase, AuditBase):
475
501
  verify (bool, optional): True to verify logs consistency after response.
476
502
  sign_local (bool, optional): True to sign event with local key.
477
503
  verbose (bool, optional): True to get a more verbose response.
504
+
478
505
  Raises:
479
506
  AuditException: If an audit based api exception happens
480
507
  PangeaAPIException: If an API Error happens
@@ -485,17 +512,13 @@ class Audit(ServiceBase, AuditBase):
485
512
  Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
486
513
 
487
514
  Examples:
488
- try:
489
- log_response = audit.log({"message": "hello world"}, verbose=True)
490
- print(f"Response. Hash: {log_response.result.hash}")
491
- except pe.PangeaAPIException as e:
492
- print(f"Request Error: {e.response.summary}")
493
- for err in e.errors:
494
- print(f"\\t{err.detail} \\n")
515
+ response = audit.log_event({"message": "hello world"}, verbose=True)
495
516
  """
496
517
 
497
518
  input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
498
- response: PangeaResponse[LogResult] = self.request.post("v1/log", LogResult, data=input.dict(exclude_none=True))
519
+ response: PangeaResponse[LogResult] = self.request.post(
520
+ "v1/log", LogResult, data=input.model_dump(exclude_none=True)
521
+ )
499
522
  if response.success and response.result is not None:
500
523
  self._process_log_result(response.result, verify=verify)
501
524
  return response
@@ -535,7 +558,7 @@ class Audit(ServiceBase, AuditBase):
535
558
 
536
559
  input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
537
560
  response: PangeaResponse[LogBulkResult] = self.request.post(
538
- "v2/log", LogBulkResult, data=input.dict(exclude_none=True)
561
+ "v2/log", LogBulkResult, data=input.model_dump(exclude_none=True)
539
562
  )
540
563
 
541
564
  if response.success and response.result is not None:
@@ -579,7 +602,7 @@ class Audit(ServiceBase, AuditBase):
579
602
  try:
580
603
  # Calling to v2 methods will return always a 202.
581
604
  response: PangeaResponse[LogBulkResult] = self.request.post(
582
- "v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
605
+ "v2/log_async", LogBulkResult, data=input.model_dump(exclude_none=True), poll_result=False
583
606
  )
584
607
  except pexc.AcceptedRequestException as e:
585
608
  return e.response
@@ -598,10 +621,11 @@ class Audit(ServiceBase, AuditBase):
598
621
  end: Optional[Union[datetime.datetime, str]] = None,
599
622
  limit: Optional[int] = None,
600
623
  max_results: Optional[int] = None,
601
- search_restriction: Optional[dict] = None,
624
+ search_restriction: Optional[Dict[str, Sequence[str]]] = None,
602
625
  verbose: Optional[bool] = None,
603
626
  verify_consistency: bool = False,
604
627
  verify_events: bool = True,
628
+ return_context: Optional[bool] = None,
605
629
  ) -> PangeaResponse[SearchOutput]:
606
630
  """
607
631
  Search the log
@@ -627,10 +651,11 @@ class Audit(ServiceBase, AuditBase):
627
651
  end (datetime, optional): An RFC-3339 formatted timestamp, or relative time adjustment from the current time.
628
652
  limit (int, optional): Optional[int] = None,
629
653
  max_results (int, optional): Maximum number of results to return.
630
- search_restriction (dict, optional): A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
654
+ search_restriction (Dict[str, Sequence[str]], optional): A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
631
655
  verbose (bool, optional): If true, response include root and membership and consistency proofs.
632
656
  verify_consistency (bool): True to verify logs consistency
633
657
  verify_events (bool): True to verify hash events and signatures
658
+ return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
634
659
 
635
660
  Raises:
636
661
  AuditException: If an audit based api exception happens
@@ -664,10 +689,11 @@ class Audit(ServiceBase, AuditBase):
664
689
  max_results=max_results,
665
690
  search_restriction=search_restriction,
666
691
  verbose=verbose,
692
+ return_context=return_context,
667
693
  )
668
694
 
669
695
  response: PangeaResponse[SearchOutput] = self.request.post(
670
- "v1/search", SearchOutput, data=input.dict(exclude_none=True)
696
+ "v1/search", SearchOutput, data=input.model_dump(exclude_none=True)
671
697
  )
672
698
  if verify_consistency and response.result is not None:
673
699
  self.update_published_roots(response.result)
@@ -678,8 +704,10 @@ class Audit(ServiceBase, AuditBase):
678
704
  id: str,
679
705
  limit: Optional[int] = 20,
680
706
  offset: Optional[int] = 0,
707
+ assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None,
681
708
  verify_consistency: bool = False,
682
709
  verify_events: bool = True,
710
+ return_context: Optional[bool] = None,
683
711
  ) -> PangeaResponse[SearchResultOutput]:
684
712
  """
685
713
  Results of a search
@@ -692,8 +720,10 @@ class Audit(ServiceBase, AuditBase):
692
720
  id (string): the id of a search action, found in `response.result.id`
693
721
  limit (integer, optional): the maximum number of results to return, default is 20
694
722
  offset (integer, optional): the position of the first result to return, default is 0
723
+ assert_search_restriction (Dict[str, Sequence[str]], optional): Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
695
724
  verify_consistency (bool): True to verify logs consistency
696
725
  verify_events (bool): True to verify hash events and signatures
726
+ return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
697
727
  Raises:
698
728
  AuditException: If an audit based api exception happens
699
729
  PangeaAPIException: If an API Error happens
@@ -716,14 +746,117 @@ class Audit(ServiceBase, AuditBase):
716
746
  id=id,
717
747
  limit=limit,
718
748
  offset=offset,
749
+ assert_search_restriction=assert_search_restriction,
750
+ return_context=return_context,
719
751
  )
720
752
  response: PangeaResponse[SearchResultOutput] = self.request.post(
721
- "v1/results", SearchResultOutput, data=input.dict(exclude_none=True)
753
+ "v1/results", SearchResultOutput, data=input.model_dump(exclude_none=True)
722
754
  )
723
755
  if verify_consistency and response.result is not None:
724
756
  self.update_published_roots(response.result)
725
757
  return self.handle_results_response(response, verify_consistency, verify_events)
726
758
 
759
+ def export(
760
+ self,
761
+ *,
762
+ format: DownloadFormat = DownloadFormat.CSV,
763
+ start: Optional[datetime.datetime] = None,
764
+ end: Optional[datetime.datetime] = None,
765
+ order: Optional[SearchOrder] = None,
766
+ order_by: Optional[str] = None,
767
+ verbose: bool = True,
768
+ ) -> PangeaResponse[PangeaResponseResult]:
769
+ """
770
+ Export from the audit log
771
+
772
+ Bulk export of data from the Secure Audit Log, with optional filtering.
773
+
774
+ OperationId: audit_post_v1_export
775
+
776
+ Args:
777
+ format: Format for the records.
778
+ start: The start of the time range to perform the search on.
779
+ end: The end of the time range to perform the search on. If omitted,
780
+ then all records up to the latest will be searched.
781
+ order: Specify the sort order of the response.
782
+ order_by: Name of column to sort the results by.
783
+ verbose: Whether or not to include the root hash of the tree and the
784
+ membership proof for each record.
785
+
786
+ Raises:
787
+ AuditException: If an audit based api exception happens
788
+ PangeaAPIException: If an API Error happens
789
+
790
+ Examples:
791
+ export_res = audit.export(verbose=False)
792
+
793
+ # Export may take several dozens of minutes, so polling for the result
794
+ # should be done in a loop. That is omitted here for brevity's sake.
795
+ try:
796
+ audit.poll_result(request_id=export_res.request_id)
797
+ except AcceptedRequestException:
798
+ # Retry later.
799
+
800
+ # Download the result when it's ready.
801
+ download_res = audit.download_results(request_id=export_res.request_id)
802
+ download_res.result.dest_url
803
+ # => https://pangea-runtime.s3.amazonaws.com/audit/xxxxx/search_results_[...]
804
+ """
805
+ input = ExportRequest(
806
+ format=format,
807
+ start=start,
808
+ end=end,
809
+ order=order,
810
+ order_by=order_by,
811
+ verbose=verbose,
812
+ )
813
+ try:
814
+ return self.request.post(
815
+ "v1/export", PangeaResponseResult, data=input.model_dump(exclude_none=True), poll_result=False
816
+ )
817
+ except pexc.AcceptedRequestException as e:
818
+ return e.response
819
+
820
+ def log_stream(self, data: dict) -> PangeaResponse[PangeaResponseResult]:
821
+ """
822
+ Log streaming endpoint
823
+
824
+ This API allows 3rd party vendors (like Auth0) to stream events to this
825
+ endpoint where the structure of the payload varies across different
826
+ vendors.
827
+
828
+ OperationId: audit_post_v1_log_stream
829
+
830
+ Args:
831
+ data: Event data. The exact schema of this will vary by vendor.
832
+
833
+ Raises:
834
+ AuditException: If an audit based api exception happens
835
+ PangeaAPIException: If an API Error happens
836
+
837
+ Examples:
838
+ data = {
839
+ "logs": [
840
+ {
841
+ "log_id": "some log ID",
842
+ "data": {
843
+ "date": "2024-03-29T17:26:50.193Z",
844
+ "type": "sapi",
845
+ "description": "Create a log stream",
846
+ "client_id": "some client ID",
847
+ "ip": "127.0.0.1",
848
+ "user_agent": "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0",
849
+ "user_id": "some user ID",
850
+ },
851
+ }
852
+ # ...
853
+ ]
854
+ }
855
+
856
+ response = audit.log_stream(data)
857
+ """
858
+ return self.request.post("v1/log_stream", PangeaResponseResult, data=data)
859
+
727
860
  def root(self, tree_size: Optional[int] = None) -> PangeaResponse[RootResult]:
728
861
  """
729
862
  Tamperproof verification
@@ -746,10 +879,14 @@ class Audit(ServiceBase, AuditBase):
746
879
  response = audit.root(tree_size=7)
747
880
  """
748
881
  input = RootRequest(tree_size=tree_size)
749
- return self.request.post("v1/root", RootResult, data=input.dict(exclude_none=True))
882
+ return self.request.post("v1/root", RootResult, data=input.model_dump(exclude_none=True))
750
883
 
751
884
  def download_results(
752
- self, result_id: str, format: Optional[DownloadFormat] = None
885
+ self,
886
+ result_id: Optional[str] = None,
887
+ format: DownloadFormat = DownloadFormat.CSV,
888
+ request_id: Optional[str] = None,
889
+ return_context: Optional[bool] = None,
753
890
  ) -> PangeaResponse[DownloadResult]:
754
891
  """
755
892
  Download search results
@@ -761,6 +898,8 @@ class Audit(ServiceBase, AuditBase):
761
898
  Args:
762
899
  result_id: ID returned by the search API.
763
900
  format: Format for the records.
901
+ request_id: ID returned by the export API.
902
+ return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
764
903
 
765
904
  Returns:
766
905
  URL where search results can be downloaded.
@@ -776,8 +915,13 @@ class Audit(ServiceBase, AuditBase):
776
915
  )
777
916
  """
778
917
 
779
- input = DownloadRequest(result_id=result_id, format=format)
780
- return self.request.post("v1/download_results", DownloadResult, data=input.dict(exclude_none=True))
918
+ if request_id is None and result_id is None:
919
+ raise ValueError("must pass one of `request_id` or `result_id`")
920
+
921
+ input = DownloadRequest(
922
+ request_id=request_id, result_id=result_id, format=format, return_context=return_context
923
+ )
924
+ return self.request.post("v1/download_results", DownloadResult, data=input.model_dump(exclude_none=True))
781
925
 
782
926
  def update_published_roots(self, result: SearchResultOutput):
783
927
  """Fetches series of published root hashes from Arweave
@@ -802,12 +946,31 @@ class Audit(ServiceBase, AuditBase):
802
946
  for tree_size in tree_sizes:
803
947
  pub_root = None
804
948
  if tree_size in arweave_roots:
805
- pub_root = PublishedRoot(**arweave_roots[tree_size].dict(exclude_none=True))
949
+ pub_root = PublishedRoot(**arweave_roots[tree_size].model_dump(exclude_none=True))
806
950
  pub_root.source = RootSource.ARWEAVE
807
951
  elif self.allow_server_roots:
808
952
  resp = self.root(tree_size=tree_size)
809
953
  if resp.success and resp.result is not None:
810
- pub_root = PublishedRoot(**resp.result.data.dict(exclude_none=True))
954
+ pub_root = PublishedRoot(**resp.result.data.model_dump(exclude_none=True))
811
955
  pub_root.source = RootSource.PANGEA
812
956
  if pub_root is not None:
813
957
  self.pub_roots[tree_size] = pub_root
958
+
959
+ self._fix_consistency_proofs(tree_sizes)
960
+
961
+ def _fix_consistency_proofs(self, tree_sizes: Iterable[int]) -> None:
962
+ # on very rare occasions, the consistency proof in Arweave may be wrong
963
+ # override it with the proof from pangea (not the root hash, just the proof)
964
+ for tree_size in tree_sizes:
965
+ if tree_size not in self.pub_roots or tree_size - 1 not in self.pub_roots:
966
+ continue
967
+
968
+ if self.pub_roots[tree_size].source == RootSource.PANGEA:
969
+ continue
970
+
971
+ if self.verify_consistency_proof(tree_size):
972
+ continue
973
+
974
+ resp = self.root(tree_size=tree_size)
975
+ if resp.success and resp.result is not None and resp.result.data is not None:
976
+ self.pub_roots[tree_size].consistency_proof = resp.result.data.consistency_proof
@@ -1,10 +1,12 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
+ from __future__ import annotations
4
+
3
5
  import datetime
4
6
  import enum
5
- from typing import Any, Dict, List, Optional, Union
7
+ from typing import Any, Dict, List, Optional, Sequence, Union
6
8
 
7
- from pangea.response import APIRequestModel, APIResponseModel, PangeaResponseResult
9
+ from pangea.response import APIRequestModel, APIResponseModel, PangeaDateTime, PangeaResponseResult
8
10
 
9
11
 
10
12
  class EventVerification(str, enum.Enum):
@@ -19,14 +21,14 @@ class EventVerification(str, enum.Enum):
19
21
  return str(self.value)
20
22
 
21
23
 
22
- class Event(dict):
24
+ class Event(Dict[str, Any]):
23
25
  """
24
26
  Event to perform an auditable activity
25
27
 
26
28
  Auxiliary class to be compatible with older SDKs
27
29
  """
28
30
 
29
- def __init__(self, **data):
31
+ def __init__(self, **data) -> None:
30
32
  super().__init__(**data)
31
33
 
32
34
  @property
@@ -124,7 +126,7 @@ class EventEnvelope(APIResponseModel):
124
126
  event: Dict[str, Any]
125
127
  signature: Optional[str] = None
126
128
  public_key: Optional[str] = None
127
- received_at: datetime.datetime
129
+ received_at: PangeaDateTime
128
130
 
129
131
 
130
132
  class LogRequest(APIRequestModel):
@@ -269,6 +271,7 @@ class SearchRequest(APIRequestModel):
269
271
  max_results -- Maximum number of results to return.
270
272
  search_restriction -- A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
271
273
  verbose -- If true, include root, membership and consistency proofs in response.
274
+ return_context -- Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
272
275
  """
273
276
 
274
277
  query: str
@@ -279,8 +282,9 @@ class SearchRequest(APIRequestModel):
279
282
  end: Optional[str] = None
280
283
  limit: Optional[int] = None
281
284
  max_results: Optional[int] = None
282
- search_restriction: Optional[dict] = None
285
+ search_restriction: Optional[Dict[str, Sequence[str]]] = None
283
286
  verbose: Optional[bool] = None
287
+ return_context: Optional[bool] = None
284
288
 
285
289
 
286
290
  class RootRequest(APIRequestModel):
@@ -361,6 +365,7 @@ class SearchEvent(APIResponseModel):
361
365
  consistency_verification -- Consistency verification calculated if required.
362
366
  membership_verification -- Membership verification calculated if required.
363
367
  signature_verification -- Signature verification calculated if required.
368
+ fpe_context -- The context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
364
369
  """
365
370
 
366
371
  envelope: EventEnvelope
@@ -371,6 +376,7 @@ class SearchEvent(APIResponseModel):
371
376
  consistency_verification: EventVerification = EventVerification.NONE
372
377
  membership_verification: EventVerification = EventVerification.NONE
373
378
  signature_verification: EventVerification = EventVerification.NONE
379
+ fpe_context: Optional[str] = None
374
380
 
375
381
 
376
382
  class SearchResultOutput(PangeaResponseResult):
@@ -404,7 +410,7 @@ class SearchOutput(SearchResultOutput):
404
410
  """
405
411
 
406
412
  id: str
407
- expires_at: datetime.datetime
413
+ expires_at: PangeaDateTime
408
414
 
409
415
 
410
416
  class SearchResultRequest(APIRequestModel):
@@ -415,11 +421,15 @@ class SearchResultRequest(APIRequestModel):
415
421
  id -- A search results identifier returned by the search call.
416
422
  limit -- Number of audit records to include from the first page of the results.
417
423
  offset -- Offset from the start of the result set to start returning results from.
424
+ assert_search_restriction -- Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
425
+ return_context -- Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
418
426
  """
419
427
 
420
428
  id: str
421
429
  limit: Optional[int] = 20
422
430
  offset: Optional[int] = 0
431
+ assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None
432
+ return_context: Optional[bool] = None
423
433
 
424
434
 
425
435
  class DownloadFormat(str, enum.Enum):
@@ -437,13 +447,51 @@ class DownloadFormat(str, enum.Enum):
437
447
 
438
448
 
439
449
  class DownloadRequest(APIRequestModel):
440
- result_id: str
450
+ request_id: Optional[str] = None
451
+ """ID returned by the export API."""
452
+
453
+ result_id: Optional[str] = None
441
454
  """ID returned by the search API."""
442
455
 
443
456
  format: Optional[str] = None
444
457
  """Format for the records."""
445
458
 
459
+ return_context: Optional[bool] = None
460
+ """Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption."""
461
+
446
462
 
447
463
  class DownloadResult(PangeaResponseResult):
448
464
  dest_url: str
449
465
  """URL where search results can be downloaded."""
466
+
467
+ expires_at: str
468
+ """
469
+ The time when the results will no longer be available to page through via
470
+ the results API.
471
+ """
472
+
473
+
474
+ class ExportRequest(APIRequestModel):
475
+ format: DownloadFormat = DownloadFormat.CSV
476
+ """Format for the records."""
477
+
478
+ start: Optional[datetime.datetime] = None
479
+ """The start of the time range to perform the search on."""
480
+
481
+ end: Optional[datetime.datetime] = None
482
+ """
483
+ The end of the time range to perform the search on. If omitted, then all
484
+ records up to the latest will be searched.
485
+ """
486
+
487
+ order_by: Optional[str] = None
488
+ """Name of column to sort the results by."""
489
+
490
+ order: Optional[SearchOrder] = None
491
+ """Specify the sort order of the response."""
492
+
493
+ verbose: bool = True
494
+ """
495
+ Whether or not to include the root hash of the tree and the membership proof
496
+ for each record.
497
+ """
@@ -1,5 +1,7 @@
1
1
  # Copyright 2022 Pangea Cyber Corporation
2
2
  # Author: Pangea Cyber Corporation
3
+ from __future__ import annotations
4
+
3
5
  from abc import ABC, abstractmethod
4
6
  from typing import Optional
5
7
 
@@ -10,7 +12,7 @@ from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes, Pub
10
12
 
11
13
  from pangea.exceptions import PangeaException
12
14
  from pangea.services.audit.util import b64decode, b64decode_ascii, b64encode_ascii
13
- from pangea.services.vault.models.common import AsymmetricAlgorithm
15
+ from pangea.services.vault.models.asymmetric import AsymmetricKeySigningAlgorithm
14
16
 
15
17
 
16
18
  class AlgorithmSigner(ABC):
@@ -43,7 +45,7 @@ class ED25519Signer(AlgorithmSigner):
43
45
  )
44
46
 
45
47
  def get_algorithm(self) -> str:
46
- return AsymmetricAlgorithm.Ed25519.value
48
+ return AsymmetricKeySigningAlgorithm.ED25519.value
47
49
 
48
50
 
49
51
  signers = {
@@ -96,7 +98,7 @@ class Signer:
96
98
 
97
99
  for func in (serialization.load_pem_private_key, serialization.load_ssh_private_key):
98
100
  try:
99
- return func(private_key, None) # type: ignore[operator]
101
+ return func(private_key, None)
100
102
  except exceptions.UnsupportedAlgorithm as e:
101
103
  raise e
102
104
  except ValueError:
@@ -144,8 +146,7 @@ class Verifier:
144
146
  for cls, verifier in verifiers.items():
145
147
  if isinstance(pubkey, cls):
146
148
  return verifier(pubkey).verify(message_bytes, signature_bytes)
147
- else:
148
- raise PangeaException(f"Not supported public key type: {type(pubkey)}")
149
+ raise PangeaException(f"Not supported public key type: {type(pubkey)}")
149
150
 
150
151
  def _decode_public_key(self, public_key: bytes):
151
152
  """Parse a public key in PEM or ssh format"""
@@ -7,7 +7,7 @@ from binascii import hexlify, unhexlify
7
7
  from dataclasses import dataclass
8
8
  from datetime import datetime
9
9
  from hashlib import sha256
10
- from typing import Dict, List, Optional
10
+ from typing import Collection, Dict, List, Optional
11
11
 
12
12
  import requests
13
13
 
@@ -61,7 +61,7 @@ def verify_hash(hash1: str, hash2: str) -> bool:
61
61
 
62
62
 
63
63
  def verify_envelope_hash(envelope: EventEnvelope, hash: str):
64
- return verify_hash(hash_dict(normalize_log(envelope.dict(exclude_none=True))), hash)
64
+ return verify_hash(hash_dict(normalize_log(envelope.model_dump(exclude_none=True))), hash)
65
65
 
66
66
 
67
67
  def canonicalize_event(event: Event) -> bytes:
@@ -192,7 +192,7 @@ def arweave_graphql_url():
192
192
  return f"{ARWEAVE_BASE_URL}/graphql"
193
193
 
194
194
 
195
- def get_arweave_published_roots(tree_name: str, tree_sizes: List[int]) -> Dict[int, PublishedRoot]:
195
+ def get_arweave_published_roots(tree_name: str, tree_sizes: Collection[int]) -> Dict[int, PublishedRoot]:
196
196
  if len(tree_sizes) == 0:
197
197
  return {}
198
198