pangea-sdk 3.8.0__py3-none-any.whl → 5.3.0__py3-none-any.whl

pangea/services/audit/audit.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import datetime
 import json
-from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
 
 import pangea.exceptions as pexc
 from pangea.config import PangeaConfig
@@ -199,7 +199,7 @@ class AuditBase:
 
                 # verify consistency proofs
                 if self.can_verify_consistency_proof(search_event):
-                    if self.verify_consistency_proof(self.pub_roots, search_event):
+                    if search_event.leaf_index is not None and self.verify_consistency_proof(search_event.leaf_index):
                         search_event.consistency_verification = EventVerification.PASS
                     else:
                         search_event.consistency_verification = EventVerification.FAIL
@@ -222,7 +222,7 @@ class AuditBase:
         tree_sizes.difference_update(self.pub_roots.keys())
 
         if tree_sizes:
-            arweave_roots = get_arweave_published_roots(result.root.tree_name, list(tree_sizes))
+            arweave_roots = get_arweave_published_roots(result.root.tree_name, tree_sizes)
         else:
             arweave_roots = {}
 
@@ -284,7 +284,7 @@ class AuditBase:
         """
         return event.published and event.leaf_index is not None and event.leaf_index >= 0  # type: ignore[return-value]
 
-    def verify_consistency_proof(self, pub_roots: Dict[int, PublishedRoot], event: SearchEvent) -> bool:
+    def verify_consistency_proof(self, tree_size: int) -> bool:
         """
         Verify consistency proof
 
@@ -293,18 +293,17 @@ class AuditBase:
         Read more at: [What is a consistency proof?](https://pangea.cloud/docs/audit/merkle-trees#what-is-a-consistency-proof)
 
         Args:
-            pub_roots (dict[int, Root]): list of published root hashes across time
-            event (SearchEvent): Audit event to be verified.
+            leaf_index (int): The tree size of the root to be verified.
 
         Returns:
             bool: True if consistency proof is verified, False otherwise.
         """
 
-        if event.leaf_index == 0:
+        if tree_size == 0:
             return True
 
-        curr_root = pub_roots.get(event.leaf_index + 1)  # type: ignore[operator]
-        prev_root = pub_roots.get(event.leaf_index)  # type: ignore[arg-type]
+        curr_root = self.pub_roots.get(tree_size + 1)
+        prev_root = self.pub_roots.get(tree_size)
 
         if not curr_root or not prev_root:
             return False
@@ -314,9 +313,12 @@ class AuditBase:
         ):
             return False
 
+        if curr_root.consistency_proof is None:
+            return False
+
         curr_root_hash = decode_hash(curr_root.root_hash)
         prev_root_hash = decode_hash(prev_root.root_hash)
-        proof = decode_consistency_proof(curr_root.consistency_proof)  # type: ignore[arg-type]
+        proof = decode_consistency_proof(curr_root.consistency_proof)
 
         return verify_consistency_proof(curr_root_hash, prev_root_hash, proof)
 
@@ -490,7 +492,7 @@ class Audit(ServiceBase, AuditBase):
         verbose: Optional[bool] = None,
     ) -> PangeaResponse[LogResult]:
         """
-        Log an entry
+        Log an event
 
         Create a log entry in the Secure Audit Log.
 
@@ -499,6 +501,7 @@ class Audit(ServiceBase, AuditBase):
             verify (bool, optional): True to verify logs consistency after response.
             sign_local (bool, optional): True to sign event with local key.
             verbose (bool, optional): True to get a more verbose response.
+
         Raises:
             AuditException: If an audit based api exception happens
             PangeaAPIException: If an API Error happens
@@ -509,17 +512,13 @@ class Audit(ServiceBase, AuditBase):
                 Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
 
         Examples:
-            try:
-                log_response = audit.log({"message": "hello world"}, verbose=True)
-                print(f"Response. Hash: {log_response.result.hash}")
-            except pe.PangeaAPIException as e:
-                print(f"Request Error: {e.response.summary}")
-                for err in e.errors:
-                    print(f"\\t{err.detail} \\n")
+            response = audit.log_event({"message": "hello world"}, verbose=True)
         """
 
         input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
-        response: PangeaResponse[LogResult] = self.request.post("v1/log", LogResult, data=input.dict(exclude_none=True))
+        response: PangeaResponse[LogResult] = self.request.post(
+            "v1/log", LogResult, data=input.model_dump(exclude_none=True)
+        )
         if response.success and response.result is not None:
             self._process_log_result(response.result, verify=verify)
         return response
@@ -559,7 +558,7 @@ class Audit(ServiceBase, AuditBase):
 
         input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
         response: PangeaResponse[LogBulkResult] = self.request.post(
-            "v2/log", LogBulkResult, data=input.dict(exclude_none=True)
+            "v2/log", LogBulkResult, data=input.model_dump(exclude_none=True)
         )
 
         if response.success and response.result is not None:
@@ -603,7 +602,7 @@ class Audit(ServiceBase, AuditBase):
         try:
             # Calling to v2 methods will return always a 202.
             response: PangeaResponse[LogBulkResult] = self.request.post(
-                "v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
+                "v2/log_async", LogBulkResult, data=input.model_dump(exclude_none=True), poll_result=False
             )
         except pexc.AcceptedRequestException as e:
             return e.response
@@ -626,6 +625,7 @@ class Audit(ServiceBase, AuditBase):
         verbose: Optional[bool] = None,
         verify_consistency: bool = False,
         verify_events: bool = True,
+        return_context: Optional[bool] = None,
     ) -> PangeaResponse[SearchOutput]:
         """
         Search the log
@@ -655,6 +655,7 @@ class Audit(ServiceBase, AuditBase):
             verbose (bool, optional): If true, response include root and membership and consistency proofs.
             verify_consistency (bool): True to verify logs consistency
             verify_events (bool): True to verify hash events and signatures
+            return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
 
         Raises:
             AuditException: If an audit based api exception happens
@@ -688,10 +689,11 @@ class Audit(ServiceBase, AuditBase):
             max_results=max_results,
             search_restriction=search_restriction,
             verbose=verbose,
+            return_context=return_context,
         )
 
         response: PangeaResponse[SearchOutput] = self.request.post(
-            "v1/search", SearchOutput, data=input.dict(exclude_none=True)
+            "v1/search", SearchOutput, data=input.model_dump(exclude_none=True)
         )
         if verify_consistency and response.result is not None:
             self.update_published_roots(response.result)
@@ -705,6 +707,7 @@ class Audit(ServiceBase, AuditBase):
         assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None,
         verify_consistency: bool = False,
         verify_events: bool = True,
+        return_context: Optional[bool] = None,
     ) -> PangeaResponse[SearchResultOutput]:
         """
         Results of a search
@@ -720,6 +723,7 @@ class Audit(ServiceBase, AuditBase):
             assert_search_restriction (Dict[str, Sequence[str]], optional): Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
             verify_consistency (bool): True to verify logs consistency
             verify_events (bool): True to verify hash events and signatures
+            return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
         Raises:
             AuditException: If an audit based api exception happens
             PangeaAPIException: If an API Error happens
@@ -743,9 +747,10 @@ class Audit(ServiceBase, AuditBase):
             limit=limit,
             offset=offset,
             assert_search_restriction=assert_search_restriction,
+            return_context=return_context,
         )
         response: PangeaResponse[SearchResultOutput] = self.request.post(
-            "v1/results", SearchResultOutput, data=input.dict(exclude_none=True)
+            "v1/results", SearchResultOutput, data=input.model_dump(exclude_none=True)
         )
         if verify_consistency and response.result is not None:
             self.update_published_roots(response.result)
@@ -807,7 +812,7 @@ class Audit(ServiceBase, AuditBase):
         )
         try:
             return self.request.post(
-                "v1/export", PangeaResponseResult, data=input.dict(exclude_none=True), poll_result=False
+                "v1/export", PangeaResponseResult, data=input.model_dump(exclude_none=True), poll_result=False
             )
         except pexc.AcceptedRequestException as e:
             return e.response
@@ -874,13 +879,14 @@ class Audit(ServiceBase, AuditBase):
             response = audit.root(tree_size=7)
         """
         input = RootRequest(tree_size=tree_size)
-        return self.request.post("v1/root", RootResult, data=input.dict(exclude_none=True))
+        return self.request.post("v1/root", RootResult, data=input.model_dump(exclude_none=True))
 
     def download_results(
         self,
         result_id: Optional[str] = None,
         format: DownloadFormat = DownloadFormat.CSV,
         request_id: Optional[str] = None,
+        return_context: Optional[bool] = None,
     ) -> PangeaResponse[DownloadResult]:
         """
         Download search results
@@ -893,6 +899,7 @@ class Audit(ServiceBase, AuditBase):
             result_id: ID returned by the search API.
             format: Format for the records.
             request_id: ID returned by the export API.
+            return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
 
         Returns:
             URL where search results can be downloaded.
@@ -911,8 +918,10 @@ class Audit(ServiceBase, AuditBase):
         if request_id is None and result_id is None:
             raise ValueError("must pass one of `request_id` or `result_id`")
 
-        input = DownloadRequest(request_id=request_id, result_id=result_id, format=format)
-        return self.request.post("v1/download_results", DownloadResult, data=input.dict(exclude_none=True))
+        input = DownloadRequest(
+            request_id=request_id, result_id=result_id, format=format, return_context=return_context
+        )
+        return self.request.post("v1/download_results", DownloadResult, data=input.model_dump(exclude_none=True))
 
     def update_published_roots(self, result: SearchResultOutput):
         """Fetches series of published root hashes from Arweave
@@ -937,12 +946,31 @@ class Audit(ServiceBase, AuditBase):
         for tree_size in tree_sizes:
             pub_root = None
             if tree_size in arweave_roots:
-                pub_root = PublishedRoot(**arweave_roots[tree_size].dict(exclude_none=True))
+                pub_root = PublishedRoot(**arweave_roots[tree_size].model_dump(exclude_none=True))
                 pub_root.source = RootSource.ARWEAVE
             elif self.allow_server_roots:
                 resp = self.root(tree_size=tree_size)
                 if resp.success and resp.result is not None:
-                    pub_root = PublishedRoot(**resp.result.data.dict(exclude_none=True))
+                    pub_root = PublishedRoot(**resp.result.data.model_dump(exclude_none=True))
                     pub_root.source = RootSource.PANGEA
             if pub_root is not None:
                 self.pub_roots[tree_size] = pub_root
+
+        self._fix_consistency_proofs(tree_sizes)
+
+    def _fix_consistency_proofs(self, tree_sizes: Iterable[int]) -> None:
+        # on very rare occasions, the consistency proof in Arweave may be wrong
+        # override it with the proof from pangea (not the root hash, just the proof)
+        for tree_size in tree_sizes:
+            if tree_size not in self.pub_roots or tree_size - 1 not in self.pub_roots:
+                continue
+
+            if self.pub_roots[tree_size].source == RootSource.PANGEA:
+                continue
+
+            if self.verify_consistency_proof(tree_size):
+                continue
+
+            resp = self.root(tree_size=tree_size)
+            if resp.success and resp.result is not None and resp.result.data is not None:
+                self.pub_roots[tree_size].consistency_proof = resp.result.data.consistency_proof

pangea/services/audit/models.py

@@ -6,7 +6,7 @@ import datetime
 import enum
 from typing import Any, Dict, List, Optional, Sequence, Union
 
-from pangea.response import APIRequestModel, APIResponseModel, PangeaResponseResult
+from pangea.response import APIRequestModel, APIResponseModel, PangeaDateTime, PangeaResponseResult
 
 
 class EventVerification(str, enum.Enum):
@@ -126,7 +126,7 @@ class EventEnvelope(APIResponseModel):
     event: Dict[str, Any]
     signature: Optional[str] = None
     public_key: Optional[str] = None
-    received_at: datetime.datetime
+    received_at: PangeaDateTime
 
 
 class LogRequest(APIRequestModel):
@@ -271,6 +271,7 @@ class SearchRequest(APIRequestModel):
     max_results -- Maximum number of results to return.
     search_restriction -- A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
     verbose -- If true, include root, membership and consistency proofs in response.
+    return_context -- Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
     """
 
     query: str
@@ -283,6 +284,7 @@ class SearchRequest(APIRequestModel):
     max_results: Optional[int] = None
     search_restriction: Optional[Dict[str, Sequence[str]]] = None
     verbose: Optional[bool] = None
+    return_context: Optional[bool] = None
 
 
 class RootRequest(APIRequestModel):
@@ -363,6 +365,7 @@ class SearchEvent(APIResponseModel):
     consistency_verification -- Consistency verification calculated if required.
     membership_verification -- Membership verification calculated if required.
     signature_verification -- Signature verification calculated if required.
+    fpe_context -- The context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
     """
 
     envelope: EventEnvelope
@@ -373,6 +376,7 @@ class SearchEvent(APIResponseModel):
     consistency_verification: EventVerification = EventVerification.NONE
     membership_verification: EventVerification = EventVerification.NONE
     signature_verification: EventVerification = EventVerification.NONE
+    fpe_context: Optional[str] = None
 
 
 class SearchResultOutput(PangeaResponseResult):
@@ -406,7 +410,7 @@ class SearchOutput(SearchResultOutput):
     """
 
     id: str
-    expires_at: datetime.datetime
+    expires_at: PangeaDateTime
 
 
 class SearchResultRequest(APIRequestModel):
@@ -418,12 +422,14 @@ class SearchResultRequest(APIRequestModel):
     limit -- Number of audit records to include from the first page of the results.
     offset -- Offset from the start of the result set to start returning results from.
     assert_search_restriction -- Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
+    return_context -- Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
     """
 
     id: str
     limit: Optional[int] = 20
     offset: Optional[int] = 0
     assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None
+    return_context: Optional[bool] = None
 
 
 class DownloadFormat(str, enum.Enum):
@@ -450,6 +456,9 @@ class DownloadRequest(APIRequestModel):
     format: Optional[str] = None
     """Format for the records."""
 
+    return_context: Optional[bool] = None
+    """Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption."""
+
 
 class DownloadResult(PangeaResponseResult):
     dest_url: str

pangea/services/audit/signing.py

@@ -1,5 +1,7 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from typing import Optional
 
@@ -10,7 +12,7 @@ from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes, Pub
 
 from pangea.exceptions import PangeaException
 from pangea.services.audit.util import b64decode, b64decode_ascii, b64encode_ascii
-from pangea.services.vault.models.common import AsymmetricAlgorithm
+from pangea.services.vault.models.asymmetric import AsymmetricKeySigningAlgorithm
 
 
 class AlgorithmSigner(ABC):
@@ -43,7 +45,7 @@ class ED25519Signer(AlgorithmSigner):
         )
 
     def get_algorithm(self) -> str:
-        return AsymmetricAlgorithm.Ed25519.value
+        return AsymmetricKeySigningAlgorithm.ED25519.value
 
 
 signers = {
@@ -96,7 +98,7 @@ class Signer:
 
         for func in (serialization.load_pem_private_key, serialization.load_ssh_private_key):
             try:
-                return func(private_key, None)  # type: ignore[operator]
+                return func(private_key, None)
             except exceptions.UnsupportedAlgorithm as e:
                 raise e
             except ValueError:
@@ -144,8 +146,7 @@ class Verifier:
         for cls, verifier in verifiers.items():
             if isinstance(pubkey, cls):
                 return verifier(pubkey).verify(message_bytes, signature_bytes)
-        else:
-            raise PangeaException(f"Not supported public key type: {type(pubkey)}")
+        raise PangeaException(f"Not supported public key type: {type(pubkey)}")
 
     def _decode_public_key(self, public_key: bytes):
         """Parse a public key in PEM or ssh format"""

pangea/services/audit/util.py

@@ -7,7 +7,7 @@ from binascii import hexlify, unhexlify
 from dataclasses import dataclass
 from datetime import datetime
 from hashlib import sha256
-from typing import Dict, List, Optional
+from typing import Collection, Dict, List, Optional
 
 import requests
 
@@ -61,7 +61,7 @@ def verify_hash(hash1: str, hash2: str) -> bool:
 
 
 def verify_envelope_hash(envelope: EventEnvelope, hash: str):
-    return verify_hash(hash_dict(normalize_log(envelope.dict(exclude_none=True))), hash)
+    return verify_hash(hash_dict(normalize_log(envelope.model_dump(exclude_none=True))), hash)
 
 
 def canonicalize_event(event: Event) -> bytes:
@@ -192,7 +192,7 @@ def arweave_graphql_url():
     return f"{ARWEAVE_BASE_URL}/graphql"
 
 
-def get_arweave_published_roots(tree_name: str, tree_sizes: List[int]) -> Dict[int, PublishedRoot]:
+def get_arweave_published_roots(tree_name: str, tree_sizes: Collection[int]) -> Dict[int, PublishedRoot]:
     if len(tree_sizes) == 0:
         return {}