pangea-sdk 3.8.0__py3-none-any.whl → 5.3.0__py3-none-any.whl

pangea/verify_audit.py

@@ -12,9 +12,16 @@ In the latter case, all the events are verified.
 import argparse
 import json
 import logging
+import os
 import sys
-from typing import Dict, List, Optional
-
+from collections.abc import Set
+from enum import Enum
+from typing import Dict, Iterable, List, Optional, Union
+
+from pangea.config import PangeaConfig
+from pangea.exceptions import TreeNotFoundException
+from pangea.services import Audit
+from pangea.services.audit.models import PublishedRoot, Root
 from pangea.services.audit.signing import Verifier
 from pangea.services.audit.util import (
     canonicalize_json,
@@ -29,7 +36,16 @@ from pangea.services.audit.util import (
 )
 
 logger = logging.getLogger("audit")
-pub_roots: Dict[int, Dict] = {}
+arweave_roots: Dict[int, PublishedRoot] = {}  # roots fetched from Arweave
+pangea_roots: Dict[int, Root] = {}  # roots fetched from Pangea
+audit: Optional[Audit] = None
+
+
+class Status(Enum):
+    SUCCEEDED = "succeeded"
+    SUCCEEDED_PANGEA = "succeeded_pangea"  # succeeded with data fetched from Pangea instead of Arweave
+    FAILED = "failed"
+    SKIPPED = "skipped"
 
 
 class VerifierLogFormatter(logging.Formatter):
@@ -40,9 +56,11 @@ class VerifierLogFormatter(logging.Formatter):
 
     def format(self, record):
         if hasattr(record, "is_result"):
-            if record.succeeded:
+            if record.status == Status.SUCCEEDED:
                 point = "🟢"
-            elif record.succeeded is None:
+            elif record.status == Status.SUCCEEDED_PANGEA:
+                point = "🟡"
+            elif record.status == Status.SKIPPED:
                 point = "⚪️"
             else:
                 point = "🔴"
@@ -61,14 +79,16 @@ class VerifierLogFormatter(logging.Formatter):
             return f"{pre}{record.msg}"
 
 
-def log_result(msg: str, succeeded: Optional[bool]):
-    if succeeded is True:
+def log_result(msg: str, status: Status):
+    if status == Status.SUCCEEDED:
         msg += " succeeded"
-    elif succeeded is False:
+    elif status == Status.SUCCEEDED_PANGEA:
+        msg += " succeeded (with data fetched from Pangea)"
+    elif status == Status.FAILED:
         msg += " failed"
     else:
-        msg += " could not be performed"
-    logger.log(logging.INFO, msg, extra={"is_result": True, "succeeded": succeeded})
+        msg += " skipped"
+    logger.log(logging.INFO, msg, extra={"is_result": True, "status": status})
 
 
 def log_section(msg: str):
@@ -77,9 +97,37 @@ def log_section(msg: str):
 
 formatter = VerifierLogFormatter()
 
+InvalidTokenError = ValueError("Invalid Pangea Token provided")
+
+
+def get_pangea_roots(tree_name: str, tree_sizes: Iterable[int]) -> Dict[int, Root]:
+    ans: Dict[int, Root] = {}
+    if audit is None:
+        return ans
+
+    for size in tree_sizes:
+        try:
+            resp = audit.root(size)
+
+            if resp.status != "Success":
+                raise ValueError(resp.status)
+            elif resp.result is None or resp.result.data is None:
+                raise ValueError("No result")
+            elif resp.result.data.tree_name != tree_name:
+                raise InvalidTokenError
+
+            ans[int(size)] = resp.result.data
+        except TreeNotFoundException as e:
+            ex = InvalidTokenError
+            logger.error(f"Error fetching root from Pangea for size {size}: {str(ex)}")
+        except Exception as e:
+            logger.error(f"Error fetching root from Pangea for size {size}: {str(e)}")
+    return ans
+
 
-def _verify_hash(data: Dict, data_hash: str) -> Optional[bool]:
+def _verify_hash(data: Dict, data_hash: str) -> Status:
     log_section("Checking data hash")
+    status = Status.SKIPPED
     try:
         logger.debug("Canonicalizing data")
         data_canon = canonicalize_json(data)
@@ -89,22 +137,20 @@ def _verify_hash(data: Dict, data_hash: str) -> Optional[bool]:
         logger.debug("Comparing calculated hash with server hash")
         if computed_hash_dec != data_hash_dec:
             raise ValueError("Hash does not match")
-        succeeded = True
+        status = Status.SUCCEEDED
     except Exception:
-        succeeded = False
+        status = Status.FAILED
 
-    log_result("Data hash verification", succeeded)
+    log_result("Data hash verification", status)
     logger.info("")
-    return succeeded
+    return status
 
 
-def _verify_unpublished_membership_proof(root_hash, node_hash: str, proof: Optional[str]) -> Optional[bool]:
-    global pub_roots
-
+def _verify_unpublished_membership_proof(root_hash, node_hash: str, proof: Optional[str]) -> Status:
     log_section("Checking unpublished membership proof")
 
     if proof is None:
-        succeeded = None
+        status = Status.SKIPPED
         logger.debug("Proof not found")
     else:
         try:
@@ -116,94 +162,209 @@ def _verify_unpublished_membership_proof(root_hash, node_hash: str, proof: Optio
             proof_dec = decode_membership_proof(proof)
 
             logger.debug("Comparing the unpublished root hash with the proof hash")
-            succeeded = verify_membership_proof(node_hash_dec, root_hash_dec, proof_dec)
+            if verify_membership_proof(node_hash_dec, root_hash_dec, proof_dec):
+                status = Status.SUCCEEDED
+            else:
+                status = Status.FAILED
 
         except Exception as e:
-            succeeded = False
+            status = Status.FAILED
             logger.debug(str(e))
 
-    log_result("Unpublished membership proof verification", succeeded)
+    log_result("Unpublished membership proof verification", status)
+    logger.info("")
+    return status
+
+
+def _fetch_roots(tree_name: str, tree_size: int, leaf_index: Optional[int]) -> Status:
+    global arweave_roots, pangea_roots
+
+    log_section("Fetching published roots")
+
+    needed_roots = {tree_size}
+    if leaf_index:
+        needed_roots |= {leaf_index, leaf_index + 1}
+
+    pending_roots = set()
+
+    def update_pending_roots():
+        nonlocal pending_roots
+        pending_roots = needed_roots - set(arweave_roots.keys()) - set(pangea_roots.keys())
+
+    def comma_sep(s: Set):
+        return ",".join(map(str, s))
+
+    status = Status.SUCCEEDED
+    update_pending_roots()
+
+    # print message for roots already fetched
+    arweave_fetched_roots = needed_roots & set(arweave_roots.keys())
+    if arweave_fetched_roots:
+        logger.debug(f"Roots {comma_sep(arweave_fetched_roots)} already fetched from Arweave")
+
+    pangea_fetched_roots = needed_roots & set(pangea_roots.keys())
+    if pangea_fetched_roots:
+        logger.debug(f"Roots {comma_sep(pangea_fetched_roots)} already fetched from Pangea")
+        status = Status.SUCCEEDED_PANGEA
+
+    if pending_roots:
+        # try Arweave first
+        try:
+            logger.debug(f"Fetching root(s) {comma_sep(pending_roots)} from Arweave")
+            arweave_roots |= {int(k): v for k, v in get_arweave_published_roots(tree_name, pending_roots).items()}
+            update_pending_roots()
+        except:
+            pass
+
+    if pending_roots:
+        logger.debug(f"Published root(s) {comma_sep(pending_roots)} could not be fetched from Arweave")
+
+    if pending_roots:
+        if audit:
+            # and then Pangea (if we've set an audit client)
+            try:
+                logger.debug(f"Fetching root(s) {comma_sep(pending_roots)} from Pangea")
+                pangea_roots |= {int(k): v for k, v in get_pangea_roots(tree_name, pending_roots).items()}
+                update_pending_roots()
+                status = Status.SUCCEEDED_PANGEA
+            except:
+                pass
+
+            if pending_roots:
+                logger.debug(f"Roots {comma_sep(pending_roots)} could not be fetched")
+        else:
+            logger.debug("Set Pangea token and domain (from envvars or script parameters) to fetch roots from Pangea")
+
+    if pending_roots:
+        status = Status.FAILED
+
+    log_result("Fetching published roots", status)
     logger.info("")
-    return succeeded
+    return status
 
 
-def _verify_membership_proof(tree_name: str, tree_size: int, node_hash: str, proof: Optional[str]) -> Optional[bool]:
-    global pub_roots
+def _verify_membership_proof(tree_size: int, node_hash: str, proof: Optional[str]) -> Status:
+    pub_roots: Dict[int, Union[Root, PublishedRoot]] = arweave_roots | pangea_roots
 
     log_section("Checking membership proof")
 
-    if proof is None:
-        succeeded = None
+    if tree_size not in pub_roots:
+        status = Status.SKIPPED
+        logger.debug("Published root not found")
+    elif proof is None:
+        status = Status.SKIPPED
         logger.debug("Proof not found (event not published yet)")
     else:
         try:
-            logger.debug("Fetching published roots from Arweave")
-            if tree_size not in pub_roots:
-                pub_roots |= {int(k): v for k, v in get_arweave_published_roots(tree_name, [tree_size]).items()}  # type: ignore[operator]
-            if tree_size not in pub_roots:
-                raise ValueError("Published root could was not found")
-
-            root_hash_dec = decode_hash(pub_roots[tree_size].root_hash)  # type: ignore[attr-defined]
+            root_hash_dec = decode_hash(pub_roots[tree_size].root_hash)
             node_hash_dec = decode_hash(node_hash)
             logger.debug("Calculating the proof")
+            if proof is None:
+                logger.debug("Consistency proof is missing")
+                return False
             proof_dec = decode_membership_proof(proof)
             logger.debug("Comparing the root hash with the proof hash")
-            succeeded = verify_membership_proof(node_hash_dec, root_hash_dec, proof_dec)
+            if verify_membership_proof(node_hash_dec, root_hash_dec, proof_dec):
+                status = Status.SUCCEEDED
+            else:
+                status = Status.FAILED
         except Exception as e:
-            succeeded = False
+            status = Status.FAILED
             logger.debug(str(e))
 
-    log_result("Membership proof verification", succeeded)
+    log_result("Membership proof verification", status)
     logger.info("")
-    return succeeded
+    return status
+
+
+def _consistency_proof_ok(pub_roots: Dict[int, Union[Root, PublishedRoot]], leaf_index: int) -> bool:
+    """returns true if a consistency proof is correct"""
+
+    curr_root = pub_roots[leaf_index + 1]
+    prev_root = pub_roots[leaf_index]
+    curr_root_hash = decode_hash(curr_root.root_hash)
+    prev_root_hash = decode_hash(prev_root.root_hash)
+    if curr_root.consistency_proof is None:
+        logger.debug("Consistency proof is missing")
+        return False
+
+    logger.debug("Calculating the proof")
+    proof = decode_consistency_proof(curr_root.consistency_proof)
+    return verify_consistency_proof(curr_root_hash, prev_root_hash, proof)
 
 
-def _verify_consistency_proof(tree_name: str, leaf_index: Optional[int]) -> Optional[bool]:
-    global pub_roots
+# Due to an (already fixed) bug, some proofs from Arweave may be wrong.
+# Try the proof from Pangea instead. If the root hash in both Arweave and Pangea is the same,
+# it doesn't matter where the proof came from.
+def _fix_consistency_proof(pub_roots: Dict[int, Union[Root, PublishedRoot]], tree_name: str, leaf_index: int):
+    logger.debug("Consistency proof from Arweave failed to verify")
+    size = leaf_index + 1
+    logger.debug(f"Fetching root from Pangea for size {size}")
+    new_roots = get_pangea_roots(tree_name, [size])
+    if size not in new_roots:
+        raise ValueError("Error fetching root from Pangea")
+    pangea_roots[size] = new_roots[size]
+    pub_roots[size] = pangea_roots[size]
+    logger.debug(f"Comparing Arweave root hash with Pangea root hash")
+    if pangea_roots[size].root_hash != arweave_roots[size].root_hash:
+        raise ValueError("Hash does not match")
+
+
+def _verify_consistency_proof(tree_name: str, leaf_index: Optional[int]) -> Status:
+    pub_roots: Dict[int, Union[Root, PublishedRoot]] = arweave_roots | pangea_roots
+
     log_section("Checking consistency proof")
 
     if leaf_index is None:
-        succeeded = None
+        status = Status.SKIPPED
         logger.debug("Proof not found (event was not published yet)")
 
     elif leaf_index == 0:
-        succeeded = None
+        status = Status.SKIPPED
         logger.debug("Proof not found (event was published in the first leaf)")
+
+    elif leaf_index not in pub_roots:
+        status = Status.SKIPPED
+        logger.debug("Published root not found")
+
     else:
         try:
-            logger.debug("Fetching published roots from Arweave")
-            pub_roots |= {  # type: ignore[operator]
-                int(k): v for k, v in get_arweave_published_roots(tree_name, [leaf_index + 1, leaf_index]).items()
-            }
-            if leaf_index + 1 not in pub_roots or leaf_index not in pub_roots:
-                raise ValueError("Published roots could not be retrieved")
-
-            curr_root = pub_roots[leaf_index + 1]
-            prev_root = pub_roots[leaf_index]
-            curr_root_hash = decode_hash(curr_root.root_hash)  # type: ignore[attr-defined]
-            prev_root_hash = decode_hash(prev_root.root_hash)  # type: ignore[attr-defined]
-            logger.debug("Calculating the proof")
-            proof = decode_consistency_proof(curr_root.consistency_proof)  # type: ignore[attr-defined]
-            succeeded = verify_consistency_proof(curr_root_hash, prev_root_hash, proof)
+            if _consistency_proof_ok(pub_roots, leaf_index):
+                status = Status.SUCCEEDED
+
+            elif audit:
+                _fix_consistency_proof(pub_roots, tree_name, leaf_index)
+
+                # check again
+                if _consistency_proof_ok(pub_roots, leaf_index):
+                    status = Status.SUCCEEDED
+                else:
+                    status = Status.FAILED
+
+            else:
+                logger.debug(
+                    "Set Pangea token and domain (from envvars or script parameters) to fetch roots from Pangea"
+                )
+                status = Status.FAILED
 
         except Exception as e:
-            succeeded = False
+            status = Status.FAILED
             logger.debug(str(e))
 
-    log_result("Consistency proof verification", succeeded)
+    log_result("Consistency proof verification", status)
     logger.info("")
-    return succeeded
+    return status
 
 
 def create_signed_event(event: Dict) -> Dict:
     return {k: v for k, v in event.items() if v is not None}
 
 
-def _verify_signature(data: Dict) -> Optional[bool]:
+def _verify_signature(data: Dict) -> Status:
     log_section("Checking signature")
     if "signature" not in data:
         logger.debug("Signature is not present")
-        succeeded = None
+        status = Status.SKIPPED
     else:
         try:
             logger.debug("Obtaining signature and public key from the event")
@@ -213,30 +374,36 @@ def _verify_signature(data: Dict) -> Optional[bool]:
             logger.debug("Checking the signature")
             if not sign_verifier.verify_signature(data["signature"], canonicalize_json(sign_event), public_key):
                 raise ValueError("Signature is invalid")
-            succeeded = True
+            status = Status.SUCCEEDED
         except Exception:
-            succeeded = False
+            status = Status.FAILED
 
-    log_result("Data signature verification", succeeded)
+    log_result("Data signature verification", status)
     logger.info("")
-    return succeeded
+    return status
 
 
-def verify_multiple(root: Dict, unpublished_root: Dict, events: List[Dict]) -> Optional[bool]:
+def verify_multiple(root: Dict, unpublished_root: Dict, events: List[Dict]) -> Status:
     """
     Verify a list of events.
     Returns a status.
     """
 
-    succeeded = []
+    statuses: List[Status] = []
     for counter, event in enumerate(events):
         event.update({"root": root, "unpublished_root": unpublished_root})
-        event_succeeded = verify_single(event, counter + 1)
-        succeeded.append(event_succeeded)
-    return not any(event_succeeded is False for event_succeeded in succeeded)
+        event_status = verify_single(event, counter + 1)
+        statuses.append(event_status)
+
+    for event_status in statuses:
+        if event_status == Status.FAILED:
+            return Status.FAILED
+        elif event_status == Status.SKIPPED:
+            return Status.SKIPPED
+    return Status.SUCCEEDED
 
 
-def verify_single(data: Dict, counter: Optional[int] = None) -> Optional[bool]:
+def verify_single(data: Dict, counter: Optional[int] = None) -> Status:
     """
     Verify a single event.
     Returns a status.
@@ -248,11 +415,15 @@ def verify_single(data: Dict, counter: Optional[int] = None) -> Optional[bool]:
     ok_hash = _verify_hash(data["envelope"], data["hash"])
     ok_signature = _verify_signature(data["envelope"])
 
+    if data.get("root"):
+        ok_roots = _fetch_roots(data["root"]["tree_name"], data["root"]["size"], data.get("leaf_index"))
+    else:
+        ok_roots = Status.SKIPPED
+
     if data["published"]:
         if not data.get("root"):
             raise ValueError("Missing 'root' element")
         ok_membership = _verify_membership_proof(
-            data["root"]["tree_name"],
             data["root"]["size"],
             data["hash"],
             data.get("membership_proof"),
@@ -267,25 +438,31 @@ def verify_single(data: Dict, counter: Optional[int] = None) -> Optional[bool]:
     if data["published"]:
         ok_consistency = _verify_consistency_proof(data["root"]["tree_name"], data.get("leaf_index"))
     else:
-        ok_consistency = True
+        ok_consistency = Status.SUCCEEDED
 
     all_ok = (
-        ok_hash is True
-        and (ok_signature is True or ok_signature is None)
-        and ok_membership is True
-        and ok_consistency is True
+        ok_hash == Status.SUCCEEDED
+        and (ok_signature in (Status.SUCCEEDED, Status.SKIPPED))
+        and (ok_roots in (Status.SKIPPED, Status.SUCCEEDED, Status.SUCCEEDED_PANGEA))
+        and ok_membership == Status.SUCCEEDED
+        and (ok_consistency in (Status.SUCCEEDED, Status.SKIPPED))
+    )
+    any_failed = (
+        ok_hash == Status.FAILED
+        or (ok_signature == Status.FAILED)
+        or (ok_membership == Status.FAILED)
+        or (ok_consistency == Status.FAILED)
     )
-    any_failed = ok_hash is False or ok_signature is False or ok_membership is False or ok_consistency is False
 
     if counter:
         formatter.indent = 0
 
     if all_ok:
-        return True
+        return Status.SUCCEEDED
     elif any_failed:
-        return False
+        return Status.FAILED
     else:
-        return None
+        return Status.SKIPPED
 
 
 def main():
@@ -303,6 +480,12 @@ def main():
         metavar="PATH",
         help="Input file (default: standard input).",
     )
+    parser.add_argument(
+        "--token", "-t", default=os.getenv("PANGEA_TOKEN"), help="Pangea token (default: env PANGEA_TOKEN)"
+    )
+    parser.add_argument(
+        "--domain", "-d", default=os.getenv("PANGEA_DOMAIN"), help="Pangea domain (default: env PANGEA_DOMAIN)"
+    )
     args = parser.parse_args()
 
     data = json.load(args.file)
@@ -311,15 +494,19 @@ def main():
     logger.info("Pangea Audit - Verification Tool")
     logger.info("")
 
+    if args.token and args.domain:
+        global audit
+        audit = Audit(token=args.token, config=PangeaConfig(domain=args.domain))
+
     if events:
         status = verify_multiple(data["result"].get("root"), data["result"].get("unpublished_root"), events)
     else:
         status = verify_single(data)
 
     logger.info("")
-    if status is True:
+    if status == Status.SUCCEEDED:
         logger.info("🟢 Verification succeeded 🟢")
-    elif status is False:
+    elif status == Status.FAILED:
         logger.info("🔴 Verification failed 🔴")
     else:
         logger.info("⚪️ Verification could not be finished ⚪️")

{pangea_sdk-3.8.0.dist-info → pangea_sdk-5.3.0.dist-info}/METADATA

@@ -1,32 +1,23 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: pangea-sdk
-Version: 3.8.0
+Version: 5.3.0
 Summary: Pangea API SDK
-Home-page: https://pangea.cloud/docs/sdk/python/
 License: MIT
 Keywords: Pangea,SDK,Audit
 Author: Glenn Gallien
 Author-email: glenn.gallien@pangea.cloud
-Requires-Python: >=3.7.2,<4.0.0
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.9
 Classifier: Topic :: Software Development
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: aiohttp (>=3.8.6,<4.0.0)
-Requires-Dist: asyncio (>=3.4.3,<4.0.0)
-Requires-Dist: cryptography (>=42.0.7,<43.0.0)
-Requires-Dist: deprecated (>=1.2.14,<2.0.0)
-Requires-Dist: google-crc32c (>=1.5.0,<2.0.0)
-Requires-Dist: pydantic (>=1.10.15,<2.0.0)
-Requires-Dist: python-dateutil (>=2.9.0,<3.0.0)
-Requires-Dist: requests (>=2.31.0,<3.0.0)
-Requires-Dist: requests-toolbelt (>=1.0.0,<2.0.0)
-Requires-Dist: typing-extensions (>=4.7.1,<5.0.0)
-Project-URL: Repository, https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk
+Requires-Dist: aiohttp
+Requires-Dist: cryptography
+Requires-Dist: deprecated
+Requires-Dist: google-crc32c
+Requires-Dist: pydantic
+Requires-Dist: python-dateutil
+Requires-Dist: requests
+Requires-Dist: requests-toolbelt
+Requires-Dist: typing-extensions
 Description-Content-Type: text/markdown
 
 <a href="https://pangea.cloud?utm_source=github&utm_medium=python-sdk" target="_blank" rel="noopener noreferrer">
@@ -36,11 +27,11 @@ Description-Content-Type: text/markdown
 <br />
 
 [![documentation](https://img.shields.io/badge/documentation-pangea-blue?style=for-the-badge&labelColor=551B76)][Documentation]
-[![Slack](https://img.shields.io/badge/Slack-4A154B?style=for-the-badge&logo=slack&logoColor=white)][Slack]
+[![Discourse](https://img.shields.io/badge/Discourse-4A154B?style=for-the-badge&logo=discourse&logoColor=white)][Discourse]
 
 # Pangea Python SDK
 
-A Python SDK for integrating with Pangea services. Supports Python v3.7 and
+A Python SDK for integrating with Pangea services. Supports Python v3.9 and
 above.
 
 ## Installation
@@ -64,20 +55,21 @@ $ poetry add pangea-sdk
 #### Beta releases
 
 Pre-release versions may be available with the `b` (beta) denotation in the
-version number. These releases serve to preview beta services and APIs. Per
-Semantic Versioning, they are considered unstable and do not carry the same
-compatibility guarantees as stable releases. [Beta changelog](https://github.com/pangeacyber/pangea-python/blob/beta/CHANGELOG.md).
+version number. These releases serve to preview Beta and Early Access services
+and APIs. Per Semantic Versioning, they are considered unstable and do not carry
+the same compatibility guarantees as stable releases.
+[Beta changelog](https://github.com/pangeacyber/pangea-python/blob/beta/CHANGELOG.md).
 
 Via pip:
 
 ```bash
-$ pip3 install pangea-sdk==3.8.0b2
+$ pip3 install pangea-sdk==5.2.0b2
 ```
 
 Via poetry:
 
 ```bash
-$ poetry add pangea-sdk==3.8.0b2
+$ poetry add pangea-sdk==5.2.0b2
 ```
 
 ## Usage
@@ -231,6 +223,6 @@ It accepts multiple file formats:
    [GA Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
    [Beta Examples]: https://github.com/pangeacyber/pangea-python/tree/beta/examples
    [Pangea Console]: https://console.pangea.cloud/
-   [Slack]: https://pangea.cloud/join-slack/
+   [Discourse]: https://l.pangea.cloud/Jd4wlGs
    [Secure Audit Log]: https://pangea.cloud/docs/audit