pangea-sdk 3.8.0__py3-none-any.whl → 5.3.0__py3-none-any.whl

pangea/__init__.py

@@ -1,6 +1,7 @@
-__version__ = "3.8.0"
+__version__ = "5.3.0"
 
 from pangea.asyncio.request import PangeaRequestAsync
 from pangea.config import PangeaConfig
+from pangea.file_uploader import FileUploader
 from pangea.request import PangeaRequest
 from pangea.response import PangeaResponse

pangea/asyncio/__init__.py

@@ -0,0 +1 @@
+from .file_uploader import FileUploaderAsync

pangea/asyncio/file_uploader.py

@@ -0,0 +1,39 @@
+# Copyright 2022 Pangea Cyber Corporation
+# Author: Pangea Cyber Corporation
+import io
+import logging
+from typing import Dict, Optional
+
+from pangea.asyncio.request import PangeaRequestAsync
+from pangea.request import PangeaConfig
+from pangea.response import TransferMethod
+
+
+class FileUploaderAsync:
+    def __init__(self) -> None:
+        self.logger = logging.getLogger("pangea")
+        self._request: PangeaRequestAsync = PangeaRequestAsync(
+            config=PangeaConfig(),
+            token="",
+            service="FileUploader",
+            logger=self.logger,
+        )
+
+    async def upload_file(
+        self,
+        url: str,
+        file: io.BufferedReader,
+        transfer_method: TransferMethod = TransferMethod.PUT_URL,
+        file_details: Optional[Dict] = None,
+    ) -> None:
+        if transfer_method == TransferMethod.PUT_URL:
+            files = [("file", ("filename", file, "application/octet-stream"))]
+            await self._request.put_presigned_url(url=url, files=files)
+        elif transfer_method == TransferMethod.POST_URL:
+            files = [("file", ("filename", file, "application/octet-stream"))]
+            await self._request.post_presigned_url(url=url, data=file_details, files=files)  # type: ignore[arg-type]
+        else:
+            raise ValueError(f"Transfer method not supported: {transfer_method}")
+
+    async def close(self) -> None:
+        await self._request.session.close()

pangea/asyncio/request.py

@@ -1,21 +1,23 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+from __future__ import annotations
 
 import asyncio
 import json
 import time
-from typing import Dict, List, Optional, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union
 
 import aiohttp
 from aiohttp import FormData
-from typing_extensions import TypeVar
+from pydantic import BaseModel
+from typing_extensions import Any, TypeVar
 
 import pangea.exceptions as pe
 from pangea.request import MultipartResponse, PangeaRequestBase
 from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
 from pangea.utils import default_encoder
 
-TResult = TypeVar("TResult", bound=PangeaResponseResult, default=PangeaResponseResult)
+TResult = TypeVar("TResult", bound=PangeaResponseResult)
 
 
 class PangeaRequestAsync(PangeaRequestBase):
@@ -31,8 +33,8 @@ class PangeaRequestAsync(PangeaRequestBase):
         self,
         endpoint: str,
         result_class: Type[TResult],
-        data: Union[str, Dict] = {},
-        files: List[Tuple] = [],
+        data: str | BaseModel | dict[str, Any] | None = None,
+        files: Optional[List[Tuple]] = None,
         poll_result: bool = True,
         url: Optional[str] = None,
     ) -> PangeaResponse[TResult]:
@@ -46,6 +48,13 @@ class PangeaRequestAsync(PangeaRequestBase):
             PangeaResponse which contains the response in its entirety and
                various properties to retrieve individual fields
         """
+
+        if isinstance(data, BaseModel):
+            data = data.model_dump(exclude_none=True)
+
+        if data is None:
+            data = {}
+
         if url is None:
             url = self._url(endpoint)
 
@@ -160,7 +169,7 @@ class PangeaRequestAsync(PangeaRequestBase):
         if resp.status < 200 or resp.status >= 300:
             raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
 
-    async def put_presigned_url(self, url: str, files: List[Tuple]):
+    async def put_presigned_url(self, url: str, files: Sequence[Tuple]):
         # Send put request with file as body
         resp = await self._http_put(url=url, files=files)
         self.logger.debug(
@@ -173,13 +182,27 @@ class PangeaRequestAsync(PangeaRequestBase):
         if resp.status < 200 or resp.status >= 300:
             raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status}", await resp.text())
 
-    async def download_file(self, url: str, filename: Optional[str] = None) -> AttachedFile:
+    async def download_file(self, url: str, filename: str | None = None) -> AttachedFile:
+        """
+        Download file
+
+        Download a file from the specified URL and save it with the given
+        filename.
+
+        Args:
+            url: URL of the file to download
+            filename: Name to save the downloaded file as. If not provided, the
+              filename will be determined from the Content-Disposition header or
+              the URL.
+        """
+
         self.logger.debug(
             json.dumps(
                 {
                     "service": self.service,
                     "action": "download_file",
                     "url": url,
+                    "filename": filename,
                     "status": "start",
                 }
             )
@@ -208,16 +231,16 @@ class PangeaRequestAsync(PangeaRequestBase):
                 )
 
                 return AttachedFile(filename=filename, file=await response.read(), content_type=content_type)
-            else:
-                raise pe.DownloadFileError(f"Failed to download file. Status: {response.status}", await response.text())
+            raise pe.DownloadFileError(f"Failed to download file. Status: {response.status}", await response.text())
 
-    async def _get_pangea_json(self, reader: aiohttp.MultipartReader) -> Optional[Dict]:
+    async def _get_pangea_json(self, reader: aiohttp.multipart.MultipartResponseWrapper) -> Optional[Dict[str, Any]]:
         # Iterate through parts
         async for part in reader:
-            return await part.json()
+            if isinstance(part, aiohttp.BodyPartReader):
+                return await part.json()
         return None
 
-    async def _get_attached_files(self, reader: aiohttp.MultipartReader) -> List[AttachedFile]:
+    async def _get_attached_files(self, reader: aiohttp.multipart.MultipartResponseWrapper) -> List[AttachedFile]:
         files = []
         i = 0
 
@@ -228,7 +251,7 @@ class PangeaRequestAsync(PangeaRequestBase):
             if name is None:
                 name = f"default_file_name_{i}"
                 i += 1
-            files.append(AttachedFile(name, await part.read(), content_type))
+            files.append(AttachedFile(name, await part.read(), content_type))  # type: ignore[union-attr]
 
         return files
 
@@ -236,13 +259,12 @@ class PangeaRequestAsync(PangeaRequestBase):
         # Parse the multipart response
         multipart_reader = aiohttp.MultipartReader.from_response(resp)
 
-        pangea_json = await self._get_pangea_json(multipart_reader)  # type: ignore[arg-type]
+        pangea_json = await self._get_pangea_json(multipart_reader)
         self.logger.debug(
             json.dumps({"service": self.service, "action": "multipart response", "response": pangea_json})
         )
 
-        multipart_reader = multipart_reader.__aiter__()
-        attached_files = await self._get_attached_files(multipart_reader)  # type: ignore[arg-type]
+        attached_files = await self._get_attached_files(multipart_reader)
         return MultipartResponse(pangea_json, attached_files)  # type: ignore[arg-type]
 
     async def _http_post(
@@ -250,7 +272,7 @@ class PangeaRequestAsync(PangeaRequestBase):
         url: str,
         headers: Dict = {},
         data: Union[str, Dict] = {},
-        files: List[Tuple] = [],
+        files: Optional[List[Tuple]] = [],
         presigned_url_post: bool = False,
     ) -> aiohttp.ClientResponse:
         if files:
@@ -275,7 +297,7 @@ class PangeaRequestAsync(PangeaRequestBase):
     async def _http_put(
         self,
         url: str,
-        files: List[Tuple],
+        files: Sequence[Tuple],
         headers: Dict = {},
     ) -> aiohttp.ClientResponse:
         self.logger.debug(
@@ -295,6 +317,9 @@ class PangeaRequestAsync(PangeaRequestBase):
             raise AttributeError("files attribute should have at least 1 file")
 
         response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
+        if response.success:  # This should only happen when uploading a zero bytes file
+            return response.raw_response
+
         if response.accepted_result is None:
             raise pe.PangeaException("No accepted_result field when requesting presigned url")
         if response.accepted_result.post_url is None:
@@ -314,9 +339,8 @@ class PangeaRequestAsync(PangeaRequestBase):
     ) -> PangeaResponse:
         # Send request
         try:
-            # This should return 202 (AcceptedRequestException)
-            resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
-            raise pe.PresignedURLException("Should return 202", resp)
+            # This should return 202 (AcceptedRequestException) at least zero size file is sent
+            return await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
         except pe.AcceptedRequestException as e:
             accepted_exception = e
         except Exception as e:
@@ -368,8 +392,7 @@ class PangeaRequestAsync(PangeaRequestBase):
 
         if loop_resp.accepted_result is not None and not loop_resp.accepted_result.has_upload_url:
             return loop_resp
-        else:
-            raise loop_exc
+        raise loop_exc
 
     async def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
         if self._queued_retry_enabled and response.http_status == 202:

pangea/asyncio/services/__init__.py

@@ -5,4 +5,6 @@ from .embargo import EmbargoAsync
 from .file_scan import FileScanAsync
 from .intel import DomainIntelAsync, FileIntelAsync, IpIntelAsync, UrlIntelAsync, UserIntelAsync
 from .redact import RedactAsync
+from .sanitize import SanitizeAsync
+from .share import ShareAsync
 from .vault import VaultAsync

pangea/asyncio/services/audit.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import datetime
-from typing import Any, Dict, List, Optional, Sequence, Union
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Union
 
 import pangea.exceptions as pexc
 from pangea.asyncio.services.base import ServiceBaseAsync
@@ -174,14 +174,16 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
         verbose: Optional[bool] = None,
     ) -> PangeaResponse[LogResult]:
         """
-        Log an entry
+        Log an event
 
         Create a log entry in the Secure Audit Log.
+
         Args:
             event (dict[str, Any]): event to be logged
             verify (bool, optional): True to verify logs consistency after response.
             sign_local (bool, optional): True to sign event with local key.
             verbose (bool, optional): True to get a more verbose response.
+
         Raises:
             AuditException: If an audit based api exception happens
             PangeaAPIException: If an API Error happens
@@ -192,18 +194,12 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
                 Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
 
         Examples:
-            try:
-                log_response = audit.log({"message"="Hello world"}, verbose=False)
-                print(f"Response. Hash: {log_response.result.hash}")
-            except pe.PangeaAPIException as e:
-                print(f"Request Error: {e.response.summary}")
-                for err in e.errors:
-                    print(f"\\t{err.detail} \\n")
+            response = await audit.log_event({"message": "hello world"}, verbose=True)
         """
 
         input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
         response: PangeaResponse[LogResult] = await self.request.post(
-            "v1/log", LogResult, data=input.dict(exclude_none=True)
+            "v1/log", LogResult, data=input.model_dump(exclude_none=True)
         )
         if response.success and response.result is not None:
             self._process_log_result(response.result, verify=verify)
@@ -239,7 +235,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
 
         input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
         response: PangeaResponse[LogBulkResult] = await self.request.post(
-            "v2/log", LogBulkResult, data=input.dict(exclude_none=True)
+            "v2/log", LogBulkResult, data=input.model_dump(exclude_none=True)
         )
         if response.success and response.result is not None:
             for result in response.result.results:
@@ -277,7 +273,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
         input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
         try:
             response: PangeaResponse[LogBulkResult] = await self.request.post(
-                "v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
+                "v2/log_async", LogBulkResult, data=input.model_dump(exclude_none=True), poll_result=False
             )
         except pexc.AcceptedRequestException as e:
             return e.response
@@ -299,6 +295,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
         verbose: Optional[bool] = None,
         verify_consistency: bool = False,
         verify_events: bool = True,
+        return_context: Optional[bool] = None,
     ) -> PangeaResponse[SearchOutput]:
         """
         Search the log
@@ -328,6 +325,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
             verbose (bool, optional): If true, response include root and membership and consistency proofs.
             verify_consistency (bool): True to verify logs consistency
             verify_events (bool): True to verify hash events and signatures
+            return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
 
         Raises:
             AuditException: If an audit based api exception happens
@@ -355,10 +353,11 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
             max_results=max_results,
             search_restriction=search_restriction,
             verbose=verbose,
+            return_context=return_context,
         )
 
         response: PangeaResponse[SearchOutput] = await self.request.post(
-            "v1/search", SearchOutput, data=input.dict(exclude_none=True)
+            "v1/search", SearchOutput, data=input.model_dump(exclude_none=True)
         )
         if verify_consistency:
             await self.update_published_roots(response.result)  # type: ignore[arg-type]
@@ -373,6 +372,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
         assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None,
         verify_consistency: bool = False,
         verify_events: bool = True,
+        return_context: Optional[bool] = None,
     ) -> PangeaResponse[SearchResultOutput]:
         """
         Results of a search
@@ -388,6 +388,8 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
             assert_search_restriction (Dict[str, Sequence[str]], optional): Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
             verify_consistency (bool): True to verify logs consistency
             verify_events (bool): True to verify hash events and signatures
+            return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
+
         Raises:
             AuditException: If an audit based api exception happens
             PangeaAPIException: If an API Error happens
@@ -418,8 +420,9 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
             limit=limit,
             offset=offset,
             assert_search_restriction=assert_search_restriction,
+            return_context=return_context,
         )
-        response = await self.request.post("v1/results", SearchResultOutput, data=input.dict(exclude_none=True))
+        response = await self.request.post("v1/results", SearchResultOutput, data=input.model_dump(exclude_none=True))
         if verify_consistency and response.result is not None:
             await self.update_published_roots(response.result)
 
@@ -481,7 +484,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
         )
         try:
             return await self.request.post(
-                "v1/export", PangeaResponseResult, data=input.dict(exclude_none=True), poll_result=False
+                "v1/export", PangeaResponseResult, data=input.model_dump(exclude_none=True), poll_result=False
             )
         except pexc.AcceptedRequestException as e:
             return e.response
@@ -548,13 +551,14 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
             response = audit.root(tree_size=7)
         """
         input = RootRequest(tree_size=tree_size)
-        return await self.request.post("v1/root", RootResult, data=input.dict(exclude_none=True))
+        return await self.request.post("v1/root", RootResult, data=input.model_dump(exclude_none=True))
 
     async def download_results(
         self,
         result_id: Optional[str] = None,
         format: DownloadFormat = DownloadFormat.CSV,
         request_id: Optional[str] = None,
+        return_context: Optional[bool] = None,
     ) -> PangeaResponse[DownloadResult]:
         """
         Download search results
@@ -567,6 +571,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
             result_id: ID returned by the search API.
             format: Format for the records.
             request_id: ID returned by the export API.
+            return_context (bool): Return the context data needed to decrypt secure audit events that have been redacted with format preserving encryption.
 
         Returns:
             URL where search results can be downloaded.
@@ -585,8 +590,10 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
         if request_id is None and result_id is None:
             raise ValueError("must pass one of `request_id` or `result_id`")
 
-        input = DownloadRequest(request_id=request_id, result_id=result_id, format=format)
-        return await self.request.post("v1/download_results", DownloadResult, data=input.dict(exclude_none=True))
+        input = DownloadRequest(
+            request_id=request_id, result_id=result_id, format=format, return_context=return_context
+        )
+        return await self.request.post("v1/download_results", DownloadResult, data=input.model_dump(exclude_none=True))
 
     async def update_published_roots(self, result: SearchResultOutput):
         """Fetches series of published root hashes from Arweave
@@ -611,12 +618,31 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
         for tree_size in tree_sizes:
             pub_root = None
             if tree_size in arweave_roots:
-                pub_root = PublishedRoot(**arweave_roots[tree_size].dict(exclude_none=True))
+                pub_root = PublishedRoot(**arweave_roots[tree_size].model_dump(exclude_none=True))
                 pub_root.source = RootSource.ARWEAVE
             elif self.allow_server_roots:
                 resp = await self.root(tree_size=tree_size)
                 if resp.success and resp.result is not None:
-                    pub_root = PublishedRoot(**resp.result.data.dict(exclude_none=True))
+                    pub_root = PublishedRoot(**resp.result.data.model_dump(exclude_none=True))
                     pub_root.source = RootSource.PANGEA
             if pub_root is not None:
                 self.pub_roots[tree_size] = pub_root
+
+        await self._fix_consistency_proofs(tree_sizes)
+
+    async def _fix_consistency_proofs(self, tree_sizes: Iterable[int]) -> None:
+        # on very rare occasions, the consistency proof in Arweave may be wrong
+        # override it with the proof from pangea (not the root hash, just the proof)
+        for tree_size in tree_sizes:
+            if tree_size not in self.pub_roots or tree_size - 1 not in self.pub_roots:
+                continue
+
+            if self.pub_roots[tree_size].source == RootSource.PANGEA:
+                continue
+
+            if self.verify_consistency_proof(tree_size):
+                continue
+
+            resp = await self.root(tree_size=tree_size)
+            if resp.success and resp.result is not None and resp.result.data is not None:
+                self.pub_roots[tree_size].consistency_proof = resp.result.data.consistency_proof