pangea-sdk 3.8.0__py3-none-any.whl → 3.8.0b2__py3-none-any.whl

pangea/request.py

@@ -5,12 +5,12 @@ import copy
 import json
 import logging
 import time
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type, Union
+from typing import Dict, List, Optional, Tuple, Type, Union
 
+import aiohttp
 import requests
 from requests.adapters import HTTPAdapter, Retry
 from requests_toolbelt import MultipartDecoder  # type: ignore
-from typing_extensions import TypeVar
 
 import pangea
 import pangea.exceptions as pe
@@ -18,9 +18,6 @@ from pangea.config import PangeaConfig
 from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
 from pangea.utils import default_encoder
 
-if TYPE_CHECKING:
-    import aiohttp
-
 
 class MultipartResponse(object):
     pangea_json: Dict[str, str]
@@ -190,9 +187,6 @@ class PangeaRequestBase(object):
         raise pe.PangeaAPIException(f"{summary} ", response)
 
 
-TResult = TypeVar("TResult", bound=PangeaResponseResult, default=PangeaResponseResult)
-
-
 class PangeaRequest(PangeaRequestBase):
     """An object that makes direct calls to Pangea Service APIs.
 
@@ -208,12 +202,12 @@ class PangeaRequest(PangeaRequestBase):
     def post(
         self,
         endpoint: str,
-        result_class: Type[TResult],
+        result_class: Type[PangeaResponseResult],
         data: Union[str, Dict] = {},
         files: Optional[List[Tuple]] = None,
         poll_result: bool = True,
         url: Optional[str] = None,
-    ) -> PangeaResponse[TResult]:
+    ) -> PangeaResponse:
         """Makes the POST call to a Pangea Service endpoint.
 
         Args:
@@ -349,7 +343,7 @@ class PangeaRequest(PangeaRequestBase):
 
         return data, files
 
-    def _handle_queued_result(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
+    def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse[Type[PangeaResponseResult]]:
         if self._queued_retry_enabled and response.http_status == 202:
             self.logger.debug(
                 json.dumps(
@@ -361,7 +355,7 @@ class PangeaRequest(PangeaRequestBase):
 
         return response
 
-    def get(self, path: str, result_class: Type[TResult], check_response: bool = True) -> PangeaResponse[TResult]:
+    def get(self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True) -> PangeaResponse:
         """Makes the GET call to a Pangea Service endpoint.
 
         Args:
@@ -433,21 +427,21 @@ class PangeaRequest(PangeaRequestBase):
             raise pe.DownloadFileError(f"Failed to download file. Status: {response.status_code}", response.text)
 
     def poll_result_by_id(
-        self, request_id: str, result_class: Type[TResult], check_response: bool = True
-    ) -> PangeaResponse[TResult]:
+        self, request_id: str, result_class: Union[Type[PangeaResponseResult], Type[dict]], check_response: bool = True
+    ):
         path = self._get_poll_path(request_id)
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
-        return self.get(path, result_class, check_response=check_response)
+        return self.get(path, result_class, check_response=check_response)  # type: ignore[arg-type]
 
     def poll_result_once(
-        self, response: PangeaResponse[TResult], check_response: bool = True
-    ) -> PangeaResponse[TResult]:
+        self, response: PangeaResponse, check_response: bool = True
+    ) -> PangeaResponse[Type[PangeaResponseResult]]:
         request_id = response.request_id
         if not request_id:
             raise pe.PangeaException("Poll result error: response did not include a 'request_id'")
 
         if response.status != ResponseStatus.ACCEPTED.value:
-            raise pe.PangeaException("Response already processed")
+            raise pe.PangeaException("Response already proccesed")
 
         return self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
 
@@ -459,10 +453,8 @@ class PangeaRequest(PangeaRequestBase):
     ) -> PangeaResponse:
         # Send request
         try:
-            # This should return 202 (AcceptedRequestException)
-            resp = self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
-            raise pe.PresignedURLException("Should return 202", resp)
-
+            # This should return 202 (AcceptedRequestException) at least zero size file is sent
+            return self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
         except pe.AcceptedRequestException as e:
             accepted_exception = e
         except Exception as e:
@@ -520,6 +512,9 @@ class PangeaRequest(PangeaRequestBase):
             raise AttributeError("files attribute should have at least 1 file")
 
         response = self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
+
+        if response.success:  # This should only happen when uploading a zero bytes file
+            return response.raw_response
         if response.accepted_result is None:
             raise pe.PangeaException("No accepted_result field when requesting presigned url")
         if response.accepted_result.post_url is None:
@@ -531,7 +526,7 @@ class PangeaRequest(PangeaRequestBase):
         self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
         return response.raw_response
 
-    def _poll_result_retry(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
+    def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse[Type[PangeaResponseResult]]:
         retry_count = 1
         start = time.time()
 
@@ -543,7 +538,9 @@ class PangeaRequest(PangeaRequestBase):
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_retry", "step": "exit"}))
         return self._check_response(response)
 
-    def _poll_presigned_url(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
+    def _poll_presigned_url(
+        self, response: PangeaResponse[Type[PangeaResponseResult]]
+    ) -> PangeaResponse[Type[PangeaResponseResult]]:
         if response.http_status != 202:
             raise AttributeError("Response should be 202")
 

pangea/response.py

@@ -3,15 +3,16 @@
 import datetime
 import enum
 import os
-from typing import Any, Dict, Generic, List, Optional, Type, Union
+from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, Union
 
 import aiohttp
 import requests
 from pydantic import BaseModel
-from typing_extensions import TypeVar
 
 from pangea.utils import format_datetime
 
+T = TypeVar("T")
+
 
 class AttachedFile(object):
     filename: str
@@ -28,6 +29,7 @@ class AttachedFile(object):
             filename = self.filename if self.filename else "default_save_filename"
 
         filepath = os.path.join(dest_folder, filename)
+        filepath = self._find_available_file(filepath)
         directory = os.path.dirname(filepath)
         if not os.path.exists(directory):
             os.makedirs(directory)
@@ -35,6 +37,17 @@ class AttachedFile(object):
         with open(filepath, "wb") as file:
             file.write(self.file)
 
+    def _find_available_file(self, file_path):
+        base_name, ext = os.path.splitext(file_path)
+        counter = 1
+        while os.path.exists(file_path):
+            if ext:
+                file_path = f"{base_name}_{counter}{ext}"
+            else:
+                file_path = f"{base_name}_{counter}"
+            counter += 1
+        return file_path
+
 
 class TransferMethod(str, enum.Enum):
     MULTIPART = "multipart"
@@ -142,34 +155,22 @@ class ResponseStatus(str, enum.Enum):
 
 
 class ResponseHeader(APIResponseModel):
-    """Pangea response API header."""
-
-    request_id: str
-    """A unique identifier assigned to each request made to the API."""
-
-    request_time: str
     """
-    Timestamp indicating the exact moment when a request is made to the API.
+    Pangea response API header.
+
+    Arguments:
+    request_id -- The request ID.
+    request_time -- The time the request was issued, ISO8601.
+    response_time -- The time the response was issued, ISO8601.
+    status -- Pangea response status
+    summary -- The summary of the response.
     """
 
+    request_id: str
+    request_time: str
     response_time: str
-    """
-    Duration it takes for the API to process a request and generate a response.
-    """
-
     status: str
-    """
-    Represents the status or outcome of the API request.
-    """
-
     summary: str
-    """
-    Provides a concise and brief overview of the purpose or primary objective of
-    the API endpoint.
-    """
-
-
-T = TypeVar("T", bound=PangeaResponseResult, default=PangeaResponseResult)
 
 
 class PangeaResponse(Generic[T], ResponseHeader):
@@ -178,14 +179,14 @@ class PangeaResponse(Generic[T], ResponseHeader):
     result: Optional[T] = None
     pangea_error: Optional[PangeaError] = None
     accepted_result: Optional[AcceptedResult] = None
-    result_class: Type[T] = PangeaResponseResult  # type: ignore[assignment]
+    result_class: Union[Type[PangeaResponseResult], Type[dict]] = PangeaResponseResult
     _json: Any
     attached_files: List[AttachedFile] = []
 
     def __init__(
         self,
         response: requests.Response,
-        result_class: Type[T],
+        result_class: Union[Type[PangeaResponseResult], Type[dict]],
         json: dict,
         attached_files: List[AttachedFile] = [],
     ):
@@ -197,7 +198,7 @@ class PangeaResponse(Generic[T], ResponseHeader):
         self.attached_files = attached_files
 
         self.result = (
-            self.result_class(**self.raw_result)
+            self.result_class(**self.raw_result)  # type: ignore[assignment]
             if self.raw_result is not None and issubclass(self.result_class, PangeaResponseResult) and self.success
             else None
         )

pangea/services/__init__.py

@@ -1,8 +1,9 @@
 from .audit.audit import Audit
 from .authn.authn import AuthN
-from .authz import AuthZ
 from .embargo import Embargo
 from .file_scan import FileScan
 from .intel import DomainIntel, FileIntel, IpIntel, UrlIntel, UserIntel
 from .redact import Redact
+from .share.share import Share
+from .sanitize import Sanitize
 from .vault.vault import Vault

pangea/services/audit/audit.py

@@ -1,14 +1,11 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
-from __future__ import annotations
-
 import datetime
 import json
-from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 import pangea.exceptions as pexc
-from pangea.config import PangeaConfig
-from pangea.response import PangeaResponse, PangeaResponseResult
+from pangea.response import PangeaResponse
 from pangea.services.audit.exceptions import AuditException, EventCorruption
 from pangea.services.audit.models import (
     DownloadFormat,
@@ -17,7 +14,6 @@ from pangea.services.audit.models import (
     Event,
     EventEnvelope,
     EventVerification,
-    ExportRequest,
     LogBulkRequest,
     LogBulkResult,
     LogEvent,
@@ -55,8 +51,8 @@ from pangea.utils import canonicalize_nested_json
 
 class AuditBase:
     def __init__(
-        self, private_key_file: str = "", public_key_info: dict[str, str] = {}, tenant_id: str | None = None
-    ) -> None:
+        self, private_key_file: str = "", public_key_info: Dict[str, str] = {}, tenant_id: Optional[str] = None
+    ):
         self.pub_roots: Dict[int, PublishedRoot] = {}
         self.buffer_data: Optional[str] = None
         self.signer: Optional[Signer] = Signer(private_key_file) if private_key_file else None
@@ -336,9 +332,7 @@ class AuditBase:
         if audit_envelope and audit_envelope.signature and public_key:
             v = Verifier()
             verification = v.verify_signature(
-                audit_envelope.signature,
-                canonicalize_event(Event(**audit_envelope.event)),
-                public_key,
+                audit_envelope.signature, canonicalize_event(audit_envelope.event), public_key  # type: ignore[arg-type]
             )
             if verification is not None:
                 return EventVerification.PASS if verification else EventVerification.FAIL
@@ -380,32 +374,14 @@ class Audit(ServiceBase, AuditBase):
 
     def __init__(
         self,
-        token: str,
-        config: PangeaConfig | None = None,
+        token,
+        config=None,
         private_key_file: str = "",
-        public_key_info: dict[str, str] = {},
-        tenant_id: str | None = None,
-        logger_name: str = "pangea",
-        config_id: str | None = None,
-    ) -> None:
-        """
-        Audit client
-
-        Initializes a new Audit client.
-
-        Args:
-            token: Pangea API token.
-            config: Configuration.
-            private_key_file: Private key filepath.
-            public_key_info: Public key information.
-            tenant_id: Tenant ID.
-            logger_name: Logger name.
-            config_id: Configuration ID.
-
-        Examples:
-             config = PangeaConfig(domain="pangea_domain")
-             audit = Audit(token="pangea_token", config=config)
-        """
+        public_key_info: Dict[str, str] = {},
+        tenant_id: Optional[str] = None,
+        logger_name="pangea",
+        config_id: Optional[str] = None,
+    ):
         # FIXME: Temporary check to deprecate config_id from PangeaConfig.
         # Delete it when deprecate PangeaConfig.config_id
         if config_id and config is not None and config.config_id is not None:
@@ -622,7 +598,7 @@ class Audit(ServiceBase, AuditBase):
         end: Optional[Union[datetime.datetime, str]] = None,
         limit: Optional[int] = None,
         max_results: Optional[int] = None,
-        search_restriction: Optional[Dict[str, Sequence[str]]] = None,
+        search_restriction: Optional[dict] = None,
         verbose: Optional[bool] = None,
         verify_consistency: bool = False,
         verify_events: bool = True,
@@ -651,7 +627,7 @@ class Audit(ServiceBase, AuditBase):
             end (datetime, optional): An RFC-3339 formatted timestamp, or relative time adjustment from the current time.
             limit (int, optional): Optional[int] = None,
             max_results (int, optional): Maximum number of results to return.
-            search_restriction (Dict[str, Sequence[str]], optional): A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
+            search_restriction (dict, optional): A list of keys to restrict the search results to. Useful for partitioning data available to the query string.
             verbose (bool, optional): If true, response include root and membership and consistency proofs.
             verify_consistency (bool): True to verify logs consistency
             verify_events (bool): True to verify hash events and signatures
@@ -702,7 +678,6 @@ class Audit(ServiceBase, AuditBase):
         id: str,
         limit: Optional[int] = 20,
         offset: Optional[int] = 0,
-        assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None,
         verify_consistency: bool = False,
         verify_events: bool = True,
     ) -> PangeaResponse[SearchResultOutput]:
@@ -717,7 +692,6 @@ class Audit(ServiceBase, AuditBase):
             id (string): the id of a search action, found in `response.result.id`
             limit (integer, optional): the maximum number of results to return, default is 20
             offset (integer, optional): the position of the first result to return, default is 0
-            assert_search_restriction (Dict[str, Sequence[str]], optional): Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
             verify_consistency (bool): True to verify logs consistency
             verify_events (bool): True to verify hash events and signatures
         Raises:
@@ -742,7 +716,6 @@ class Audit(ServiceBase, AuditBase):
             id=id,
             limit=limit,
             offset=offset,
-            assert_search_restriction=assert_search_restriction,
         )
         response: PangeaResponse[SearchResultOutput] = self.request.post(
             "v1/results", SearchResultOutput, data=input.dict(exclude_none=True)
@@ -751,107 +724,6 @@ class Audit(ServiceBase, AuditBase):
             self.update_published_roots(response.result)
         return self.handle_results_response(response, verify_consistency, verify_events)
 
-    def export(
-        self,
-        *,
-        format: DownloadFormat = DownloadFormat.CSV,
-        start: Optional[datetime.datetime] = None,
-        end: Optional[datetime.datetime] = None,
-        order: Optional[SearchOrder] = None,
-        order_by: Optional[str] = None,
-        verbose: bool = True,
-    ) -> PangeaResponse[PangeaResponseResult]:
-        """
-        Export from the audit log
-
-        Bulk export of data from the Secure Audit Log, with optional filtering.
-
-        OperationId: audit_post_v1_export
-
-        Args:
-            format: Format for the records.
-            start: The start of the time range to perform the search on.
-            end: The end of the time range to perform the search on. If omitted,
-                then all records up to the latest will be searched.
-            order: Specify the sort order of the response.
-            order_by: Name of column to sort the results by.
-            verbose: Whether or not to include the root hash of the tree and the
-                membership proof for each record.
-
-        Raises:
-            AuditException: If an audit based api exception happens
-            PangeaAPIException: If an API Error happens
-
-        Examples:
-            export_res = audit.export(verbose=False)
-
-            # Export may take several dozens of minutes, so polling for the result
-            # should be done in a loop. That is omitted here for brevity's sake.
-            try:
-                audit.poll_result(request_id=export_res.request_id)
-            except AcceptedRequestException:
-                # Retry later.
-
-            # Download the result when it's ready.
-            download_res = audit.download_results(request_id=export_res.request_id)
-            download_res.result.dest_url
-            # => https://pangea-runtime.s3.amazonaws.com/audit/xxxxx/search_results_[...]
-        """
-        input = ExportRequest(
-            format=format,
-            start=start,
-            end=end,
-            order=order,
-            order_by=order_by,
-            verbose=verbose,
-        )
-        try:
-            return self.request.post(
-                "v1/export", PangeaResponseResult, data=input.dict(exclude_none=True), poll_result=False
-            )
-        except pexc.AcceptedRequestException as e:
-            return e.response
-
-    def log_stream(self, data: dict) -> PangeaResponse[PangeaResponseResult]:
-        """
-        Log streaming endpoint
-
-        This API allows 3rd party vendors (like Auth0) to stream events to this
-        endpoint where the structure of the payload varies across different
-        vendors.
-
-        OperationId: audit_post_v1_log_stream
-
-        Args:
-            data: Event data. The exact schema of this will vary by vendor.
-
-        Raises:
-            AuditException: If an audit based api exception happens
-            PangeaAPIException: If an API Error happens
-
-        Examples:
-            data = {
-                "logs": [
-                    {
-                        "log_id": "some log ID",
-                        "data": {
-                            "date": "2024-03-29T17:26:50.193Z",
-                            "type": "sapi",
-                            "description": "Create a log stream",
-                            "client_id": "some client ID",
-                            "ip": "127.0.0.1",
-                            "user_agent": "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0",
-                            "user_id": "some user ID",
-                        },
-                    }
-                    # ...
-                ]
-            }
-
-            response = audit.log_stream(data)
-        """
-        return self.request.post("v1/log_stream", PangeaResponseResult, data=data)
-
     def root(self, tree_size: Optional[int] = None) -> PangeaResponse[RootResult]:
         """
         Tamperproof verification
@@ -877,10 +749,7 @@ class Audit(ServiceBase, AuditBase):
         return self.request.post("v1/root", RootResult, data=input.dict(exclude_none=True))
 
     def download_results(
-        self,
-        result_id: Optional[str] = None,
-        format: DownloadFormat = DownloadFormat.CSV,
-        request_id: Optional[str] = None,
+        self, result_id: str, format: Optional[DownloadFormat] = None
     ) -> PangeaResponse[DownloadResult]:
         """
         Download search results
@@ -892,7 +761,6 @@ class Audit(ServiceBase, AuditBase):
         Args:
             result_id: ID returned by the search API.
             format: Format for the records.
-            request_id: ID returned by the export API.
 
         Returns:
             URL where search results can be downloaded.
@@ -908,10 +776,7 @@ class Audit(ServiceBase, AuditBase):
             )
         """
 
-        if request_id is None and result_id is None:
-            raise ValueError("must pass one of `request_id` or `result_id`")
-
-        input = DownloadRequest(request_id=request_id, result_id=result_id, format=format)
+        input = DownloadRequest(result_id=result_id, format=format)
         return self.request.post("v1/download_results", DownloadResult, data=input.dict(exclude_none=True))
 
     def update_published_roots(self, result: SearchResultOutput):

pangea/services/audit/models.py

@@ -1,10 +1,8 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
-from __future__ import annotations
-
 import datetime
 import enum
-from typing import Any, Dict, List, Optional, Sequence, Union
+from typing import Any, Dict, List, Optional, Union
 
 from pangea.response import APIRequestModel, APIResponseModel, PangeaResponseResult
 
@@ -21,14 +19,14 @@ class EventVerification(str, enum.Enum):
         return str(self.value)
 
 
-class Event(Dict[str, Any]):
+class Event(dict):
     """
     Event to perform an auditable activity
 
     Auxiliary class to be compatible with older SDKs
     """
 
-    def __init__(self, **data) -> None:
+    def __init__(self, **data):
         super().__init__(**data)
 
     @property
@@ -281,7 +279,7 @@ class SearchRequest(APIRequestModel):
     end: Optional[str] = None
     limit: Optional[int] = None
     max_results: Optional[int] = None
-    search_restriction: Optional[Dict[str, Sequence[str]]] = None
+    search_restriction: Optional[dict] = None
     verbose: Optional[bool] = None
 
 
@@ -417,13 +415,11 @@ class SearchResultRequest(APIRequestModel):
     id -- A search results identifier returned by the search call.
     limit -- Number of audit records to include from the first page of the results.
     offset -- Offset from the start of the result set to start returning results from.
-    assert_search_restriction -- Assert the requested search results were queried with the exact same search restrictions, to ensure the results comply to the expected restrictions.
     """
 
     id: str
     limit: Optional[int] = 20
     offset: Optional[int] = 0
-    assert_search_restriction: Optional[Dict[str, Sequence[str]]] = None
 
 
 class DownloadFormat(str, enum.Enum):
@@ -441,10 +437,7 @@ class DownloadFormat(str, enum.Enum):
 
 
 class DownloadRequest(APIRequestModel):
-    request_id: Optional[str] = None
-    """ID returned by the export API."""
-
-    result_id: Optional[str] = None
+    result_id: str
     """ID returned by the search API."""
 
     format: Optional[str] = None
@@ -454,35 +447,3 @@ class DownloadRequest(APIRequestModel):
 class DownloadResult(PangeaResponseResult):
     dest_url: str
     """URL where search results can be downloaded."""
-
-    expires_at: str
-    """
-    The time when the results will no longer be available to page through via
-    the results API.
-    """
-
-
-class ExportRequest(APIRequestModel):
-    format: DownloadFormat = DownloadFormat.CSV
-    """Format for the records."""
-
-    start: Optional[datetime.datetime] = None
-    """The start of the time range to perform the search on."""
-
-    end: Optional[datetime.datetime] = None
-    """
-    The end of the time range to perform the search on. If omitted, then all
-    records up to the latest will be searched.
-    """
-
-    order_by: Optional[str] = None
-    """Name of column to sort the results by."""
-
-    order: Optional[SearchOrder] = None
-    """Specify the sort order of the response."""
-
-    verbose: bool = True
-    """
-    Whether or not to include the root hash of the tree and the membership proof
-    for each record.
-    """

pangea/services/authn/authn.py

@@ -1,11 +1,9 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
-from __future__ import annotations
 
 from typing import Dict, List, Optional, Union
 
 import pangea.services.authn.models as m
-from pangea.config import PangeaConfig
 from pangea.response import PangeaResponse
 from pangea.services.base import ServiceBase
 
@@ -39,24 +37,10 @@ class AuthN(ServiceBase):
 
     def __init__(
         self,
-        token: str,
-        config: PangeaConfig | None = None,
-        logger_name: str = "pangea",
-    ) -> None:
-        """
-        AuthN client
-
-        Initializes a new AuthN client.
-
-        Args:
-            token: Pangea API token.
-            config: Configuration.
-            logger_name: Logger name.
-
-        Examples:
-             config = PangeaConfig(domain="pangea_domain")
-             authn = AuthN(token="pangea_token", config=config)
-        """
+        token,
+        config=None,
+        logger_name="pangea",
+    ):
         super().__init__(token, config, logger_name=logger_name)
         self.user = AuthN.User(token, config, logger_name=logger_name)
         self.flow = AuthN.Flow(token, config, logger_name=logger_name)