pangea-sdk 5.2.0b1__py3-none-any.whl → 5.2.1__py3-none-any.whl

pangea/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "5.2.0beta1"
+__version__ = "5.2.1"
 
 from pangea.asyncio.request import PangeaRequestAsync
 from pangea.config import PangeaConfig

pangea/asyncio/request.py

@@ -5,12 +5,11 @@ from __future__ import annotations
 import asyncio
 import json
 import time
-from typing import Dict, List, Optional, Sequence, Tuple, Type, Union, cast
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union
 
 import aiohttp
 from aiohttp import FormData
 from pydantic import BaseModel
-from pydantic_core import to_jsonable_python
 from typing_extensions import Any, TypeVar
 
 import pangea.exceptions as pe
@@ -56,20 +55,17 @@ class PangeaRequestAsync(PangeaRequestBase):
         if data is None:
             data = {}
 
-        # Normalize.
-        data = cast(dict[str, Any], to_jsonable_python(data))
-
         if url is None:
             url = self._url(endpoint)
 
         # Set config ID if available
-        if self.config_id and data.get("config_id", None) is None:
-            data["config_id"] = self.config_id
+        if self.config_id and data.get("config_id", None) is None:  # type: ignore[union-attr]
+            data["config_id"] = self.config_id  # type: ignore[index]
 
         self.logger.debug(
             json.dumps({"service": self.service, "action": "post", "url": url, "data": data}, default=default_encoder)
         )
-        transfer_method = data.get("transfer_method", None)
+        transfer_method = data.get("transfer_method", None)  # type: ignore[union-attr]
 
         if files and type(data) is dict and (transfer_method == TransferMethod.POST_URL.value):
             requests_response = await self._full_post_presigned_url(
@@ -186,7 +182,20 @@ class PangeaRequestAsync(PangeaRequestBase):
         if resp.status < 200 or resp.status >= 300:
             raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status}", await resp.text())
 
-    async def download_file(self, url: str, filename: Optional[str] = None) -> AttachedFile:
+    async def download_file(self, url: str, filename: str | None = None) -> AttachedFile:
+        """
+        Download file
+
+        Download a file from the specified URL and save it with the given
+        filename.
+
+        Args:
+            url: URL of the file to download
+            filename: Name to save the downloaded file as. If not provided, the
+              filename will be determined from the Content-Disposition header or
+              the URL.
+        """
+
         self.logger.debug(
             json.dumps(
                 {

pangea/asyncio/services/audit.py

@@ -174,14 +174,16 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
         verbose: Optional[bool] = None,
     ) -> PangeaResponse[LogResult]:
         """
-        Log an entry
+        Log an event
 
         Create a log entry in the Secure Audit Log.
+
         Args:
             event (dict[str, Any]): event to be logged
             verify (bool, optional): True to verify logs consistency after response.
             sign_local (bool, optional): True to sign event with local key.
             verbose (bool, optional): True to get a more verbose response.
+
         Raises:
             AuditException: If an audit based api exception happens
             PangeaAPIException: If an API Error happens
@@ -192,13 +194,7 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
                 Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
 
         Examples:
-            try:
-                log_response = audit.log({"message"="Hello world"}, verbose=False)
-                print(f"Response. Hash: {log_response.result.hash}")
-            except pe.PangeaAPIException as e:
-                print(f"Request Error: {e.response.summary}")
-                for err in e.errors:
-                    print(f"\\t{err.detail} \\n")
+            response = await audit.log_event({"message": "hello world"}, verbose=True)
         """
 
         input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
@@ -632,9 +628,9 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
             if pub_root is not None:
                 self.pub_roots[tree_size] = pub_root
 
-        await self.fix_consistency_proofs(tree_sizes)
+        await self._fix_consistency_proofs(tree_sizes)
 
-    async def fix_consistency_proofs(self, tree_sizes: Iterable[int]):
+    async def _fix_consistency_proofs(self, tree_sizes: Iterable[int]) -> None:
         # on very rare occasions, the consistency proof in Arweave may be wrong
         # override it with the proof from pangea (not the root hash, just the proof)
         for tree_size in tree_sizes:

pangea/asyncio/services/authz.py

@@ -1,9 +1,12 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
 
+from __future__ import annotations
+
 from typing import Any, Dict, List, Optional
 
 from pangea.asyncio.services.base import ServiceBaseAsync
+from pangea.config import PangeaConfig
 from pangea.response import PangeaResponse
 from pangea.services.authz import (
     CheckRequest,
@@ -36,8 +39,8 @@ class AuthZAsync(ServiceBaseAsync):
 
     Examples:
         import os
+        from pangea.asyncio.services import AuthZAsync
         from pangea.config import PangeaConfig
-        from pangea.services import AuthZ
 
         PANGEA_TOKEN = os.getenv("PANGEA_AUTHZ_TOKEN")
 
@@ -49,7 +52,25 @@ class AuthZAsync(ServiceBaseAsync):
 
     service_name = "authz"
 
-    def __init__(self, token: str, config=None, logger_name="pangea", config_id: Optional[str] = None):
+    def __init__(
+        self, token: str, config: PangeaConfig | None = None, logger_name: str = "pangea", config_id: str | None = None
+    ) -> None:
+        """
+        AuthZ client
+
+        Initializes a new AuthZ client.
+
+        Args:
+            token: Pangea API token.
+            config: Configuration.
+            logger_name: Logger name.
+            config_id: Configuration ID.
+
+        Examples:
+             config = PangeaConfig(domain="aws.us.pangea.cloud")
+             authz = AuthZAsync(token="pangea_token", config=config)
+        """
+
         super().__init__(token, config, logger_name, config_id=config_id)
 
     async def tuple_create(self, tuples: List[Tuple]) -> PangeaResponse[TupleCreateResult]:

pangea/asyncio/services/base.py

@@ -1,8 +1,11 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+from __future__ import annotations
 
 from typing import Dict, Optional, Type, Union
 
+from typing_extensions import override
+
 from pangea.asyncio.request import PangeaRequestAsync
 from pangea.exceptions import AcceptedRequestException
 from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult
@@ -23,6 +26,7 @@ class ServiceBaseAsync(ServiceBase):
 
         return self._request
 
+    @override
     async def poll_result(  # type: ignore[override]
         self,
         exception: Optional[AcceptedRequestException] = None,
@@ -36,7 +40,8 @@ class ServiceBaseAsync(ServiceBase):
         Returns request's result that has been accepted by the server
 
         Args:
-            exception (AcceptedRequestException): Exception raise by SDK on the call that is been processed.
+            exception: Exception that was previously raised by the SDK on a call
+              that is being processed.
 
         Returns:
             PangeaResponse
@@ -58,7 +63,21 @@ class ServiceBaseAsync(ServiceBase):
         else:
             raise AttributeError("Need to set exception, response or request_id")
 
-    async def download_file(self, url: str, filename: Optional[str] = None) -> AttachedFile:  # type: ignore[override]
+    @override
+    async def download_file(self, url: str, filename: str | None = None) -> AttachedFile:  # type: ignore[override]
+        """
+        Download file
+
+        Download a file from the specified URL and save it with the given
+        filename.
+
+        Args:
+            url: URL of the file to download
+            filename: Name to save the downloaded file as. If not provided, the
+              filename will be determined from the Content-Disposition header or
+              the URL.
+        """
+
         return await self.request.download_file(url=url, filename=filename)
 
     async def close(self):

pangea/asyncio/services/intel.py

@@ -790,6 +790,7 @@ class UserIntelAsync(ServiceBaseAsync):
         verbose: Optional[bool] = None,
         raw: Optional[bool] = None,
         provider: Optional[str] = None,
+        cursor: Optional[str] = None,
     ) -> PangeaResponse[m.UserBreachedResult]:
         """
         Look up breached users
@@ -808,6 +809,7 @@ class UserIntelAsync(ServiceBaseAsync):
             verbose (bool, optional): Echo the API parameters in the response
             raw (bool, optional): Include raw data from this provider
             provider (str, optional): Use reputation data from this provider: "crowdstrike"
+            cursor (str, optional): A token given in the raw response from SpyCloud. Post this back to paginate results
 
         Raises:
             PangeaAPIException: If an API Error happens
@@ -835,6 +837,7 @@ class UserIntelAsync(ServiceBaseAsync):
             end=end,
             verbose=verbose,
             raw=raw,
+            cursor=cursor,
         )
         return await self.request.post(
             "v1/user/breached", m.UserBreachedResult, data=input.model_dump(exclude_none=True)

pangea/asyncio/services/redact.py

@@ -65,6 +65,8 @@ class RedactAsync(ServiceBaseAsync):
         rulesets: Optional[List[str]] = None,
         return_result: Optional[bool] = None,
         redaction_method_overrides: Optional[m.RedactionMethodOverrides] = None,
+        llm_request: Optional[bool] = None,
+        vault_parameters: Optional[m.VaultParameters] = None,
     ) -> PangeaResponse[m.RedactResult]:
         """
         Redact
@@ -81,6 +83,8 @@ class RedactAsync(ServiceBaseAsync):
             rulesets (list[str], optional): An array of redact rulesets short names
             return_result(bool, optional): Setting this value to false will omit the redacted result only returning count
             redaction_method_overrides: A set of redaction method overrides for any enabled rule. These methods override the config declared methods
+            llm_request: Boolean flag to enable FPE redaction for LLM requests
+            vault_parameters: A set of vault parameters to use for redaction
 
         Raises:
             PangeaAPIException: If an API Error happens
@@ -101,6 +105,8 @@ class RedactAsync(ServiceBaseAsync):
             rulesets=rulesets,
             return_result=return_result,
             redaction_method_overrides=redaction_method_overrides,
+            llm_request=llm_request,
+            vault_parameters=vault_parameters,
         )
         return await self.request.post("v1/redact", m.RedactResult, data=input.model_dump(exclude_none=True))
 
@@ -114,6 +120,8 @@ class RedactAsync(ServiceBaseAsync):
         rulesets: Optional[List[str]] = None,
         return_result: Optional[bool] = None,
         redaction_method_overrides: Optional[m.RedactionMethodOverrides] = None,
+        llm_request: bool | None = None,
+        vault_parameters: m.VaultParameters | None = None,
     ) -> PangeaResponse[m.StructuredResult]:
         """
         Redact structured
@@ -134,6 +142,8 @@ class RedactAsync(ServiceBaseAsync):
             rulesets (list[str], optional): An array of redact rulesets short names
             return_result(bool, optional): Setting this value to false will omit the redacted result only returning count
             redaction_method_overrides: A set of redaction method overrides for any enabled rule. These methods override the config declared methods
+            llm_request: Boolean flag to enable FPE redaction for LLM requests
+            vault_parameters: A set of vault parameters to use for redaction
 
         Raises:
             PangeaAPIException: If an API Error happens
@@ -161,6 +171,8 @@ class RedactAsync(ServiceBaseAsync):
             rulesets=rulesets,
             return_result=return_result,
             redaction_method_overrides=redaction_method_overrides,
+            llm_request=llm_request,
+            vault_parameters=vault_parameters,
         )
         return await self.request.post(
             "v1/redact_structured", m.StructuredResult, data=input.model_dump(exclude_none=True)

pangea/asyncio/services/sanitize.py

@@ -1,10 +1,13 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+from __future__ import annotations
+
 import io
 from typing import List, Optional, Tuple
 
 import pangea.services.sanitize as m
 from pangea.asyncio.services.base import ServiceBaseAsync
+from pangea.config import PangeaConfig
 from pangea.response import PangeaResponse, TransferMethod
 from pangea.utils import FileUploadParams, get_file_upload_params
 
@@ -16,17 +19,38 @@ class SanitizeAsync(ServiceBaseAsync):
         import os
 
         # Pangea SDK
+        from pangea.asyncio.services import SanitizeAsync
         from pangea.config import PangeaConfig
-        from pangea.asyncio.services import Sanitize
 
         PANGEA_SANITIZE_TOKEN = os.getenv("PANGEA_SANITIZE_TOKEN")
         config = PangeaConfig(domain="pangea.cloud")
 
-        sanitize = Sanitize(token=PANGEA_SANITIZE_TOKEN, config=config)
+        sanitize = SanitizeAsync(token=PANGEA_SANITIZE_TOKEN, config=config)
     """
 
     service_name = "sanitize"
 
+    def __init__(
+        self, token: str, config: PangeaConfig | None = None, logger_name: str = "pangea", config_id: str | None = None
+    ) -> None:
+        """
+        Sanitize client
+
+        Initializes a new Sanitize client.
+
+        Args:
+            token: Pangea API token.
+            config: Configuration.
+            logger_name: Logger name.
+            config_id: Configuration ID.
+
+        Examples:
+             config = PangeaConfig(domain="aws.us.pangea.cloud")
+             authz = SanitizeAsync(token="pangea_token", config=config)
+        """
+
+        super().__init__(token, config, logger_name, config_id=config_id)
+
     async def sanitize(
         self,
         transfer_method: TransferMethod = TransferMethod.POST_URL,

pangea/asyncio/services/share.py

@@ -7,16 +7,38 @@ from typing import Dict, List, Optional, Tuple, Union
 
 import pangea.services.share.share as m
 from pangea.asyncio.services.base import ServiceBaseAsync
+from pangea.config import PangeaConfig
 from pangea.response import PangeaResponse, TransferMethod
 from pangea.services.share.file_format import FileFormat
 from pangea.utils import get_file_size, get_file_upload_params
 
 
 class ShareAsync(ServiceBaseAsync):
-    """Share service client."""
+    """Secure Share service client."""
 
     service_name = "share"
 
+    def __init__(
+        self, token: str, config: PangeaConfig | None = None, logger_name: str = "pangea", config_id: str | None = None
+    ) -> None:
+        """
+        Secure Share client
+
+        Initializes a new Secure Share client.
+
+        Args:
+            token: Pangea API token.
+            config: Configuration.
+            logger_name: Logger name.
+            config_id: Configuration ID.
+
+        Examples:
+             config = PangeaConfig(domain="aws.us.pangea.cloud")
+             authz = ShareAsync(token="pangea_token", config=config)
+        """
+
+        super().__init__(token, config, logger_name, config_id=config_id)
+
     async def buckets(self) -> PangeaResponse[m.BucketsResult]:
         """
         Buckets

pangea/deep_verify.py

@@ -263,8 +263,14 @@ def main():
         audit = init_audit(args.token, args.domain)
         errors = deep_verify(audit, args.file)
 
-        print("\n\nTotal errors:")
+        print("\n\nWarnings:")
+        val = errors["not_persisted"]
+        print(f"\tnot_persisted: {val}")
+
+        print("\nTotal errors:")
         for key, val in errors.items():
+            if key == "not_persisted":
+                continue
             print(f"\t{key.title()}: {val}")
         print()
 

pangea/dump_audit.py

@@ -63,11 +63,12 @@ def dump_before(audit: Audit, output: io.TextIOWrapper, start: datetime) -> int:
     cnt = 0
     if search_res.result and search_res.result.count > 0:
         leaf_index = search_res.result.events[0].leaf_index
-        for row in reversed(search_res.result.events):
-            if row.leaf_index != leaf_index:
-                break
-            dump_event(output, row, search_res)
-            cnt += 1
+        if leaf_index is not None:
+            for row in reversed(search_res.result.events):
+                if row.leaf_index != leaf_index:
+                    break
+                dump_event(output, row, search_res)
+                cnt += 1
     print(f"Dumping before... {cnt} events")
     return cnt
 
@@ -89,7 +90,7 @@ def dump_after(audit: Audit, output: io.TextIOWrapper, start: datetime, last_eve
     cnt = 0
     if search_res.result and search_res.result.count > 0:
         leaf_index = search_res.result.events[0].leaf_index
-        if leaf_index == last_leaf_index:
+        if leaf_index is not None and leaf_index == last_leaf_index:
             start_idx: int = 1 if last_event_hash == search_res.result.events[0].hash else 0
             for row in search_res.result.events[start_idx:]:
                 if row.leaf_index != leaf_index:
@@ -124,7 +125,7 @@ def dump_page(
     msg = f"Dumping... {search_res.result.count} events"
 
     if search_res.result.count <= 1:
-        return end, 0  # type: ignore[return-value]
+        return end, 0, True, "", 0
 
     offset = 0
     result_id = search_res.result.id

pangea/request.py

@@ -6,13 +6,12 @@ import copy
 import json
 import logging
 import time
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Type, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Type, Union
 
 import requests
 from pydantic import BaseModel
-from pydantic_core import to_jsonable_python
 from requests.adapters import HTTPAdapter, Retry
-from requests_toolbelt import MultipartDecoder  # type: ignore
+from requests_toolbelt import MultipartDecoder  # type: ignore[import-untyped]
 from typing_extensions import TypeVar
 
 import pangea
@@ -233,9 +232,6 @@ class PangeaRequest(PangeaRequestBase):
         if data is None:
             data = {}
 
-        # Normalize.
-        data = cast(dict[str, Any], to_jsonable_python(data))
-
         if url is None:
             url = self._url(endpoint)
 
@@ -406,7 +402,20 @@ class PangeaRequest(PangeaRequestBase):
 
         return self._check_response(pangea_response)
 
-    def download_file(self, url: str, filename: Optional[str] = None) -> AttachedFile:
+    def download_file(self, url: str, filename: str | None = None) -> AttachedFile:
+        """
+        Download file
+
+        Download a file from the specified URL and save it with the given
+        filename.
+
+        Args:
+            url: URL of the file to download
+            filename: Name to save the downloaded file as. If not provided, the
+              filename will be determined from the Content-Disposition header or
+              the URL.
+        """
+
         self.logger.debug(
             json.dumps(
                 {

pangea/services/__init__.py

@@ -1,11 +1,9 @@
 from .audit.audit import Audit
 from .authn.authn import AuthN
 from .authz import AuthZ
-from .data_guard import DataGuard
 from .embargo import Embargo
 from .file_scan import FileScan
 from .intel import DomainIntel, FileIntel, IpIntel, UrlIntel, UserIntel
-from .prompt_guard import PromptGuard
 from .redact import Redact
 from .sanitize import Sanitize
 from .share.share import Share

pangea/services/audit/audit.py

@@ -492,7 +492,7 @@ class Audit(ServiceBase, AuditBase):
         verbose: Optional[bool] = None,
     ) -> PangeaResponse[LogResult]:
         """
-        Log an entry
+        Log an event
 
         Create a log entry in the Secure Audit Log.
 
@@ -501,6 +501,7 @@ class Audit(ServiceBase, AuditBase):
             verify (bool, optional): True to verify logs consistency after response.
             sign_local (bool, optional): True to sign event with local key.
             verbose (bool, optional): True to get a more verbose response.
+
         Raises:
             AuditException: If an audit based api exception happens
             PangeaAPIException: If an API Error happens
@@ -511,13 +512,7 @@ class Audit(ServiceBase, AuditBase):
                 Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
 
         Examples:
-            try:
-                log_response = audit.log({"message": "hello world"}, verbose=True)
-                print(f"Response. Hash: {log_response.result.hash}")
-            except pe.PangeaAPIException as e:
-                print(f"Request Error: {e.response.summary}")
-                for err in e.errors:
-                    print(f"\\t{err.detail} \\n")
+            response = audit.log_event({"message": "hello world"}, verbose=True)
         """
 
         input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
@@ -961,9 +956,9 @@ class Audit(ServiceBase, AuditBase):
             if pub_root is not None:
                 self.pub_roots[tree_size] = pub_root
 
-        self.fix_consistency_proofs(tree_sizes)
+        self._fix_consistency_proofs(tree_sizes)
 
-    def fix_consistency_proofs(self, tree_sizes: Iterable[int]):
+    def _fix_consistency_proofs(self, tree_sizes: Iterable[int]) -> None:
         # on very rare occasions, the consistency proof in Arweave may be wrong
         # override it with the proof from pangea (not the root hash, just the proof)
         for tree_size in tree_sizes:

pangea/services/authz.py

@@ -1,9 +1,11 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+from __future__ import annotations
 
 import enum
 from typing import Any, Dict, List, Optional, Union
 
+from pangea.config import PangeaConfig
 from pangea.response import APIRequestModel, APIResponseModel, PangeaResponse, PangeaResponseResult
 from pangea.services.base import ServiceBase
 
@@ -171,7 +173,25 @@ class AuthZ(ServiceBase):
 
     service_name = "authz"
 
-    def __init__(self, token: str, config=None, logger_name="pangea", config_id: Optional[str] = None):
+    def __init__(
+        self, token: str, config: PangeaConfig | None = None, logger_name: str = "pangea", config_id: str | None = None
+    ) -> None:
+        """
+        AuthZ client
+
+        Initializes a new AuthZ client.
+
+        Args:
+            token: Pangea API token.
+            config: Configuration.
+            logger_name: Logger name.
+            config_id: Configuration ID.
+
+        Examples:
+             config = PangeaConfig(domain="aws.us.pangea.cloud")
+             authz = AuthZ(token="pangea_token", config=config)
+        """
+
         super().__init__(token, config, logger_name, config_id=config_id)
 
     def tuple_create(self, tuples: List[Tuple]) -> PangeaResponse[TupleCreateResult]:

pangea/services/base.py

@@ -80,7 +80,8 @@ class ServiceBase(object):
         Returns request's result that has been accepted by the server
 
         Args:
-            exception (AcceptedRequestException): Exception raise by SDK on the call that is been processed.
+            exception: Exception that was previously raised by the SDK on a call
+              that is being processed.
 
         Returns:
             PangeaResponse
@@ -100,5 +101,18 @@ class ServiceBase(object):
         else:
             raise AttributeError("Need to set exception, response or request_id")
 
-    def download_file(self, url: str, filename: Optional[str] = None) -> AttachedFile:
+    def download_file(self, url: str, filename: str | None = None) -> AttachedFile:
+        """
+        Download file
+
+        Download a file from the specified URL and save it with the given
+        filename.
+
+        Args:
+            url: URL of the file to download
+            filename: Name to save the downloaded file as. If not provided, the
+              filename will be determined from the Content-Disposition header or
+              the URL.
+        """
+
         return self.request.download_file(url=url, filename=filename)