pangea-sdk 6.1.1__py3-none-any.whl → 6.2.0__py3-none-any.whl

pangea/deep_verify.py

@@ -1,13 +1,16 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
 
+from __future__ import annotations
+
 import argparse
 import io
 import math
 import os
 import sys
+from collections.abc import Iterator
 from itertools import groupby
-from typing import Dict, Iterator, List, Optional, Set, TypedDict, Union
+from typing import Optional, TypedDict, Union
 
 import pangea.services.audit.util as audit_util
 from pangea.services import Audit
@@ -24,7 +27,7 @@ class Errors(TypedDict):
     buffer_missing: int
 
 
-root_hashes: Dict[int, str] = {}
+root_hashes: dict[int, str] = {}
 
 
 def num_lines(f: io.TextIOWrapper) -> int:
@@ -124,10 +127,7 @@ def get_root_hash(audit: Audit, tree_size: int) -> str:
 
 
 def print_error(msg: str, level: str = "error"):
-    if level == "warning":
-        dot = "🟡"
-    else:
-        dot = "🔴"
+    dot = "🟡" if level == "warning" else "🔴"
     print(f"{dot} {msg:200s}")
 
 
@@ -148,14 +148,14 @@ def deep_verify(audit: Audit, file: io.TextIOWrapper) -> Errors:
     }
 
     events = file_events(root_hashes, file)
-    events_by_idx: Union[List[Event], Iterator[Event]]
+    events_by_idx: Union[list[Event], Iterator[Event]]
     cold_indexes = SequenceFollower()
     for leaf_index, events_by_idx in groupby(events, lambda event: event.get("leaf_index")):
         events_by_idx = list(events_by_idx)
         buffer_lines = (cnt, cnt + len(events_by_idx) - 1)
         if leaf_index is None:
             print_error(
-                f"Lines {buffer_lines[0]}-{buffer_lines[1]} ({buffer_lines[1]-buffer_lines[0]+1}): Buffer was not persisted"
+                f"Lines {buffer_lines[0]}-{buffer_lines[1]} ({buffer_lines[1] - buffer_lines[0] + 1}): Buffer was not persisted"
             )
             errors["not_persisted"] += len(events_by_idx)
             cnt += len(events_by_idx)
@@ -164,8 +164,8 @@ def deep_verify(audit: Audit, file: io.TextIOWrapper) -> Errors:
         cold_indexes.add(leaf_index)
 
         cold_path_size: Optional[int] = None
-        hot_indexes: Set[int] = set()
-        for i, event in enumerate(events_by_idx):
+        hot_indexes: set[int] = set()
+        for _i, event in enumerate(events_by_idx):
             cnt += 1
             tree_size = get_tree_size(event)
             if tree_size not in root_hashes:
@@ -203,11 +203,11 @@ def deep_verify(audit: Audit, file: io.TextIOWrapper) -> Errors:
             errors["missing"] += len(hot_indexes_diff)
             print(f"missing hot indexes: {hot_indexes_diff}")
             print(f"hot_indexes: {hot_indexes} ")
-            print(f"events:")
+            print("events:")
             for e in events_by_idx:
                 print(e)
             print_error(
-                f"Lines {buffer_lines[0]}-{buffer_lines[1]} ({buffer_lines[1]-buffer_lines[0]}), Buffer #{cold_idx}: {len(hot_indexes_diff)} event(s) missing"
+                f"Lines {buffer_lines[0]}-{buffer_lines[1]} ({buffer_lines[1] - buffer_lines[0]}), Buffer #{cold_idx}: {len(hot_indexes_diff)} event(s) missing"
             )
 
     cold_holes = cold_indexes.holes()
@@ -232,7 +232,7 @@ def create_parser():
         "-f",
         required=True,
         type=argparse.FileType("r"),
-        help="Event input file. Must be a collection of " "JSON Objects separated by newlines",
+        help="Event input file. Must be a collection of JSON Objects separated by newlines",
     )
     return parser
 

pangea/deprecated.py

@@ -19,7 +19,7 @@ def pangea_deprecated(*args, **kwargs):
         def wrapper(*iargs, **ikwargs):
             return deprecated(*args, **kwargs)(f)(*iargs, **ikwargs)
 
-        setattr(wrapper, "_deprecated", kwargs)
+        wrapper._deprecated = kwargs
         return wrapper
 
     return decorator

pangea/dump_audit.py

@@ -7,7 +7,6 @@ import json
 import os
 import sys
 from datetime import datetime
-from typing import Tuple
 
 import dateutil.parser
 
@@ -103,7 +102,7 @@ def dump_after(audit: Audit, output: io.TextIOWrapper, start: datetime, last_eve
 
 def dump_page(
     audit: Audit, output: io.TextIOWrapper, start: datetime, end: datetime, first: bool = False
-) -> Tuple[datetime, int, bool, str, int]:
+) -> tuple[datetime, int, bool, str, int]:
     PAGE_SIZE = 1000
     print(start, end)
     print("Dumping...")
@@ -175,7 +174,7 @@ def parse_args(parser: argparse.ArgumentParser):
         raise ValueError("domain missing")
 
     if args.output is None:
-        args.output = open(f"dump-{datetime.now().strftime('%Y%m%d%H%M%S')}.jsonl", "w")
+        args.output = open(f"dump-{datetime.now().strftime('%Y%m%d%H%M%S')}.jsonl", "w")  # noqa: SIM115
 
     args.start = make_aware_datetime(args.start)
     args.end = make_aware_datetime(args.end)

pangea/exceptions.py

@@ -1,6 +1,9 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
 
+# TODO: Modernize.
+# ruff: noqa: UP006, UP035
+
 from typing import List, Optional
 
 from pangea.response import AcceptedResult, ErrorField, PangeaResponse
@@ -44,7 +47,7 @@ class PangeaAPIException(PangeaException):
     response: PangeaResponse
 
     def __init__(self, message: str, response: PangeaResponse):
-        super(PangeaAPIException, self).__init__(message)
+        super().__init__(message)
         self.response = response
 
     @property
@@ -92,7 +95,7 @@ class UnauthorizedException(PangeaAPIException):
 
     def __init__(self, service_name: str, response: PangeaResponse):
         message = f"User is not authorized to access service {service_name}"
-        super(UnauthorizedException, self).__init__(message, response)
+        super().__init__(message, response)
 
 
 class NotFound(PangeaAPIException):
@@ -100,20 +103,20 @@ class NotFound(PangeaAPIException):
 
     def __init__(self, url: str, response: PangeaResponse):
         message = f"Resource url:'{url}' not found"
-        super(NotFound, self).__init__(message, response)
+        super().__init__(message, response)
 
 
 class ServiceNotEnabledException(PangeaAPIException):
     def __init__(self, service_name: str, response: PangeaResponse):
         message = f"{service_name} is not enabled. Go to console.pangea.cloud/service/{service_name} to enable"
-        super(ServiceNotEnabledException, self).__init__(message, response)
+        super().__init__(message, response)
 
 
 class MissingConfigID(PangeaAPIException):
     """No config ID was provided in either token scopes or explicitly"""
 
     def __init__(self, service_name: str, response: PangeaResponse):
-        super(MissingConfigID, self).__init__(
+        super().__init__(
             f"Token did not contain a config scope for service {service_name}. Create a new token or provide a config ID explicitly in the service base",
             response,
         )

pangea/file_uploader.py

@@ -1,5 +1,9 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+
+# TODO: Modernize.
+# ruff: noqa: UP006, UP035
+
 import io
 import logging
 from typing import Dict, Optional

pangea/request.py

@@ -1,19 +1,24 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+
+# TODO: Modernize.
+# ruff: noqa: UP006, UP035
+
 from __future__ import annotations
 
 import copy
 import json
 import logging
 import time
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Type, Union, cast
+from collections.abc import Iterable, Mapping
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union, cast
 
 import requests
 from pydantic import BaseModel
 from pydantic_core import to_jsonable_python
 from requests.adapters import HTTPAdapter, Retry
 from requests_toolbelt import MultipartDecoder  # type: ignore[import-untyped]
-from typing_extensions import TypeVar
+from typing_extensions import TypeAlias, TypeVar, override
 from yarl import URL
 
 import pangea
@@ -26,11 +31,26 @@ if TYPE_CHECKING:
     import aiohttp
 
 
+_Data: TypeAlias = Union[Iterable[bytes], str, bytes, list[tuple[Any, Any]]]
+
+_FileName: TypeAlias = Union[str, None]
+_FileContent: TypeAlias = Union[str, bytes]
+_FileContentType: TypeAlias = str
+_FileCustomHeaders: TypeAlias = Mapping[str, str]
+_FileSpecTuple2: TypeAlias = tuple[_FileName, _FileContent]
+_FileSpecTuple3: TypeAlias = tuple[_FileName, _FileContent, _FileContentType]
+_FileSpecTuple4: TypeAlias = tuple[_FileName, _FileContent, _FileContentType, _FileCustomHeaders]
+_FileSpec: TypeAlias = Union[_FileContent, _FileSpecTuple2, _FileSpecTuple3, _FileSpecTuple4]
+_Files: TypeAlias = Union[Mapping[str, _FileSpec], Iterable[tuple[str, _FileSpec]]]
+
+_HeadersUpdateMapping: TypeAlias = Mapping[str, str]
+
+
 class MultipartResponse:
     pangea_json: Dict[str, str]
     attached_files: List = []
 
-    def __init__(self, pangea_json: Dict[str, str], attached_files: List = []):
+    def __init__(self, pangea_json: dict[str, str], attached_files: list = []):  # noqa: B006
         self.pangea_json = pangea_json
         self.attached_files = attached_files
 
@@ -48,7 +68,7 @@ class PangeaRequestBase:
         self._queued_retry_enabled = config.queued_retry_enabled
 
         # Custom headers
-        self._extra_headers: Dict = {}
+        self._extra_headers: _HeadersUpdateMapping = {}
         self._user_agent = ""
 
         self.set_custom_user_agent(config.custom_user_agent)
@@ -63,18 +83,17 @@ class PangeaRequestBase:
 
         return self._session
 
-    def set_extra_headers(self, headers: dict):
+    def set_extra_headers(self, headers: _HeadersUpdateMapping):
         """Sets any additional headers in the request.
 
         Args:
-            headers (dict): key-value pair containing extra headers to et
+            headers: key-value pairs containing extra headers to set
 
         Example:
             set_extra_headers({ "My-Header" : "foobar" })
         """
 
-        if isinstance(headers, dict):
-            self._extra_headers = headers
+        self._extra_headers = headers
 
     def set_custom_user_agent(self, user_agent: Optional[str]):
         self.config.custom_user_agent = user_agent
@@ -111,16 +130,14 @@ class PangeaRequestBase:
         url = URL(self.config.base_url_template.format(SERVICE_NAME=self.service))
         return str(url / path)
 
-    def _headers(self) -> dict:
-        headers = {
-            "User-Agent": self._user_agent,
+    def _headers(self) -> dict[str, str]:
+        return {
+            **self._extra_headers,
             "Authorization": f"Bearer {self.token}",
+            "Content-Type": "application/json",
+            "User-Agent": self._user_agent,
         }
 
-        # We want to ignore previous headers if user tried to set them, so we will overwrite them.
-        self._extra_headers.update(headers)
-        return self._extra_headers
-
     def _get_filename_from_content_disposition(self, content_disposition: str) -> Optional[str]:
         filename_parts = content_disposition.split("name=")
         if len(filename_parts) > 1:
@@ -184,6 +201,9 @@ class PangeaRequestBase:
             raise pe.AcceptedRequestException(response)
         raise pe.PangeaAPIException(f"{summary} ", response)
 
+    def _init_session(self) -> requests.Session | aiohttp.ClientSession:
+        raise NotImplementedError
+
 
 TResult = TypeVar("TResult", bound=PangeaResponseResult)
 
@@ -204,8 +224,8 @@ class PangeaRequest(PangeaRequestBase):
         self,
         endpoint: str,
         result_class: Type[TResult],
-        data: str | BaseModel | dict[str, Any] | None = None,
-        files: Optional[List[Tuple]] = None,
+        data: str | BaseModel | Mapping[str, Any] | None = None,
+        files: Optional[list[Tuple]] = None,
         poll_result: bool = True,
         url: Optional[str] = None,
     ) -> PangeaResponse[TResult]:
@@ -250,9 +270,10 @@ class PangeaRequest(PangeaRequestBase):
                 endpoint, result_class=result_class, data=data, files=files
             )
         else:
-            requests_response = self._http_post(
-                url, headers=self._headers(), data=data, files=files, multipart_post=True
-            )
+            headers = self._headers()
+            if transfer_method == TransferMethod.MULTIPART.value:
+                del headers["Content-Type"]
+            requests_response = self._http_post(url, headers=headers, data=data, files=files)
 
         self._check_http_errors(requests_response)
 
@@ -273,7 +294,7 @@ class PangeaRequest(PangeaRequestBase):
 
                 pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)
             except requests.exceptions.JSONDecodeError as e:
-                raise pe.PangeaException(f"Failed to decode json response. {e}. Body: {requests_response.text}")
+                raise pe.PangeaException(f"Failed to decode json response. {e}. Body: {requests_response.text}") from e
 
         if poll_result:
             pangea_response = self._handle_queued_result(pangea_response)
@@ -324,9 +345,9 @@ class PangeaRequest(PangeaRequestBase):
     def _http_post(
         self,
         url: str,
-        headers: Dict = {},
-        data: Union[str, Dict] = {},
-        files: Optional[List[Tuple]] = None,
+        headers: _HeadersUpdateMapping = {},  # noqa: B006
+        data: str | dict[Any, Any] = {},  # noqa: B006
+        files: _Files | None = None,
         multipart_post: bool = True,
     ) -> requests.Response:
         data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
@@ -334,31 +355,25 @@ class PangeaRequest(PangeaRequestBase):
 
     def _http_post_process(
         self,
-        data: Union[str, Dict] = {},
-        files: Optional[Sequence[Tuple[str, Tuple[Any, str, str]]]] = None,
+        data: str | dict[Any, Any] = {},  # noqa: B006
+        files: _Files | None = None,
         multipart_post: bool = True,
-    ):
+    ) -> tuple[_Data | None, _Files | None]:
         if files:
             if multipart_post is True:
                 data_send: str = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
-                multi = [("request", (None, data_send, "application/json"))]
-                multi.extend(files)
-                files = multi
-                return None, files
-            # Post to presigned url as form
-            data_send: list = []  # type: ignore[no-redef]
-            for k, v in data.items():  # type: ignore[union-attr]
-                data_send.append((k, v))  # type: ignore[attr-defined]
-            # When posting to presigned url, file key should be 'file'
-            files = {  # type: ignore[assignment]
-                "file": files[0][1],
-            }
-            return data_send, files
+                multi: list[tuple[str, _FileSpec]] = [("request", (None, data_send, "application/json")), *files]
+                return None, multi
+
+            # Post to presigned URL as form.
+            # When posting to presigned URL, file key should be 'file'.
+            assert isinstance(data, dict)
+            assert isinstance(files, list)
+            return [(k, v) for k, v in data.items()], {"file": files[0][1]}
+
         data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
         return data_send, None
 
-        return data, files
-
     def _handle_queued_result(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
         if self._queued_retry_enabled and response.http_status == 202:
             self.logger.debug(
@@ -477,7 +492,7 @@ class PangeaRequest(PangeaRequestBase):
         self,
         endpoint: str,
         result_class: Type[PangeaResponseResult],
-        data: Union[str, Dict] = {},
+        data: Union[str, Mapping[str, Any]] = {},
     ) -> PangeaResponse:
         # Send request
         try:
@@ -520,8 +535,8 @@ class PangeaRequest(PangeaRequestBase):
     def _http_put(
         self,
         url: str,
-        files: List[Tuple],
-        headers: Dict = {},
+        files: list[Tuple],
+        headers: Mapping[str, str] = {},
     ) -> requests.Response:
         self.logger.debug(
             json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
@@ -533,7 +548,7 @@ class PangeaRequest(PangeaRequestBase):
         self,
         endpoint: str,
         result_class: Type[PangeaResponseResult],
-        data: Union[str, Dict] = {},
+        data: Union[str, Mapping[str, Any]] = {},
         files: Optional[List[Tuple]] = None,
     ):
         if files is None or len(files) == 0:
@@ -603,7 +618,7 @@ class PangeaRequest(PangeaRequestBase):
                         {"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
                     )
                 )
-                raise pe.PresignedURLException("Failed to pull Presigned URL", loop_resp, e)
+                raise pe.PresignedURLException("Failed to pull Presigned URL", loop_resp, e) from e
 
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
 
@@ -611,6 +626,7 @@ class PangeaRequest(PangeaRequestBase):
             return loop_resp
         raise loop_exc
 
+    @override
     def _init_session(self) -> requests.Session:
         retry_config = Retry(
             total=self.config.request_retries,

pangea/response.py

@@ -1,19 +1,25 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+
+# TODO: Modernize.
+# ruff: noqa: UP006, UP035
+
+from __future__ import annotations
+
 import datetime
 import enum
 import os
-from typing import Any, Dict, Generic, List, Optional, Type, Union
+from typing import Annotated, Any, Dict, Generic, List, Optional, Type, Union
 
 import aiohttp
 import requests
 from pydantic import BaseModel, ConfigDict, PlainSerializer
-from typing_extensions import Annotated, TypeVar
+from typing_extensions import TypeVar
 
 from pangea.utils import format_datetime
 
 
-class AttachedFile(object):
+class AttachedFile:
     filename: str
     file: bytes
     content_type: str
@@ -40,10 +46,7 @@ class AttachedFile(object):
         base_name, ext = os.path.splitext(file_path)
         counter = 1
         while os.path.exists(file_path):
-            if ext:
-                file_path = f"{base_name}_{counter}{ext}"
-            else:
-                file_path = f"{base_name}_{counter}"
+            file_path = f"{base_name}_{counter}{ext}" if ext else f"{base_name}_{counter}"
             counter += 1
         return file_path
 
@@ -79,14 +82,12 @@ class TransferMethod(str, enum.Enum):
 PangeaDateTime = Annotated[datetime.datetime, PlainSerializer(format_datetime)]
 
 
-# API response should accept arbitrary fields to make them accept possible new parameters
-class APIResponseModel(BaseModel):
-    model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
+class APIRequestModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
 
 
-# API request models doesn't not allow arbitrary fields
-class APIRequestModel(BaseModel):
-    model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
+class APIResponseModel(BaseModel):
+    model_config = ConfigDict(extra="allow")
 
 
 class PangeaResponseResult(APIResponseModel):
@@ -192,6 +193,8 @@ T = TypeVar("T", bound=PangeaResponseResult)
 
 
 class PangeaResponse(ResponseHeader, Generic[T]):
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid")
+
     raw_result: Optional[Dict[str, Any]] = None
     raw_response: Optional[Union[requests.Response, aiohttp.ClientResponse]] = None
     result: Optional[T] = None
@@ -199,16 +202,16 @@ class PangeaResponse(ResponseHeader, Generic[T]):
     accepted_result: Optional[AcceptedResult] = None
     result_class: Type[T] = PangeaResponseResult  # type: ignore[assignment]
     _json: Any
-    attached_files: List[AttachedFile] = []
+    attached_files: list[AttachedFile] = []
 
     def __init__(
         self,
         response: requests.Response,
         result_class: Type[T],
         json: dict,
-        attached_files: List[AttachedFile] = [],
+        attached_files: list[AttachedFile] = [],  # noqa: B006
     ):
-        super(PangeaResponse, self).__init__(**json)
+        super().__init__(**json)
         self._json = json
         self.raw_response = response
         self.raw_result = self._json["result"]
@@ -241,10 +244,10 @@ class PangeaResponse(ResponseHeader, Generic[T]):
     @property
     def http_status(self) -> int:  # type: ignore[return]
         if self.raw_response:
-            if type(self.raw_response) == aiohttp.ClientResponse:
+            if isinstance(self.raw_response, aiohttp.ClientResponse):
                 return self.raw_response.status
             else:
-                return self.raw_response.status_code  # type: ignore[union-attr]
+                return self.raw_response.status_code
 
     @property
     def url(self) -> str:

pangea/services/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa: F401
+
 from .ai_guard import AIGuard
 from .audit.audit import Audit
 from .authn.authn import AuthN