pangea-sdk 3.7.0__tar.gz → 5.3.0__tar.gz

{pangea_sdk-3.7.0 → pangea_sdk-5.3.0}/PKG-INFO

@@ -1,30 +1,23 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: pangea-sdk
-Version: 3.7.0
+Version: 5.3.0
 Summary: Pangea API SDK
-Home-page: https://pangea.cloud/docs/sdk/python/
 License: MIT
 Keywords: Pangea,SDK,Audit
 Author: Glenn Gallien
 Author-email: glenn.gallien@pangea.cloud
-Requires-Python: >=3.7.2,<4.0.0
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.9
 Classifier: Topic :: Software Development
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: aiohttp (>=3.8.6,<4.0.0)
-Requires-Dist: asyncio (>=3.4.3,<4.0.0)
-Requires-Dist: cryptography (>=42.0.5,<43.0.0)
-Requires-Dist: deprecated (>=1.2.14,<2.0.0)
-Requires-Dist: google-crc32c (>=1.5.0,<2.0.0)
-Requires-Dist: pydantic (>=1.10.14,<2.0.0)
-Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
-Requires-Dist: requests (>=2.31.0,<3.0.0)
-Project-URL: Repository, https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk
+Requires-Dist: aiohttp
+Requires-Dist: cryptography
+Requires-Dist: deprecated
+Requires-Dist: google-crc32c
+Requires-Dist: pydantic
+Requires-Dist: python-dateutil
+Requires-Dist: requests
+Requires-Dist: requests-toolbelt
+Requires-Dist: typing-extensions
 Description-Content-Type: text/markdown
 
 <a href="https://pangea.cloud?utm_source=github&utm_medium=python-sdk" target="_blank" rel="noopener noreferrer">
@@ -34,15 +27,17 @@ Description-Content-Type: text/markdown
 <br />
 
 [![documentation](https://img.shields.io/badge/documentation-pangea-blue?style=for-the-badge&labelColor=551B76)][Documentation]
-[![Slack](https://img.shields.io/badge/Slack-4A154B?style=for-the-badge&logo=slack&logoColor=white)][Slack]
+[![Discourse](https://img.shields.io/badge/Discourse-4A154B?style=for-the-badge&logo=discourse&logoColor=white)][Discourse]
 
 # Pangea Python SDK
 
-A Python SDK for integrating with Pangea services. Supports Python v3.7 and
+A Python SDK for integrating with Pangea services. Supports Python v3.9 and
 above.
 
 ## Installation
 
+#### GA releases
+
 Via pip:
 
 ```bash
@@ -55,10 +50,33 @@ Via poetry:
 $ poetry add pangea-sdk
 ```
 
+<a name="beta-releases"></a>
+
+#### Beta releases
+
+Pre-release versions may be available with the `b` (beta) denotation in the
+version number. These releases serve to preview Beta and Early Access services
+and APIs. Per Semantic Versioning, they are considered unstable and do not carry
+the same compatibility guarantees as stable releases.
+[Beta changelog](https://github.com/pangeacyber/pangea-python/blob/beta/CHANGELOG.md).
+
+Via pip:
+
+```bash
+$ pip3 install pangea-sdk==5.2.0b2
+```
+
+Via poetry:
+
+```bash
+$ poetry add pangea-sdk==5.2.0b2
+```
+
 ## Usage
 
 - [Documentation][]
-- [Examples][]
+- [GA Examples][]
+- [Beta Examples][]
 
 General usage would be to create a token for a service through the
 [Pangea Console][] and then construct an API client for that respective service.
@@ -202,8 +220,9 @@ It accepts multiple file formats:
 
 
    [Documentation]: https://pangea.cloud/docs/sdk/python/
-   [Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
+   [GA Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
+   [Beta Examples]: https://github.com/pangeacyber/pangea-python/tree/beta/examples
    [Pangea Console]: https://console.pangea.cloud/
-   [Slack]: https://pangea.cloud/join-slack/
+   [Discourse]: https://l.pangea.cloud/Jd4wlGs
    [Secure Audit Log]: https://pangea.cloud/docs/audit
 

{pangea_sdk-3.7.0 → pangea_sdk-5.3.0}/README.md

@@ -5,15 +5,17 @@
 <br />
 
 [![documentation](https://img.shields.io/badge/documentation-pangea-blue?style=for-the-badge&labelColor=551B76)][Documentation]
-[![Slack](https://img.shields.io/badge/Slack-4A154B?style=for-the-badge&logo=slack&logoColor=white)][Slack]
+[![Discourse](https://img.shields.io/badge/Discourse-4A154B?style=for-the-badge&logo=discourse&logoColor=white)][Discourse]
 
 # Pangea Python SDK
 
-A Python SDK for integrating with Pangea services. Supports Python v3.7 and
+A Python SDK for integrating with Pangea services. Supports Python v3.9 and
 above.
 
 ## Installation
 
+#### GA releases
+
 Via pip:
 
 ```bash
@@ -26,10 +28,33 @@ Via poetry:
 $ poetry add pangea-sdk
 ```
 
+<a name="beta-releases"></a>
+
+#### Beta releases
+
+Pre-release versions may be available with the `b` (beta) denotation in the
+version number. These releases serve to preview Beta and Early Access services
+and APIs. Per Semantic Versioning, they are considered unstable and do not carry
+the same compatibility guarantees as stable releases.
+[Beta changelog](https://github.com/pangeacyber/pangea-python/blob/beta/CHANGELOG.md).
+
+Via pip:
+
+```bash
+$ pip3 install pangea-sdk==5.2.0b2
+```
+
+Via poetry:
+
+```bash
+$ poetry add pangea-sdk==5.2.0b2
+```
+
 ## Usage
 
 - [Documentation][]
-- [Examples][]
+- [GA Examples][]
+- [Beta Examples][]
 
 General usage would be to create a token for a service through the
 [Pangea Console][] and then construct an API client for that respective service.
@@ -173,7 +198,8 @@ It accepts multiple file formats:
 
 
    [Documentation]: https://pangea.cloud/docs/sdk/python/
-   [Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
+   [GA Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
+   [Beta Examples]: https://github.com/pangeacyber/pangea-python/tree/beta/examples
    [Pangea Console]: https://console.pangea.cloud/
-   [Slack]: https://pangea.cloud/join-slack/
+   [Discourse]: https://l.pangea.cloud/Jd4wlGs
    [Secure Audit Log]: https://pangea.cloud/docs/audit

{pangea_sdk-3.7.0 → pangea_sdk-5.3.0}/pangea/__init__.py

@@ -1,6 +1,7 @@
-__version__ = "3.7.0"
+__version__ = "5.3.0"
 
 from pangea.asyncio.request import PangeaRequestAsync
 from pangea.config import PangeaConfig
+from pangea.file_uploader import FileUploader
 from pangea.request import PangeaRequest
 from pangea.response import PangeaResponse

pangea_sdk-5.3.0/pangea/asyncio/__init__.py

@@ -0,0 +1 @@
+from .file_uploader import FileUploaderAsync

pangea_sdk-5.3.0/pangea/asyncio/file_uploader.py

@@ -0,0 +1,39 @@
+# Copyright 2022 Pangea Cyber Corporation
+# Author: Pangea Cyber Corporation
+import io
+import logging
+from typing import Dict, Optional
+
+from pangea.asyncio.request import PangeaRequestAsync
+from pangea.request import PangeaConfig
+from pangea.response import TransferMethod
+
+
+class FileUploaderAsync:
+    def __init__(self) -> None:
+        self.logger = logging.getLogger("pangea")
+        self._request: PangeaRequestAsync = PangeaRequestAsync(
+            config=PangeaConfig(),
+            token="",
+            service="FileUploader",
+            logger=self.logger,
+        )
+
+    async def upload_file(
+        self,
+        url: str,
+        file: io.BufferedReader,
+        transfer_method: TransferMethod = TransferMethod.PUT_URL,
+        file_details: Optional[Dict] = None,
+    ) -> None:
+        if transfer_method == TransferMethod.PUT_URL:
+            files = [("file", ("filename", file, "application/octet-stream"))]
+            await self._request.put_presigned_url(url=url, files=files)
+        elif transfer_method == TransferMethod.POST_URL:
+            files = [("file", ("filename", file, "application/octet-stream"))]
+            await self._request.post_presigned_url(url=url, data=file_details, files=files)  # type: ignore[arg-type]
+        else:
+            raise ValueError(f"Transfer method not supported: {transfer_method}")
+
+    async def close(self) -> None:
+        await self._request.session.close()

{pangea_sdk-3.7.0 → pangea_sdk-5.3.0}/pangea/asyncio/request.py

@@ -1,20 +1,24 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+from __future__ import annotations
 
 import asyncio
 import json
 import time
-from typing import Dict, List, Optional, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union
 
 import aiohttp
-import pangea.exceptions as pe
 from aiohttp import FormData
+from pydantic import BaseModel
+from typing_extensions import Any, TypeVar
 
-# from requests.adapters import HTTPAdapter, Retry
-from pangea.request import PangeaRequestBase
-from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
+import pangea.exceptions as pe
+from pangea.request import MultipartResponse, PangeaRequestBase
+from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
 from pangea.utils import default_encoder
 
+TResult = TypeVar("TResult", bound=PangeaResponseResult)
+
 
 class PangeaRequestAsync(PangeaRequestBase):
     """An object that makes direct calls to Pangea Service APIs.
@@ -28,12 +32,12 @@ class PangeaRequestAsync(PangeaRequestBase):
     async def post(
         self,
         endpoint: str,
-        result_class: Type[PangeaResponseResult],
-        data: Union[str, Dict] = {},
+        result_class: Type[TResult],
+        data: str | BaseModel | dict[str, Any] | None = None,
         files: Optional[List[Tuple]] = None,
         poll_result: bool = True,
         url: Optional[str] = None,
-    ) -> PangeaResponse:
+    ) -> PangeaResponse[TResult]:
         """Makes the POST call to a Pangea Service endpoint.
 
         Args:
@@ -44,6 +48,13 @@ class PangeaRequestAsync(PangeaRequestBase):
             PangeaResponse which contains the response in its entirety and
                various properties to retrieve individual fields
         """
+
+        if isinstance(data, BaseModel):
+            data = data.model_dump(exclude_none=True)
+
+        if data is None:
+            data = {}
+
         if url is None:
             url = self._url(endpoint)
 
@@ -56,7 +67,7 @@ class PangeaRequestAsync(PangeaRequestBase):
         )
         transfer_method = data.get("transfer_method", None)  # type: ignore[union-attr]
 
-        if files is not None and type(data) is dict and (transfer_method == TransferMethod.POST_URL.value):
+        if files and type(data) is dict and (transfer_method == TransferMethod.POST_URL.value):
             requests_response = await self._full_post_presigned_url(
                 endpoint, result_class=result_class, data=data, files=files
             )
@@ -66,18 +77,32 @@ class PangeaRequestAsync(PangeaRequestBase):
             )
 
         await self._check_http_errors(requests_response)
-        json_resp = await requests_response.json()
-        self.logger.debug(json.dumps({"service": self.service, "action": "post", "url": url, "response": json_resp}))
 
-        pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)  # type: ignore[var-annotated]
+        if "multipart/form-data" in requests_response.headers.get("content-type", ""):
+            multipart_response = await self._process_multipart_response(requests_response)
+            pangea_response: PangeaResponse = PangeaResponse(
+                requests_response,
+                result_class=result_class,
+                json=multipart_response.pangea_json,
+                attached_files=multipart_response.attached_files,
+            )
+        else:
+            try:
+                json_resp = await requests_response.json()
+                self.logger.debug(
+                    json.dumps({"service": self.service, "action": "post", "url": url, "response": json_resp})
+                )
+
+                pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)
+            except aiohttp.ContentTypeError as e:
+                raise pe.PangeaException(f"Failed to decode json response. {e}. Body: {await requests_response.text()}")
+
         if poll_result:
             pangea_response = await self._handle_queued_result(pangea_response)
 
         return self._check_response(pangea_response)
 
-    async def get(
-        self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
-    ) -> PangeaResponse:
+    async def get(self, path: str, result_class: Type[TResult], check_response: bool = True) -> PangeaResponse[TResult]:
         """Makes the GET call to a Pangea Service endpoint.
 
         Args:
@@ -94,7 +119,7 @@ class PangeaRequestAsync(PangeaRequestBase):
 
         async with self.session.get(url, headers=self._headers()) as requests_response:
             await self._check_http_errors(requests_response)
-            pangea_response = PangeaResponse(  # type: ignore[var-annotated]
+            pangea_response = PangeaResponse(
                 requests_response, result_class=result_class, json=await requests_response.json()
             )
 
@@ -115,11 +140,11 @@ class PangeaRequestAsync(PangeaRequestBase):
             raise pe.ServiceTemporarilyUnavailable(await resp.json())
 
     async def poll_result_by_id(
-        self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
-    ):
+        self, request_id: str, result_class: Type[TResult], check_response: bool = True
+    ) -> PangeaResponse[TResult]:
         path = self._get_poll_path(request_id)
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
-        return await self.get(path, result_class, check_response=check_response)  # type: ignore[arg-type]
+        return await self.get(path, result_class, check_response=check_response)
 
     async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
         request_id = response.request_id
@@ -144,7 +169,7 @@ class PangeaRequestAsync(PangeaRequestBase):
         if resp.status < 200 or resp.status >= 300:
             raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
 
-    async def put_presigned_url(self, url: str, files: List[Tuple]):
+    async def put_presigned_url(self, url: str, files: Sequence[Tuple]):
         # Send put request with file as body
         resp = await self._http_put(url=url, files=files)
         self.logger.debug(
@@ -157,12 +182,97 @@ class PangeaRequestAsync(PangeaRequestBase):
         if resp.status < 200 or resp.status >= 300:
             raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status}", await resp.text())
 
+    async def download_file(self, url: str, filename: str | None = None) -> AttachedFile:
+        """
+        Download file
+
+        Download a file from the specified URL and save it with the given
+        filename.
+
+        Args:
+            url: URL of the file to download
+            filename: Name to save the downloaded file as. If not provided, the
+              filename will be determined from the Content-Disposition header or
+              the URL.
+        """
+
+        self.logger.debug(
+            json.dumps(
+                {
+                    "service": self.service,
+                    "action": "download_file",
+                    "url": url,
+                    "filename": filename,
+                    "status": "start",
+                }
+            )
+        )
+        async with self.session.get(url, headers={}) as response:
+            if response.status == 200:
+                if filename is None:
+                    content_disposition = response.headers.get("Content-Disposition", "")
+                    filename = self._get_filename_from_content_disposition(content_disposition)
+                    if filename is None:
+                        filename = self._get_filename_from_url(url)
+                        if filename is None:
+                            filename = "default_filename"
+
+                content_type = response.headers.get("Content-Type", "")
+                self.logger.debug(
+                    json.dumps(
+                        {
+                            "service": self.service,
+                            "action": "download_file",
+                            "url": url,
+                            "filename": filename,
+                            "status": "success",
+                        }
+                    )
+                )
+
+                return AttachedFile(filename=filename, file=await response.read(), content_type=content_type)
+            raise pe.DownloadFileError(f"Failed to download file. Status: {response.status}", await response.text())
+
+    async def _get_pangea_json(self, reader: aiohttp.multipart.MultipartResponseWrapper) -> Optional[Dict[str, Any]]:
+        # Iterate through parts
+        async for part in reader:
+            if isinstance(part, aiohttp.BodyPartReader):
+                return await part.json()
+        return None
+
+    async def _get_attached_files(self, reader: aiohttp.multipart.MultipartResponseWrapper) -> List[AttachedFile]:
+        files = []
+        i = 0
+
+        async for part in reader:
+            content_type = part.headers.get("Content-Type", "")
+            content_disposition = part.headers.get("Content-Disposition", "")
+            name = self._get_filename_from_content_disposition(content_disposition)
+            if name is None:
+                name = f"default_file_name_{i}"
+                i += 1
+            files.append(AttachedFile(name, await part.read(), content_type))  # type: ignore[union-attr]
+
+        return files
+
+    async def _process_multipart_response(self, resp: aiohttp.ClientResponse) -> MultipartResponse:
+        # Parse the multipart response
+        multipart_reader = aiohttp.MultipartReader.from_response(resp)
+
+        pangea_json = await self._get_pangea_json(multipart_reader)
+        self.logger.debug(
+            json.dumps({"service": self.service, "action": "multipart response", "response": pangea_json})
+        )
+
+        attached_files = await self._get_attached_files(multipart_reader)
+        return MultipartResponse(pangea_json, attached_files)  # type: ignore[arg-type]
+
     async def _http_post(
         self,
         url: str,
         headers: Dict = {},
         data: Union[str, Dict] = {},
-        files: Optional[List[Tuple]] = None,
+        files: Optional[List[Tuple]] = [],
         presigned_url_post: bool = False,
     ) -> aiohttp.ClientResponse:
         if files:
@@ -187,32 +297,38 @@ class PangeaRequestAsync(PangeaRequestBase):
     async def _http_put(
         self,
         url: str,
-        files: List[Tuple],
+        files: Sequence[Tuple],
         headers: Dict = {},
     ) -> aiohttp.ClientResponse:
         self.logger.debug(
             json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
         )
-        form = FormData()
-        name, value = files[0]
-        form.add_field(name, value[1], filename=value[0], content_type=value[2])
-        return await self.session.put(url, headers=headers, data=form)
+        _, value = files[0]
+        return await self.session.put(url, headers=headers, data=value[1])
 
     async def _full_post_presigned_url(
         self,
         endpoint: str,
         result_class: Type[PangeaResponseResult],
         data: Union[str, Dict] = {},
-        files: Optional[List[Tuple]] = None,
+        files: List[Tuple] = [],
     ):
-        if len(files) == 0:  # type: ignore[arg-type]
+        if len(files) == 0:
             raise AttributeError("files attribute should have at least 1 file")
 
         response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
-        data_to_presigned = response.accepted_result.post_form_data  # type: ignore[union-attr]
-        presigned_url = response.accepted_result.post_url  # type: ignore[union-attr]
+        if response.success:  # This should only happen when uploading a zero bytes file
+            return response.raw_response
 
-        await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)  # type: ignore[arg-type]
+        if response.accepted_result is None:
+            raise pe.PangeaException("No accepted_result field when requesting presigned url")
+        if response.accepted_result.post_url is None:
+            raise pe.PresignedURLException("No presigned url", response)
+
+        data_to_presigned = response.accepted_result.post_form_data
+        presigned_url = response.accepted_result.post_url
+
+        await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
         return response.raw_response
 
     async def request_presigned_url(
@@ -223,23 +339,22 @@ class PangeaRequestAsync(PangeaRequestBase):
     ) -> PangeaResponse:
         # Send request
         try:
-            # This should return 202 (AcceptedRequestException)
-            resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
-            raise pe.PresignedURLException("Should return 202", resp)
+            # This should return 202 (AcceptedRequestException) at least zero size file is sent
+            return await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
         except pe.AcceptedRequestException as e:
             accepted_exception = e
         except Exception as e:
             raise e
 
         # Receive 202
-        return await self._poll_presigned_url(accepted_exception.response)  # type: ignore[return-value]
+        return await self._poll_presigned_url(accepted_exception.response)
 
-    async def _poll_presigned_url(self, response: PangeaResponse) -> AcceptedResult:
+    async def _poll_presigned_url(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
         if response.http_status != 202:
             raise AttributeError("Response should be 202")
 
         if response.accepted_result is not None and response.accepted_result.has_upload_url:
-            return response  # type: ignore[return-value]
+            return response
 
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
         retry_count = 1
@@ -276,9 +391,8 @@ class PangeaRequestAsync(PangeaRequestBase):
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
 
         if loop_resp.accepted_result is not None and not loop_resp.accepted_result.has_upload_url:
-            return loop_resp  # type: ignore[return-value]
-        else:
-            raise loop_exc
+            return loop_resp
+        raise loop_exc
 
     async def _handle_queued_result(self, response: PangeaResponse) -> PangeaResponse:
         if self._queued_retry_enabled and response.http_status == 202:

{pangea_sdk-3.7.0 → pangea_sdk-5.3.0}/pangea/asyncio/services/__init__.py

@@ -1,7 +1,10 @@
 from .audit import AuditAsync
 from .authn import AuthNAsync
+from .authz import AuthZAsync
 from .embargo import EmbargoAsync
 from .file_scan import FileScanAsync
 from .intel import DomainIntelAsync, FileIntelAsync, IpIntelAsync, UrlIntelAsync, UserIntelAsync
 from .redact import RedactAsync
+from .sanitize import SanitizeAsync
+from .share import ShareAsync
 from .vault import VaultAsync