pangea-sdk 3.7.0__tar.gz → 3.8.0__tar.gz

{pangea_sdk-3.7.0 → pangea_sdk-3.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pangea-sdk
-Version: 3.7.0
+Version: 3.8.0
 Summary: Pangea API SDK
 Home-page: https://pangea.cloud/docs/sdk/python/
 License: MIT
@@ -18,12 +18,14 @@ Classifier: Topic :: Software Development
 Classifier: Topic :: Software Development :: Libraries
 Requires-Dist: aiohttp (>=3.8.6,<4.0.0)
 Requires-Dist: asyncio (>=3.4.3,<4.0.0)
-Requires-Dist: cryptography (>=42.0.5,<43.0.0)
+Requires-Dist: cryptography (>=42.0.7,<43.0.0)
 Requires-Dist: deprecated (>=1.2.14,<2.0.0)
 Requires-Dist: google-crc32c (>=1.5.0,<2.0.0)
-Requires-Dist: pydantic (>=1.10.14,<2.0.0)
-Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
+Requires-Dist: pydantic (>=1.10.15,<2.0.0)
+Requires-Dist: python-dateutil (>=2.9.0,<3.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: requests-toolbelt (>=1.0.0,<2.0.0)
+Requires-Dist: typing-extensions (>=4.7.1,<5.0.0)
 Project-URL: Repository, https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk
 Description-Content-Type: text/markdown
 
@@ -43,6 +45,8 @@ above.
 
 ## Installation
 
+#### GA releases
+
 Via pip:
 
 ```bash
@@ -55,10 +59,32 @@ Via poetry:
 $ poetry add pangea-sdk
 ```
 
+<a name="beta-releases"></a>
+
+#### Beta releases
+
+Pre-release versions may be available with the `b` (beta) denotation in the
+version number. These releases serve to preview beta services and APIs. Per
+Semantic Versioning, they are considered unstable and do not carry the same
+compatibility guarantees as stable releases. [Beta changelog](https://github.com/pangeacyber/pangea-python/blob/beta/CHANGELOG.md).
+
+Via pip:
+
+```bash
+$ pip3 install pangea-sdk==3.8.0b2
+```
+
+Via poetry:
+
+```bash
+$ poetry add pangea-sdk==3.8.0b2
+```
+
 ## Usage
 
 - [Documentation][]
-- [Examples][]
+- [GA Examples][]
+- [Beta Examples][]
 
 General usage would be to create a token for a service through the
 [Pangea Console][] and then construct an API client for that respective service.
@@ -202,7 +228,8 @@ It accepts multiple file formats:
 
 
    [Documentation]: https://pangea.cloud/docs/sdk/python/
-   [Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
+   [GA Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
+   [Beta Examples]: https://github.com/pangeacyber/pangea-python/tree/beta/examples
    [Pangea Console]: https://console.pangea.cloud/
    [Slack]: https://pangea.cloud/join-slack/
    [Secure Audit Log]: https://pangea.cloud/docs/audit

{pangea_sdk-3.7.0 → pangea_sdk-3.8.0}/README.md

@@ -14,6 +14,8 @@ above.
 
 ## Installation
 
+#### GA releases
+
 Via pip:
 
 ```bash
@@ -26,10 +28,32 @@ Via poetry:
 $ poetry add pangea-sdk
 ```
 
+<a name="beta-releases"></a>
+
+#### Beta releases
+
+Pre-release versions may be available with the `b` (beta) denotation in the
+version number. These releases serve to preview beta services and APIs. Per
+Semantic Versioning, they are considered unstable and do not carry the same
+compatibility guarantees as stable releases. [Beta changelog](https://github.com/pangeacyber/pangea-python/blob/beta/CHANGELOG.md).
+
+Via pip:
+
+```bash
+$ pip3 install pangea-sdk==3.8.0b2
+```
+
+Via poetry:
+
+```bash
+$ poetry add pangea-sdk==3.8.0b2
+```
+
 ## Usage
 
 - [Documentation][]
-- [Examples][]
+- [GA Examples][]
+- [Beta Examples][]
 
 General usage would be to create a token for a service through the
 [Pangea Console][] and then construct an API client for that respective service.
@@ -173,7 +197,8 @@ It accepts multiple file formats:
 
 
    [Documentation]: https://pangea.cloud/docs/sdk/python/
-   [Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
+   [GA Examples]: https://github.com/pangeacyber/pangea-python/tree/main/examples
+   [Beta Examples]: https://github.com/pangeacyber/pangea-python/tree/beta/examples
    [Pangea Console]: https://console.pangea.cloud/
    [Slack]: https://pangea.cloud/join-slack/
    [Secure Audit Log]: https://pangea.cloud/docs/audit

{pangea_sdk-3.7.0 → pangea_sdk-3.8.0}/pangea/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.7.0"
+__version__ = "3.8.0"
 
 from pangea.asyncio.request import PangeaRequestAsync
 from pangea.config import PangeaConfig

{pangea_sdk-3.7.0 → pangea_sdk-3.8.0}/pangea/asyncio/request.py

@@ -7,14 +7,16 @@ import time
 from typing import Dict, List, Optional, Tuple, Type, Union
 
 import aiohttp
-import pangea.exceptions as pe
 from aiohttp import FormData
+from typing_extensions import TypeVar
 
-# from requests.adapters import HTTPAdapter, Retry
-from pangea.request import PangeaRequestBase
-from pangea.response import AcceptedResult, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
+import pangea.exceptions as pe
+from pangea.request import MultipartResponse, PangeaRequestBase
+from pangea.response import AttachedFile, PangeaResponse, PangeaResponseResult, ResponseStatus, TransferMethod
 from pangea.utils import default_encoder
 
+TResult = TypeVar("TResult", bound=PangeaResponseResult, default=PangeaResponseResult)
+
 
 class PangeaRequestAsync(PangeaRequestBase):
     """An object that makes direct calls to Pangea Service APIs.
@@ -28,12 +30,12 @@ class PangeaRequestAsync(PangeaRequestBase):
     async def post(
         self,
         endpoint: str,
-        result_class: Type[PangeaResponseResult],
+        result_class: Type[TResult],
         data: Union[str, Dict] = {},
-        files: Optional[List[Tuple]] = None,
+        files: List[Tuple] = [],
         poll_result: bool = True,
         url: Optional[str] = None,
-    ) -> PangeaResponse:
+    ) -> PangeaResponse[TResult]:
         """Makes the POST call to a Pangea Service endpoint.
 
         Args:
@@ -56,7 +58,7 @@ class PangeaRequestAsync(PangeaRequestBase):
         )
         transfer_method = data.get("transfer_method", None)  # type: ignore[union-attr]
 
-        if files is not None and type(data) is dict and (transfer_method == TransferMethod.POST_URL.value):
+        if files and type(data) is dict and (transfer_method == TransferMethod.POST_URL.value):
             requests_response = await self._full_post_presigned_url(
                 endpoint, result_class=result_class, data=data, files=files
             )
@@ -66,18 +68,32 @@ class PangeaRequestAsync(PangeaRequestBase):
             )
 
         await self._check_http_errors(requests_response)
-        json_resp = await requests_response.json()
-        self.logger.debug(json.dumps({"service": self.service, "action": "post", "url": url, "response": json_resp}))
 
-        pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)  # type: ignore[var-annotated]
+        if "multipart/form-data" in requests_response.headers.get("content-type", ""):
+            multipart_response = await self._process_multipart_response(requests_response)
+            pangea_response: PangeaResponse = PangeaResponse(
+                requests_response,
+                result_class=result_class,
+                json=multipart_response.pangea_json,
+                attached_files=multipart_response.attached_files,
+            )
+        else:
+            try:
+                json_resp = await requests_response.json()
+                self.logger.debug(
+                    json.dumps({"service": self.service, "action": "post", "url": url, "response": json_resp})
+                )
+
+                pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)
+            except aiohttp.ContentTypeError as e:
+                raise pe.PangeaException(f"Failed to decode json response. {e}. Body: {await requests_response.text()}")
+
         if poll_result:
             pangea_response = await self._handle_queued_result(pangea_response)
 
         return self._check_response(pangea_response)
 
-    async def get(
-        self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
-    ) -> PangeaResponse:
+    async def get(self, path: str, result_class: Type[TResult], check_response: bool = True) -> PangeaResponse[TResult]:
         """Makes the GET call to a Pangea Service endpoint.
 
         Args:
@@ -94,7 +110,7 @@ class PangeaRequestAsync(PangeaRequestBase):
 
         async with self.session.get(url, headers=self._headers()) as requests_response:
             await self._check_http_errors(requests_response)
-            pangea_response = PangeaResponse(  # type: ignore[var-annotated]
+            pangea_response = PangeaResponse(
                 requests_response, result_class=result_class, json=await requests_response.json()
             )
 
@@ -115,11 +131,11 @@ class PangeaRequestAsync(PangeaRequestBase):
             raise pe.ServiceTemporarilyUnavailable(await resp.json())
 
     async def poll_result_by_id(
-        self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
-    ):
+        self, request_id: str, result_class: Type[TResult], check_response: bool = True
+    ) -> PangeaResponse[TResult]:
         path = self._get_poll_path(request_id)
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
-        return await self.get(path, result_class, check_response=check_response)  # type: ignore[arg-type]
+        return await self.get(path, result_class, check_response=check_response)
 
     async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
         request_id = response.request_id
@@ -157,12 +173,84 @@ class PangeaRequestAsync(PangeaRequestBase):
         if resp.status < 200 or resp.status >= 300:
             raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status}", await resp.text())
 
+    async def download_file(self, url: str, filename: Optional[str] = None) -> AttachedFile:
+        self.logger.debug(
+            json.dumps(
+                {
+                    "service": self.service,
+                    "action": "download_file",
+                    "url": url,
+                    "status": "start",
+                }
+            )
+        )
+        async with self.session.get(url, headers={}) as response:
+            if response.status == 200:
+                if filename is None:
+                    content_disposition = response.headers.get("Content-Disposition", "")
+                    filename = self._get_filename_from_content_disposition(content_disposition)
+                    if filename is None:
+                        filename = self._get_filename_from_url(url)
+                        if filename is None:
+                            filename = "default_filename"
+
+                content_type = response.headers.get("Content-Type", "")
+                self.logger.debug(
+                    json.dumps(
+                        {
+                            "service": self.service,
+                            "action": "download_file",
+                            "url": url,
+                            "filename": filename,
+                            "status": "success",
+                        }
+                    )
+                )
+
+                return AttachedFile(filename=filename, file=await response.read(), content_type=content_type)
+            else:
+                raise pe.DownloadFileError(f"Failed to download file. Status: {response.status}", await response.text())
+
+    async def _get_pangea_json(self, reader: aiohttp.MultipartReader) -> Optional[Dict]:
+        # Iterate through parts
+        async for part in reader:
+            return await part.json()
+        return None
+
+    async def _get_attached_files(self, reader: aiohttp.MultipartReader) -> List[AttachedFile]:
+        files = []
+        i = 0
+
+        async for part in reader:
+            content_type = part.headers.get("Content-Type", "")
+            content_disposition = part.headers.get("Content-Disposition", "")
+            name = self._get_filename_from_content_disposition(content_disposition)
+            if name is None:
+                name = f"default_file_name_{i}"
+                i += 1
+            files.append(AttachedFile(name, await part.read(), content_type))
+
+        return files
+
+    async def _process_multipart_response(self, resp: aiohttp.ClientResponse) -> MultipartResponse:
+        # Parse the multipart response
+        multipart_reader = aiohttp.MultipartReader.from_response(resp)
+
+        pangea_json = await self._get_pangea_json(multipart_reader)  # type: ignore[arg-type]
+        self.logger.debug(
+            json.dumps({"service": self.service, "action": "multipart response", "response": pangea_json})
+        )
+
+        multipart_reader = multipart_reader.__aiter__()
+        attached_files = await self._get_attached_files(multipart_reader)  # type: ignore[arg-type]
+        return MultipartResponse(pangea_json, attached_files)  # type: ignore[arg-type]
+
     async def _http_post(
         self,
         url: str,
         headers: Dict = {},
         data: Union[str, Dict] = {},
-        files: Optional[List[Tuple]] = None,
+        files: List[Tuple] = [],
         presigned_url_post: bool = False,
     ) -> aiohttp.ClientResponse:
         if files:
@@ -193,26 +281,29 @@ class PangeaRequestAsync(PangeaRequestBase):
         self.logger.debug(
             json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
         )
-        form = FormData()
-        name, value = files[0]
-        form.add_field(name, value[1], filename=value[0], content_type=value[2])
-        return await self.session.put(url, headers=headers, data=form)
+        _, value = files[0]
+        return await self.session.put(url, headers=headers, data=value[1])
 
     async def _full_post_presigned_url(
         self,
         endpoint: str,
         result_class: Type[PangeaResponseResult],
         data: Union[str, Dict] = {},
-        files: Optional[List[Tuple]] = None,
+        files: List[Tuple] = [],
     ):
-        if len(files) == 0:  # type: ignore[arg-type]
+        if len(files) == 0:
             raise AttributeError("files attribute should have at least 1 file")
 
         response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
-        data_to_presigned = response.accepted_result.post_form_data  # type: ignore[union-attr]
-        presigned_url = response.accepted_result.post_url  # type: ignore[union-attr]
+        if response.accepted_result is None:
+            raise pe.PangeaException("No accepted_result field when requesting presigned url")
+        if response.accepted_result.post_url is None:
+            raise pe.PresignedURLException("No presigned url", response)
+
+        data_to_presigned = response.accepted_result.post_form_data
+        presigned_url = response.accepted_result.post_url
 
-        await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)  # type: ignore[arg-type]
+        await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
         return response.raw_response
 
     async def request_presigned_url(
@@ -232,14 +323,14 @@ class PangeaRequestAsync(PangeaRequestBase):
             raise e
 
         # Receive 202
-        return await self._poll_presigned_url(accepted_exception.response)  # type: ignore[return-value]
+        return await self._poll_presigned_url(accepted_exception.response)
 
-    async def _poll_presigned_url(self, response: PangeaResponse) -> AcceptedResult:
+    async def _poll_presigned_url(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
         if response.http_status != 202:
             raise AttributeError("Response should be 202")
 
         if response.accepted_result is not None and response.accepted_result.has_upload_url:
-            return response  # type: ignore[return-value]
+            return response
 
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
         retry_count = 1
@@ -276,7 +367,7 @@ class PangeaRequestAsync(PangeaRequestBase):
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
 
         if loop_resp.accepted_result is not None and not loop_resp.accepted_result.has_upload_url:
-            return loop_resp  # type: ignore[return-value]
+            return loop_resp
         else:
             raise loop_exc
 

{pangea_sdk-3.7.0 → pangea_sdk-3.8.0}/pangea/asyncio/services/__init__.py

@@ -1,5 +1,6 @@
 from .audit import AuditAsync
 from .authn import AuthNAsync
+from .authz import AuthZAsync
 from .embargo import EmbargoAsync
 from .file_scan import FileScanAsync
 from .intel import DomainIntelAsync, FileIntelAsync, IpIntelAsync, UrlIntelAsync, UserIntelAsync