pangea-sdk 3.1.0__py3-none-any.whl → 3.3.0__py3-none-any.whl

pangea/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.1.0"
+__version__ = "3.3.0"
 
 from pangea.asyncio.request import PangeaRequestAsync
 from pangea.config import PangeaConfig

pangea/asyncio/request.py

@@ -51,12 +51,14 @@ class PangeaRequestAsync(PangeaRequestBase):
         if self.config_id and data.get("config_id", None) is None:
             data["config_id"] = self.config_id
 
+        transfer_method = data.get("transfer_method", None)
+
         if (
             files is not None
             and type(data) is dict
-            and data.get("transfer_method", None) == TransferMethod.DIRECT.value
+            and (transfer_method == TransferMethod.DIRECT.value or transfer_method == TransferMethod.POST_URL.value)
         ):
-            requests_response = await self._post_presigned_url(
+            requests_response = await self._full_post_presigned_url(
                 endpoint, result_class=result_class, data=data, files=files
             )
         else:
@@ -72,6 +74,83 @@ class PangeaRequestAsync(PangeaRequestBase):
 
         return self._check_response(pangea_response)
 
+    async def get(
+        self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
+    ) -> PangeaResponse:
+        """Makes the GET call to a Pangea Service endpoint.
+
+        Args:
+            endpoint(str): The Pangea Service API endpoint.
+            path(str): Additional URL path
+
+        Returns:
+            PangeaResponse which contains the response in its entirety and
+               various properties to retrieve individual fields
+        """
+
+        url = self._url(path)
+        self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
+
+        async with self.session.get(url, headers=self._headers()) as requests_response:
+            pangea_response = PangeaResponse(
+                requests_response, result_class=result_class, json=await requests_response.json()
+            )
+
+        self.logger.debug(
+            json.dumps(
+                {"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
+                default=default_encoder,
+            )
+        )
+
+        if check_response is False:
+            return pangea_response
+
+        return self._check_response(pangea_response)
+
+    async def poll_result_by_id(
+        self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
+    ):
+        path = self._get_poll_path(request_id)
+        self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
+        return await self.get(path, result_class, check_response=check_response)
+
+    async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
+        request_id = response.request_id
+        if not request_id:
+            raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
+
+        if response.status != ResponseStatus.ACCEPTED.value:
+            raise pe.PangeaException("Response already proccesed")
+
+        return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
+
+    async def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
+        # Send form request with file and upload_details as body
+        resp = await self._http_post(url=url, data=data, files=files, presigned_url_post=True)
+        self.logger.debug(
+            json.dumps(
+                {"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
+                default=default_encoder,
+            )
+        )
+
+        if resp.status < 200 or resp.status >= 300:
+            raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", resp.text)
+
+    async def put_presigned_url(self, url: str, files: List[Tuple]):
+        # Send put request with file as body
+        resp = await self._http_put(url=url, files=files)
+        self.logger.debug(
+            json.dumps(
+                {"service": self.service, "action": "put presigned", "url": url, "response": resp.text},
+                default=default_encoder,
+            )
+        )
+
+        if resp.status_code < 200 or resp.status_code >= 300:
+            raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status_code}", resp.text)
+
     async def _http_post(
         self,
         url: str,
@@ -105,7 +184,21 @@ class PangeaRequestAsync(PangeaRequestBase):
 
         return await self.session.post(url, headers=headers, data=data_send)
 
-    async def _post_presigned_url(
+    async def _http_put(
+        self,
+        url: str,
+        files: List[Tuple],
+        headers: Dict = {},
+    ) -> aiohttp.ClientResponse:
+        self.logger.debug(
+            json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
+        )
+        form = FormData()
+        name, value = files[0]
+        form.add_field(name, value[1], filename=value[0], content_type=value[2])
+        return self.session.put(url, headers=headers, data=form)
+
+    async def _full_post_presigned_url(
         self,
         endpoint: str,
         result_class: Type[PangeaResponseResult],
@@ -115,61 +208,52 @@ class PangeaRequestAsync(PangeaRequestBase):
         if len(files) == 0:
             raise AttributeError("files attribute should have at least 1 file")
 
+        response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
+        data_to_presigned = response.accepted_result.accepted_status.upload_details
+        presigned_url = response.accepted_result.accepted_status.upload_url
+
+        await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
+        return response.raw_response
+
+    async def request_presigned_url(
+        self,
+        endpoint: str,
+        result_class: Type[PangeaResponseResult],
+        data: Union[str, Dict] = {},
+    ) -> PangeaResponse:
         # Send request
         try:
             # This should return 202 (AcceptedRequestException)
             resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
             raise pe.PresignedURLException("Should return 202", resp)
-
         except pe.AcceptedRequestException as e:
             accepted_exception = e
         except Exception as e:
             raise e
 
         # Receive 202 with accepted_status
-        result = await self._poll_presigned_url(accepted_exception)
-        data_to_presigned = result.accepted_status.upload_details
-        presigned_url = result.accepted_status.upload_url
-
-        # Send multipart request with file and upload_details as body
-        resp = await self._http_post(url=presigned_url, data=data_to_presigned, files=files, presigned_url_post=True)
-        self.logger.debug(
-            json.dumps(
-                {
-                    "service": self.service,
-                    "action": "post presigned",
-                    "url": presigned_url,
-                    "response": await resp.text(),
-                },
-                default=default_encoder,
-            )
-        )
-
-        if resp.status < 200 or resp.status >= 300:
-            raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
-
-        return accepted_exception.response.raw_response
+        return await self._poll_presigned_url(accepted_exception.response)
 
-    async def _poll_presigned_url(self, initial_exc: pe.AcceptedRequestException) -> AcceptedResult:
-        if type(initial_exc) is not pe.AcceptedRequestException:
-            raise AttributeError("Exception should be of type AcceptedRequestException")
+    async def _poll_presigned_url(self, response: PangeaResponse) -> AcceptedResult:
+        if response.http_status != 202:
+            raise AttributeError("Response should be 202")
 
-        if initial_exc.accepted_result.accepted_status.upload_url:
-            return initial_exc.accepted_result
+        if response.accepted_result.accepted_status.upload_url:
+            return response
 
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
         retry_count = 1
         start = time.time()
-        loop_exc = initial_exc
+        loop_resp = response
 
         while (
-            loop_exc.accepted_result is not None
-            and not loop_exc.accepted_result.accepted_status.upload_url
+            loop_resp.accepted_result is not None
+            and not loop_resp.accepted_result.accepted_status.upload_url
             and not self._reach_timeout(start)
         ):
             await asyncio.sleep(self._get_delay(retry_count, start))
             try:
-                await self.poll_result_once(initial_exc.response, check_response=False)
+                await self.poll_result_once(response, check_response=False)
                 msg = "Polling presigned url return 200 instead of 202"
                 self.logger.debug(
                     json.dumps(
@@ -179,6 +263,7 @@ class PangeaRequestAsync(PangeaRequestBase):
                 raise pe.PangeaException(msg)
             except pe.AcceptedRequestException as e:
                 retry_count += 1
+                loop_resp = e.response
                 loop_exc = e
             except Exception as e:
                 self.logger.debug(
@@ -190,8 +275,8 @@ class PangeaRequestAsync(PangeaRequestBase):
 
         self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
 
-        if loop_exc.accepted_result is not None and not loop_exc.accepted_result.accepted_status.upload_url:
-            return loop_exc.accepted_result
+        if loop_resp.accepted_result is not None and not loop_resp.accepted_result.accepted_status.upload_url:
+            return loop_resp
         else:
             raise loop_exc
 
@@ -207,57 +292,6 @@ class PangeaRequestAsync(PangeaRequestBase):
 
         return response
 
-    async def get(
-        self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
-    ) -> PangeaResponse:
-        """Makes the GET call to a Pangea Service endpoint.
-
-        Args:
-            endpoint(str): The Pangea Service API endpoint.
-            path(str): Additional URL path
-
-        Returns:
-            PangeaResponse which contains the response in its entirety and
-               various properties to retrieve individual fields
-        """
-
-        url = self._url(path)
-        self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
-
-        async with self.session.get(url, headers=self._headers()) as requests_response:
-            pangea_response = PangeaResponse(
-                requests_response, result_class=result_class, json=await requests_response.json()
-            )
-
-        self.logger.debug(
-            json.dumps(
-                {"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
-                default=default_encoder,
-            )
-        )
-
-        if check_response is False:
-            return pangea_response
-
-        return self._check_response(pangea_response)
-
-    async def poll_result_by_id(
-        self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
-    ):
-        path = self._get_poll_path(request_id)
-        self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
-        return await self.get(path, result_class, check_response=check_response)
-
-    async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
-        request_id = response.request_id
-        if not request_id:
-            raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
-
-        if response.status != ResponseStatus.ACCEPTED.value:
-            raise pe.PangeaException("Response already proccesed")
-
-        return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
-
     async def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
         retry_count = 1
         start = time.time()

pangea/asyncio/services/audit.py

@@ -1,13 +1,15 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
 import datetime
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
+import pangea.exceptions as pexc
 from pangea.response import PangeaResponse
 from pangea.services.audit.audit import AuditBase
 from pangea.services.audit.exceptions import AuditException
 from pangea.services.audit.models import (
     Event,
+    LogBulkResult,
     LogResult,
     RootRequest,
     RootResult,
@@ -172,9 +174,86 @@ class AuditAsync(ServiceBaseAsync, AuditBase):
                     print(f"\\t{err.detail} \\n")
         """
 
-        input = self._pre_log_process(event, sign_local=sign_local, verify=verify, verbose=verbose)
+        input = self._get_log_request(event, sign_local=sign_local, verify=verify, verbose=verbose)
         response = await self.request.post("v1/log", LogResult, data=input.dict(exclude_none=True))
-        return self.handle_log_response(response, verify=verify)
+        if response.success:
+            self._process_log_result(response.result, verify=verify)
+        return response
+
+    async def log_bulk(
+        self,
+        events: List[Dict[str, Any]],
+        sign_local: bool = False,
+        verbose: Optional[bool] = None,
+    ) -> PangeaResponse[LogBulkResult]:
+        """
+        Log an entry
+
+        Create a log entry in the Secure Audit Log.
+        Args:
+            events (List[dict[str, Any]]): events to be logged
+            verify (bool, optional): True to verify logs consistency after response.
+            sign_local (bool, optional): True to sign event with local key.
+            verbose (bool, optional): True to get a more verbose response.
+        Raises:
+            AuditException: If an audit based api exception happens
+            PangeaAPIException: If an API Error happens
+
+        Returns:
+            A PangeaResponse where the hash of event data and optional verbose
+                results are returned in the response.result field.
+                Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
+
+        Examples:
+            FIXME:
+        """
+
+        input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
+        response = await self.request.post("v2/log", LogBulkResult, data=input.dict(exclude_none=True))
+        if response.success:
+            for result in response.result.results:
+                self._process_log_result(result, verify=True)
+        return response
+
+    async def log_bulk_async(
+        self,
+        events: List[Dict[str, Any]],
+        sign_local: bool = False,
+        verbose: Optional[bool] = None,
+    ) -> PangeaResponse[LogBulkResult]:
+        """
+        Log an entry
+
+        Create a log entry in the Secure Audit Log.
+        Args:
+            events (List[dict[str, Any]]): events to be logged
+            verify (bool, optional): True to verify logs consistency after response.
+            sign_local (bool, optional): True to sign event with local key.
+            verbose (bool, optional): True to get a more verbose response.
+        Raises:
+            AuditException: If an audit based api exception happens
+            PangeaAPIException: If an API Error happens
+
+        Returns:
+            A PangeaResponse where the hash of event data and optional verbose
+                results are returned in the response.result field.
+                Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#log-an-entry).
+
+        Examples:
+            FIXME:
+        """
+
+        input = self._get_log_request(events, sign_local=sign_local, verify=False, verbose=verbose)
+        try:
+            response = await self.request.post(
+                "v2/log_async", LogBulkResult, data=input.dict(exclude_none=True), poll_result=False
+            )
+        except pexc.AcceptedRequestException as e:
+            return e.response
+        if response.success:
+            for result in response.result.results:
+                self._process_log_result(result, verify=True)
+        return response
 
     async def search(
         self,

pangea/asyncio/services/base.py

@@ -1,9 +1,11 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
 
+from typing import Optional, Type, Union
+
 from pangea.asyncio.request import PangeaRequestAsync
 from pangea.exceptions import AcceptedRequestException
-from pangea.response import PangeaResponse
+from pangea.response import PangeaResponse, PangeaResponseResult
 from pangea.services.base import ServiceBase
 
 
@@ -21,7 +23,13 @@ class ServiceBaseAsync(ServiceBase):
 
         return self._request
 
-    async def poll_result(self, exception: AcceptedRequestException) -> PangeaResponse:
+    async def poll_result(
+        self,
+        exception: Optional[AcceptedRequestException] = None,
+        response: Optional[PangeaResponse] = None,
+        request_id: Optional[str] = None,
+        result_class: Union[Type[PangeaResponseResult], dict] = dict,
+    ) -> PangeaResponse:
         """
         Poll result
 
@@ -39,7 +47,16 @@ class ServiceBaseAsync(ServiceBase):
         Examples:
             response = service.poll_result(exception)
         """
-        return await self.request.poll_result_once(exception.response, check_response=True)
+        if exception is not None:
+            return await self.request.poll_result_once(exception.response, check_response=True)
+        elif response is not None:
+            return await self.request.poll_result_once(response, check_response=True)
+        elif request_id is not None:
+            return await self.request.poll_result_by_id(
+                request_id=request_id, result_class=result_class, check_response=True
+            )
+        else:
+            raise AttributeError("Need to set exception, response or request_id")
 
     async def close(self):
         await self.request.session.close()

pangea/asyncio/services/file_scan.py

@@ -1,11 +1,14 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
 import io
-from typing import Optional
+import logging
+from typing import Dict, Optional
 
 import pangea.services.file_scan as m
-from pangea.response import PangeaResponse
-from pangea.utils import get_presigned_url_upload_params
+from pangea.asyncio.request import PangeaRequestAsync
+from pangea.request import PangeaConfig
+from pangea.response import PangeaResponse, TransferMethod
+from pangea.utils import FileUploadParams, get_file_upload_params
 
 from .base import ServiceBaseAsync
 
@@ -46,6 +49,7 @@ class FileScanAsync(ServiceBaseAsync):
         raw: Optional[bool] = None,
         provider: Optional[str] = None,
         sync_call: bool = True,
+        transfer_method: TransferMethod = TransferMethod.DIRECT,
     ) -> PangeaResponse[m.FileScanResult]:
         """
         Scan
@@ -84,13 +88,79 @@ class FileScanAsync(ServiceBaseAsync):
         if file or file_path:
             if file_path:
                 file = open(file_path, "rb")
-            crc, sha, size, _ = get_presigned_url_upload_params(file)
+            if transfer_method == TransferMethod.DIRECT or transfer_method == TransferMethod.POST_URL:
+                params = get_file_upload_params(file)
+                crc = params.crc_hex
+                sha = params.sha256_hex
+                size = params.size
+            else:
+                crc, sha, size = None, None, None
             files = [("upload", ("filename", file, "application/octet-stream"))]
         else:
             raise ValueError("Need to set file_path or file arguments")
 
         input = m.FileScanRequest(
-            verbose=verbose, raw=raw, provider=provider, transfer_crc32c=crc, transfer_sha256=sha, transfer_size=size
+            verbose=verbose,
+            raw=raw,
+            provider=provider,
+            transfer_crc32c=crc,
+            transfer_sha256=sha,
+            transfer_size=size,
+            transfer_method=transfer_method,
         )
         data = input.dict(exclude_none=True)
         return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
+
+    async def request_upload_url(
+        self,
+        transfer_method: TransferMethod = TransferMethod.PUT_URL,
+        params: Optional[FileUploadParams] = None,
+        verbose: Optional[bool] = None,
+        raw: Optional[bool] = None,
+        provider: Optional[str] = None,
+    ) -> PangeaResponse[m.FileScanResult]:
+        input = m.FileScanRequest(
+            verbose=verbose,
+            raw=raw,
+            provider=provider,
+            transfer_method=transfer_method,
+        )
+        if params is not None and (
+            transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT
+        ):
+            input.transfer_crc32c = params.crc_hex
+            input.transfer_sha256 = params.sha256_hex
+            input.transfer_size = params.size
+
+        data = input.dict(exclude_none=True)
+        return await self.request.request_presigned_url("v1/scan", m.FileScanResult, data=data)
+
+
+class FileUploaderAsync:
+    def __init__(self):
+        self.logger = logging.getLogger("pangea")
+        self._request: PangeaRequestAsync = PangeaRequestAsync(
+            config=PangeaConfig(),
+            token="",
+            service="FileScanUploader",
+            logger=self.logger,
+        )
+
+    async def upload_file(
+        self,
+        url: str,
+        file: io.BufferedReader,
+        transfer_method: TransferMethod = TransferMethod.PUT_URL,
+        file_details: Optional[Dict] = None,
+    ):
+        if transfer_method == TransferMethod.PUT_URL:
+            files = [("file", ("filename", file, "application/octet-stream"))]
+            await self._request.put_presigned_url(url=url, files=files)
+        elif transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT:
+            files = [("file", ("filename", file, "application/octet-stream"))]
+            await self._request.post_presigned_url(url=url, data=file_details, files=files)
+        else:
+            raise ValueError(f"Transfer method not supported: {transfer_method}")
+
+    async def close(self):
+        await self._request.session.close()

pangea/asyncio/services/redact.py

@@ -36,13 +36,8 @@ class RedactAsync(ServiceBaseAsync):
 
     service_name = "redact"
 
-    def __init__(
-        self,
-        token,
-        config=None,
-        logger_name="pangea",
-    ):
-        super().__init__(token, config, logger_name)
+    def __init__(self, token, config=None, logger_name="pangea", config_id: Optional[str] = None):
+        super().__init__(token, config, logger_name, config_id=config_id)
 
     async def redact(
         self,

pangea/config.py

@@ -10,7 +10,7 @@ class PangeaConfig:
     """Holds run time configuration information used by SDK components."""
 
     """
-    Used to set pangea domain (and port if needed), it should not include service subdomain
+    Used to set Pangea domain (and port if needed), it should not include service subdomain
     just for particular use cases when environment = "local", domain could be set to an url including:
     scheme (http:// or https://), subdomain, domain and port.
 
@@ -19,7 +19,7 @@ class PangeaConfig:
 
     """
     Used to generate service url.
-    It should be only 'production' or 'local' in case of particular services that can run locally as Redact
+    It should be only 'production' or 'local' in cases of particular services that can run locally as Redact.
 
     """
     environment: str = "production"

pangea/exceptions.py

@@ -119,7 +119,7 @@ class AcceptedRequestException(PangeaAPIException):
     """Accepted request exception. Async response"""
 
     request_id: str
-    accepted_result: Optional[AcceptedResult]
+    accepted_result: Optional[AcceptedResult] = None
 
     def __init__(self, response: PangeaResponse):
         message = f"summary: {response.summary}. request_id: {response.request_id}."