pangea-sdk 6.1.0__py3-none-any.whl → 6.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +9 -1
- pangea/asyncio/__init__.py +1 -0
- pangea/asyncio/file_uploader.py +4 -2
- pangea/asyncio/request.py +53 -18
- pangea/asyncio/services/__init__.py +2 -0
- pangea/asyncio/services/ai_guard.py +9 -12
- pangea/asyncio/services/audit.py +12 -7
- pangea/asyncio/services/authn.py +36 -25
- pangea/asyncio/services/authz.py +6 -6
- pangea/asyncio/services/base.py +4 -0
- pangea/asyncio/services/file_scan.py +8 -2
- pangea/asyncio/services/intel.py +26 -28
- pangea/asyncio/services/redact.py +7 -3
- pangea/asyncio/services/sanitize.py +5 -1
- pangea/asyncio/services/share.py +5 -1
- pangea/asyncio/services/vault.py +19 -15
- pangea/audit_logger.py +3 -1
- pangea/deep_verify.py +13 -13
- pangea/deprecated.py +1 -1
- pangea/dump_audit.py +2 -3
- pangea/exceptions.py +8 -5
- pangea/file_uploader.py +4 -0
- pangea/request.py +64 -48
- pangea/response.py +21 -18
- pangea/services/__init__.py +2 -0
- pangea/services/ai_guard.py +35 -24
- pangea/services/audit/audit.py +16 -13
- pangea/services/audit/models.py +71 -34
- pangea/services/audit/signing.py +1 -1
- pangea/services/audit/util.py +10 -10
- pangea/services/authn/authn.py +36 -25
- pangea/services/authn/models.py +10 -56
- pangea/services/authz.py +10 -6
- pangea/services/base.py +7 -4
- pangea/services/embargo.py +6 -0
- pangea/services/file_scan.py +8 -2
- pangea/services/intel.py +36 -19
- pangea/services/redact.py +7 -3
- pangea/services/sanitize.py +5 -1
- pangea/services/share/share.py +13 -7
- pangea/services/vault/models/asymmetric.py +4 -0
- pangea/services/vault/models/common.py +4 -0
- pangea/services/vault/models/symmetric.py +4 -0
- pangea/services/vault/vault.py +17 -19
- pangea/tools.py +13 -9
- pangea/utils.py +3 -5
- pangea/verify_audit.py +23 -27
- {pangea_sdk-6.1.0.dist-info → pangea_sdk-6.2.0.dist-info}/METADATA +36 -17
- pangea_sdk-6.2.0.dist-info/RECORD +60 -0
- {pangea_sdk-6.1.0.dist-info → pangea_sdk-6.2.0.dist-info}/WHEEL +1 -1
- pangea_sdk-6.1.0.dist-info/RECORD +0 -60
pangea/request.py
CHANGED
@@ -1,19 +1,24 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
3
7
|
from __future__ import annotations
|
4
8
|
|
5
9
|
import copy
|
6
10
|
import json
|
7
11
|
import logging
|
8
12
|
import time
|
9
|
-
from
|
13
|
+
from collections.abc import Iterable, Mapping
|
14
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union, cast
|
10
15
|
|
11
16
|
import requests
|
12
17
|
from pydantic import BaseModel
|
13
18
|
from pydantic_core import to_jsonable_python
|
14
19
|
from requests.adapters import HTTPAdapter, Retry
|
15
20
|
from requests_toolbelt import MultipartDecoder # type: ignore[import-untyped]
|
16
|
-
from typing_extensions import TypeVar
|
21
|
+
from typing_extensions import TypeAlias, TypeVar, override
|
17
22
|
from yarl import URL
|
18
23
|
|
19
24
|
import pangea
|
@@ -26,11 +31,26 @@ if TYPE_CHECKING:
|
|
26
31
|
import aiohttp
|
27
32
|
|
28
33
|
|
34
|
+
_Data: TypeAlias = Union[Iterable[bytes], str, bytes, list[tuple[Any, Any]]]
|
35
|
+
|
36
|
+
_FileName: TypeAlias = Union[str, None]
|
37
|
+
_FileContent: TypeAlias = Union[str, bytes]
|
38
|
+
_FileContentType: TypeAlias = str
|
39
|
+
_FileCustomHeaders: TypeAlias = Mapping[str, str]
|
40
|
+
_FileSpecTuple2: TypeAlias = tuple[_FileName, _FileContent]
|
41
|
+
_FileSpecTuple3: TypeAlias = tuple[_FileName, _FileContent, _FileContentType]
|
42
|
+
_FileSpecTuple4: TypeAlias = tuple[_FileName, _FileContent, _FileContentType, _FileCustomHeaders]
|
43
|
+
_FileSpec: TypeAlias = Union[_FileContent, _FileSpecTuple2, _FileSpecTuple3, _FileSpecTuple4]
|
44
|
+
_Files: TypeAlias = Union[Mapping[str, _FileSpec], Iterable[tuple[str, _FileSpec]]]
|
45
|
+
|
46
|
+
_HeadersUpdateMapping: TypeAlias = Mapping[str, str]
|
47
|
+
|
48
|
+
|
29
49
|
class MultipartResponse:
|
30
50
|
pangea_json: Dict[str, str]
|
31
51
|
attached_files: List = []
|
32
52
|
|
33
|
-
def __init__(self, pangea_json:
|
53
|
+
def __init__(self, pangea_json: dict[str, str], attached_files: list = []): # noqa: B006
|
34
54
|
self.pangea_json = pangea_json
|
35
55
|
self.attached_files = attached_files
|
36
56
|
|
@@ -48,7 +68,7 @@ class PangeaRequestBase:
|
|
48
68
|
self._queued_retry_enabled = config.queued_retry_enabled
|
49
69
|
|
50
70
|
# Custom headers
|
51
|
-
self._extra_headers:
|
71
|
+
self._extra_headers: _HeadersUpdateMapping = {}
|
52
72
|
self._user_agent = ""
|
53
73
|
|
54
74
|
self.set_custom_user_agent(config.custom_user_agent)
|
@@ -63,18 +83,17 @@ class PangeaRequestBase:
|
|
63
83
|
|
64
84
|
return self._session
|
65
85
|
|
66
|
-
def set_extra_headers(self, headers:
|
86
|
+
def set_extra_headers(self, headers: _HeadersUpdateMapping):
|
67
87
|
"""Sets any additional headers in the request.
|
68
88
|
|
69
89
|
Args:
|
70
|
-
headers
|
90
|
+
headers: key-value pairs containing extra headers to set
|
71
91
|
|
72
92
|
Example:
|
73
93
|
set_extra_headers({ "My-Header" : "foobar" })
|
74
94
|
"""
|
75
95
|
|
76
|
-
|
77
|
-
self._extra_headers = headers
|
96
|
+
self._extra_headers = headers
|
78
97
|
|
79
98
|
def set_custom_user_agent(self, user_agent: Optional[str]):
|
80
99
|
self.config.custom_user_agent = user_agent
|
@@ -111,16 +130,14 @@ class PangeaRequestBase:
|
|
111
130
|
url = URL(self.config.base_url_template.format(SERVICE_NAME=self.service))
|
112
131
|
return str(url / path)
|
113
132
|
|
114
|
-
def _headers(self) -> dict:
|
115
|
-
|
116
|
-
|
133
|
+
def _headers(self) -> dict[str, str]:
|
134
|
+
return {
|
135
|
+
**self._extra_headers,
|
117
136
|
"Authorization": f"Bearer {self.token}",
|
137
|
+
"Content-Type": "application/json",
|
138
|
+
"User-Agent": self._user_agent,
|
118
139
|
}
|
119
140
|
|
120
|
-
# We want to ignore previous headers if user tried to set them, so we will overwrite them.
|
121
|
-
self._extra_headers.update(headers)
|
122
|
-
return self._extra_headers
|
123
|
-
|
124
141
|
def _get_filename_from_content_disposition(self, content_disposition: str) -> Optional[str]:
|
125
142
|
filename_parts = content_disposition.split("name=")
|
126
143
|
if len(filename_parts) > 1:
|
@@ -184,6 +201,9 @@ class PangeaRequestBase:
|
|
184
201
|
raise pe.AcceptedRequestException(response)
|
185
202
|
raise pe.PangeaAPIException(f"{summary} ", response)
|
186
203
|
|
204
|
+
def _init_session(self) -> requests.Session | aiohttp.ClientSession:
|
205
|
+
raise NotImplementedError
|
206
|
+
|
187
207
|
|
188
208
|
TResult = TypeVar("TResult", bound=PangeaResponseResult)
|
189
209
|
|
@@ -204,8 +224,8 @@ class PangeaRequest(PangeaRequestBase):
|
|
204
224
|
self,
|
205
225
|
endpoint: str,
|
206
226
|
result_class: Type[TResult],
|
207
|
-
data: str | BaseModel |
|
208
|
-
files: Optional[
|
227
|
+
data: str | BaseModel | Mapping[str, Any] | None = None,
|
228
|
+
files: Optional[list[Tuple]] = None,
|
209
229
|
poll_result: bool = True,
|
210
230
|
url: Optional[str] = None,
|
211
231
|
) -> PangeaResponse[TResult]:
|
@@ -231,7 +251,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
231
251
|
data = {}
|
232
252
|
|
233
253
|
# Normalize.
|
234
|
-
data = cast(dict[str, Any], to_jsonable_python(data))
|
254
|
+
data = cast(dict[str, Any], to_jsonable_python(data, exclude_none=True))
|
235
255
|
|
236
256
|
if url is None:
|
237
257
|
url = self._url(endpoint)
|
@@ -250,9 +270,10 @@ class PangeaRequest(PangeaRequestBase):
|
|
250
270
|
endpoint, result_class=result_class, data=data, files=files
|
251
271
|
)
|
252
272
|
else:
|
253
|
-
|
254
|
-
|
255
|
-
|
273
|
+
headers = self._headers()
|
274
|
+
if transfer_method == TransferMethod.MULTIPART.value:
|
275
|
+
del headers["Content-Type"]
|
276
|
+
requests_response = self._http_post(url, headers=headers, data=data, files=files)
|
256
277
|
|
257
278
|
self._check_http_errors(requests_response)
|
258
279
|
|
@@ -273,7 +294,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
273
294
|
|
274
295
|
pangea_response = PangeaResponse(requests_response, result_class=result_class, json=json_resp)
|
275
296
|
except requests.exceptions.JSONDecodeError as e:
|
276
|
-
raise pe.PangeaException(f"Failed to decode json response. {e}. Body: {requests_response.text}")
|
297
|
+
raise pe.PangeaException(f"Failed to decode json response. {e}. Body: {requests_response.text}") from e
|
277
298
|
|
278
299
|
if poll_result:
|
279
300
|
pangea_response = self._handle_queued_result(pangea_response)
|
@@ -324,9 +345,9 @@ class PangeaRequest(PangeaRequestBase):
|
|
324
345
|
def _http_post(
|
325
346
|
self,
|
326
347
|
url: str,
|
327
|
-
headers:
|
328
|
-
data:
|
329
|
-
files:
|
348
|
+
headers: _HeadersUpdateMapping = {}, # noqa: B006
|
349
|
+
data: str | dict[Any, Any] = {}, # noqa: B006
|
350
|
+
files: _Files | None = None,
|
330
351
|
multipart_post: bool = True,
|
331
352
|
) -> requests.Response:
|
332
353
|
data_send, files = self._http_post_process(data=data, files=files, multipart_post=multipart_post)
|
@@ -334,31 +355,25 @@ class PangeaRequest(PangeaRequestBase):
|
|
334
355
|
|
335
356
|
def _http_post_process(
|
336
357
|
self,
|
337
|
-
data:
|
338
|
-
files:
|
358
|
+
data: str | dict[Any, Any] = {}, # noqa: B006
|
359
|
+
files: _Files | None = None,
|
339
360
|
multipart_post: bool = True,
|
340
|
-
):
|
361
|
+
) -> tuple[_Data | None, _Files | None]:
|
341
362
|
if files:
|
342
363
|
if multipart_post is True:
|
343
364
|
data_send: str = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
344
|
-
multi = [("request", (None, data_send, "application/json"))]
|
345
|
-
multi
|
346
|
-
|
347
|
-
|
348
|
-
#
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
files = { # type: ignore[assignment]
|
354
|
-
"file": files[0][1],
|
355
|
-
}
|
356
|
-
return data_send, files
|
365
|
+
multi: list[tuple[str, _FileSpec]] = [("request", (None, data_send, "application/json")), *files]
|
366
|
+
return None, multi
|
367
|
+
|
368
|
+
# Post to presigned URL as form.
|
369
|
+
# When posting to presigned URL, file key should be 'file'.
|
370
|
+
assert isinstance(data, dict)
|
371
|
+
assert isinstance(files, list)
|
372
|
+
return [(k, v) for k, v in data.items()], {"file": files[0][1]}
|
373
|
+
|
357
374
|
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
358
375
|
return data_send, None
|
359
376
|
|
360
|
-
return data, files
|
361
|
-
|
362
377
|
def _handle_queued_result(self, response: PangeaResponse[TResult]) -> PangeaResponse[TResult]:
|
363
378
|
if self._queued_retry_enabled and response.http_status == 202:
|
364
379
|
self.logger.debug(
|
@@ -477,7 +492,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
477
492
|
self,
|
478
493
|
endpoint: str,
|
479
494
|
result_class: Type[PangeaResponseResult],
|
480
|
-
data: Union[str,
|
495
|
+
data: Union[str, Mapping[str, Any]] = {},
|
481
496
|
) -> PangeaResponse:
|
482
497
|
# Send request
|
483
498
|
try:
|
@@ -520,8 +535,8 @@ class PangeaRequest(PangeaRequestBase):
|
|
520
535
|
def _http_put(
|
521
536
|
self,
|
522
537
|
url: str,
|
523
|
-
files:
|
524
|
-
headers:
|
538
|
+
files: list[Tuple],
|
539
|
+
headers: Mapping[str, str] = {},
|
525
540
|
) -> requests.Response:
|
526
541
|
self.logger.debug(
|
527
542
|
json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
|
@@ -533,7 +548,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
533
548
|
self,
|
534
549
|
endpoint: str,
|
535
550
|
result_class: Type[PangeaResponseResult],
|
536
|
-
data: Union[str,
|
551
|
+
data: Union[str, Mapping[str, Any]] = {},
|
537
552
|
files: Optional[List[Tuple]] = None,
|
538
553
|
):
|
539
554
|
if files is None or len(files) == 0:
|
@@ -603,7 +618,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
603
618
|
{"service": self.service, "action": "poll_presigned_url", "step": "exit", "cause": {str(e)}}
|
604
619
|
)
|
605
620
|
)
|
606
|
-
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_resp, e)
|
621
|
+
raise pe.PresignedURLException("Failed to pull Presigned URL", loop_resp, e) from e
|
607
622
|
|
608
623
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
609
624
|
|
@@ -611,6 +626,7 @@ class PangeaRequest(PangeaRequestBase):
|
|
611
626
|
return loop_resp
|
612
627
|
raise loop_exc
|
613
628
|
|
629
|
+
@override
|
614
630
|
def _init_session(self) -> requests.Session:
|
615
631
|
retry_config = Retry(
|
616
632
|
total=self.config.request_retries,
|
pangea/response.py
CHANGED
@@ -1,19 +1,25 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
7
|
+
from __future__ import annotations
|
8
|
+
|
3
9
|
import datetime
|
4
10
|
import enum
|
5
11
|
import os
|
6
|
-
from typing import Any, Dict, Generic, List, Optional, Type, Union
|
12
|
+
from typing import Annotated, Any, Dict, Generic, List, Optional, Type, Union
|
7
13
|
|
8
14
|
import aiohttp
|
9
15
|
import requests
|
10
16
|
from pydantic import BaseModel, ConfigDict, PlainSerializer
|
11
|
-
from typing_extensions import
|
17
|
+
from typing_extensions import TypeVar
|
12
18
|
|
13
19
|
from pangea.utils import format_datetime
|
14
20
|
|
15
21
|
|
16
|
-
class AttachedFile
|
22
|
+
class AttachedFile:
|
17
23
|
filename: str
|
18
24
|
file: bytes
|
19
25
|
content_type: str
|
@@ -40,10 +46,7 @@ class AttachedFile(object):
|
|
40
46
|
base_name, ext = os.path.splitext(file_path)
|
41
47
|
counter = 1
|
42
48
|
while os.path.exists(file_path):
|
43
|
-
if ext
|
44
|
-
file_path = f"{base_name}_{counter}{ext}"
|
45
|
-
else:
|
46
|
-
file_path = f"{base_name}_{counter}"
|
49
|
+
file_path = f"{base_name}_{counter}{ext}" if ext else f"{base_name}_{counter}"
|
47
50
|
counter += 1
|
48
51
|
return file_path
|
49
52
|
|
@@ -79,14 +82,12 @@ class TransferMethod(str, enum.Enum):
|
|
79
82
|
PangeaDateTime = Annotated[datetime.datetime, PlainSerializer(format_datetime)]
|
80
83
|
|
81
84
|
|
82
|
-
|
83
|
-
|
84
|
-
model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
|
85
|
+
class APIRequestModel(BaseModel):
|
86
|
+
model_config = ConfigDict(extra="forbid")
|
85
87
|
|
86
88
|
|
87
|
-
|
88
|
-
|
89
|
-
model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
|
89
|
+
class APIResponseModel(BaseModel):
|
90
|
+
model_config = ConfigDict(extra="allow")
|
90
91
|
|
91
92
|
|
92
93
|
class PangeaResponseResult(APIResponseModel):
|
@@ -192,6 +193,8 @@ T = TypeVar("T", bound=PangeaResponseResult)
|
|
192
193
|
|
193
194
|
|
194
195
|
class PangeaResponse(ResponseHeader, Generic[T]):
|
196
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid")
|
197
|
+
|
195
198
|
raw_result: Optional[Dict[str, Any]] = None
|
196
199
|
raw_response: Optional[Union[requests.Response, aiohttp.ClientResponse]] = None
|
197
200
|
result: Optional[T] = None
|
@@ -199,16 +202,16 @@ class PangeaResponse(ResponseHeader, Generic[T]):
|
|
199
202
|
accepted_result: Optional[AcceptedResult] = None
|
200
203
|
result_class: Type[T] = PangeaResponseResult # type: ignore[assignment]
|
201
204
|
_json: Any
|
202
|
-
attached_files:
|
205
|
+
attached_files: list[AttachedFile] = []
|
203
206
|
|
204
207
|
def __init__(
|
205
208
|
self,
|
206
209
|
response: requests.Response,
|
207
210
|
result_class: Type[T],
|
208
211
|
json: dict,
|
209
|
-
attached_files:
|
212
|
+
attached_files: list[AttachedFile] = [], # noqa: B006
|
210
213
|
):
|
211
|
-
super(
|
214
|
+
super().__init__(**json)
|
212
215
|
self._json = json
|
213
216
|
self.raw_response = response
|
214
217
|
self.raw_result = self._json["result"]
|
@@ -241,10 +244,10 @@ class PangeaResponse(ResponseHeader, Generic[T]):
|
|
241
244
|
@property
|
242
245
|
def http_status(self) -> int: # type: ignore[return]
|
243
246
|
if self.raw_response:
|
244
|
-
if
|
247
|
+
if isinstance(self.raw_response, aiohttp.ClientResponse):
|
245
248
|
return self.raw_response.status
|
246
249
|
else:
|
247
|
-
return self.raw_response.status_code
|
250
|
+
return self.raw_response.status_code
|
248
251
|
|
249
252
|
@property
|
250
253
|
def url(self) -> str:
|
pangea/services/__init__.py
CHANGED
pangea/services/ai_guard.py
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
from
|
3
|
+
from collections.abc import Sequence
|
4
|
+
from typing import Generic, Literal, Optional, overload
|
5
|
+
|
6
|
+
from pydantic import BaseModel, ConfigDict
|
7
|
+
from typing_extensions import TypeVar
|
4
8
|
|
5
9
|
from pangea.config import PangeaConfig
|
6
10
|
from pangea.response import APIRequestModel, APIResponseModel, PangeaResponse, PangeaResponseResult
|
@@ -17,6 +21,13 @@ MaliciousEntityAction = Literal["report", "defang", "disabled", "block"]
|
|
17
21
|
PiiEntityAction = Literal["disabled", "report", "block", "mask", "partial_masking", "replacement", "hash", "fpe"]
|
18
22
|
|
19
23
|
|
24
|
+
class Message(BaseModel):
|
25
|
+
model_config = ConfigDict(extra="forbid")
|
26
|
+
|
27
|
+
role: str
|
28
|
+
content: str
|
29
|
+
|
30
|
+
|
20
31
|
class CodeDetectionOverride(APIRequestModel):
|
21
32
|
disabled: Optional[bool] = None
|
22
33
|
action: Optional[Literal["report", "block"]] = None
|
@@ -24,14 +35,14 @@ class CodeDetectionOverride(APIRequestModel):
|
|
24
35
|
|
25
36
|
class LanguageDetectionOverride(APIRequestModel):
|
26
37
|
disabled: Optional[bool] = None
|
27
|
-
allow: Optional[
|
28
|
-
block: Optional[
|
29
|
-
report: Optional[
|
38
|
+
allow: Optional[list[str]] = None
|
39
|
+
block: Optional[list[str]] = None
|
40
|
+
report: Optional[list[str]] = None
|
30
41
|
|
31
42
|
|
32
43
|
class TopicDetectionOverride(APIRequestModel):
|
33
44
|
disabled: Optional[bool] = None
|
34
|
-
block: Optional[
|
45
|
+
block: Optional[list[str]] = None
|
35
46
|
|
36
47
|
|
37
48
|
class PromptInjectionOverride(APIRequestModel):
|
@@ -179,7 +190,7 @@ class PromptInjectionResult(APIResponseModel):
|
|
179
190
|
action: str
|
180
191
|
"""The action taken by this Detector"""
|
181
192
|
|
182
|
-
analyzer_responses:
|
193
|
+
analyzer_responses: list[AnalyzerResponse]
|
183
194
|
"""Triggered prompt injection analyzers."""
|
184
195
|
|
185
196
|
|
@@ -192,20 +203,20 @@ class PiiEntity(APIResponseModel):
|
|
192
203
|
|
193
204
|
|
194
205
|
class PiiEntityResult(APIResponseModel):
|
195
|
-
entities:
|
206
|
+
entities: list[PiiEntity]
|
196
207
|
"""Detected redaction rules."""
|
197
208
|
|
198
209
|
|
199
210
|
class MaliciousEntity(APIResponseModel):
|
200
211
|
type: str
|
201
212
|
value: str
|
202
|
-
action: str
|
213
|
+
action: Optional[str] = None
|
203
214
|
start_pos: Optional[int] = None
|
204
|
-
raw: Optional[
|
215
|
+
raw: Optional[dict[str, object]] = None
|
205
216
|
|
206
217
|
|
207
218
|
class MaliciousEntityResult(APIResponseModel):
|
208
|
-
entities:
|
219
|
+
entities: list[MaliciousEntity]
|
209
220
|
"""Detected harmful items."""
|
210
221
|
|
211
222
|
|
@@ -215,11 +226,11 @@ class CustomEntity(APIResponseModel):
|
|
215
226
|
action: str
|
216
227
|
"""The action taken on this Entity"""
|
217
228
|
start_pos: Optional[int] = None
|
218
|
-
raw: Optional[
|
229
|
+
raw: Optional[dict[str, object]] = None
|
219
230
|
|
220
231
|
|
221
232
|
class CustomEntityResult(APIResponseModel):
|
222
|
-
entities:
|
233
|
+
entities: list[CustomEntity]
|
223
234
|
"""Detected redaction rules."""
|
224
235
|
|
225
236
|
|
@@ -233,7 +244,7 @@ class SecretsEntity(APIResponseModel):
|
|
233
244
|
|
234
245
|
|
235
246
|
class SecretsEntityResult(APIResponseModel):
|
236
|
-
entities:
|
247
|
+
entities: list[SecretsEntity]
|
237
248
|
"""Detected redaction rules."""
|
238
249
|
|
239
250
|
|
@@ -266,22 +277,22 @@ class TextGuardDetectors(APIResponseModel):
|
|
266
277
|
prompt_injection: Optional[TextGuardDetector[PromptInjectionResult]] = None
|
267
278
|
pii_entity: Optional[TextGuardDetector[PiiEntityResult]] = None
|
268
279
|
malicious_entity: Optional[TextGuardDetector[MaliciousEntityResult]] = None
|
269
|
-
custom_entity: Optional[TextGuardDetector[
|
280
|
+
custom_entity: Optional[TextGuardDetector[object]] = None
|
270
281
|
secrets_detection: Optional[TextGuardDetector[SecretsEntityResult]] = None
|
271
|
-
profanity_and_toxicity: Optional[TextGuardDetector[
|
282
|
+
profanity_and_toxicity: Optional[TextGuardDetector[object]] = None
|
272
283
|
language_detection: Optional[TextGuardDetector[LanguageDetectionResult]] = None
|
273
284
|
topic_detection: Optional[TextGuardDetector[TopicDetectionResult]] = None
|
274
285
|
code_detection: Optional[TextGuardDetector[CodeDetectionResult]] = None
|
275
286
|
|
276
287
|
|
277
|
-
class TextGuardResult(PangeaResponseResult
|
288
|
+
class TextGuardResult(PangeaResponseResult):
|
278
289
|
detectors: TextGuardDetectors
|
279
290
|
"""Result of the recipe analyzing and input prompt."""
|
280
291
|
|
281
292
|
prompt_text: Optional[str] = None
|
282
293
|
"""Updated prompt text, if applicable."""
|
283
294
|
|
284
|
-
prompt_messages: Optional[
|
295
|
+
prompt_messages: Optional[object] = None
|
285
296
|
"""Updated structured prompt, if applicable."""
|
286
297
|
|
287
298
|
blocked: bool
|
@@ -345,7 +356,7 @@ class AIGuard(ServiceBase):
|
|
345
356
|
debug: bool | None = None,
|
346
357
|
overrides: Overrides | None = None,
|
347
358
|
log_fields: LogFields | None = None,
|
348
|
-
) -> PangeaResponse[TextGuardResult
|
359
|
+
) -> PangeaResponse[TextGuardResult]:
|
349
360
|
"""
|
350
361
|
Text Guard for scanning LLM inputs and outputs
|
351
362
|
|
@@ -373,12 +384,12 @@ class AIGuard(ServiceBase):
|
|
373
384
|
def guard_text(
|
374
385
|
self,
|
375
386
|
*,
|
376
|
-
messages:
|
387
|
+
messages: Sequence[Message],
|
377
388
|
recipe: str | None = None,
|
378
389
|
debug: bool | None = None,
|
379
390
|
overrides: Overrides | None = None,
|
380
391
|
log_fields: LogFields | None = None,
|
381
|
-
) -> PangeaResponse[TextGuardResult
|
392
|
+
) -> PangeaResponse[TextGuardResult]:
|
382
393
|
"""
|
383
394
|
Text Guard for scanning LLM inputs and outputs
|
384
395
|
|
@@ -400,19 +411,19 @@ class AIGuard(ServiceBase):
|
|
400
411
|
log_field: Additional fields to include in activity log
|
401
412
|
|
402
413
|
Examples:
|
403
|
-
response = ai_guard.guard_text(messages=[
|
414
|
+
response = ai_guard.guard_text(messages=[Message(role="user", content="hello world")])
|
404
415
|
"""
|
405
416
|
|
406
|
-
def guard_text(
|
417
|
+
def guard_text(
|
407
418
|
self,
|
408
419
|
text: str | None = None,
|
409
420
|
*,
|
410
|
-
messages:
|
421
|
+
messages: Sequence[Message] | None = None,
|
411
422
|
recipe: str | None = None,
|
412
423
|
debug: bool | None = None,
|
413
424
|
overrides: Overrides | None = None,
|
414
425
|
log_fields: LogFields | None = None,
|
415
|
-
) -> PangeaResponse[TextGuardResult
|
426
|
+
) -> PangeaResponse[TextGuardResult]:
|
416
427
|
"""
|
417
428
|
Text Guard for scanning LLM inputs and outputs
|
418
429
|
|
pangea/services/audit/audit.py
CHANGED
@@ -1,9 +1,14 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
3
7
|
from __future__ import annotations
|
4
8
|
|
5
9
|
import datetime
|
6
10
|
import json
|
11
|
+
from collections.abc import Mapping
|
7
12
|
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
|
8
13
|
|
9
14
|
import pangea.exceptions as pexc
|
@@ -55,7 +60,7 @@ from pangea.utils import canonicalize_nested_json
|
|
55
60
|
|
56
61
|
class AuditBase:
|
57
62
|
def __init__(
|
58
|
-
self, private_key_file: str = "", public_key_info:
|
63
|
+
self, private_key_file: str = "", public_key_info: Mapping[str, str] = {}, tenant_id: str | None = None
|
59
64
|
) -> None:
|
60
65
|
self.pub_roots: Dict[int, PublishedRoot] = {}
|
61
66
|
self.buffer_data: Optional[str] = None
|
@@ -90,7 +95,7 @@ class AuditBase:
|
|
90
95
|
return input # type: ignore[return-value]
|
91
96
|
|
92
97
|
def _process_log(self, event: dict, sign_local: bool) -> LogEvent:
|
93
|
-
if event.get("tenant_id"
|
98
|
+
if event.get("tenant_id") is None and self.tenant_id:
|
94
99
|
event["tenant_id"] = self.tenant_id
|
95
100
|
|
96
101
|
event = {k: v for k, v in event.items() if v is not None}
|
@@ -221,10 +226,7 @@ class AuditBase:
|
|
221
226
|
tree_sizes.add(result.root.size)
|
222
227
|
tree_sizes.difference_update(self.pub_roots.keys())
|
223
228
|
|
224
|
-
if tree_sizes
|
225
|
-
arweave_roots = get_arweave_published_roots(result.root.tree_name, tree_sizes)
|
226
|
-
else:
|
227
|
-
arweave_roots = {}
|
229
|
+
arweave_roots = get_arweave_published_roots(result.root.tree_name, tree_sizes) if tree_sizes else {}
|
228
230
|
|
229
231
|
return tree_sizes, arweave_roots
|
230
232
|
|
@@ -336,6 +338,7 @@ class AuditBase:
|
|
336
338
|
public_key = get_public_key(audit_envelope.public_key)
|
337
339
|
|
338
340
|
if audit_envelope and audit_envelope.signature and public_key:
|
341
|
+
assert audit_envelope.event
|
339
342
|
v = Verifier()
|
340
343
|
verification = v.verify_signature(
|
341
344
|
audit_envelope.signature,
|
@@ -385,7 +388,7 @@ class Audit(ServiceBase, AuditBase):
|
|
385
388
|
token: str,
|
386
389
|
config: PangeaConfig | None = None,
|
387
390
|
private_key_file: str = "",
|
388
|
-
public_key_info:
|
391
|
+
public_key_info: Mapping[str, str] = {},
|
389
392
|
tenant_id: str | None = None,
|
390
393
|
logger_name: str = "pangea",
|
391
394
|
config_id: str | None = None,
|
@@ -457,7 +460,7 @@ class Audit(ServiceBase, AuditBase):
|
|
457
460
|
A PangeaResponse where the hash of event data and optional verbose
|
458
461
|
results are returned in the response.result field.
|
459
462
|
Available response fields can be found in our
|
460
|
-
[API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
|
463
|
+
[API documentation](https://pangea.cloud/docs/api/audit#/v1/log-post).
|
461
464
|
|
462
465
|
Examples:
|
463
466
|
log_response = audit.log(
|
@@ -505,7 +508,7 @@ class Audit(ServiceBase, AuditBase):
|
|
505
508
|
Returns:
|
506
509
|
A PangeaResponse where the hash of event data and optional verbose
|
507
510
|
results are returned in the response.result field.
|
508
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/log).
|
511
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/log-post).
|
509
512
|
|
510
513
|
Examples:
|
511
514
|
response = audit.log_event({"message": "hello world"}, verbose=True)
|
@@ -543,7 +546,7 @@ class Audit(ServiceBase, AuditBase):
|
|
543
546
|
Returns:
|
544
547
|
A PangeaResponse where the hash of event data and optional verbose
|
545
548
|
results are returned in the response.result field.
|
546
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v2/log).
|
549
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v2/log-post).
|
547
550
|
|
548
551
|
Examples:
|
549
552
|
log_response = audit.log_bulk(
|
@@ -584,7 +587,7 @@ class Audit(ServiceBase, AuditBase):
|
|
584
587
|
Returns:
|
585
588
|
A PangeaResponse where the hash of event data and optional verbose
|
586
589
|
results are returned in the response.result field.
|
587
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v2/log_async).
|
590
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v2/log_async-post).
|
588
591
|
|
589
592
|
Examples:
|
590
593
|
log_response = audit.log_bulk_async(
|
@@ -659,8 +662,8 @@ class Audit(ServiceBase, AuditBase):
|
|
659
662
|
|
660
663
|
Returns:
|
661
664
|
A PangeaResponse[SearchOutput] where the first page of matched events is returned in the
|
662
|
-
response.result field. Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/search).
|
663
|
-
Pagination can be found in the [search results endpoint](https://pangea.cloud/docs/api/audit#/v1/results).
|
665
|
+
response.result field. Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/audit#/v1/search-post).
|
666
|
+
Pagination can be found in the [search results endpoint](https://pangea.cloud/docs/api/audit#/v1/results-post).
|
664
667
|
|
665
668
|
Examples:
|
666
669
|
response = audit.search(
|