pangea-sdk 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +1 -1
- pangea/deep_verify.py +2 -2
- pangea/dump_audit.py +3 -9
- pangea/exceptions.py +40 -2
- pangea/request.py +40 -7
- pangea/response.py +8 -0
- pangea/services/__init__.py +1 -0
- pangea/services/audit/audit.py +28 -7
- pangea/services/audit/models.py +12 -0
- pangea/services/audit/util.py +6 -1
- pangea/services/intel.py +270 -82
- pangea/services/vault/models/asymmetric.py +67 -0
- pangea/services/vault/models/common.py +337 -0
- pangea/services/vault/models/secret.py +24 -0
- pangea/services/vault/models/symmetric.py +61 -0
- pangea/services/vault/vault.py +458 -0
- pangea/{tools_util.py → tools.py} +7 -9
- pangea/utils.py +22 -0
- {pangea_sdk-1.3.0.dist-info → pangea_sdk-1.5.0.dist-info}/METADATA +4 -3
- pangea_sdk-1.5.0.dist-info/RECORD +30 -0
- {pangea_sdk-1.3.0.dist-info → pangea_sdk-1.5.0.dist-info}/WHEEL +1 -1
- pangea_sdk-1.3.0.dist-info/RECORD +0 -24
pangea/__init__.py
CHANGED
pangea/deep_verify.py
CHANGED
@@ -12,7 +12,7 @@ from itertools import groupby
|
|
12
12
|
import pangea.services.audit.util as audit_util
|
13
13
|
from pangea.services import Audit
|
14
14
|
from pangea.services.audit.models import EventEnvelope
|
15
|
-
from pangea.
|
15
|
+
from pangea.tools import Event, SequenceFollower, exit_with_error, file_events, init_audit, print_progress_bar
|
16
16
|
|
17
17
|
|
18
18
|
class Errors(t.TypedDict):
|
@@ -161,7 +161,7 @@ def deep_verify(audit: Audit, file: io.TextIOWrapper) -> Errors:
|
|
161
161
|
}
|
162
162
|
|
163
163
|
events = file_events(root_hashes, file)
|
164
|
-
events_by_idx: list[Event]
|
164
|
+
events_by_idx: t.Union[list[Event], t.Iterator[Event]]
|
165
165
|
cold_indexes = SequenceFollower()
|
166
166
|
for leaf_index, events_by_idx in groupby(events, lambda event: event.get("leaf_index")):
|
167
167
|
events_by_idx = list(events_by_idx)
|
pangea/dump_audit.py
CHANGED
@@ -11,21 +11,15 @@ from datetime import datetime
|
|
11
11
|
import dateutil.parser
|
12
12
|
from pangea.response import PangeaResponse
|
13
13
|
from pangea.services import Audit
|
14
|
-
from pangea.
|
15
|
-
|
16
|
-
get_script_name,
|
17
|
-
init_audit,
|
18
|
-
json_defaults,
|
19
|
-
make_aware_datetime,
|
20
|
-
print_progress_bar,
|
21
|
-
)
|
14
|
+
from pangea.tools import filter_deep_none, get_script_name, init_audit, make_aware_datetime, print_progress_bar
|
15
|
+
from pangea.utils import default_encoder
|
22
16
|
|
23
17
|
|
24
18
|
def dump_event(output: io.TextIOWrapper, row: dict, resp: PangeaResponse):
|
25
19
|
row_data = filter_deep_none(row.dict())
|
26
20
|
if resp.result.root:
|
27
21
|
row_data["tree_size"] = resp.result.root.size
|
28
|
-
output.write(json.dumps(row_data, default=
|
22
|
+
output.write(json.dumps(row_data, default=default_encoder) + "\n")
|
29
23
|
|
30
24
|
|
31
25
|
def dump_audit(audit: Audit, output: io.TextIOWrapper, start: datetime, end: datetime) -> int:
|
pangea/exceptions.py
CHANGED
@@ -27,6 +27,22 @@ class PangeaAPIException(PangeaException):
|
|
27
27
|
def errors(self) -> List[ErrorField]:
|
28
28
|
return self.response.errors
|
29
29
|
|
30
|
+
def __repr__(self) -> str:
|
31
|
+
ret = f"Summary: {self.response.summary}\n"
|
32
|
+
if self.response.errors:
|
33
|
+
ret += "Errors: \n"
|
34
|
+
for ef in self.response.errors:
|
35
|
+
ret += f"\t {ef.detail}\n"
|
36
|
+
return ret
|
37
|
+
|
38
|
+
def __str__(self) -> str:
|
39
|
+
ret = f"Summary: {self.response.summary}\n"
|
40
|
+
if self.response.errors:
|
41
|
+
ret += "Errors: \n"
|
42
|
+
for ef in self.response.errors:
|
43
|
+
ret += f"\t {ef.detail}\n"
|
44
|
+
return ret
|
45
|
+
|
30
46
|
|
31
47
|
class ValidationException(PangeaAPIException):
|
32
48
|
"""Pangea Validation Errors denoting issues with an API request"""
|
@@ -76,8 +92,12 @@ class ProviderErrorException(PangeaAPIException):
|
|
76
92
|
"""Downstream provider error"""
|
77
93
|
|
78
94
|
|
79
|
-
class
|
80
|
-
"""A pangea
|
95
|
+
class InternalServerError(PangeaAPIException):
|
96
|
+
"""A pangea server error"""
|
97
|
+
|
98
|
+
def __init__(self, response: PangeaResponse):
|
99
|
+
message = f"summary: {response.summary}. request_id: {response.request_id}. request_time: {response.request_time}. response_time: ${response.response_time}"
|
100
|
+
super().__init__(message, response)
|
81
101
|
|
82
102
|
|
83
103
|
class ServiceNotAvailableException(PangeaAPIException):
|
@@ -103,3 +123,21 @@ class TreeNotFoundException(AuditAPIException):
|
|
103
123
|
|
104
124
|
class BadOffsetException(AuditAPIException):
|
105
125
|
"""Bad offset in results search"""
|
126
|
+
|
127
|
+
|
128
|
+
# Vault SDK specific exceptions
|
129
|
+
class VaultException(PangeaException):
|
130
|
+
"""Vault SDK specific exceptions"""
|
131
|
+
|
132
|
+
|
133
|
+
# Vault API specific exceptions
|
134
|
+
class VaultAPIException(PangeaAPIException):
|
135
|
+
"""Vault service specific exceptions"""
|
136
|
+
|
137
|
+
|
138
|
+
class ForbiddenVaultOperation(VaultAPIException):
|
139
|
+
"""Forbiden Vault operation"""
|
140
|
+
|
141
|
+
|
142
|
+
class VaultItemNotFound(VaultAPIException):
|
143
|
+
"""Vault item not found"""
|
pangea/request.py
CHANGED
@@ -4,13 +4,14 @@
|
|
4
4
|
import json
|
5
5
|
import logging
|
6
6
|
import time
|
7
|
-
from typing import
|
7
|
+
from typing import Dict, Union
|
8
8
|
|
9
9
|
import pangea
|
10
10
|
import requests
|
11
11
|
from pangea import exceptions
|
12
12
|
from pangea.config import PangeaConfig
|
13
13
|
from pangea.response import PangeaResponse, ResponseStatus
|
14
|
+
from pangea.utils import default_encoder
|
14
15
|
from requests.adapters import HTTPAdapter, Retry
|
15
16
|
|
16
17
|
|
@@ -76,7 +77,7 @@ class PangeaRequest(object):
|
|
76
77
|
|
77
78
|
return self._queued_retry_enabled
|
78
79
|
|
79
|
-
def post(self, endpoint: str = "", data:
|
80
|
+
def post(self, endpoint: str = "", data: Union[str, Dict] = {}) -> PangeaResponse:
|
80
81
|
"""Makes the POST call to a Pangea Service endpoint.
|
81
82
|
|
82
83
|
If queued_support mode is enabled, progress checks will be made for
|
@@ -92,14 +93,21 @@ class PangeaRequest(object):
|
|
92
93
|
various properties to retrieve individual fields
|
93
94
|
"""
|
94
95
|
url = self._url(endpoint)
|
95
|
-
data_send = json.dumps(data)
|
96
|
-
|
97
|
-
|
96
|
+
data_send = json.dumps(data, default=default_encoder) if isinstance(data, dict) else data
|
97
|
+
self.logger.debug(
|
98
|
+
json.dumps({"service": self.service, "action": "post", "url": url, "data": data}, default=default_encoder)
|
99
|
+
)
|
98
100
|
|
99
101
|
requests_response = self.session.post(url, headers=self._headers(), data=data_send)
|
100
102
|
|
101
103
|
if self._queued_retry_enabled and requests_response.status_code == 202:
|
102
104
|
response_json = requests_response.json()
|
105
|
+
self.logger.debug(
|
106
|
+
json.dumps(
|
107
|
+
{"service": self.service, "action": "post", "url": url, "response": response_json},
|
108
|
+
default=default_encoder,
|
109
|
+
)
|
110
|
+
)
|
103
111
|
request_id = response_json.get("request_id", None)
|
104
112
|
|
105
113
|
if not request_id:
|
@@ -109,6 +117,12 @@ class PangeaRequest(object):
|
|
109
117
|
else:
|
110
118
|
pangea_response = PangeaResponse(requests_response)
|
111
119
|
|
120
|
+
self.logger.debug(
|
121
|
+
json.dumps(
|
122
|
+
{"service": self.service, "action": "post", "url": url, "result": pangea_response.raw_result},
|
123
|
+
default=default_encoder,
|
124
|
+
)
|
125
|
+
)
|
112
126
|
self._check_response(pangea_response)
|
113
127
|
return pangea_response
|
114
128
|
|
@@ -126,9 +140,16 @@ class PangeaRequest(object):
|
|
126
140
|
url = self._url(f"{endpoint}/{path}")
|
127
141
|
|
128
142
|
self.logger.debug(json.dupms({"service": self.service, "action": "get", "url": url}))
|
129
|
-
|
130
143
|
requests_response = self.session.get(url, headers=self._headers())
|
144
|
+
|
131
145
|
pangea_response = PangeaResponse(requests_response)
|
146
|
+
|
147
|
+
self.logger.debug(
|
148
|
+
json.dumps(
|
149
|
+
{"service": self.service, "action": "post", "url": url, "result": pangea_response.raw_result},
|
150
|
+
default=default_encoder,
|
151
|
+
)
|
152
|
+
)
|
132
153
|
self._check_response(pangea_response)
|
133
154
|
return pangea_response
|
134
155
|
|
@@ -189,7 +210,13 @@ class PangeaRequest(object):
|
|
189
210
|
|
190
211
|
self.logger.error(
|
191
212
|
json.dumps(
|
192
|
-
{
|
213
|
+
{
|
214
|
+
"service": self.service,
|
215
|
+
"action": "api_error",
|
216
|
+
"url": response.raw_response.url,
|
217
|
+
"summary": summary,
|
218
|
+
"result": response.raw_result,
|
219
|
+
}
|
193
220
|
)
|
194
221
|
)
|
195
222
|
|
@@ -215,6 +242,12 @@ class PangeaRequest(object):
|
|
215
242
|
raise exceptions.IPNotFoundException(summary)
|
216
243
|
elif status == ResponseStatus.BAD_OFFSET.value:
|
217
244
|
raise exceptions.BadOffsetException(summary, response)
|
245
|
+
elif status == ResponseStatus.FORBIDDEN_VAULT_OPERATION.value:
|
246
|
+
raise exceptions.ForbiddenVaultOperation(summary, response)
|
247
|
+
elif status == ResponseStatus.VAULT_ITEM_NOT_FOUND.value:
|
248
|
+
raise exceptions.VaultItemNotFound(summary, response)
|
218
249
|
elif status == ResponseStatus.NOT_FOUND.value:
|
219
250
|
raise exceptions.NotFound(response.raw_response.url if response.raw_response is not None else "", response)
|
251
|
+
elif status == ResponseStatus.INTERNAL_SERVER_ERROR.value:
|
252
|
+
raise exceptions.InternalServerError(response)
|
220
253
|
raise exceptions.PangeaAPIException(f"{summary} ", response)
|
pangea/response.py
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
import datetime
|
3
4
|
import enum
|
4
5
|
from typing import Any, Dict, Generic, List, Optional, TypeVar
|
5
6
|
|
6
7
|
import requests
|
8
|
+
from pangea.utils import format_datetime
|
7
9
|
from pydantic import BaseModel
|
8
10
|
|
9
11
|
T = TypeVar("T")
|
@@ -21,6 +23,9 @@ class APIResponseModel(BaseModel):
|
|
21
23
|
class APIRequestModel(BaseModel):
|
22
24
|
class Config:
|
23
25
|
arbitrary_types_allowed = True
|
26
|
+
json_encoders = {
|
27
|
+
datetime.datetime: format_datetime,
|
28
|
+
}
|
24
29
|
|
25
30
|
|
26
31
|
class PangeaResponseResult(APIResponseModel):
|
@@ -63,7 +68,10 @@ class ResponseStatus(str, enum.Enum):
|
|
63
68
|
TREE_NOT_FOUND = "TreeNotFound"
|
64
69
|
IP_NOT_FOUND = "IPNotFound"
|
65
70
|
BAD_OFFSET = "BadOffset"
|
71
|
+
FORBIDDEN_VAULT_OPERATION = "ForbiddenVaultOperation"
|
72
|
+
VAULT_ITEM_NOT_FOUND = "VaultItemNotFound"
|
66
73
|
NOT_FOUND = "NotFound"
|
74
|
+
INTERNAL_SERVER_ERROR = "InternalError"
|
67
75
|
|
68
76
|
|
69
77
|
class ResponseHeader(APIResponseModel):
|
pangea/services/__init__.py
CHANGED
pangea/services/audit/audit.py
CHANGED
@@ -1,10 +1,30 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
-
|
3
|
+
import datetime
|
4
|
+
from typing import Dict, Optional, Union
|
4
5
|
|
5
6
|
from pangea.response import PangeaResponse
|
6
7
|
from pangea.services.audit.exceptions import AuditException, EventCorruption
|
7
|
-
from pangea.services.audit.models import
|
8
|
+
from pangea.services.audit.models import (
|
9
|
+
Event,
|
10
|
+
EventEnvelope,
|
11
|
+
EventSigning,
|
12
|
+
EventVerification,
|
13
|
+
LogRequest,
|
14
|
+
LogResult,
|
15
|
+
PublishedRoot,
|
16
|
+
Root,
|
17
|
+
RootRequest,
|
18
|
+
RootResult,
|
19
|
+
RootSource,
|
20
|
+
SearchEvent,
|
21
|
+
SearchOrder,
|
22
|
+
SearchOrderBy,
|
23
|
+
SearchOutput,
|
24
|
+
SearchRequest,
|
25
|
+
SearchResultOutput,
|
26
|
+
SearchResultRequest,
|
27
|
+
)
|
8
28
|
from pangea.services.audit.signing import Signer, Verifier
|
9
29
|
from pangea.services.audit.util import (
|
10
30
|
b64encode_ascii,
|
@@ -99,6 +119,7 @@ class Audit(ServiceBase):
|
|
99
119
|
verify (bool, optional): True to verify logs consistency after response.
|
100
120
|
signing (bool, optional): True to sign event.
|
101
121
|
verbose (bool, optional): True to get a more verbose response.
|
122
|
+
tenant_id (string, optional): Used to record the tenant associated with this activity.
|
102
123
|
Raises:
|
103
124
|
AuditException: If an audit based api exception happens
|
104
125
|
PangeaAPIException: If an API Error happens
|
@@ -169,7 +190,7 @@ class Audit(ServiceBase):
|
|
169
190
|
# verify event hash
|
170
191
|
if response.result.hash and not verify_envelope_hash(response.result.envelope, response.result.hash):
|
171
192
|
# it's a extreme case, it's OK to raise an exception
|
172
|
-
raise EventCorruption(
|
193
|
+
raise EventCorruption("Error: Event hash failed.", response.result.envelope)
|
173
194
|
|
174
195
|
response.result.signature_verification = self.verify_signature(response.result.envelope)
|
175
196
|
|
@@ -208,8 +229,8 @@ class Audit(ServiceBase):
|
|
208
229
|
query: str,
|
209
230
|
order: Optional[SearchOrder] = None,
|
210
231
|
order_by: Optional[SearchOrderBy] = None,
|
211
|
-
start: Optional[datetime.datetime] = None,
|
212
|
-
end: Optional[datetime.datetime] = None,
|
232
|
+
start: Optional[Union[datetime.datetime, str]] = None,
|
233
|
+
end: Optional[Union[datetime.datetime, str]] = None,
|
213
234
|
limit: Optional[int] = None,
|
214
235
|
max_results: Optional[int] = None,
|
215
236
|
search_restriction: Optional[dict] = None,
|
@@ -266,8 +287,8 @@ class Audit(ServiceBase):
|
|
266
287
|
query=query,
|
267
288
|
order=order,
|
268
289
|
order_by=order_by,
|
269
|
-
start=
|
270
|
-
end=
|
290
|
+
start=format_datetime(start) if isinstance(start, datetime.datetime) else start,
|
291
|
+
end=format_datetime(end) if isinstance(end, datetime.datetime) else end,
|
271
292
|
limit=limit,
|
272
293
|
max_results=max_results,
|
273
294
|
search_restriction=search_restriction,
|
pangea/services/audit/models.py
CHANGED
@@ -173,6 +173,12 @@ class SearchOrder(str, enum.Enum):
|
|
173
173
|
ASC = "desc"
|
174
174
|
DESC = "asc"
|
175
175
|
|
176
|
+
def __str__(self):
|
177
|
+
return str(self.value)
|
178
|
+
|
179
|
+
def __repr__(self):
|
180
|
+
return str(self.value)
|
181
|
+
|
176
182
|
|
177
183
|
class SearchOrderBy(str, enum.Enum):
|
178
184
|
ACTOR = "actor"
|
@@ -184,6 +190,12 @@ class SearchOrderBy(str, enum.Enum):
|
|
184
190
|
TARGET = "target"
|
185
191
|
TIMESTAMP = "timestamp"
|
186
192
|
|
193
|
+
def __str__(self):
|
194
|
+
return str(self.value)
|
195
|
+
|
196
|
+
def __repr__(self):
|
197
|
+
return str(self.value)
|
198
|
+
|
187
199
|
|
188
200
|
class SearchRequest(APIRequestModel):
|
189
201
|
"""
|
pangea/services/audit/util.py
CHANGED
@@ -13,6 +13,7 @@ from typing import Dict, List, Optional
|
|
13
13
|
|
14
14
|
import requests
|
15
15
|
from pangea.services.audit.models import Event, EventEnvelope, PublishedRoot
|
16
|
+
from pangea.utils import format_datetime
|
16
17
|
|
17
18
|
Hash = bytes
|
18
19
|
|
@@ -140,7 +141,11 @@ def format_datetime(dt: datetime):
|
|
140
141
|
"""
|
141
142
|
Format a datetime in ISO format, using Z instead of +00:00
|
142
143
|
"""
|
143
|
-
|
144
|
+
ret = dt.isoformat()
|
145
|
+
if dt.tzinfo is not None:
|
146
|
+
return ret.replace("+00:00", "Z")
|
147
|
+
else:
|
148
|
+
return ret + "Z"
|
144
149
|
|
145
150
|
|
146
151
|
def normalize_log(audit: dict) -> dict:
|