pangea-sdk 6.1.0__py3-none-any.whl → 6.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pangea/__init__.py +9 -1
- pangea/asyncio/__init__.py +1 -0
- pangea/asyncio/file_uploader.py +4 -2
- pangea/asyncio/request.py +53 -18
- pangea/asyncio/services/__init__.py +2 -0
- pangea/asyncio/services/ai_guard.py +9 -12
- pangea/asyncio/services/audit.py +12 -7
- pangea/asyncio/services/authn.py +36 -25
- pangea/asyncio/services/authz.py +6 -6
- pangea/asyncio/services/base.py +4 -0
- pangea/asyncio/services/file_scan.py +8 -2
- pangea/asyncio/services/intel.py +26 -28
- pangea/asyncio/services/redact.py +7 -3
- pangea/asyncio/services/sanitize.py +5 -1
- pangea/asyncio/services/share.py +5 -1
- pangea/asyncio/services/vault.py +19 -15
- pangea/audit_logger.py +3 -1
- pangea/deep_verify.py +13 -13
- pangea/deprecated.py +1 -1
- pangea/dump_audit.py +2 -3
- pangea/exceptions.py +8 -5
- pangea/file_uploader.py +4 -0
- pangea/request.py +64 -48
- pangea/response.py +21 -18
- pangea/services/__init__.py +2 -0
- pangea/services/ai_guard.py +35 -24
- pangea/services/audit/audit.py +16 -13
- pangea/services/audit/models.py +71 -34
- pangea/services/audit/signing.py +1 -1
- pangea/services/audit/util.py +10 -10
- pangea/services/authn/authn.py +36 -25
- pangea/services/authn/models.py +10 -56
- pangea/services/authz.py +10 -6
- pangea/services/base.py +7 -4
- pangea/services/embargo.py +6 -0
- pangea/services/file_scan.py +8 -2
- pangea/services/intel.py +36 -19
- pangea/services/redact.py +7 -3
- pangea/services/sanitize.py +5 -1
- pangea/services/share/share.py +13 -7
- pangea/services/vault/models/asymmetric.py +4 -0
- pangea/services/vault/models/common.py +4 -0
- pangea/services/vault/models/symmetric.py +4 -0
- pangea/services/vault/vault.py +17 -19
- pangea/tools.py +13 -9
- pangea/utils.py +3 -5
- pangea/verify_audit.py +23 -27
- {pangea_sdk-6.1.0.dist-info → pangea_sdk-6.2.0.dist-info}/METADATA +36 -17
- pangea_sdk-6.2.0.dist-info/RECORD +60 -0
- {pangea_sdk-6.1.0.dist-info → pangea_sdk-6.2.0.dist-info}/WHEEL +1 -1
- pangea_sdk-6.1.0.dist-info/RECORD +0 -60
pangea/asyncio/services/base.py
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
7
|
+
from __future__ import annotations
|
8
|
+
|
3
9
|
import io
|
4
10
|
import logging
|
5
11
|
from typing import Dict, List, Optional, Tuple
|
@@ -16,7 +22,7 @@ class FileScanAsync(ServiceBaseAsync):
|
|
16
22
|
"""FileScan service client.
|
17
23
|
|
18
24
|
Provides methods to interact with Pangea FileScan Service:
|
19
|
-
https://pangea.cloud/docs/api/
|
25
|
+
https://pangea.cloud/docs/api/file-scan
|
20
26
|
|
21
27
|
The following information is needed:
|
22
28
|
PANGEA_TOKEN - service token which can be found on the Pangea User
|
@@ -98,7 +104,7 @@ class FileScanAsync(ServiceBaseAsync):
|
|
98
104
|
files: Optional[List[Tuple]] = None
|
99
105
|
if file or file_path:
|
100
106
|
if file_path:
|
101
|
-
file = open(file_path, "rb")
|
107
|
+
file = open(file_path, "rb") # noqa: SIM115
|
102
108
|
if transfer_method == TransferMethod.POST_URL:
|
103
109
|
params = get_file_upload_params(file) # type: ignore[arg-type]
|
104
110
|
crc = params.crc_hex
|
pangea/asyncio/services/intel.py
CHANGED
@@ -1,7 +1,12 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
from __future__ import annotations
|
7
|
+
|
3
8
|
import hashlib
|
4
|
-
from typing import List, Optional
|
9
|
+
from typing import List, Literal, Optional
|
5
10
|
|
6
11
|
import pangea.services.intel as m
|
7
12
|
from pangea.asyncio.services.base import ServiceBaseAsync
|
@@ -73,11 +78,11 @@ class FileIntelAsync(ServiceBaseAsync):
|
|
73
78
|
|
74
79
|
async def hash_reputation_bulk(
|
75
80
|
self,
|
76
|
-
hashes:
|
77
|
-
hash_type:
|
78
|
-
provider:
|
79
|
-
verbose:
|
80
|
-
raw:
|
81
|
+
hashes: list[str],
|
82
|
+
hash_type: Literal["sha256", "sha", "md5"],
|
83
|
+
provider: Literal["reversinglabs", "crowdstrike"] | None = None,
|
84
|
+
verbose: bool | None = None,
|
85
|
+
raw: bool | None = None,
|
81
86
|
) -> PangeaResponse[m.FileReputationBulkResult]:
|
82
87
|
"""
|
83
88
|
Reputation check
|
@@ -85,11 +90,11 @@ class FileIntelAsync(ServiceBaseAsync):
|
|
85
90
|
Retrieve hash-based file reputation from a provider, including an optional detailed report.
|
86
91
|
|
87
92
|
Args:
|
88
|
-
hashes
|
89
|
-
hash_type
|
90
|
-
provider
|
91
|
-
verbose
|
92
|
-
raw
|
93
|
+
hashes: The hash of each file to be looked up
|
94
|
+
hash_type: One of "sha256", "sha", "md5"
|
95
|
+
provider: Use reputation data from these providers: "reversinglabs" or "crowdstrike"
|
96
|
+
verbose: Echo the API parameters in the response
|
97
|
+
raw: Include raw data from this provider
|
93
98
|
|
94
99
|
Raises:
|
95
100
|
PangeaAPIException: If an API Error happens
|
@@ -97,12 +102,8 @@ class FileIntelAsync(ServiceBaseAsync):
|
|
97
102
|
Returns:
|
98
103
|
A PangeaResponse where the sanctioned source(s) are in the
|
99
104
|
response.result field. Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/file-intel).
|
100
|
-
|
101
|
-
Examples:
|
102
|
-
FIXME:
|
103
|
-
|
104
105
|
"""
|
105
|
-
input = m.FileReputationBulkRequest(
|
106
|
+
input = m.FileReputationBulkRequest(
|
106
107
|
hashes=hashes, hash_type=hash_type, verbose=verbose, raw=raw, provider=provider
|
107
108
|
)
|
108
109
|
return await self.request.post(
|
@@ -144,8 +145,8 @@ class FileIntelAsync(ServiceBaseAsync):
|
|
144
145
|
)
|
145
146
|
"""
|
146
147
|
|
147
|
-
|
148
|
-
|
148
|
+
with open(filepath, "rb") as data:
|
149
|
+
hash = hashlib.sha256(data.read()).hexdigest()
|
149
150
|
|
150
151
|
input = m.FileReputationRequest(hash=hash, hash_type="sha256", verbose=verbose, raw=raw, provider=provider)
|
151
152
|
return await self.request.post(
|
@@ -154,8 +155,8 @@ class FileIntelAsync(ServiceBaseAsync):
|
|
154
155
|
|
155
156
|
async def filepath_reputation_bulk(
|
156
157
|
self,
|
157
|
-
filepaths:
|
158
|
-
provider:
|
158
|
+
filepaths: list[str],
|
159
|
+
provider: Literal["reversinglabs", "crowdstrike"] | None = None,
|
159
160
|
verbose: Optional[bool] = None,
|
160
161
|
raw: Optional[bool] = None,
|
161
162
|
) -> PangeaResponse[m.FileReputationBulkResult]:
|
@@ -168,10 +169,10 @@ class FileIntelAsync(ServiceBaseAsync):
|
|
168
169
|
OperationId: file_intel_post_v1_reputation
|
169
170
|
|
170
171
|
Args:
|
171
|
-
filepaths
|
172
|
-
provider
|
173
|
-
verbose
|
174
|
-
raw
|
172
|
+
filepaths: The path list to the files to be looked up
|
173
|
+
provider: Use reputation data from these providers: "reversinglabs" or "crowdstrike"
|
174
|
+
verbose: Echo the API parameters in the response
|
175
|
+
raw: Include raw data from this provider
|
175
176
|
|
176
177
|
Raises:
|
177
178
|
PangeaAPIException: If an API Error happens
|
@@ -179,9 +180,6 @@ class FileIntelAsync(ServiceBaseAsync):
|
|
179
180
|
Returns:
|
180
181
|
A PangeaResponse where the sanctioned source(s) are in the
|
181
182
|
response.result field. Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/file-intel).
|
182
|
-
|
183
|
-
Examples:
|
184
|
-
FIXME:
|
185
183
|
"""
|
186
184
|
hashes = []
|
187
185
|
for filepath in filepaths:
|
@@ -317,7 +315,7 @@ class DomainIntelAsync(ServiceBaseAsync):
|
|
317
315
|
provider="whoisxml",
|
318
316
|
)
|
319
317
|
"""
|
320
|
-
input = m.DomainWhoIsRequest(domain=domain, verbose=verbose, provider=provider, raw=raw)
|
318
|
+
input = m.DomainWhoIsRequest(domain=domain, verbose=verbose, provider=provider, raw=raw)
|
321
319
|
return await self.request.post("v1/whois", m.DomainWhoIsResult, data=input.model_dump(exclude_none=True))
|
322
320
|
|
323
321
|
|
@@ -1,5 +1,9 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
3
7
|
from __future__ import annotations
|
4
8
|
|
5
9
|
from typing import Dict, List, Optional, Union
|
@@ -92,7 +96,7 @@ class RedactAsync(ServiceBaseAsync):
|
|
92
96
|
Returns:
|
93
97
|
Pangea Response with redacted text in the response.result property,
|
94
98
|
available response fields can be found in our
|
95
|
-
[API Documentation](https://pangea.cloud/docs/api/redact#redact).
|
99
|
+
[API Documentation](https://pangea.cloud/docs/api/redact#redact-post).
|
96
100
|
|
97
101
|
Examples:
|
98
102
|
response = redact.redact(text="Jenny Jenny... 555-867-5309")
|
@@ -151,7 +155,7 @@ class RedactAsync(ServiceBaseAsync):
|
|
151
155
|
Returns:
|
152
156
|
Pangea Response with redacted data in the response.result field,
|
153
157
|
available response fields can be found in our
|
154
|
-
[API Documentation](https://pangea.cloud/docs/api/redact#redact-structured)
|
158
|
+
[API Documentation](https://pangea.cloud/docs/api/redact#redact-structured-post)
|
155
159
|
|
156
160
|
Examples:
|
157
161
|
data = {
|
@@ -196,7 +200,7 @@ class RedactAsync(ServiceBaseAsync):
|
|
196
200
|
Returns:
|
197
201
|
Pangea Response with redacted data in the response.result field,
|
198
202
|
available response fields can be found in our
|
199
|
-
[API Documentation](https://pangea.cloud/docs/api/redact#unredact)
|
203
|
+
[API Documentation](https://pangea.cloud/docs/api/redact#unredact-post)
|
200
204
|
"""
|
201
205
|
input = m.UnredactRequest(redacted_data=redacted_data, fpe_context=fpe_context)
|
202
206
|
return await self.request.post("v1/unredact", m.UnredactResult, data=input.model_dump(exclude_none=True))
|
@@ -1,5 +1,9 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
3
7
|
from __future__ import annotations
|
4
8
|
|
5
9
|
import io
|
@@ -116,7 +120,7 @@ class SanitizeAsync(ServiceBaseAsync):
|
|
116
120
|
files: Optional[List[Tuple]] = None
|
117
121
|
if file or file_path:
|
118
122
|
if file_path:
|
119
|
-
file = open(file_path, "rb")
|
123
|
+
file = open(file_path, "rb") # noqa: SIM115
|
120
124
|
if transfer_method == TransferMethod.POST_URL and (sha256 is None or crc32c is None or size is None):
|
121
125
|
params = get_file_upload_params(file) # type: ignore[arg-type]
|
122
126
|
crc32c = params.crc_hex if crc32c is None else crc32c
|
pangea/asyncio/services/share.py
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
3
7
|
from __future__ import annotations
|
4
8
|
|
5
9
|
import io
|
@@ -194,7 +198,7 @@ class ShareAsync(ServiceBaseAsync):
|
|
194
198
|
|
195
199
|
async def get_archive(
|
196
200
|
self,
|
197
|
-
ids:
|
201
|
+
ids: list[str],
|
198
202
|
format: Optional[m.ArchiveFormat] = None,
|
199
203
|
transfer_method: Optional[TransferMethod] = None,
|
200
204
|
bucket_id: Optional[str] = None,
|
pangea/asyncio/services/vault.py
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
3
7
|
from __future__ import annotations
|
4
8
|
|
5
9
|
from collections.abc import Mapping
|
@@ -164,7 +168,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
164
168
|
Returns:
|
165
169
|
A PangeaResponse where the id of the deleted secret or key
|
166
170
|
is returned in the response.result field.
|
167
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault
|
171
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault/v1-general#/v1/delete-post).
|
168
172
|
|
169
173
|
Raises:
|
170
174
|
PangeaAPIException: If an API Error happens
|
@@ -197,7 +201,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
197
201
|
Returns:
|
198
202
|
A PangeaResponse where the secret or key
|
199
203
|
is returned in the response.result field.
|
200
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault
|
204
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault/v1-general#/v1/get-post).
|
201
205
|
|
202
206
|
Raises:
|
203
207
|
PangeaAPIException: If an API Error happens
|
@@ -283,7 +287,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
283
287
|
Returns:
|
284
288
|
A PangeaResponse where a list of secrets or keys
|
285
289
|
is returned in the response.result field.
|
286
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault
|
290
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault/v1-general#/v1/list-post).
|
287
291
|
|
288
292
|
Raises:
|
289
293
|
PangeaAPIException: If an API Error happens
|
@@ -345,7 +349,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
345
349
|
Returns:
|
346
350
|
A PangeaResponse where the item ID is returned in the
|
347
351
|
response.result field. Available response fields can be found in our
|
348
|
-
[API documentation](https://pangea.cloud/docs/api/vault
|
352
|
+
[API documentation](https://pangea.cloud/docs/api/vault/v1-general#/v1/update-post).
|
349
353
|
|
350
354
|
Raises:
|
351
355
|
PangeaAPIException: If an API Error happens
|
@@ -1680,7 +1684,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1680
1684
|
Returns:
|
1681
1685
|
A PangeaResponse where the encrypted message in base64 is returned
|
1682
1686
|
in the response.result field. Available response fields can be found
|
1683
|
-
in our [API documentation](https://pangea.cloud/docs/api/vault
|
1687
|
+
in our [API documentation](https://pangea.cloud/docs/api/vault/v1-keys#/v1/key/encrypt-post).
|
1684
1688
|
|
1685
1689
|
Raises:
|
1686
1690
|
PangeaAPIException: If an API Error happens
|
@@ -1716,7 +1720,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1716
1720
|
|
1717
1721
|
Returns:
|
1718
1722
|
A PangeaResponse where the decrypted message in base64 is returned
|
1719
|
-
in the response.result field. Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault
|
1723
|
+
in the response.result field. Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault/v1-keys#/v1/key/decrypt-post).
|
1720
1724
|
|
1721
1725
|
Raises:
|
1722
1726
|
PangeaAPIException: If an API Error happens
|
@@ -1750,7 +1754,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1750
1754
|
Returns:
|
1751
1755
|
A PangeaResponse where the signature of the message in base64 is
|
1752
1756
|
returned in the response.result field. Available response fields can
|
1753
|
-
be found in our [API documentation](https://pangea.cloud/docs/api/vault
|
1757
|
+
be found in our [API documentation](https://pangea.cloud/docs/api/vault/v1-keys#/v1/key/sign-post).
|
1754
1758
|
|
1755
1759
|
Raises:
|
1756
1760
|
PangeaAPIException: If an API Error happens
|
@@ -1788,7 +1792,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1788
1792
|
Returns:
|
1789
1793
|
A PangeaResponse where the signature is valid
|
1790
1794
|
is returned in the response.result field.
|
1791
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault
|
1795
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault/v1-keys#/v1/key/verify-post).
|
1792
1796
|
|
1793
1797
|
Examples:
|
1794
1798
|
response = await vault.verify(
|
@@ -1826,7 +1830,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1826
1830
|
Returns:
|
1827
1831
|
A PangeaResponse where the signature is valid
|
1828
1832
|
is returned in the response.result field.
|
1829
|
-
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault
|
1833
|
+
Available response fields can be found in our [API documentation](https://pangea.cloud/docs/api/vault/v1-jwt#/v1/key/verify/jwt-post).
|
1830
1834
|
|
1831
1835
|
Examples:
|
1832
1836
|
response = await vault.jwt_verify(jws="ewogICJhbGciO...")
|
@@ -1851,7 +1855,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1851
1855
|
Returns:
|
1852
1856
|
A PangeaResponse where the signed JSON Web Token (JWS) is returned
|
1853
1857
|
in the response.result field. Available response fields can be found
|
1854
|
-
in our [API documentation](https://pangea.cloud/docs/api/vault
|
1858
|
+
in our [API documentation](https://pangea.cloud/docs/api/vault/v1-jwt#/v1/key/sign/jwt-post).
|
1855
1859
|
|
1856
1860
|
Examples:
|
1857
1861
|
response = await vault.jwt_sign(
|
@@ -1881,7 +1885,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1881
1885
|
Returns:
|
1882
1886
|
A PangeaResponse where the JSON Web Key Set (JWKS) object is
|
1883
1887
|
returned in the response.result field. Available response fields can
|
1884
|
-
be found in our [API documentation](https://pangea.cloud/docs/api/vault
|
1888
|
+
be found in our [API documentation](https://pangea.cloud/docs/api/vault/v1-jwt#/v1/get/jwk-post).
|
1885
1889
|
|
1886
1890
|
Examples:
|
1887
1891
|
response = await vault.jwk_get("pvi_p6g5i3gtbvqvc3u6zugab6qs6r63tqf5")
|
@@ -1913,7 +1917,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
1913
1917
|
Returns:
|
1914
1918
|
A PangeaResponse where the state change object is returned in the
|
1915
1919
|
response.result field. Available response fields can be found in our
|
1916
|
-
[API documentation](https://pangea.cloud/docs/api/vault
|
1920
|
+
[API documentation](https://pangea.cloud/docs/api/vault/v1-general#/v1/state/change-post).
|
1917
1921
|
|
1918
1922
|
Raises:
|
1919
1923
|
PangeaAPIException: If an API Error happens
|
@@ -2014,7 +2018,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
2014
2018
|
Returns:
|
2015
2019
|
A `PangeaResponse` where the encrypted object is returned in the
|
2016
2020
|
`response.result` field. Available response fields can be found in
|
2017
|
-
our [API documentation](https://pangea.cloud/docs/api/vault
|
2021
|
+
our [API documentation](https://pangea.cloud/docs/api/vault/v1-keys#/v1/key/encrypt/structured-post).
|
2018
2022
|
|
2019
2023
|
Raises:
|
2020
2024
|
PangeaAPIException: If an API error happens.
|
@@ -2070,7 +2074,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
2070
2074
|
Returns:
|
2071
2075
|
A `PangeaResponse` where the decrypted object is returned in the
|
2072
2076
|
`response.result` field. Available response fields can be found in
|
2073
|
-
our [API documentation](https://pangea.cloud/docs/api/vault
|
2077
|
+
our [API documentation](https://pangea.cloud/docs/api/vault/v1-keys#/v1/key/decrypt/structured-post).
|
2074
2078
|
|
2075
2079
|
Examples:
|
2076
2080
|
data = {"field1": [1, 2, "kxcbC9E9IlgVaSCChPWUMgUC3ko=", "6FfI/LCzatLRLNAc8SuBK/TDnGxp"], "field2": "data2"}
|
@@ -2214,7 +2218,7 @@ class VaultAsync(ServiceBaseAsync):
|
|
2214
2218
|
Returns:
|
2215
2219
|
A `PangeaResponse` where the exported key is returned in the
|
2216
2220
|
`response.result` field. Available response fields can be found in
|
2217
|
-
our [API documentation](https://pangea.cloud/docs/api/vault
|
2221
|
+
our [API documentation](https://pangea.cloud/docs/api/vault/v1-general#/v1/export-post).
|
2218
2222
|
|
2219
2223
|
Raises:
|
2220
2224
|
PangeaAPIException: If an API error happens.
|
pangea/audit_logger.py
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
|
+
from __future__ import annotations
|
4
|
+
|
3
5
|
import logging
|
4
6
|
|
5
7
|
import pangea.exceptions as pe
|
@@ -20,7 +22,7 @@ class AuditLogger(logging.Logger):
|
|
20
22
|
"""
|
21
23
|
|
22
24
|
def __init__(self, *args, **kwargs):
|
23
|
-
super(
|
25
|
+
super().__init__(*args, **kwargs)
|
24
26
|
|
25
27
|
def set_auditor(self, auditor: Audit):
|
26
28
|
"""Sets the internal Pangea Audit Service client instance
|
pangea/deep_verify.py
CHANGED
@@ -1,13 +1,16 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
|
4
|
+
from __future__ import annotations
|
5
|
+
|
4
6
|
import argparse
|
5
7
|
import io
|
6
8
|
import math
|
7
9
|
import os
|
8
10
|
import sys
|
11
|
+
from collections.abc import Iterator
|
9
12
|
from itertools import groupby
|
10
|
-
from typing import
|
13
|
+
from typing import Optional, TypedDict, Union
|
11
14
|
|
12
15
|
import pangea.services.audit.util as audit_util
|
13
16
|
from pangea.services import Audit
|
@@ -24,7 +27,7 @@ class Errors(TypedDict):
|
|
24
27
|
buffer_missing: int
|
25
28
|
|
26
29
|
|
27
|
-
root_hashes:
|
30
|
+
root_hashes: dict[int, str] = {}
|
28
31
|
|
29
32
|
|
30
33
|
def num_lines(f: io.TextIOWrapper) -> int:
|
@@ -124,10 +127,7 @@ def get_root_hash(audit: Audit, tree_size: int) -> str:
|
|
124
127
|
|
125
128
|
|
126
129
|
def print_error(msg: str, level: str = "error"):
|
127
|
-
if level == "warning"
|
128
|
-
dot = "🟡"
|
129
|
-
else:
|
130
|
-
dot = "🔴"
|
130
|
+
dot = "🟡" if level == "warning" else "🔴"
|
131
131
|
print(f"{dot} {msg:200s}")
|
132
132
|
|
133
133
|
|
@@ -148,14 +148,14 @@ def deep_verify(audit: Audit, file: io.TextIOWrapper) -> Errors:
|
|
148
148
|
}
|
149
149
|
|
150
150
|
events = file_events(root_hashes, file)
|
151
|
-
events_by_idx: Union[
|
151
|
+
events_by_idx: Union[list[Event], Iterator[Event]]
|
152
152
|
cold_indexes = SequenceFollower()
|
153
153
|
for leaf_index, events_by_idx in groupby(events, lambda event: event.get("leaf_index")):
|
154
154
|
events_by_idx = list(events_by_idx)
|
155
155
|
buffer_lines = (cnt, cnt + len(events_by_idx) - 1)
|
156
156
|
if leaf_index is None:
|
157
157
|
print_error(
|
158
|
-
f"Lines {buffer_lines[0]}-{buffer_lines[1]} ({buffer_lines[1]-buffer_lines[0]+1}): Buffer was not persisted"
|
158
|
+
f"Lines {buffer_lines[0]}-{buffer_lines[1]} ({buffer_lines[1] - buffer_lines[0] + 1}): Buffer was not persisted"
|
159
159
|
)
|
160
160
|
errors["not_persisted"] += len(events_by_idx)
|
161
161
|
cnt += len(events_by_idx)
|
@@ -164,8 +164,8 @@ def deep_verify(audit: Audit, file: io.TextIOWrapper) -> Errors:
|
|
164
164
|
cold_indexes.add(leaf_index)
|
165
165
|
|
166
166
|
cold_path_size: Optional[int] = None
|
167
|
-
hot_indexes:
|
168
|
-
for
|
167
|
+
hot_indexes: set[int] = set()
|
168
|
+
for _i, event in enumerate(events_by_idx):
|
169
169
|
cnt += 1
|
170
170
|
tree_size = get_tree_size(event)
|
171
171
|
if tree_size not in root_hashes:
|
@@ -203,11 +203,11 @@ def deep_verify(audit: Audit, file: io.TextIOWrapper) -> Errors:
|
|
203
203
|
errors["missing"] += len(hot_indexes_diff)
|
204
204
|
print(f"missing hot indexes: {hot_indexes_diff}")
|
205
205
|
print(f"hot_indexes: {hot_indexes} ")
|
206
|
-
print(
|
206
|
+
print("events:")
|
207
207
|
for e in events_by_idx:
|
208
208
|
print(e)
|
209
209
|
print_error(
|
210
|
-
f"Lines {buffer_lines[0]}-{buffer_lines[1]} ({buffer_lines[1]-buffer_lines[0]}), Buffer #{cold_idx}: {len(hot_indexes_diff)} event(s) missing"
|
210
|
+
f"Lines {buffer_lines[0]}-{buffer_lines[1]} ({buffer_lines[1] - buffer_lines[0]}), Buffer #{cold_idx}: {len(hot_indexes_diff)} event(s) missing"
|
211
211
|
)
|
212
212
|
|
213
213
|
cold_holes = cold_indexes.holes()
|
@@ -232,7 +232,7 @@ def create_parser():
|
|
232
232
|
"-f",
|
233
233
|
required=True,
|
234
234
|
type=argparse.FileType("r"),
|
235
|
-
help="Event input file. Must be a collection of
|
235
|
+
help="Event input file. Must be a collection of JSON Objects separated by newlines",
|
236
236
|
)
|
237
237
|
return parser
|
238
238
|
|
pangea/deprecated.py
CHANGED
pangea/dump_audit.py
CHANGED
@@ -7,7 +7,6 @@ import json
|
|
7
7
|
import os
|
8
8
|
import sys
|
9
9
|
from datetime import datetime
|
10
|
-
from typing import Tuple
|
11
10
|
|
12
11
|
import dateutil.parser
|
13
12
|
|
@@ -103,7 +102,7 @@ def dump_after(audit: Audit, output: io.TextIOWrapper, start: datetime, last_eve
|
|
103
102
|
|
104
103
|
def dump_page(
|
105
104
|
audit: Audit, output: io.TextIOWrapper, start: datetime, end: datetime, first: bool = False
|
106
|
-
) ->
|
105
|
+
) -> tuple[datetime, int, bool, str, int]:
|
107
106
|
PAGE_SIZE = 1000
|
108
107
|
print(start, end)
|
109
108
|
print("Dumping...")
|
@@ -175,7 +174,7 @@ def parse_args(parser: argparse.ArgumentParser):
|
|
175
174
|
raise ValueError("domain missing")
|
176
175
|
|
177
176
|
if args.output is None:
|
178
|
-
args.output = open(f"dump-{datetime.now().strftime('%Y%m%d%H%M%S')}.jsonl", "w")
|
177
|
+
args.output = open(f"dump-{datetime.now().strftime('%Y%m%d%H%M%S')}.jsonl", "w") # noqa: SIM115
|
179
178
|
|
180
179
|
args.start = make_aware_datetime(args.start)
|
181
180
|
args.end = make_aware_datetime(args.end)
|
pangea/exceptions.py
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
|
4
|
+
# TODO: Modernize.
|
5
|
+
# ruff: noqa: UP006, UP035
|
6
|
+
|
4
7
|
from typing import List, Optional
|
5
8
|
|
6
9
|
from pangea.response import AcceptedResult, ErrorField, PangeaResponse
|
@@ -44,7 +47,7 @@ class PangeaAPIException(PangeaException):
|
|
44
47
|
response: PangeaResponse
|
45
48
|
|
46
49
|
def __init__(self, message: str, response: PangeaResponse):
|
47
|
-
super(
|
50
|
+
super().__init__(message)
|
48
51
|
self.response = response
|
49
52
|
|
50
53
|
@property
|
@@ -92,7 +95,7 @@ class UnauthorizedException(PangeaAPIException):
|
|
92
95
|
|
93
96
|
def __init__(self, service_name: str, response: PangeaResponse):
|
94
97
|
message = f"User is not authorized to access service {service_name}"
|
95
|
-
super(
|
98
|
+
super().__init__(message, response)
|
96
99
|
|
97
100
|
|
98
101
|
class NotFound(PangeaAPIException):
|
@@ -100,20 +103,20 @@ class NotFound(PangeaAPIException):
|
|
100
103
|
|
101
104
|
def __init__(self, url: str, response: PangeaResponse):
|
102
105
|
message = f"Resource url:'{url}' not found"
|
103
|
-
super(
|
106
|
+
super().__init__(message, response)
|
104
107
|
|
105
108
|
|
106
109
|
class ServiceNotEnabledException(PangeaAPIException):
|
107
110
|
def __init__(self, service_name: str, response: PangeaResponse):
|
108
111
|
message = f"{service_name} is not enabled. Go to console.pangea.cloud/service/{service_name} to enable"
|
109
|
-
super(
|
112
|
+
super().__init__(message, response)
|
110
113
|
|
111
114
|
|
112
115
|
class MissingConfigID(PangeaAPIException):
|
113
116
|
"""No config ID was provided in either token scopes or explicitly"""
|
114
117
|
|
115
118
|
def __init__(self, service_name: str, response: PangeaResponse):
|
116
|
-
super(
|
119
|
+
super().__init__(
|
117
120
|
f"Token did not contain a config scope for service {service_name}. Create a new token or provide a config ID explicitly in the service base",
|
118
121
|
response,
|
119
122
|
)
|