pangea-sdk 3.2.0__tar.gz → 3.3.0__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/PKG-INFO +13 -13
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/README.md +10 -10
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/__init__.py +1 -1
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/request.py +122 -88
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/base.py +20 -3
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/file_scan.py +75 -5
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/redact.py +2 -7
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/config.py +2 -2
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/request.py +131 -91
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/response.py +4 -1
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/audit.py +7 -7
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/authn/authn.py +4 -1
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/authn/models.py +2 -1
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/base.py +17 -4
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/file_scan.py +59 -3
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/redact.py +2 -7
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/utils.py +9 -2
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pyproject.toml +3 -3
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/__init__.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/audit.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/authn.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/embargo.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/intel.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/asyncio/services/vault.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/audit_logger.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/deep_verify.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/deprecated.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/dump_audit.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/exceptions.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/__init__.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/exceptions.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/models.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/signing.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/audit/util.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/embargo.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/intel.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/models/asymmetric.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/models/common.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/models/secret.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/models/symmetric.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/services/vault/vault.py +2 -2
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/tools.py +0 -0
- {pangea_sdk-3.2.0 → pangea_sdk-3.3.0}/pangea/verify_audit.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: pangea-sdk
|
3
|
-
Version: 3.
|
3
|
+
Version: 3.3.0
|
4
4
|
Summary: Pangea API SDK
|
5
5
|
License: MIT
|
6
6
|
Keywords: Pangea,SDK,Audit
|
@@ -15,10 +15,10 @@ Classifier: Programming Language :: Python :: 3.10
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
16
16
|
Classifier: Topic :: Software Development
|
17
17
|
Classifier: Topic :: Software Development :: Libraries
|
18
|
-
Requires-Dist: aiohttp (>=3.8.
|
18
|
+
Requires-Dist: aiohttp (>=3.8.6,<4.0.0)
|
19
19
|
Requires-Dist: alive-progress (>=2.4.1,<3.0.0)
|
20
20
|
Requires-Dist: asyncio (>=3.4.3,<4.0.0)
|
21
|
-
Requires-Dist: cryptography (==41.0.
|
21
|
+
Requires-Dist: cryptography (==41.0.5)
|
22
22
|
Requires-Dist: deprecated (>=1.2.13,<2.0.0)
|
23
23
|
Requires-Dist: google-crc32c (>=1.5.0,<2.0.0)
|
24
24
|
Requires-Dist: pydantic (>=1.10.2,<2.0.0)
|
@@ -59,18 +59,18 @@ poetry add pangea-sdk
|
|
59
59
|
|
60
60
|
## Usage
|
61
61
|
|
62
|
-
For
|
62
|
+
For sample apps, look at the [/examples](https://github.com/pangeacyber/pangea-python/tree/main/examples) folder in this repository. There you will find basic sample apps for each of the services supported on this SDK. Each service folder has a README.md with instructions to install, setup, and run the sample app.
|
63
63
|
|
64
64
|
|
65
65
|
## Asyncio support
|
66
66
|
|
67
|
-
We have added support to asyncio library using aiohttp in order to support async/await calls to all our services.
|
67
|
+
We have added support to the asyncio library using aiohttp in order to support async/await calls to all our services.
|
68
68
|
Async services classes are inside [pangea/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk/pangea/asyncio) folder, and examples about how to use them are in [/examples/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/examples/asyncio).
|
69
69
|
|
70
70
|
|
71
71
|
### Secure Audit Service - Integrity Tools
|
72
72
|
|
73
|
-
Python Pangea SDK
|
73
|
+
The Python Pangea SDK also includes some extra features to validate Audit Service log's integrity. Here we explain how to run them.
|
74
74
|
|
75
75
|
#### Verify audit data
|
76
76
|
|
@@ -101,7 +101,7 @@ curl -H "Authorization: Bearer ${PANGEA_TOKEN}" -X POST -H 'Content-Type: applic
|
|
101
101
|
|
102
102
|
Download all audit logs for a given time range. Start and end date should be provided,
|
103
103
|
a variety of formats is supported, including ISO-8601. The result is stored in a
|
104
|
-
|
104
|
+
json file (one json per line).
|
105
105
|
|
106
106
|
```
|
107
107
|
usage: python -m pangea.dump_audit [-h] [--token TOKEN] [--domain DOMAIN] [--output OUTPUT] start end
|
@@ -126,8 +126,8 @@ options:
|
|
126
126
|
|
127
127
|
#### Perform Exhaustive Verification of Audit Data
|
128
128
|
|
129
|
-
This script performs extensive verification on a range of events of the log stream.
|
130
|
-
and the membership proof, it checks that there
|
129
|
+
This script performs extensive verification on a range of events of the log stream. Apart from verifying the hash
|
130
|
+
and the membership proof, it checks that there are no omissions in the stream, i.e. all the events are present and properly located.
|
131
131
|
|
132
132
|
```
|
133
133
|
usage: python -m pangea.deep_verify [-h] [--token TOKEN] [--domain DOMAIN] --file FILE
|
@@ -153,9 +153,9 @@ It accepts multiple file formats:
|
|
153
153
|
|
154
154
|
## Reporting issues and new features
|
155
155
|
|
156
|
-
If
|
157
|
-
We would need you to provide some basic information
|
158
|
-
Also feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/)
|
156
|
+
If you run into an issue using or testing this SDK or if you have a new feature request, feel free to open an issue by [clicking here](https://github.com/pangeacyber/pangea-python/issues).
|
157
|
+
We would need you to provide some basic information, such as what SDK version you are using, the stack trace if you got it, the framework used, and steps to reproduce the issue.
|
158
|
+
Also, feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/).
|
159
159
|
|
160
160
|
|
161
161
|
## Contributing
|
@@ -168,5 +168,5 @@ These linters will run on every `git commit` operation.
|
|
168
168
|
|
169
169
|
### Send a PR
|
170
170
|
|
171
|
-
If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in code or an error in documents we
|
171
|
+
If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in the code or an error in documents, we really appreciate it and after review and approval, you will be included in our [contributors list](https://github.com/pangeacyber/pangea-python/blob/main/packages/pangea-sdk/CONTRIBUTING.md).
|
172
172
|
|
@@ -29,18 +29,18 @@ poetry add pangea-sdk
|
|
29
29
|
|
30
30
|
## Usage
|
31
31
|
|
32
|
-
For
|
32
|
+
For sample apps, look at the [/examples](https://github.com/pangeacyber/pangea-python/tree/main/examples) folder in this repository. There you will find basic sample apps for each of the services supported on this SDK. Each service folder has a README.md with instructions to install, setup, and run the sample app.
|
33
33
|
|
34
34
|
|
35
35
|
## Asyncio support
|
36
36
|
|
37
|
-
We have added support to asyncio library using aiohttp in order to support async/await calls to all our services.
|
37
|
+
We have added support to the asyncio library using aiohttp in order to support async/await calls to all our services.
|
38
38
|
Async services classes are inside [pangea/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/packages/pangea-sdk/pangea/asyncio) folder, and examples about how to use them are in [/examples/asyncio](https://github.com/pangeacyber/pangea-python/tree/main/examples/asyncio).
|
39
39
|
|
40
40
|
|
41
41
|
### Secure Audit Service - Integrity Tools
|
42
42
|
|
43
|
-
Python Pangea SDK
|
43
|
+
The Python Pangea SDK also includes some extra features to validate Audit Service log's integrity. Here we explain how to run them.
|
44
44
|
|
45
45
|
#### Verify audit data
|
46
46
|
|
@@ -71,7 +71,7 @@ curl -H "Authorization: Bearer ${PANGEA_TOKEN}" -X POST -H 'Content-Type: applic
|
|
71
71
|
|
72
72
|
Download all audit logs for a given time range. Start and end date should be provided,
|
73
73
|
a variety of formats is supported, including ISO-8601. The result is stored in a
|
74
|
-
|
74
|
+
json file (one json per line).
|
75
75
|
|
76
76
|
```
|
77
77
|
usage: python -m pangea.dump_audit [-h] [--token TOKEN] [--domain DOMAIN] [--output OUTPUT] start end
|
@@ -96,8 +96,8 @@ options:
|
|
96
96
|
|
97
97
|
#### Perform Exhaustive Verification of Audit Data
|
98
98
|
|
99
|
-
This script performs extensive verification on a range of events of the log stream.
|
100
|
-
and the membership proof, it checks that there
|
99
|
+
This script performs extensive verification on a range of events of the log stream. Apart from verifying the hash
|
100
|
+
and the membership proof, it checks that there are no omissions in the stream, i.e. all the events are present and properly located.
|
101
101
|
|
102
102
|
```
|
103
103
|
usage: python -m pangea.deep_verify [-h] [--token TOKEN] [--domain DOMAIN] --file FILE
|
@@ -123,9 +123,9 @@ It accepts multiple file formats:
|
|
123
123
|
|
124
124
|
## Reporting issues and new features
|
125
125
|
|
126
|
-
If
|
127
|
-
We would need you to provide some basic information
|
128
|
-
Also feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/)
|
126
|
+
If you run into an issue using or testing this SDK or if you have a new feature request, feel free to open an issue by [clicking here](https://github.com/pangeacyber/pangea-python/issues).
|
127
|
+
We would need you to provide some basic information, such as what SDK version you are using, the stack trace if you got it, the framework used, and steps to reproduce the issue.
|
128
|
+
Also, feel free to contact [Pangea support](mailto:support@pangea.cloud) by email or send us a message on [Slack](https://pangea.cloud/join-slack/).
|
129
129
|
|
130
130
|
|
131
131
|
## Contributing
|
@@ -138,4 +138,4 @@ These linters will run on every `git commit` operation.
|
|
138
138
|
|
139
139
|
### Send a PR
|
140
140
|
|
141
|
-
If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in code or an error in documents we
|
141
|
+
If you would like to [send a PR](https://github.com/pangeacyber/pangea-python/pulls) including a new feature or fixing a bug in the code or an error in documents, we really appreciate it and after review and approval, you will be included in our [contributors list](https://github.com/pangeacyber/pangea-python/blob/main/packages/pangea-sdk/CONTRIBUTING.md).
|
@@ -51,12 +51,14 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
51
51
|
if self.config_id and data.get("config_id", None) is None:
|
52
52
|
data["config_id"] = self.config_id
|
53
53
|
|
54
|
+
transfer_method = data.get("transfer_method", None)
|
55
|
+
|
54
56
|
if (
|
55
57
|
files is not None
|
56
58
|
and type(data) is dict
|
57
|
-
and
|
59
|
+
and (transfer_method == TransferMethod.DIRECT.value or transfer_method == TransferMethod.POST_URL.value)
|
58
60
|
):
|
59
|
-
requests_response = await self.
|
61
|
+
requests_response = await self._full_post_presigned_url(
|
60
62
|
endpoint, result_class=result_class, data=data, files=files
|
61
63
|
)
|
62
64
|
else:
|
@@ -72,6 +74,83 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
72
74
|
|
73
75
|
return self._check_response(pangea_response)
|
74
76
|
|
77
|
+
async def get(
|
78
|
+
self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
|
79
|
+
) -> PangeaResponse:
|
80
|
+
"""Makes the GET call to a Pangea Service endpoint.
|
81
|
+
|
82
|
+
Args:
|
83
|
+
endpoint(str): The Pangea Service API endpoint.
|
84
|
+
path(str): Additional URL path
|
85
|
+
|
86
|
+
Returns:
|
87
|
+
PangeaResponse which contains the response in its entirety and
|
88
|
+
various properties to retrieve individual fields
|
89
|
+
"""
|
90
|
+
|
91
|
+
url = self._url(path)
|
92
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
93
|
+
|
94
|
+
async with self.session.get(url, headers=self._headers()) as requests_response:
|
95
|
+
pangea_response = PangeaResponse(
|
96
|
+
requests_response, result_class=result_class, json=await requests_response.json()
|
97
|
+
)
|
98
|
+
|
99
|
+
self.logger.debug(
|
100
|
+
json.dumps(
|
101
|
+
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
102
|
+
default=default_encoder,
|
103
|
+
)
|
104
|
+
)
|
105
|
+
|
106
|
+
if check_response is False:
|
107
|
+
return pangea_response
|
108
|
+
|
109
|
+
return self._check_response(pangea_response)
|
110
|
+
|
111
|
+
async def poll_result_by_id(
|
112
|
+
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
113
|
+
):
|
114
|
+
path = self._get_poll_path(request_id)
|
115
|
+
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
116
|
+
return await self.get(path, result_class, check_response=check_response)
|
117
|
+
|
118
|
+
async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
119
|
+
request_id = response.request_id
|
120
|
+
if not request_id:
|
121
|
+
raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
|
122
|
+
|
123
|
+
if response.status != ResponseStatus.ACCEPTED.value:
|
124
|
+
raise pe.PangeaException("Response already proccesed")
|
125
|
+
|
126
|
+
return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
127
|
+
|
128
|
+
async def post_presigned_url(self, url: str, data: Dict, files: List[Tuple]):
|
129
|
+
# Send form request with file and upload_details as body
|
130
|
+
resp = await self._http_post(url=url, data=data, files=files, presigned_url_post=True)
|
131
|
+
self.logger.debug(
|
132
|
+
json.dumps(
|
133
|
+
{"service": self.service, "action": "post presigned", "url": url, "response": resp.text},
|
134
|
+
default=default_encoder,
|
135
|
+
)
|
136
|
+
)
|
137
|
+
|
138
|
+
if resp.status < 200 or resp.status >= 300:
|
139
|
+
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", resp.text)
|
140
|
+
|
141
|
+
async def put_presigned_url(self, url: str, files: List[Tuple]):
|
142
|
+
# Send put request with file as body
|
143
|
+
resp = await self._http_put(url=url, files=files)
|
144
|
+
self.logger.debug(
|
145
|
+
json.dumps(
|
146
|
+
{"service": self.service, "action": "put presigned", "url": url, "response": resp.text},
|
147
|
+
default=default_encoder,
|
148
|
+
)
|
149
|
+
)
|
150
|
+
|
151
|
+
if resp.status_code < 200 or resp.status_code >= 300:
|
152
|
+
raise pe.PresignedUploadError(f"presigned PUT failure: {resp.status_code}", resp.text)
|
153
|
+
|
75
154
|
async def _http_post(
|
76
155
|
self,
|
77
156
|
url: str,
|
@@ -105,7 +184,21 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
105
184
|
|
106
185
|
return await self.session.post(url, headers=headers, data=data_send)
|
107
186
|
|
108
|
-
async def
|
187
|
+
async def _http_put(
|
188
|
+
self,
|
189
|
+
url: str,
|
190
|
+
files: List[Tuple],
|
191
|
+
headers: Dict = {},
|
192
|
+
) -> aiohttp.ClientResponse:
|
193
|
+
self.logger.debug(
|
194
|
+
json.dumps({"service": self.service, "action": "http_put", "url": url}, default=default_encoder)
|
195
|
+
)
|
196
|
+
form = FormData()
|
197
|
+
name, value = files[0]
|
198
|
+
form.add_field(name, value[1], filename=value[0], content_type=value[2])
|
199
|
+
return self.session.put(url, headers=headers, data=form)
|
200
|
+
|
201
|
+
async def _full_post_presigned_url(
|
109
202
|
self,
|
110
203
|
endpoint: str,
|
111
204
|
result_class: Type[PangeaResponseResult],
|
@@ -115,61 +208,52 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
115
208
|
if len(files) == 0:
|
116
209
|
raise AttributeError("files attribute should have at least 1 file")
|
117
210
|
|
211
|
+
response = await self.request_presigned_url(endpoint=endpoint, result_class=result_class, data=data)
|
212
|
+
data_to_presigned = response.accepted_result.accepted_status.upload_details
|
213
|
+
presigned_url = response.accepted_result.accepted_status.upload_url
|
214
|
+
|
215
|
+
await self.post_presigned_url(url=presigned_url, data=data_to_presigned, files=files)
|
216
|
+
return response.raw_response
|
217
|
+
|
218
|
+
async def request_presigned_url(
|
219
|
+
self,
|
220
|
+
endpoint: str,
|
221
|
+
result_class: Type[PangeaResponseResult],
|
222
|
+
data: Union[str, Dict] = {},
|
223
|
+
) -> PangeaResponse:
|
118
224
|
# Send request
|
119
225
|
try:
|
120
226
|
# This should return 202 (AcceptedRequestException)
|
121
227
|
resp = await self.post(endpoint=endpoint, result_class=result_class, data=data, poll_result=False)
|
122
228
|
raise pe.PresignedURLException("Should return 202", resp)
|
123
|
-
|
124
229
|
except pe.AcceptedRequestException as e:
|
125
230
|
accepted_exception = e
|
126
231
|
except Exception as e:
|
127
232
|
raise e
|
128
233
|
|
129
234
|
# Receive 202 with accepted_status
|
130
|
-
|
131
|
-
data_to_presigned = result.accepted_status.upload_details
|
132
|
-
presigned_url = result.accepted_status.upload_url
|
133
|
-
|
134
|
-
# Send multipart request with file and upload_details as body
|
135
|
-
resp = await self._http_post(url=presigned_url, data=data_to_presigned, files=files, presigned_url_post=True)
|
136
|
-
self.logger.debug(
|
137
|
-
json.dumps(
|
138
|
-
{
|
139
|
-
"service": self.service,
|
140
|
-
"action": "post presigned",
|
141
|
-
"url": presigned_url,
|
142
|
-
"response": await resp.text(),
|
143
|
-
},
|
144
|
-
default=default_encoder,
|
145
|
-
)
|
146
|
-
)
|
147
|
-
|
148
|
-
if resp.status < 200 or resp.status >= 300:
|
149
|
-
raise pe.PresignedUploadError(f"presigned POST failure: {resp.status}", await resp.text())
|
150
|
-
|
151
|
-
return accepted_exception.response.raw_response
|
235
|
+
return await self._poll_presigned_url(accepted_exception.response)
|
152
236
|
|
153
|
-
async def _poll_presigned_url(self,
|
154
|
-
if
|
155
|
-
raise AttributeError("
|
237
|
+
async def _poll_presigned_url(self, response: PangeaResponse) -> AcceptedResult:
|
238
|
+
if response.http_status != 202:
|
239
|
+
raise AttributeError("Response should be 202")
|
156
240
|
|
157
|
-
if
|
158
|
-
return
|
241
|
+
if response.accepted_result.accepted_status.upload_url:
|
242
|
+
return response
|
159
243
|
|
160
244
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "start"}))
|
161
245
|
retry_count = 1
|
162
246
|
start = time.time()
|
163
|
-
|
247
|
+
loop_resp = response
|
164
248
|
|
165
249
|
while (
|
166
|
-
|
167
|
-
and not
|
250
|
+
loop_resp.accepted_result is not None
|
251
|
+
and not loop_resp.accepted_result.accepted_status.upload_url
|
168
252
|
and not self._reach_timeout(start)
|
169
253
|
):
|
170
254
|
await asyncio.sleep(self._get_delay(retry_count, start))
|
171
255
|
try:
|
172
|
-
await self.poll_result_once(
|
256
|
+
await self.poll_result_once(response, check_response=False)
|
173
257
|
msg = "Polling presigned url return 200 instead of 202"
|
174
258
|
self.logger.debug(
|
175
259
|
json.dumps(
|
@@ -179,6 +263,7 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
179
263
|
raise pe.PangeaException(msg)
|
180
264
|
except pe.AcceptedRequestException as e:
|
181
265
|
retry_count += 1
|
266
|
+
loop_resp = e.response
|
182
267
|
loop_exc = e
|
183
268
|
except Exception as e:
|
184
269
|
self.logger.debug(
|
@@ -190,8 +275,8 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
190
275
|
|
191
276
|
self.logger.debug(json.dumps({"service": self.service, "action": "poll_presigned_url", "step": "exit"}))
|
192
277
|
|
193
|
-
if
|
194
|
-
return
|
278
|
+
if loop_resp.accepted_result is not None and not loop_resp.accepted_result.accepted_status.upload_url:
|
279
|
+
return loop_resp
|
195
280
|
else:
|
196
281
|
raise loop_exc
|
197
282
|
|
@@ -207,57 +292,6 @@ class PangeaRequestAsync(PangeaRequestBase):
|
|
207
292
|
|
208
293
|
return response
|
209
294
|
|
210
|
-
async def get(
|
211
|
-
self, path: str, result_class: Type[PangeaResponseResult], check_response: bool = True
|
212
|
-
) -> PangeaResponse:
|
213
|
-
"""Makes the GET call to a Pangea Service endpoint.
|
214
|
-
|
215
|
-
Args:
|
216
|
-
endpoint(str): The Pangea Service API endpoint.
|
217
|
-
path(str): Additional URL path
|
218
|
-
|
219
|
-
Returns:
|
220
|
-
PangeaResponse which contains the response in its entirety and
|
221
|
-
various properties to retrieve individual fields
|
222
|
-
"""
|
223
|
-
|
224
|
-
url = self._url(path)
|
225
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "get", "url": url}))
|
226
|
-
|
227
|
-
async with self.session.get(url, headers=self._headers()) as requests_response:
|
228
|
-
pangea_response = PangeaResponse(
|
229
|
-
requests_response, result_class=result_class, json=await requests_response.json()
|
230
|
-
)
|
231
|
-
|
232
|
-
self.logger.debug(
|
233
|
-
json.dumps(
|
234
|
-
{"service": self.service, "action": "get", "url": url, "response": pangea_response.json},
|
235
|
-
default=default_encoder,
|
236
|
-
)
|
237
|
-
)
|
238
|
-
|
239
|
-
if check_response is False:
|
240
|
-
return pangea_response
|
241
|
-
|
242
|
-
return self._check_response(pangea_response)
|
243
|
-
|
244
|
-
async def poll_result_by_id(
|
245
|
-
self, request_id: str, result_class: Union[Type[PangeaResponseResult], dict], check_response: bool = True
|
246
|
-
):
|
247
|
-
path = self._get_poll_path(request_id)
|
248
|
-
self.logger.debug(json.dumps({"service": self.service, "action": "poll_result_once", "url": path}))
|
249
|
-
return await self.get(path, result_class, check_response=check_response)
|
250
|
-
|
251
|
-
async def poll_result_once(self, response: PangeaResponse, check_response: bool = True):
|
252
|
-
request_id = response.request_id
|
253
|
-
if not request_id:
|
254
|
-
raise pe.PangeaException("Poll result error error: response did not include a 'request_id'")
|
255
|
-
|
256
|
-
if response.status != ResponseStatus.ACCEPTED.value:
|
257
|
-
raise pe.PangeaException("Response already proccesed")
|
258
|
-
|
259
|
-
return await self.poll_result_by_id(request_id, response.result_class, check_response=check_response)
|
260
|
-
|
261
295
|
async def _poll_result_retry(self, response: PangeaResponse) -> PangeaResponse:
|
262
296
|
retry_count = 1
|
263
297
|
start = time.time()
|
@@ -1,9 +1,11 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
|
4
|
+
from typing import Optional, Type, Union
|
5
|
+
|
4
6
|
from pangea.asyncio.request import PangeaRequestAsync
|
5
7
|
from pangea.exceptions import AcceptedRequestException
|
6
|
-
from pangea.response import PangeaResponse
|
8
|
+
from pangea.response import PangeaResponse, PangeaResponseResult
|
7
9
|
from pangea.services.base import ServiceBase
|
8
10
|
|
9
11
|
|
@@ -21,7 +23,13 @@ class ServiceBaseAsync(ServiceBase):
|
|
21
23
|
|
22
24
|
return self._request
|
23
25
|
|
24
|
-
async def poll_result(
|
26
|
+
async def poll_result(
|
27
|
+
self,
|
28
|
+
exception: Optional[AcceptedRequestException] = None,
|
29
|
+
response: Optional[PangeaResponse] = None,
|
30
|
+
request_id: Optional[str] = None,
|
31
|
+
result_class: Union[Type[PangeaResponseResult], dict] = dict,
|
32
|
+
) -> PangeaResponse:
|
25
33
|
"""
|
26
34
|
Poll result
|
27
35
|
|
@@ -39,7 +47,16 @@ class ServiceBaseAsync(ServiceBase):
|
|
39
47
|
Examples:
|
40
48
|
response = service.poll_result(exception)
|
41
49
|
"""
|
42
|
-
|
50
|
+
if exception is not None:
|
51
|
+
return await self.request.poll_result_once(exception.response, check_response=True)
|
52
|
+
elif response is not None:
|
53
|
+
return await self.request.poll_result_once(response, check_response=True)
|
54
|
+
elif request_id is not None:
|
55
|
+
return await self.request.poll_result_by_id(
|
56
|
+
request_id=request_id, result_class=result_class, check_response=True
|
57
|
+
)
|
58
|
+
else:
|
59
|
+
raise AttributeError("Need to set exception, response or request_id")
|
43
60
|
|
44
61
|
async def close(self):
|
45
62
|
await self.request.session.close()
|
@@ -1,11 +1,14 @@
|
|
1
1
|
# Copyright 2022 Pangea Cyber Corporation
|
2
2
|
# Author: Pangea Cyber Corporation
|
3
3
|
import io
|
4
|
-
|
4
|
+
import logging
|
5
|
+
from typing import Dict, Optional
|
5
6
|
|
6
7
|
import pangea.services.file_scan as m
|
7
|
-
from pangea.
|
8
|
-
from pangea.
|
8
|
+
from pangea.asyncio.request import PangeaRequestAsync
|
9
|
+
from pangea.request import PangeaConfig
|
10
|
+
from pangea.response import PangeaResponse, TransferMethod
|
11
|
+
from pangea.utils import FileUploadParams, get_file_upload_params
|
9
12
|
|
10
13
|
from .base import ServiceBaseAsync
|
11
14
|
|
@@ -46,6 +49,7 @@ class FileScanAsync(ServiceBaseAsync):
|
|
46
49
|
raw: Optional[bool] = None,
|
47
50
|
provider: Optional[str] = None,
|
48
51
|
sync_call: bool = True,
|
52
|
+
transfer_method: TransferMethod = TransferMethod.DIRECT,
|
49
53
|
) -> PangeaResponse[m.FileScanResult]:
|
50
54
|
"""
|
51
55
|
Scan
|
@@ -84,13 +88,79 @@ class FileScanAsync(ServiceBaseAsync):
|
|
84
88
|
if file or file_path:
|
85
89
|
if file_path:
|
86
90
|
file = open(file_path, "rb")
|
87
|
-
|
91
|
+
if transfer_method == TransferMethod.DIRECT or transfer_method == TransferMethod.POST_URL:
|
92
|
+
params = get_file_upload_params(file)
|
93
|
+
crc = params.crc_hex
|
94
|
+
sha = params.sha256_hex
|
95
|
+
size = params.size
|
96
|
+
else:
|
97
|
+
crc, sha, size = None, None, None
|
88
98
|
files = [("upload", ("filename", file, "application/octet-stream"))]
|
89
99
|
else:
|
90
100
|
raise ValueError("Need to set file_path or file arguments")
|
91
101
|
|
92
102
|
input = m.FileScanRequest(
|
93
|
-
verbose=verbose,
|
103
|
+
verbose=verbose,
|
104
|
+
raw=raw,
|
105
|
+
provider=provider,
|
106
|
+
transfer_crc32c=crc,
|
107
|
+
transfer_sha256=sha,
|
108
|
+
transfer_size=size,
|
109
|
+
transfer_method=transfer_method,
|
94
110
|
)
|
95
111
|
data = input.dict(exclude_none=True)
|
96
112
|
return await self.request.post("v1/scan", m.FileScanResult, data=data, files=files, poll_result=sync_call)
|
113
|
+
|
114
|
+
async def request_upload_url(
|
115
|
+
self,
|
116
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
117
|
+
params: Optional[FileUploadParams] = None,
|
118
|
+
verbose: Optional[bool] = None,
|
119
|
+
raw: Optional[bool] = None,
|
120
|
+
provider: Optional[str] = None,
|
121
|
+
) -> PangeaResponse[m.FileScanResult]:
|
122
|
+
input = m.FileScanRequest(
|
123
|
+
verbose=verbose,
|
124
|
+
raw=raw,
|
125
|
+
provider=provider,
|
126
|
+
transfer_method=transfer_method,
|
127
|
+
)
|
128
|
+
if params is not None and (
|
129
|
+
transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT
|
130
|
+
):
|
131
|
+
input.transfer_crc32c = params.crc_hex
|
132
|
+
input.transfer_sha256 = params.sha256_hex
|
133
|
+
input.transfer_size = params.size
|
134
|
+
|
135
|
+
data = input.dict(exclude_none=True)
|
136
|
+
return await self.request.request_presigned_url("v1/scan", m.FileScanResult, data=data)
|
137
|
+
|
138
|
+
|
139
|
+
class FileUploaderAsync:
|
140
|
+
def __init__(self):
|
141
|
+
self.logger = logging.getLogger("pangea")
|
142
|
+
self._request: PangeaRequestAsync = PangeaRequestAsync(
|
143
|
+
config=PangeaConfig(),
|
144
|
+
token="",
|
145
|
+
service="FileScanUploader",
|
146
|
+
logger=self.logger,
|
147
|
+
)
|
148
|
+
|
149
|
+
async def upload_file(
|
150
|
+
self,
|
151
|
+
url: str,
|
152
|
+
file: io.BufferedReader,
|
153
|
+
transfer_method: TransferMethod = TransferMethod.PUT_URL,
|
154
|
+
file_details: Optional[Dict] = None,
|
155
|
+
):
|
156
|
+
if transfer_method == TransferMethod.PUT_URL:
|
157
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
158
|
+
await self._request.put_presigned_url(url=url, files=files)
|
159
|
+
elif transfer_method == TransferMethod.POST_URL or transfer_method == TransferMethod.DIRECT:
|
160
|
+
files = [("file", ("filename", file, "application/octet-stream"))]
|
161
|
+
await self._request.post_presigned_url(url=url, data=file_details, files=files)
|
162
|
+
else:
|
163
|
+
raise ValueError(f"Transfer method not supported: {transfer_method}")
|
164
|
+
|
165
|
+
async def close(self):
|
166
|
+
await self._request.session.close()
|
@@ -36,13 +36,8 @@ class RedactAsync(ServiceBaseAsync):
|
|
36
36
|
|
37
37
|
service_name = "redact"
|
38
38
|
|
39
|
-
def __init__(
|
40
|
-
|
41
|
-
token,
|
42
|
-
config=None,
|
43
|
-
logger_name="pangea",
|
44
|
-
):
|
45
|
-
super().__init__(token, config, logger_name)
|
39
|
+
def __init__(self, token, config=None, logger_name="pangea", config_id: Optional[str] = None):
|
40
|
+
super().__init__(token, config, logger_name, config_id=config_id)
|
46
41
|
|
47
42
|
async def redact(
|
48
43
|
self,
|
@@ -10,7 +10,7 @@ class PangeaConfig:
|
|
10
10
|
"""Holds run time configuration information used by SDK components."""
|
11
11
|
|
12
12
|
"""
|
13
|
-
Used to set
|
13
|
+
Used to set Pangea domain (and port if needed), it should not include service subdomain
|
14
14
|
just for particular use cases when environment = "local", domain could be set to an url including:
|
15
15
|
scheme (http:// or https://), subdomain, domain and port.
|
16
16
|
|
@@ -19,7 +19,7 @@ class PangeaConfig:
|
|
19
19
|
|
20
20
|
"""
|
21
21
|
Used to generate service url.
|
22
|
-
It should be only 'production' or 'local' in
|
22
|
+
It should be only 'production' or 'local' in cases of particular services that can run locally as Redact.
|
23
23
|
|
24
24
|
"""
|
25
25
|
environment: str = "production"
|