anaplan-sdk 0.4.5__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anaplan_sdk/__init__.py +2 -0
- anaplan_sdk/_async_clients/__init__.py +4 -0
- anaplan_sdk/_async_clients/_alm.py +257 -44
- anaplan_sdk/_async_clients/_audit.py +31 -21
- anaplan_sdk/_async_clients/_bulk.py +270 -173
- anaplan_sdk/_async_clients/_cloud_works.py +61 -41
- anaplan_sdk/_async_clients/_cw_flow.py +26 -18
- anaplan_sdk/_async_clients/_scim.py +148 -0
- anaplan_sdk/_async_clients/_transactional.py +265 -56
- anaplan_sdk/_clients/__init__.py +12 -1
- anaplan_sdk/_clients/_alm.py +255 -45
- anaplan_sdk/_clients/_audit.py +32 -22
- anaplan_sdk/_clients/_bulk.py +265 -157
- anaplan_sdk/_clients/_cloud_works.py +59 -40
- anaplan_sdk/_clients/_cw_flow.py +24 -16
- anaplan_sdk/_clients/_scim.py +145 -0
- anaplan_sdk/_clients/_transactional.py +260 -50
- anaplan_sdk/_services.py +277 -0
- anaplan_sdk/_utils.py +188 -0
- anaplan_sdk/models/__init__.py +49 -2
- anaplan_sdk/models/_alm.py +64 -6
- anaplan_sdk/models/_bulk.py +22 -13
- anaplan_sdk/models/_transactional.py +221 -4
- anaplan_sdk/models/cloud_works.py +6 -2
- anaplan_sdk/models/scim.py +282 -0
- {anaplan_sdk-0.4.5.dist-info → anaplan_sdk-0.5.0.dist-info}/METADATA +2 -2
- anaplan_sdk-0.5.0.dist-info/RECORD +34 -0
- anaplan_sdk/_base.py +0 -297
- anaplan_sdk-0.4.5.dist-info/RECORD +0 -30
- {anaplan_sdk-0.4.5.dist-info → anaplan_sdk-0.5.0.dist-info}/WHEEL +0 -0
- {anaplan_sdk-0.4.5.dist-info → anaplan_sdk-0.5.0.dist-info}/licenses/LICENSE +0 -0
anaplan_sdk/_clients/_bulk.py
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
import logging
|
2
2
|
import multiprocessing
|
3
|
-
import time
|
4
3
|
from concurrent.futures import ThreadPoolExecutor
|
5
4
|
from copy import copy
|
6
|
-
from typing import Iterator
|
5
|
+
from typing import Iterator, Literal
|
7
6
|
|
8
7
|
import httpx
|
9
8
|
from typing_extensions import Self
|
10
9
|
|
11
10
|
from anaplan_sdk._auth import _create_auth
|
12
|
-
from anaplan_sdk.
|
11
|
+
from anaplan_sdk._services import _HttpService
|
12
|
+
from anaplan_sdk._utils import action_url, models_url, sort_params
|
13
13
|
from anaplan_sdk.exceptions import AnaplanActionError, InvalidIdentifierException
|
14
14
|
from anaplan_sdk.models import (
|
15
15
|
Action,
|
@@ -17,6 +17,7 @@ from anaplan_sdk.models import (
|
|
17
17
|
File,
|
18
18
|
Import,
|
19
19
|
Model,
|
20
|
+
ModelDeletionResult,
|
20
21
|
Process,
|
21
22
|
TaskStatus,
|
22
23
|
TaskSummary,
|
@@ -26,13 +27,15 @@ from anaplan_sdk.models import (
|
|
26
27
|
from ._alm import _AlmClient
|
27
28
|
from ._audit import _AuditClient
|
28
29
|
from ._cloud_works import _CloudWorksClient
|
30
|
+
from ._scim import _ScimClient
|
29
31
|
from ._transactional import _TransactionalClient
|
30
32
|
|
31
|
-
|
33
|
+
SortBy = Literal["id", "name"] | None
|
34
|
+
|
32
35
|
logger = logging.getLogger("anaplan_sdk")
|
33
36
|
|
34
37
|
|
35
|
-
class Client
|
38
|
+
class Client:
|
36
39
|
"""
|
37
40
|
Synchronous Anaplan Client. For guides and examples
|
38
41
|
refer to https://vinzenzklass.github.io/anaplan-sdk.
|
@@ -51,10 +54,14 @@ class Client(_BaseClient):
|
|
51
54
|
auth: httpx.Auth | None = None,
|
52
55
|
timeout: float | httpx.Timeout = 30,
|
53
56
|
retry_count: int = 2,
|
57
|
+
backoff: float = 1.0,
|
58
|
+
backoff_factor: float = 2.0,
|
59
|
+
page_size: int = 5_000,
|
54
60
|
status_poll_delay: int = 1,
|
55
61
|
upload_parallel: bool = True,
|
56
62
|
upload_chunk_size: int = 25_000_000,
|
57
63
|
allow_file_creation: bool = False,
|
64
|
+
**httpx_kwargs,
|
58
65
|
) -> None:
|
59
66
|
"""
|
60
67
|
Synchronous Anaplan Client. For guides and examples
|
@@ -86,7 +93,15 @@ class Client(_BaseClient):
|
|
86
93
|
an instance of `httpx.Timeout` to set the timeout for the HTTP requests.
|
87
94
|
:param retry_count: The number of times to retry an HTTP request if it fails. Set this to 0
|
88
95
|
to never retry. Defaults to 2, meaning each HTTP Operation will be tried a total
|
96
|
+
:param backoff: The initial backoff time in seconds for the retry mechanism. This is the
|
97
|
+
time to wait before the first retry.
|
98
|
+
:param backoff_factor: The factor by which the backoff time is multiplied after each retry.
|
99
|
+
For example, if the initial backoff is 1 second and the factor is 2, the second
|
100
|
+
retry will wait 2 seconds, the third retry will wait 4 seconds, and so on.
|
89
101
|
number of 2 times.
|
102
|
+
:param page_size: The number of items to return per page when paginating through results.
|
103
|
+
Defaults to 5000. This is the maximum number of items that can be returned per
|
104
|
+
request. If you pass a value greater than 5000, it will be capped to 5000.
|
90
105
|
:param status_poll_delay: The delay between polling the status of a task.
|
91
106
|
:param upload_parallel: Whether to upload chunks in parallel when uploading files.
|
92
107
|
:param upload_chunk_size: The size of the chunks to upload. This is the maximum size of
|
@@ -96,54 +111,63 @@ class Client(_BaseClient):
|
|
96
111
|
altogether. A file that is created this way will not be referenced by any action in
|
97
112
|
anaplan until manually assigned so there is typically no value in dynamically
|
98
113
|
creating new files and uploading content to them.
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
114
|
+
:param httpx_kwargs: Additional keyword arguments to pass to the `httpx.Client`.
|
115
|
+
This can be used to set additional options such as proxies, headers, etc. See
|
116
|
+
https://www.python-httpx.org/api/#client for the full list of arguments.
|
117
|
+
"""
|
118
|
+
auth = auth or _create_auth(
|
119
|
+
token=token,
|
120
|
+
user_email=user_email,
|
121
|
+
password=password,
|
122
|
+
certificate=certificate,
|
123
|
+
private_key=private_key,
|
124
|
+
private_key_password=private_key_password,
|
125
|
+
)
|
126
|
+
_client = httpx.Client(auth=auth, timeout=timeout, **httpx_kwargs)
|
127
|
+
self._http = _HttpService(
|
128
|
+
_client,
|
129
|
+
retry_count=retry_count,
|
130
|
+
backoff=backoff,
|
131
|
+
backoff_factor=backoff_factor,
|
132
|
+
page_size=page_size,
|
133
|
+
poll_delay=status_poll_delay,
|
113
134
|
)
|
114
135
|
self._retry_count = retry_count
|
136
|
+
self._workspace_id = workspace_id
|
137
|
+
self._model_id = model_id
|
115
138
|
self._url = f"https://api.anaplan.com/2/0/workspaces/{workspace_id}/models/{model_id}"
|
116
139
|
self._transactional_client = (
|
117
|
-
_TransactionalClient(
|
140
|
+
_TransactionalClient(self._http, model_id) if model_id else None
|
118
141
|
)
|
119
|
-
self._alm_client = _AlmClient(
|
120
|
-
self.
|
142
|
+
self._alm_client = _AlmClient(self._http, model_id) if model_id else None
|
143
|
+
self._audit_client = _AuditClient(self._http)
|
144
|
+
self._scim_client = _ScimClient(self._http)
|
145
|
+
self._cloud_works = _CloudWorksClient(self._http)
|
121
146
|
self._thread_count = multiprocessing.cpu_count()
|
122
|
-
self._audit = _AuditClient(_client, self._retry_count, self._thread_count)
|
123
147
|
self.status_poll_delay = status_poll_delay
|
124
148
|
self.upload_parallel = upload_parallel
|
125
149
|
self.upload_chunk_size = upload_chunk_size
|
126
150
|
self.allow_file_creation = allow_file_creation
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
update the relevant attributes to the new workspace and model.
|
136
|
-
:param existing: The existing instance to copy.
|
137
|
-
:param workspace_id: The workspace Id to use.
|
138
|
-
:param model_id: The model Id to use.
|
151
|
+
logger.debug(f"Initialized Client with workspace_id={workspace_id}, model_id={model_id}")
|
152
|
+
|
153
|
+
def with_model(self, model_id: str | None = None, workspace_id: str | None = None) -> Self:
|
154
|
+
"""
|
155
|
+
Create a new instance of the Client with the given model and workspace Ids. **This creates
|
156
|
+
a copy of the current client. The current instance remains unchanged.**
|
157
|
+
:param workspace_id: The workspace Id to use or None to use the existing workspace Id.
|
158
|
+
:param model_id: The model Id to use or None to use the existing model Id.
|
139
159
|
:return: A new instance of the Client.
|
140
160
|
"""
|
141
|
-
client = copy(
|
142
|
-
|
143
|
-
|
144
|
-
|
161
|
+
client = copy(self)
|
162
|
+
new_ws_id = workspace_id or self._workspace_id
|
163
|
+
new_model_id = model_id or self._model_id
|
164
|
+
logger.debug(
|
165
|
+
f"Creating a new AsyncClient from existing instance "
|
166
|
+
f"with workspace_id={new_ws_id}, model_id={new_model_id}."
|
145
167
|
)
|
146
|
-
client.
|
168
|
+
client._url = f"https://api.anaplan.com/2/0/workspaces/{new_ws_id}/models/{new_model_id}"
|
169
|
+
client._transactional_client = _TransactionalClient(self._http, new_model_id)
|
170
|
+
client._alm_client = _AlmClient(self._http, new_model_id)
|
147
171
|
return client
|
148
172
|
|
149
173
|
@property
|
@@ -152,7 +176,7 @@ class Client(_BaseClient):
|
|
152
176
|
The Audit Client provides access to the Anaplan Audit API.
|
153
177
|
For details, see https://vinzenzklass.github.io/anaplan-sdk/guides/audit/.
|
154
178
|
"""
|
155
|
-
return self.
|
179
|
+
return self._audit_client
|
156
180
|
|
157
181
|
@property
|
158
182
|
def cw(self) -> _CloudWorksClient:
|
@@ -163,14 +187,14 @@ class Client(_BaseClient):
|
|
163
187
|
return self._cloud_works
|
164
188
|
|
165
189
|
@property
|
166
|
-
def
|
190
|
+
def tr(self) -> _TransactionalClient:
|
167
191
|
"""
|
168
192
|
The Transactional Client provides access to the Anaplan Transactional API. This is useful
|
169
193
|
for more advanced use cases where you need to interact with the Anaplan Model in a more
|
170
194
|
granular way.
|
171
195
|
|
172
196
|
If you instantiated the client without the field `model_id`, this will raise a
|
173
|
-
|
197
|
+
`ValueError`, since none of the endpoints can be invoked without the model Id.
|
174
198
|
:return: The Transactional Client.
|
175
199
|
"""
|
176
200
|
if not self._transactional_client:
|
@@ -201,114 +225,174 @@ class Client(_BaseClient):
|
|
201
225
|
)
|
202
226
|
return self._alm_client
|
203
227
|
|
204
|
-
|
228
|
+
@property
|
229
|
+
def scim(self) -> _ScimClient:
|
230
|
+
"""
|
231
|
+
To use the SCIM API, you must be User Admin. The SCIM API allows managing internal users.
|
232
|
+
Visiting users are excluded from the SCIM API.
|
233
|
+
:return: The SCIM Client.
|
234
|
+
"""
|
235
|
+
return self._scim_client
|
236
|
+
|
237
|
+
def get_workspaces(
|
238
|
+
self,
|
239
|
+
search_pattern: str | None = None,
|
240
|
+
sort_by: Literal["size_allowance", "name"] | None = None,
|
241
|
+
descending: bool = False,
|
242
|
+
) -> list[Workspace]:
|
205
243
|
"""
|
206
244
|
Lists all the Workspaces the authenticated user has access to.
|
207
|
-
:param search_pattern:
|
208
|
-
|
209
|
-
|
245
|
+
:param search_pattern: **Caution: This is an undocumented Feature and may behave
|
246
|
+
unpredictably. It requires the Tenant Admin role. For non-admin users, it is
|
247
|
+
ignored.** Optionally filter for specific workspaces. When provided,
|
248
|
+
case-insensitive matches workspaces with names containing this string.
|
249
|
+
You can use the wildcards `%` for 0-n characters, and `_` for exactly 1 character.
|
250
|
+
When None (default), returns all users.
|
251
|
+
:param sort_by: The field to sort the results by.
|
252
|
+
:param descending: If True, the results will be sorted in descending order.
|
210
253
|
:return: The List of Workspaces.
|
211
254
|
"""
|
212
|
-
params = {"tenantDetails": "true"}
|
255
|
+
params = {"tenantDetails": "true"} | sort_params(sort_by, descending)
|
213
256
|
if search_pattern:
|
214
257
|
params["s"] = search_pattern
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
)
|
220
|
-
]
|
258
|
+
res = self._http.get_paginated(
|
259
|
+
"https://api.anaplan.com/2/0/workspaces", "workspaces", params=params
|
260
|
+
)
|
261
|
+
return [Workspace.model_validate(e) for e in res]
|
221
262
|
|
222
|
-
def
|
263
|
+
def get_models(
|
264
|
+
self,
|
265
|
+
only_in_workspace: bool | str = False,
|
266
|
+
search_pattern: str | None = None,
|
267
|
+
sort_by: Literal["active_state", "name"] | None = None,
|
268
|
+
descending: bool = False,
|
269
|
+
) -> list[Model]:
|
223
270
|
"""
|
224
271
|
Lists all the Models the authenticated user has access to.
|
225
|
-
:param
|
226
|
-
|
272
|
+
:param only_in_workspace: If True, only lists models in the workspace provided when
|
273
|
+
instantiating the client. If a string is provided, only lists models in the workspace
|
274
|
+
with the given Id. If False (default), lists models in all workspaces the user
|
275
|
+
:param search_pattern: **Caution: This is an undocumented Feature and may behave
|
276
|
+
unpredictably. It requires the Tenant Admin role. For non-admin users, it is
|
277
|
+
ignored.** Optionally filter for specific models. When provided,
|
278
|
+
case-insensitive matches model names containing this string.
|
279
|
+
You can use the wildcards `%` for 0-n characters, and `_` for exactly 1 character.
|
280
|
+
When None (default), returns all models.
|
281
|
+
:param sort_by: The field to sort the results by.
|
282
|
+
:param descending: If True, the results will be sorted in descending order.
|
227
283
|
:return: The List of Models.
|
228
284
|
"""
|
229
|
-
params = {"modelDetails": "true"}
|
285
|
+
params = {"modelDetails": "true"} | sort_params(sort_by, descending)
|
230
286
|
if search_pattern:
|
231
287
|
params["s"] = search_pattern
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
288
|
+
res = self._http.get_paginated(
|
289
|
+
models_url(only_in_workspace, self._workspace_id), "models", params=params
|
290
|
+
)
|
291
|
+
return [Model.model_validate(e) for e in res]
|
292
|
+
|
293
|
+
def delete_models(self, model_ids: list[str]) -> ModelDeletionResult:
|
294
|
+
"""
|
295
|
+
Delete the given Models. Models need to be closed before they can be deleted. If one of the
|
296
|
+
deletions fails, the other deletions will still be attempted and may complete.
|
297
|
+
:param model_ids: The list of Model identifiers to delete.
|
298
|
+
:return:
|
299
|
+
"""
|
300
|
+
logger.info(f"Deleting Models: {', '.join(model_ids)}.")
|
301
|
+
res = self._http.post(
|
302
|
+
f"https://api.anaplan.com/2/0/workspaces/{self._workspace_id}/bulkDeleteModels",
|
303
|
+
json={"modelIdsToDelete": model_ids},
|
304
|
+
)
|
305
|
+
return ModelDeletionResult.model_validate(res)
|
238
306
|
|
239
|
-
def
|
307
|
+
def get_files(
|
308
|
+
self, sort_by: Literal["id", "name"] = "id", descending: bool = False
|
309
|
+
) -> list[File]:
|
240
310
|
"""
|
241
311
|
Lists all the Files in the Model.
|
312
|
+
:param sort_by: The field to sort the results by.
|
313
|
+
:param descending: If True, the results will be sorted in descending order.
|
242
314
|
:return: The List of Files.
|
243
315
|
"""
|
244
|
-
|
316
|
+
res = self._http.get_paginated(
|
317
|
+
f"{self._url}/files", "files", params=sort_params(sort_by, descending)
|
318
|
+
)
|
319
|
+
return [File.model_validate(e) for e in res]
|
245
320
|
|
246
|
-
def
|
321
|
+
def get_actions(self, sort_by: SortBy = None, descending: bool = False) -> list[Action]:
|
247
322
|
"""
|
248
323
|
Lists all the Actions in the Model. This will only return the Actions listed under
|
249
324
|
`Other Actions` in Anaplan. For Imports, exports, and processes, see their respective
|
250
325
|
methods instead.
|
326
|
+
:param sort_by: The field to sort the results by.
|
327
|
+
:param descending: If True, the results will be sorted in descending order.
|
251
328
|
:return: The List of Actions.
|
252
329
|
"""
|
253
|
-
|
254
|
-
|
255
|
-
|
330
|
+
res = self._http.get_paginated(
|
331
|
+
f"{self._url}/actions", "actions", params=sort_params(sort_by, descending)
|
332
|
+
)
|
333
|
+
return [Action.model_validate(e) for e in res]
|
256
334
|
|
257
|
-
def
|
335
|
+
def get_processes(self, sort_by: SortBy = None, descending: bool = False) -> list[Process]:
|
258
336
|
"""
|
259
337
|
Lists all the Processes in the Model.
|
338
|
+
:param sort_by: The field to sort the results by.
|
339
|
+
:param descending: If True, the results will be sorted in descending order.
|
260
340
|
:return: The List of Processes.
|
261
341
|
"""
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
]
|
342
|
+
res = self._http.get_paginated(
|
343
|
+
f"{self._url}/processes", "processes", params=sort_params(sort_by, descending)
|
344
|
+
)
|
345
|
+
return [Process.model_validate(e) for e in res]
|
266
346
|
|
267
|
-
def
|
347
|
+
def get_imports(self, sort_by: SortBy = None, descending: bool = False) -> list[Import]:
|
268
348
|
"""
|
269
349
|
Lists all the Imports in the Model.
|
350
|
+
:param sort_by: The field to sort the results by.
|
351
|
+
:param descending: If True, the results will be sorted in descending order.
|
270
352
|
:return: The List of Imports.
|
271
353
|
"""
|
272
|
-
|
273
|
-
|
274
|
-
|
354
|
+
res = self._http.get_paginated(
|
355
|
+
f"{self._url}/imports", "imports", params=sort_params(sort_by, descending)
|
356
|
+
)
|
357
|
+
return [Import.model_validate(e) for e in res]
|
275
358
|
|
276
|
-
def
|
359
|
+
def get_exports(self, sort_by: SortBy = None, descending: bool = False) -> list[Export]:
|
277
360
|
"""
|
278
361
|
Lists all the Exports in the Model.
|
362
|
+
:param sort_by: The field to sort the results by.
|
363
|
+
:param descending: If True, the results will be sorted in descending order.
|
279
364
|
:return: The List of Exports.
|
280
365
|
"""
|
281
|
-
|
282
|
-
|
283
|
-
|
366
|
+
res = self._http.get_paginated(
|
367
|
+
f"{self._url}/exports", "exports", params=sort_params(sort_by, descending)
|
368
|
+
)
|
369
|
+
return [Export.model_validate(e) for e in res]
|
284
370
|
|
285
|
-
def run_action(self, action_id: int) -> TaskStatus:
|
371
|
+
def run_action(self, action_id: int, wait_for_completion: bool = True) -> TaskStatus:
|
286
372
|
"""
|
287
|
-
Runs the
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
raise an Exception to handle - if you for e.g. think that one of the uploaded chunks may
|
292
|
-
have been dropped and simply retrying with new data may help - and not return the task
|
293
|
-
status information that needs to be handled by the caller.
|
294
|
-
|
295
|
-
If you need more information or control, you can use `invoke_action()` and
|
296
|
-
`get_task_status()`.
|
373
|
+
Runs the Action and validates the spawned task. If the Action fails or completes with
|
374
|
+
errors, this will raise an AnaplanActionError. Failed Tasks are often not something you
|
375
|
+
can recover from at runtime and often require manual changes in Anaplan, i.e. updating the
|
376
|
+
mapping of an Import or similar.
|
297
377
|
:param action_id: The identifier of the Action to run. Can be any Anaplan Invokable;
|
298
|
-
|
378
|
+
Processes, Imports, Exports, Other Actions.
|
379
|
+
:param wait_for_completion: If True, the method will poll the task status and not return
|
380
|
+
until the task is complete. If False, it will spawn the task and return immediately.
|
299
381
|
"""
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
time.sleep(self.status_poll_delay)
|
305
|
-
task_status = self.get_task_status(action_id, task_id)
|
382
|
+
body = {"localeName": "en_US"}
|
383
|
+
res = self._http.post(f"{self._url}/{action_url(action_id)}/{action_id}/tasks", json=body)
|
384
|
+
task_id = res["task"]["taskId"]
|
385
|
+
logger.info(f"Invoked Action '{action_id}', spawned Task: '{task_id}'.")
|
306
386
|
|
307
|
-
if
|
387
|
+
if not wait_for_completion:
|
388
|
+
return TaskStatus.model_validate(self.get_task_status(action_id, task_id))
|
389
|
+
status = self._http.poll_task(self.get_task_status, action_id, task_id)
|
390
|
+
if status.task_state == "COMPLETE" and not status.result.successful:
|
391
|
+
logger.error(f"Task '{task_id}' completed with errors.")
|
308
392
|
raise AnaplanActionError(f"Task '{task_id}' completed with errors.")
|
309
393
|
|
310
|
-
logger.info(f"Task '{task_id}' completed successfully.")
|
311
|
-
return
|
394
|
+
logger.info(f"Task '{task_id}' of Action '{action_id}' completed successfully.")
|
395
|
+
return status
|
312
396
|
|
313
397
|
def get_file(self, file_id: int) -> bytes:
|
314
398
|
"""
|
@@ -317,43 +401,53 @@ class Client(_BaseClient):
|
|
317
401
|
:return: The content of the file.
|
318
402
|
"""
|
319
403
|
chunk_count = self._file_pre_check(file_id)
|
320
|
-
if chunk_count <= 1:
|
321
|
-
return self._get_binary(f"{self._url}/files/{file_id}")
|
322
404
|
logger.info(f"File {file_id} has {chunk_count} chunks.")
|
405
|
+
if chunk_count <= 1:
|
406
|
+
return self._http.get_binary(f"{self._url}/files/{file_id}")
|
323
407
|
with ThreadPoolExecutor(max_workers=self._thread_count) as executor:
|
324
408
|
chunks = executor.map(
|
325
|
-
self.
|
326
|
-
|
409
|
+
self._http.get_binary,
|
410
|
+
(f"{self._url}/files/{file_id}/chunks/{i}" for i in range(chunk_count)),
|
327
411
|
)
|
328
412
|
return b"".join(chunks)
|
329
413
|
|
330
|
-
def get_file_stream(self, file_id: int) -> Iterator[bytes]:
|
414
|
+
def get_file_stream(self, file_id: int, batch_size: int = 1) -> Iterator[bytes]:
|
331
415
|
"""
|
332
416
|
Retrieves the content of the specified file as a stream of chunks. The chunks are yielded
|
333
417
|
one by one, so you can process them as they arrive. This is useful for large files where
|
334
418
|
you don't want to or cannot load the entire file into memory at once.
|
335
419
|
:param file_id: The identifier of the file to retrieve.
|
420
|
+
:param batch_size: Number of chunks to fetch concurrently. If > 1, n chunks will be fetched
|
421
|
+
concurrently. This still yields each chunk individually, only the requests are
|
422
|
+
batched. If 1 (default), each chunk is fetched sequentially.
|
336
423
|
:return: A generator yielding the chunks of the file.
|
337
424
|
"""
|
338
425
|
chunk_count = self._file_pre_check(file_id)
|
426
|
+
logger.info(f"File {file_id} has {chunk_count} chunks.")
|
339
427
|
if chunk_count <= 1:
|
340
|
-
yield self.
|
428
|
+
yield self._http.get_binary(f"{self._url}/files/{file_id}")
|
341
429
|
return
|
342
|
-
|
343
|
-
|
344
|
-
|
430
|
+
|
431
|
+
with ThreadPoolExecutor(max_workers=batch_size) as executor:
|
432
|
+
for batch_start in range(0, chunk_count, batch_size):
|
433
|
+
batch_chunks = executor.map(
|
434
|
+
self._http.get_binary,
|
435
|
+
(
|
436
|
+
f"{self._url}/files/{file_id}/chunks/{i}"
|
437
|
+
for i in range(batch_start, min(batch_start + batch_size, chunk_count))
|
438
|
+
),
|
439
|
+
)
|
440
|
+
for chunk in batch_chunks:
|
441
|
+
yield chunk
|
345
442
|
|
346
443
|
def upload_file(self, file_id: int, content: str | bytes) -> None:
|
347
444
|
"""
|
348
|
-
Uploads the content to the specified file. If
|
349
|
-
|
350
|
-
better performance. If you are network bound or are experiencing rate limiting issues, set
|
351
|
-
`upload_parallel` to False.
|
445
|
+
Uploads the content to the specified file. If there are several chunks, upload of
|
446
|
+
individual chunks are uploaded concurrently.
|
352
447
|
|
353
448
|
:param file_id: The identifier of the file to upload to.
|
354
449
|
:param content: The content to upload. **This Content will be compressed before uploading.
|
355
|
-
|
356
|
-
redundant work.**
|
450
|
+
If you are passing the Input as bytes, pass it uncompressed.**
|
357
451
|
"""
|
358
452
|
if isinstance(content, str):
|
359
453
|
content = content.encode()
|
@@ -361,7 +455,7 @@ class Client(_BaseClient):
|
|
361
455
|
content[i : i + self.upload_chunk_size]
|
362
456
|
for i in range(0, len(content), self.upload_chunk_size)
|
363
457
|
]
|
364
|
-
logger.info(f"Content will be uploaded in {len(chunks)} chunks.")
|
458
|
+
logger.info(f"Content for file '{file_id}' will be uploaded in {len(chunks)} chunks.")
|
365
459
|
self._set_chunk_count(file_id, len(chunks))
|
366
460
|
if self.upload_parallel:
|
367
461
|
with ThreadPoolExecutor(max_workers=self._thread_count) as executor:
|
@@ -371,36 +465,66 @@ class Client(_BaseClient):
|
|
371
465
|
else:
|
372
466
|
for index, chunk in enumerate(chunks):
|
373
467
|
self._upload_chunk(file_id, index, chunk)
|
468
|
+
logger.info(f"Completed upload for file '{file_id}'.")
|
374
469
|
|
375
|
-
def upload_file_stream(
|
470
|
+
def upload_file_stream(
|
471
|
+
self, file_id: int, content: Iterator[str | bytes], batch_size: int = 1
|
472
|
+
) -> None:
|
376
473
|
"""
|
377
474
|
Uploads the content to the specified file as a stream of chunks. This is useful either for
|
378
475
|
large files where you don't want to or cannot load the entire file into memory at once, or
|
379
476
|
if you simply do not know the number of chunks ahead of time and instead just want to pass
|
380
477
|
on chunks i.e. consumed from a queue until it is exhausted. In this case, you can pass a
|
381
478
|
generator that yields the chunks of the file one by one to this method.
|
479
|
+
|
382
480
|
:param file_id: The identifier of the file to upload to.
|
383
|
-
:param content: An Iterator yielding the chunks of the file.
|
481
|
+
:param content: An Iterator or AsyncIterator yielding the chunks of the file. You can pass
|
482
|
+
any Iterator, but you will most likely want to pass a Generator.
|
483
|
+
:param batch_size: Number of chunks to upload concurrently. If > 1, n chunks will be
|
484
|
+
uploaded concurrently. This can be useful if you either do not control the chunk
|
485
|
+
size, or if you want to keep the chunk size small but still want some concurrency.
|
384
486
|
"""
|
487
|
+
logger.info(f"Starting upload stream for file '{file_id}' with batch size {batch_size}.")
|
385
488
|
self._set_chunk_count(file_id, -1)
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
489
|
+
indices, chunks = [], []
|
490
|
+
with ThreadPoolExecutor(max_workers=batch_size) as executor:
|
491
|
+
for index, chunk in enumerate(content):
|
492
|
+
indices.append(index)
|
493
|
+
chunks.append(chunk)
|
494
|
+
if len(indices) == max(batch_size, 1):
|
495
|
+
list(
|
496
|
+
executor.map(self._upload_chunk, (file_id,) * len(indices), indices, chunks)
|
497
|
+
)
|
498
|
+
logger.info(
|
499
|
+
f"Completed upload stream batch of size {batch_size} for file {file_id}."
|
500
|
+
)
|
501
|
+
indices, chunks = [], []
|
502
|
+
|
503
|
+
if indices:
|
504
|
+
executor.map(self._upload_chunk, (file_id,) * len(indices), indices, chunks)
|
505
|
+
logger.info(
|
506
|
+
f"Completed final upload stream batch of size {len(indices)} for file {file_id}."
|
507
|
+
)
|
508
|
+
self._http.post(f"{self._url}/files/{file_id}/complete", json={"id": file_id})
|
509
|
+
logger.info(f"Completed upload stream for '{file_id}'.")
|
391
510
|
|
392
|
-
def upload_and_import(
|
511
|
+
def upload_and_import(
|
512
|
+
self, file_id: int, content: str | bytes, action_id: int, wait_for_completion: bool = True
|
513
|
+
) -> TaskStatus:
|
393
514
|
"""
|
394
515
|
Convenience wrapper around `upload_file()` and `run_action()` to upload content to a file
|
395
516
|
and run an import action in one call.
|
396
517
|
:param file_id: The identifier of the file to upload to.
|
397
518
|
:param content: The content to upload. **This Content will be compressed before uploading.
|
398
|
-
|
399
|
-
|
519
|
+
If you are passing the Input as bytes, pass it uncompressed to avoid redundant
|
520
|
+
work.**
|
400
521
|
:param action_id: The identifier of the action to run after uploading the content.
|
522
|
+
:param wait_for_completion: If True, the method will poll the import task status and not
|
523
|
+
return until the task is complete. If False, it will spawn the import task and
|
524
|
+
return immediately.
|
401
525
|
"""
|
402
526
|
self.upload_file(file_id, content)
|
403
|
-
self.run_action(action_id)
|
527
|
+
return self.run_action(action_id, wait_for_completion)
|
404
528
|
|
405
529
|
def export_and_download(self, action_id: int) -> bytes:
|
406
530
|
"""
|
@@ -412,7 +536,7 @@ class Client(_BaseClient):
|
|
412
536
|
self.run_action(action_id)
|
413
537
|
return self.get_file(action_id)
|
414
538
|
|
415
|
-
def
|
539
|
+
def get_task_summaries(self, action_id: int) -> list[TaskSummary]:
|
416
540
|
"""
|
417
541
|
Retrieves the status of all tasks spawned by the specified action.
|
418
542
|
:param action_id: The identifier of the action that was invoked.
|
@@ -420,7 +544,7 @@ class Client(_BaseClient):
|
|
420
544
|
"""
|
421
545
|
return [
|
422
546
|
TaskSummary.model_validate(e)
|
423
|
-
for e in self.
|
547
|
+
for e in self._http.get_paginated(
|
424
548
|
f"{self._url}/{action_url(action_id)}/{action_id}/tasks", "tasks"
|
425
549
|
)
|
426
550
|
]
|
@@ -433,7 +557,7 @@ class Client(_BaseClient):
|
|
433
557
|
:return: The status of the task.
|
434
558
|
"""
|
435
559
|
return TaskStatus.model_validate(
|
436
|
-
self.
|
560
|
+
self._http.get(f"{self._url}/{action_url(action_id)}/{action_id}/tasks/{task_id}").get(
|
437
561
|
"task"
|
438
562
|
)
|
439
563
|
)
|
@@ -445,45 +569,29 @@ class Client(_BaseClient):
|
|
445
569
|
:param task_id: The Task identifier, sometimes also referred to as the Correlation Id.
|
446
570
|
:return: The content of the solution logs.
|
447
571
|
"""
|
448
|
-
return self.
|
572
|
+
return self._http.get_binary(
|
449
573
|
f"{self._url}/optimizeActions/{action_id}/tasks/{task_id}/solutionLogs"
|
450
574
|
)
|
451
575
|
|
452
|
-
def invoke_action(self, action_id: int) -> str:
|
453
|
-
"""
|
454
|
-
You may want to consider using `run_action()` instead.
|
455
|
-
|
456
|
-
Invokes the specified Anaplan Action and returns the spawned Task identifier. This is
|
457
|
-
useful if you want to handle the Task status yourself or if you want to run multiple
|
458
|
-
Actions in parallel.
|
459
|
-
:param action_id: The identifier of the Action to run. Can be any Anaplan Invokable.
|
460
|
-
:return: The identifier of the spawned Task.
|
461
|
-
"""
|
462
|
-
response = self._post(
|
463
|
-
f"{self._url}/{action_url(action_id)}/{action_id}/tasks", json={"localeName": "en_US"}
|
464
|
-
)
|
465
|
-
task_id = response.get("task").get("taskId")
|
466
|
-
logger.info(f"Invoked Action '{action_id}', spawned Task: '{task_id}'.")
|
467
|
-
return task_id
|
468
|
-
|
469
576
|
def _file_pre_check(self, file_id: int) -> int:
|
470
|
-
file = next(
|
577
|
+
file = next((f for f in self.get_files() if f.id == file_id), None)
|
471
578
|
if not file:
|
472
579
|
raise InvalidIdentifierException(f"File {file_id} not found.")
|
473
580
|
return file.chunk_count
|
474
581
|
|
475
|
-
def _upload_chunk(self, file_id: int, index: int, chunk: bytes) -> None:
|
476
|
-
self.
|
477
|
-
logger.
|
582
|
+
def _upload_chunk(self, file_id: int, index: int, chunk: str | bytes) -> None:
|
583
|
+
self._http.put_binary_gzip(f"{self._url}/files/{file_id}/chunks/{index}", chunk)
|
584
|
+
logger.debug(f"Chunk {index} loaded to file '{file_id}'.")
|
478
585
|
|
479
586
|
def _set_chunk_count(self, file_id: int, num_chunks: int) -> None:
|
587
|
+
logger.debug(f"Setting chunk count for file '{file_id}' to {num_chunks}.")
|
480
588
|
if not self.allow_file_creation and not (113000000000 <= file_id <= 113999999999):
|
481
589
|
raise InvalidIdentifierException(
|
482
590
|
f"File with Id {file_id} does not exist. If you want to dynamically create files "
|
483
591
|
"to avoid this error, set `allow_file_creation=True` on the calling instance. "
|
484
592
|
"Make sure you have understood the implications of this before doing so. "
|
485
593
|
)
|
486
|
-
response = self.
|
594
|
+
response = self._http.post(f"{self._url}/files/{file_id}", json={"chunkCount": num_chunks})
|
487
595
|
optionally_new_file = int(response.get("file").get("id"))
|
488
596
|
if optionally_new_file != file_id:
|
489
597
|
if self.allow_file_creation:
|