cognite-toolkit 0.6.84__py3-none-any.whl → 0.6.85__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-toolkit might be problematic. Click here for more details.
- cognite_toolkit/_cdf_tk/apps/_purge.py +84 -4
- cognite_toolkit/_cdf_tk/commands/_purge.py +171 -348
- cognite_toolkit/_cdf_tk/utils/aggregators.py +15 -4
- cognite_toolkit/_cdf_tk/utils/validate_access.py +205 -43
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.6.84.dist-info → cognite_toolkit-0.6.85.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.6.84.dist-info → cognite_toolkit-0.6.85.dist-info}/RECORD +13 -13
- {cognite_toolkit-0.6.84.dist-info → cognite_toolkit-0.6.85.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.6.84.dist-info → cognite_toolkit-0.6.85.dist-info}/entry_points.txt +0 -0
- {cognite_toolkit-0.6.84.dist-info → cognite_toolkit-0.6.85.dist-info}/licenses/LICENSE +0 -0
|
@@ -3,11 +3,10 @@ from abc import ABC, abstractmethod
|
|
|
3
3
|
from collections.abc import Callable, Hashable, Iterable, Sequence
|
|
4
4
|
from dataclasses import dataclass
|
|
5
5
|
from functools import partial
|
|
6
|
-
from graphlib import TopologicalSorter
|
|
7
6
|
from typing import Literal, cast
|
|
8
7
|
|
|
9
8
|
import questionary
|
|
10
|
-
from cognite.client.data_classes import
|
|
9
|
+
from cognite.client.data_classes import DataSetUpdate
|
|
11
10
|
from cognite.client.data_classes._base import CogniteResourceList
|
|
12
11
|
from cognite.client.data_classes.data_modeling import (
|
|
13
12
|
EdgeList,
|
|
@@ -20,45 +19,49 @@ from cognite.client.utils._identifier import InstanceId
|
|
|
20
19
|
from rich import print
|
|
21
20
|
from rich.console import Console
|
|
22
21
|
from rich.panel import Panel
|
|
23
|
-
from rich.status import Status
|
|
24
22
|
|
|
25
23
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
26
24
|
from cognite_toolkit._cdf_tk.cruds import (
|
|
27
|
-
RESOURCE_CRUD_LIST,
|
|
28
25
|
AssetCRUD,
|
|
29
26
|
ContainerCRUD,
|
|
30
27
|
DataModelCRUD,
|
|
31
|
-
DataSetsCRUD,
|
|
32
28
|
EdgeCRUD,
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
HostedExtractorDestinationCRUD,
|
|
38
|
-
InfieldV1CRUD,
|
|
39
|
-
LocationFilterCRUD,
|
|
29
|
+
EventCRUD,
|
|
30
|
+
ExtractionPipelineCRUD,
|
|
31
|
+
FileMetadataCRUD,
|
|
32
|
+
LabelCRUD,
|
|
40
33
|
NodeCRUD,
|
|
34
|
+
RelationshipCRUD,
|
|
41
35
|
ResourceCRUD,
|
|
36
|
+
SequenceCRUD,
|
|
42
37
|
SpaceCRUD,
|
|
43
|
-
|
|
38
|
+
ThreeDModelCRUD,
|
|
39
|
+
TimeSeriesCRUD,
|
|
40
|
+
TransformationCRUD,
|
|
44
41
|
ViewCRUD,
|
|
42
|
+
WorkflowCRUD,
|
|
45
43
|
)
|
|
46
44
|
from cognite_toolkit._cdf_tk.data_classes import DeployResults, ResourceDeployResult
|
|
47
45
|
from cognite_toolkit._cdf_tk.exceptions import (
|
|
48
46
|
AuthorizationError,
|
|
49
|
-
CDFAPIError,
|
|
50
47
|
ToolkitMissingResourceError,
|
|
51
|
-
ToolkitRequiredValueError,
|
|
52
|
-
ToolkitValueError,
|
|
53
48
|
)
|
|
54
49
|
from cognite_toolkit._cdf_tk.storageio import InstanceIO
|
|
55
50
|
from cognite_toolkit._cdf_tk.storageio.selectors import InstanceSelector
|
|
56
51
|
from cognite_toolkit._cdf_tk.tk_warnings import (
|
|
57
52
|
HighSeverityWarning,
|
|
58
53
|
LimitedAccessWarning,
|
|
59
|
-
MediumSeverityWarning,
|
|
60
54
|
)
|
|
61
55
|
from cognite_toolkit._cdf_tk.utils import humanize_collection
|
|
56
|
+
from cognite_toolkit._cdf_tk.utils.aggregators import (
|
|
57
|
+
AssetAggregator,
|
|
58
|
+
EventAggregator,
|
|
59
|
+
FileAggregator,
|
|
60
|
+
LabelCountAggregator,
|
|
61
|
+
RelationshipAggregator,
|
|
62
|
+
SequenceAggregator,
|
|
63
|
+
TimeSeriesAggregator,
|
|
64
|
+
)
|
|
62
65
|
from cognite_toolkit._cdf_tk.utils.http_client import (
|
|
63
66
|
FailedRequestItems,
|
|
64
67
|
FailedResponseItems,
|
|
@@ -107,6 +110,9 @@ class ToDelete(ABC):
|
|
|
107
110
|
) -> Callable[[CogniteResourceList], list[JsonVal]]:
|
|
108
111
|
raise NotImplementedError()
|
|
109
112
|
|
|
113
|
+
def get_extra_fields(self) -> dict[str, JsonVal]:
|
|
114
|
+
return {}
|
|
115
|
+
|
|
110
116
|
|
|
111
117
|
@dataclass
|
|
112
118
|
class DataModelingToDelete(ToDelete):
|
|
@@ -165,6 +171,36 @@ class NodesToDelete(ToDelete):
|
|
|
165
171
|
return check_for_data
|
|
166
172
|
|
|
167
173
|
|
|
174
|
+
@dataclass
|
|
175
|
+
class IdResourceToDelete(ToDelete):
|
|
176
|
+
def get_process_function(
|
|
177
|
+
self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
|
|
178
|
+
) -> Callable[[CogniteResourceList], list[JsonVal]]:
|
|
179
|
+
def as_id(chunk: CogniteResourceList) -> list[JsonVal]:
|
|
180
|
+
return [{"id": item.id} for item in chunk]
|
|
181
|
+
|
|
182
|
+
return as_id
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@dataclass
|
|
186
|
+
class ExternalIdToDelete(ToDelete):
|
|
187
|
+
def get_process_function(
|
|
188
|
+
self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
|
|
189
|
+
) -> Callable[[CogniteResourceList], list[JsonVal]]:
|
|
190
|
+
def as_external_id(chunk: CogniteResourceList) -> list[JsonVal]:
|
|
191
|
+
return [{"externalId": item.external_id} for item in chunk]
|
|
192
|
+
|
|
193
|
+
return as_external_id
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@dataclass
|
|
197
|
+
class AssetToDelete(IdResourceToDelete):
|
|
198
|
+
recursive: bool
|
|
199
|
+
|
|
200
|
+
def get_extra_fields(self) -> dict[str, JsonVal]:
|
|
201
|
+
return {"recursive": self.recursive}
|
|
202
|
+
|
|
203
|
+
|
|
168
204
|
class PurgeCommand(ToolkitCommand):
|
|
169
205
|
BATCH_SIZE_DM = 1000
|
|
170
206
|
|
|
@@ -218,8 +254,9 @@ class PurgeCommand(ToolkitCommand):
|
|
|
218
254
|
print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
|
|
219
255
|
return results
|
|
220
256
|
|
|
257
|
+
@staticmethod
|
|
221
258
|
def _create_to_delete_list_purge_space(
|
|
222
|
-
|
|
259
|
+
client: ToolkitClient, delete_datapoints: bool, delete_file_content: bool, stats: SpaceStatistics
|
|
223
260
|
) -> list[ToDelete]:
|
|
224
261
|
config = client.config
|
|
225
262
|
to_delete = [
|
|
@@ -289,7 +326,7 @@ class PurgeCommand(ToolkitCommand):
|
|
|
289
326
|
item.crud, space, data_set_external_id, batch_size=self.BATCH_SIZE_DM
|
|
290
327
|
),
|
|
291
328
|
process=item.get_process_function(client, console, verbose, process_results),
|
|
292
|
-
write=self._purge_batch(item
|
|
329
|
+
write=self._purge_batch(item, item.delete_url, delete_client, write_results),
|
|
293
330
|
max_queue_size=10,
|
|
294
331
|
iteration_count=iteration_count,
|
|
295
332
|
download_description=f"Downloading {item.display_name}",
|
|
@@ -323,14 +360,24 @@ class PurgeCommand(ToolkitCommand):
|
|
|
323
360
|
|
|
324
361
|
@staticmethod
|
|
325
362
|
def _purge_batch(
|
|
326
|
-
|
|
363
|
+
delete_item: ToDelete, delete_url: str, delete_client: HTTPClient, result: ResourceDeployResult
|
|
327
364
|
) -> Callable[[list[JsonVal]], None]:
|
|
365
|
+
crud = delete_item.crud
|
|
366
|
+
|
|
367
|
+
def as_id(item: JsonVal) -> Hashable:
|
|
368
|
+
try:
|
|
369
|
+
return crud.get_id(item)
|
|
370
|
+
except KeyError:
|
|
371
|
+
# Fallback to internal ID
|
|
372
|
+
return crud.get_internal_id(item)
|
|
373
|
+
|
|
328
374
|
def process(items: list[JsonVal]) -> None:
|
|
329
375
|
responses = delete_client.request_with_retries(
|
|
330
376
|
ItemsRequest(
|
|
331
377
|
endpoint_url=delete_url,
|
|
332
378
|
method="POST",
|
|
333
|
-
items=[DeleteItem(item=item, as_id_fun=
|
|
379
|
+
items=[DeleteItem(item=item, as_id_fun=as_id) for item in items],
|
|
380
|
+
extra_body_fields=delete_item.get_extra_fields(),
|
|
334
381
|
)
|
|
335
382
|
)
|
|
336
383
|
for response in responses:
|
|
@@ -341,27 +388,35 @@ class PurgeCommand(ToolkitCommand):
|
|
|
341
388
|
|
|
342
389
|
return process
|
|
343
390
|
|
|
344
|
-
|
|
345
|
-
def _get_dependencies(
|
|
346
|
-
loader_cls: type[ResourceCRUD], exclude: set[type[ResourceCRUD]] | None = None
|
|
347
|
-
) -> dict[type[ResourceCRUD], frozenset[type[ResourceCRUD]]]:
|
|
348
|
-
return {
|
|
349
|
-
dep_cls: dep_cls.dependencies
|
|
350
|
-
for dep_cls in RESOURCE_CRUD_LIST
|
|
351
|
-
if loader_cls in dep_cls.dependencies and (exclude is None or dep_cls not in exclude)
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
def dataset_v2(
|
|
391
|
+
def dataset(
|
|
355
392
|
self,
|
|
356
393
|
client: ToolkitClient,
|
|
357
394
|
selected_data_set_external_id: str,
|
|
358
|
-
|
|
395
|
+
archive_dataset: bool = False,
|
|
359
396
|
include_data: bool = True,
|
|
360
397
|
include_configurations: bool = False,
|
|
398
|
+
asset_recursive: bool = False,
|
|
361
399
|
dry_run: bool = False,
|
|
362
400
|
auto_yes: bool = False,
|
|
363
401
|
verbose: bool = False,
|
|
364
402
|
) -> DeployResults:
|
|
403
|
+
"""Purge a dataset and all its content
|
|
404
|
+
|
|
405
|
+
Args:
|
|
406
|
+
client: The ToolkitClient to use
|
|
407
|
+
selected_data_set_external_id: The external ID of the dataset to purge
|
|
408
|
+
archive_dataset: Whether to archive the dataset itself after the purge
|
|
409
|
+
include_data: Whether to include data (assets, events, time series, files, sequences, 3D models, relationships, labels) in the purge
|
|
410
|
+
include_configurations: Whether to include configurations (workflows, transformations, extraction pipelines) in the purge
|
|
411
|
+
asset_recursive: Whether to recursively delete assets.
|
|
412
|
+
dry_run: Whether to perform a dry run
|
|
413
|
+
auto_yes: Whether to automatically confirm the purge
|
|
414
|
+
verbose: Whether to print verbose output
|
|
415
|
+
|
|
416
|
+
Returns:
|
|
417
|
+
DeployResults: The results of the purge operation
|
|
418
|
+
|
|
419
|
+
"""
|
|
365
420
|
# Warning Messages
|
|
366
421
|
if not dry_run:
|
|
367
422
|
self._print_panel("dataSet", selected_data_set_external_id)
|
|
@@ -389,6 +444,8 @@ class PurgeCommand(ToolkitCommand):
|
|
|
389
444
|
client,
|
|
390
445
|
include_data,
|
|
391
446
|
include_configurations,
|
|
447
|
+
selected_data_set_external_id,
|
|
448
|
+
asset_recursive,
|
|
392
449
|
)
|
|
393
450
|
if dry_run:
|
|
394
451
|
results = DeployResults([], "purge", dry_run=True)
|
|
@@ -397,11 +454,12 @@ class PurgeCommand(ToolkitCommand):
|
|
|
397
454
|
else:
|
|
398
455
|
results = self._delete_resources(to_delete, client, verbose, None, selected_data_set_external_id)
|
|
399
456
|
print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
|
|
400
|
-
if
|
|
457
|
+
if archive_dataset and not dry_run:
|
|
401
458
|
self._archive_dataset(client, selected_data_set_external_id)
|
|
402
459
|
return results
|
|
403
460
|
|
|
404
|
-
|
|
461
|
+
@staticmethod
|
|
462
|
+
def _archive_dataset(client: ToolkitClient, data_set: str) -> None:
|
|
405
463
|
archived = (
|
|
406
464
|
DataSetUpdate(external_id=data_set)
|
|
407
465
|
.external_id.set(str(uuid.uuid4()))
|
|
@@ -411,77 +469,92 @@ class PurgeCommand(ToolkitCommand):
|
|
|
411
469
|
client.data_sets.update(archived)
|
|
412
470
|
print(f"DataSet {data_set} archived")
|
|
413
471
|
|
|
472
|
+
@staticmethod
|
|
414
473
|
def _create_to_delete_list_purge_dataset(
|
|
415
|
-
self,
|
|
416
474
|
client: ToolkitClient,
|
|
417
475
|
include_data: bool,
|
|
418
476
|
include_configurations: bool,
|
|
477
|
+
data_set_external_id: str,
|
|
478
|
+
asset_recursive: bool,
|
|
419
479
|
) -> list[ToDelete]:
|
|
420
|
-
|
|
480
|
+
config = client.config
|
|
481
|
+
to_delete: list[ToDelete] = []
|
|
421
482
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
.external_id.set(str(uuid.uuid4()))
|
|
469
|
-
.metadata.add({"archived": "true"})
|
|
470
|
-
.write_protected.set(True)
|
|
471
|
-
)
|
|
472
|
-
client.data_sets.update(archived)
|
|
473
|
-
print(f"DataSet {selected_dataset} archived")
|
|
474
|
-
elif include_dataset:
|
|
475
|
-
self.warn(
|
|
476
|
-
HighSeverityWarning(f"DataSet {selected_dataset} was not archived due to errors during the purge")
|
|
483
|
+
if include_data:
|
|
484
|
+
three_d_crud = ThreeDModelCRUD.create_loader(client)
|
|
485
|
+
to_delete.extend(
|
|
486
|
+
[
|
|
487
|
+
ExternalIdToDelete(
|
|
488
|
+
RelationshipCRUD.create_loader(client),
|
|
489
|
+
RelationshipAggregator(client).count(data_set_external_id=data_set_external_id),
|
|
490
|
+
config.create_api_url("/relationships/delete"),
|
|
491
|
+
),
|
|
492
|
+
IdResourceToDelete(
|
|
493
|
+
EventCRUD.create_loader(client),
|
|
494
|
+
EventAggregator(client).count(data_set_external_id=data_set_external_id),
|
|
495
|
+
config.create_api_url("/events/delete"),
|
|
496
|
+
),
|
|
497
|
+
IdResourceToDelete(
|
|
498
|
+
FileMetadataCRUD.create_loader(client),
|
|
499
|
+
FileAggregator(client).count(data_set_external_id=data_set_external_id),
|
|
500
|
+
config.create_api_url("/files/delete"),
|
|
501
|
+
),
|
|
502
|
+
IdResourceToDelete(
|
|
503
|
+
TimeSeriesCRUD.create_loader(client),
|
|
504
|
+
TimeSeriesAggregator(client).count(data_set_external_id=data_set_external_id),
|
|
505
|
+
config.create_api_url("/timeseries/delete"),
|
|
506
|
+
),
|
|
507
|
+
IdResourceToDelete(
|
|
508
|
+
SequenceCRUD.create_loader(client),
|
|
509
|
+
SequenceAggregator(client).count(data_set_external_id=data_set_external_id),
|
|
510
|
+
config.create_api_url("/sequences/delete"),
|
|
511
|
+
),
|
|
512
|
+
IdResourceToDelete(
|
|
513
|
+
three_d_crud,
|
|
514
|
+
sum(1 for _ in three_d_crud.iterate(data_set_external_id=data_set_external_id)),
|
|
515
|
+
config.create_api_url("/3d/models/delete"),
|
|
516
|
+
),
|
|
517
|
+
AssetToDelete(
|
|
518
|
+
AssetCRUD.create_loader(client),
|
|
519
|
+
AssetAggregator(client).count(data_set_external_id=data_set_external_id),
|
|
520
|
+
config.create_api_url("/assets/delete"),
|
|
521
|
+
recursive=asset_recursive,
|
|
522
|
+
),
|
|
523
|
+
ExternalIdToDelete(
|
|
524
|
+
LabelCRUD.create_loader(client),
|
|
525
|
+
LabelCountAggregator(client).count(data_set_external_id=data_set_external_id),
|
|
526
|
+
config.create_api_url("/labels/delete"),
|
|
527
|
+
),
|
|
528
|
+
]
|
|
477
529
|
)
|
|
530
|
+
if include_configurations:
|
|
531
|
+
transformation_crud = TransformationCRUD.create_loader(client)
|
|
532
|
+
workflow_crud = WorkflowCRUD.create_loader(client)
|
|
533
|
+
extraction_pipeline_crud = ExtractionPipelineCRUD.create_loader(client)
|
|
534
|
+
|
|
535
|
+
to_delete.extend(
|
|
536
|
+
[
|
|
537
|
+
IdResourceToDelete(
|
|
538
|
+
transformation_crud,
|
|
539
|
+
sum(1 for _ in transformation_crud.iterate(data_set_external_id=data_set_external_id)),
|
|
540
|
+
config.create_api_url("/transformations/delete"),
|
|
541
|
+
),
|
|
542
|
+
ExternalIdToDelete(
|
|
543
|
+
workflow_crud,
|
|
544
|
+
sum(1 for _ in workflow_crud.iterate(data_set_external_id=data_set_external_id)),
|
|
545
|
+
config.create_api_url("/workflows/delete"),
|
|
546
|
+
),
|
|
547
|
+
IdResourceToDelete(
|
|
548
|
+
extraction_pipeline_crud,
|
|
549
|
+
sum(1 for _ in extraction_pipeline_crud.iterate(data_set_external_id=data_set_external_id)),
|
|
550
|
+
config.create_api_url("/extpipes/delete"),
|
|
551
|
+
),
|
|
552
|
+
]
|
|
553
|
+
)
|
|
554
|
+
return to_delete
|
|
478
555
|
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
elif not dry_run:
|
|
482
|
-
print(f"Purged dataset {selected_dataset!r} partly completed. See warnings for details.")
|
|
483
|
-
|
|
484
|
-
def _print_panel(self, resource_type: str, resource: str) -> None:
|
|
556
|
+
@staticmethod
|
|
557
|
+
def _print_panel(resource_type: str, resource: str) -> None:
|
|
485
558
|
print(
|
|
486
559
|
Panel(
|
|
487
560
|
f"[red]WARNING:[/red] This operation [bold]cannot be undone[/bold]! "
|
|
@@ -494,256 +567,6 @@ class PurgeCommand(ToolkitCommand):
|
|
|
494
567
|
)
|
|
495
568
|
)
|
|
496
569
|
|
|
497
|
-
@staticmethod
|
|
498
|
-
def _get_selected_dataset(external_id: str | None, client: ToolkitClient) -> str:
|
|
499
|
-
if external_id is None:
|
|
500
|
-
datasets = client.data_sets.list(limit=-1)
|
|
501
|
-
selected_dataset: str = questionary.select(
|
|
502
|
-
"Which space are you going to purge (delete all resources in dataset)?",
|
|
503
|
-
sorted([dataset.external_id for dataset in datasets if dataset.external_id]),
|
|
504
|
-
).ask()
|
|
505
|
-
else:
|
|
506
|
-
retrieved = client.data_sets.retrieve(external_id=external_id)
|
|
507
|
-
if retrieved is None:
|
|
508
|
-
raise ToolkitMissingResourceError(f"DataSet {external_id!r} does not exist")
|
|
509
|
-
selected_dataset = external_id
|
|
510
|
-
|
|
511
|
-
if selected_dataset is None:
|
|
512
|
-
raise ToolkitValueError("No space selected")
|
|
513
|
-
return selected_dataset
|
|
514
|
-
|
|
515
|
-
def _purge(
|
|
516
|
-
self,
|
|
517
|
-
client: ToolkitClient,
|
|
518
|
-
loaders: dict[type[ResourceCRUD], frozenset[type[ResourceCRUD]]],
|
|
519
|
-
selected_space: str | None = None,
|
|
520
|
-
selected_data_set: str | None = None,
|
|
521
|
-
dry_run: bool = False,
|
|
522
|
-
verbose: bool = False,
|
|
523
|
-
batch_size: int = 1000,
|
|
524
|
-
) -> bool:
|
|
525
|
-
is_purged = True
|
|
526
|
-
results = DeployResults([], "purge", dry_run=dry_run)
|
|
527
|
-
loader_cls: type[ResourceCRUD]
|
|
528
|
-
has_purged_views = False
|
|
529
|
-
with Console().status("...", spinner="aesthetic", speed=0.4) as status:
|
|
530
|
-
for loader_cls in reversed(list(TopologicalSorter(loaders).static_order())):
|
|
531
|
-
if loader_cls not in loaders:
|
|
532
|
-
# Dependency that is included
|
|
533
|
-
continue
|
|
534
|
-
loader = loader_cls.create_loader(client, console=status.console)
|
|
535
|
-
status_prefix = "Would have deleted" if dry_run else "Deleted"
|
|
536
|
-
if isinstance(loader, ViewCRUD) and not dry_run:
|
|
537
|
-
status_prefix = "Expected deleted" # Views are not always deleted immediately
|
|
538
|
-
has_purged_views = True
|
|
539
|
-
|
|
540
|
-
if not dry_run and isinstance(loader, AssetCRUD):
|
|
541
|
-
# Special handling of assets as we must ensure all children are deleted before the parent.
|
|
542
|
-
# In dry-run mode, we are not deleting the assets, so we can skip this.
|
|
543
|
-
deleted_assets = self._purge_assets(loader, status, selected_data_set)
|
|
544
|
-
results[loader.display_name] = ResourceDeployResult(
|
|
545
|
-
loader.display_name, deleted=deleted_assets, total=deleted_assets
|
|
546
|
-
)
|
|
547
|
-
continue
|
|
548
|
-
|
|
549
|
-
# Child loaders are, for example, WorkflowTriggerLoader, WorkflowVersionLoader for WorkflowLoader
|
|
550
|
-
# These must delete all resources that are connected to the resource that the loader is deleting
|
|
551
|
-
# Exclude loaders that we are already iterating over
|
|
552
|
-
child_loader_classes = self._get_dependencies(loader_cls, exclude=set(loaders))
|
|
553
|
-
child_loaders = [
|
|
554
|
-
child_loader.create_loader(client)
|
|
555
|
-
for child_loader in reversed(list(TopologicalSorter(child_loader_classes).static_order()))
|
|
556
|
-
# Necessary as the topological sort includes dependencies that are not in the loaders
|
|
557
|
-
if child_loader in child_loader_classes
|
|
558
|
-
]
|
|
559
|
-
count = 0
|
|
560
|
-
status.update(f"{status_prefix} {count:,} {loader.display_name}...")
|
|
561
|
-
batch_ids: list[Hashable] = []
|
|
562
|
-
for resource in loader.iterate(data_set_external_id=selected_data_set, space=selected_space):
|
|
563
|
-
try:
|
|
564
|
-
batch_ids.append(loader.get_id(resource))
|
|
565
|
-
except (ToolkitRequiredValueError, KeyError) as e:
|
|
566
|
-
try:
|
|
567
|
-
batch_ids.append(loader.get_internal_id(resource))
|
|
568
|
-
except (AttributeError, NotImplementedError):
|
|
569
|
-
self.warn(
|
|
570
|
-
HighSeverityWarning(
|
|
571
|
-
f"Cannot delete {type(resource).__name__}. Failed to obtain ID: {e}"
|
|
572
|
-
),
|
|
573
|
-
console=status.console,
|
|
574
|
-
)
|
|
575
|
-
is_purged = False
|
|
576
|
-
continue
|
|
577
|
-
|
|
578
|
-
if len(batch_ids) >= batch_size:
|
|
579
|
-
child_deletion = self._delete_children(
|
|
580
|
-
batch_ids, child_loaders, dry_run, status.console, verbose
|
|
581
|
-
)
|
|
582
|
-
batch_delete, batch_size = self._delete_batch(
|
|
583
|
-
batch_ids, dry_run, loader, batch_size, status.console, verbose
|
|
584
|
-
)
|
|
585
|
-
count += batch_delete
|
|
586
|
-
status.update(f"{status_prefix} {count:,} {loader.display_name}...")
|
|
587
|
-
batch_ids = []
|
|
588
|
-
# The DeployResults is overloaded such that the below accumulates the counts
|
|
589
|
-
for name, child_count in child_deletion.items():
|
|
590
|
-
results[name] = ResourceDeployResult(name, deleted=child_count, total=child_count)
|
|
591
|
-
|
|
592
|
-
if batch_ids:
|
|
593
|
-
child_deletion = self._delete_children(batch_ids, child_loaders, dry_run, status.console, verbose)
|
|
594
|
-
batch_delete, batch_size = self._delete_batch(
|
|
595
|
-
batch_ids, dry_run, loader, batch_size, status.console, verbose
|
|
596
|
-
)
|
|
597
|
-
count += batch_delete
|
|
598
|
-
status.update(f"{status_prefix} {count:,} {loader.display_name}...")
|
|
599
|
-
for name, child_count in child_deletion.items():
|
|
600
|
-
results[name] = ResourceDeployResult(name, deleted=child_count, total=child_count)
|
|
601
|
-
if count > 0:
|
|
602
|
-
status.console.print(f"{status_prefix} {count:,} {loader.display_name}.")
|
|
603
|
-
results[loader.display_name] = ResourceDeployResult(
|
|
604
|
-
name=loader.display_name,
|
|
605
|
-
deleted=count,
|
|
606
|
-
total=count,
|
|
607
|
-
)
|
|
608
|
-
print(results.counts_table(exclude_columns={"Created", "Changed", "Untouched", "Total"}))
|
|
609
|
-
if has_purged_views:
|
|
610
|
-
print("You might need to run the purge command multiple times to delete all views.")
|
|
611
|
-
return is_purged
|
|
612
|
-
|
|
613
|
-
def _delete_batch(
|
|
614
|
-
self,
|
|
615
|
-
batch_ids: list[Hashable],
|
|
616
|
-
dry_run: bool,
|
|
617
|
-
loader: ResourceCRUD,
|
|
618
|
-
batch_size: int,
|
|
619
|
-
console: Console,
|
|
620
|
-
verbose: bool,
|
|
621
|
-
) -> tuple[int, int]:
|
|
622
|
-
if dry_run:
|
|
623
|
-
deleted = len(batch_ids)
|
|
624
|
-
else:
|
|
625
|
-
try:
|
|
626
|
-
deleted = loader.delete(batch_ids)
|
|
627
|
-
except CogniteAPIError as delete_error:
|
|
628
|
-
if (
|
|
629
|
-
delete_error.code == 408
|
|
630
|
-
and "timed out" in delete_error.message.casefold()
|
|
631
|
-
and batch_size > 1
|
|
632
|
-
and (len(batch_ids) > 1)
|
|
633
|
-
):
|
|
634
|
-
self.warn(
|
|
635
|
-
MediumSeverityWarning(
|
|
636
|
-
f"Timed out deleting {loader.display_name}. Trying again with a smaller batch size."
|
|
637
|
-
),
|
|
638
|
-
include_timestamp=True,
|
|
639
|
-
console=console,
|
|
640
|
-
)
|
|
641
|
-
new_batch_size = len(batch_ids) // 2
|
|
642
|
-
first = batch_ids[:new_batch_size]
|
|
643
|
-
second = batch_ids[new_batch_size:]
|
|
644
|
-
first_deleted, first_batch_size = self._delete_batch(
|
|
645
|
-
first, dry_run, loader, new_batch_size, console, verbose
|
|
646
|
-
)
|
|
647
|
-
second_deleted, second_batch_size = self._delete_batch(
|
|
648
|
-
second, dry_run, loader, new_batch_size, console, verbose
|
|
649
|
-
)
|
|
650
|
-
return first_deleted + second_deleted, min(first_batch_size, second_batch_size)
|
|
651
|
-
else:
|
|
652
|
-
raise delete_error
|
|
653
|
-
|
|
654
|
-
if verbose:
|
|
655
|
-
prefix = "Would delete" if dry_run else "Finished purging"
|
|
656
|
-
console.print(f"{prefix} {deleted:,} {loader.display_name}")
|
|
657
|
-
return deleted, batch_size
|
|
658
|
-
|
|
659
|
-
@staticmethod
|
|
660
|
-
def _delete_children(
|
|
661
|
-
parent_ids: list[Hashable], child_loaders: list[ResourceCRUD], dry_run: bool, console: Console, verbose: bool
|
|
662
|
-
) -> dict[str, int]:
|
|
663
|
-
child_deletion: dict[str, int] = {}
|
|
664
|
-
for child_loader in child_loaders:
|
|
665
|
-
child_ids = set()
|
|
666
|
-
for child in child_loader.iterate(parent_ids=parent_ids):
|
|
667
|
-
child_ids.add(child_loader.get_id(child))
|
|
668
|
-
count = 0
|
|
669
|
-
if child_ids:
|
|
670
|
-
if dry_run:
|
|
671
|
-
count = len(child_ids)
|
|
672
|
-
else:
|
|
673
|
-
count = child_loader.delete(list(child_ids))
|
|
674
|
-
|
|
675
|
-
if verbose:
|
|
676
|
-
prefix = "Would delete" if dry_run else "Deleted"
|
|
677
|
-
console.print(f"{prefix} {count:,} {child_loader.display_name}")
|
|
678
|
-
child_deletion[child_loader.display_name] = count
|
|
679
|
-
return child_deletion
|
|
680
|
-
|
|
681
|
-
def _purge_assets(
|
|
682
|
-
self,
|
|
683
|
-
loader: AssetCRUD,
|
|
684
|
-
status: Status,
|
|
685
|
-
selected_data_set: str | None = None,
|
|
686
|
-
batch_size: int = 1000,
|
|
687
|
-
) -> int:
|
|
688
|
-
# Using sets to avoid duplicates
|
|
689
|
-
children_ids: set[int] = set()
|
|
690
|
-
parent_ids: set[int] = set()
|
|
691
|
-
is_first = True
|
|
692
|
-
last_failed = False
|
|
693
|
-
count = level = depth = last_parent_count = 0
|
|
694
|
-
total_asset_count: int | None = None
|
|
695
|
-
# Iterate through the asset hierarchy once per depth level. This is to delete all children before the parent.
|
|
696
|
-
while is_first or level < depth:
|
|
697
|
-
for asset in loader.iterate(data_set_external_id=selected_data_set):
|
|
698
|
-
aggregates = cast(AggregateResultItem, asset.aggregates)
|
|
699
|
-
if is_first and aggregates.depth is not None:
|
|
700
|
-
depth = max(depth, aggregates.depth)
|
|
701
|
-
|
|
702
|
-
if aggregates.child_count == 0:
|
|
703
|
-
children_ids.add(asset.id)
|
|
704
|
-
else:
|
|
705
|
-
parent_ids.add(asset.id)
|
|
706
|
-
|
|
707
|
-
if len(children_ids) >= batch_size:
|
|
708
|
-
count += loader.delete(list(children_ids))
|
|
709
|
-
if total_asset_count:
|
|
710
|
-
status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
|
|
711
|
-
else:
|
|
712
|
-
status.update(f"Deleted {count:,} {loader.display_name}...")
|
|
713
|
-
children_ids = set()
|
|
714
|
-
|
|
715
|
-
if is_first:
|
|
716
|
-
total_asset_count = count + len(parent_ids) + len(children_ids)
|
|
717
|
-
if children_ids:
|
|
718
|
-
count += loader.delete(list(children_ids))
|
|
719
|
-
status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
|
|
720
|
-
children_ids = set()
|
|
721
|
-
|
|
722
|
-
if len(parent_ids) == last_parent_count and last_failed:
|
|
723
|
-
try:
|
|
724
|
-
# Just try to delete them all at once
|
|
725
|
-
count += loader.delete(list(parent_ids))
|
|
726
|
-
except CogniteAPIError as e:
|
|
727
|
-
raise CDFAPIError(
|
|
728
|
-
f"Failed to delete {len(parent_ids)} assets. This could be due to a parent-child cycle or an "
|
|
729
|
-
"eventual consistency issue. Wait a few seconds and try again. An alternative is to use the "
|
|
730
|
-
"Python-SDK to delete the asset hierarchy "
|
|
731
|
-
"`client.assets.delete(external_id='my_root_asset', recursive=True)`"
|
|
732
|
-
) from e
|
|
733
|
-
else:
|
|
734
|
-
status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
|
|
735
|
-
break
|
|
736
|
-
elif len(parent_ids) == last_parent_count:
|
|
737
|
-
last_failed = True
|
|
738
|
-
else:
|
|
739
|
-
last_failed = False
|
|
740
|
-
level += 1
|
|
741
|
-
last_parent_count = len(parent_ids)
|
|
742
|
-
parent_ids.clear()
|
|
743
|
-
is_first = False
|
|
744
|
-
status.console.print(f"Finished purging {loader.display_name}.")
|
|
745
|
-
return count
|
|
746
|
-
|
|
747
570
|
def instances(
|
|
748
571
|
self,
|
|
749
572
|
client: ToolkitClient,
|