cognite-toolkit 0.6.84__py3-none-any.whl → 0.6.86__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-toolkit might be problematic. Click here for more details.

@@ -3,11 +3,10 @@ from abc import ABC, abstractmethod
3
3
  from collections.abc import Callable, Hashable, Iterable, Sequence
4
4
  from dataclasses import dataclass
5
5
  from functools import partial
6
- from graphlib import TopologicalSorter
7
6
  from typing import Literal, cast
8
7
 
9
8
  import questionary
10
- from cognite.client.data_classes import AggregateResultItem, DataSetUpdate
9
+ from cognite.client.data_classes import DataSetUpdate
11
10
  from cognite.client.data_classes._base import CogniteResourceList
12
11
  from cognite.client.data_classes.data_modeling import (
13
12
  EdgeList,
@@ -20,45 +19,49 @@ from cognite.client.utils._identifier import InstanceId
20
19
  from rich import print
21
20
  from rich.console import Console
22
21
  from rich.panel import Panel
23
- from rich.status import Status
24
22
 
25
23
  from cognite_toolkit._cdf_tk.client import ToolkitClient
26
24
  from cognite_toolkit._cdf_tk.cruds import (
27
- RESOURCE_CRUD_LIST,
28
25
  AssetCRUD,
29
26
  ContainerCRUD,
30
27
  DataModelCRUD,
31
- DataSetsCRUD,
32
28
  EdgeCRUD,
33
- FunctionCRUD,
34
- GroupAllScopedCRUD,
35
- GroupCRUD,
36
- GroupResourceScopedCRUD,
37
- HostedExtractorDestinationCRUD,
38
- InfieldV1CRUD,
39
- LocationFilterCRUD,
29
+ EventCRUD,
30
+ ExtractionPipelineCRUD,
31
+ FileMetadataCRUD,
32
+ LabelCRUD,
40
33
  NodeCRUD,
34
+ RelationshipCRUD,
41
35
  ResourceCRUD,
36
+ SequenceCRUD,
42
37
  SpaceCRUD,
43
- StreamlitCRUD,
38
+ ThreeDModelCRUD,
39
+ TimeSeriesCRUD,
40
+ TransformationCRUD,
44
41
  ViewCRUD,
42
+ WorkflowCRUD,
45
43
  )
46
44
  from cognite_toolkit._cdf_tk.data_classes import DeployResults, ResourceDeployResult
47
45
  from cognite_toolkit._cdf_tk.exceptions import (
48
46
  AuthorizationError,
49
- CDFAPIError,
50
47
  ToolkitMissingResourceError,
51
- ToolkitRequiredValueError,
52
- ToolkitValueError,
53
48
  )
54
49
  from cognite_toolkit._cdf_tk.storageio import InstanceIO
55
50
  from cognite_toolkit._cdf_tk.storageio.selectors import InstanceSelector
56
51
  from cognite_toolkit._cdf_tk.tk_warnings import (
57
52
  HighSeverityWarning,
58
53
  LimitedAccessWarning,
59
- MediumSeverityWarning,
60
54
  )
61
55
  from cognite_toolkit._cdf_tk.utils import humanize_collection
56
+ from cognite_toolkit._cdf_tk.utils.aggregators import (
57
+ AssetAggregator,
58
+ EventAggregator,
59
+ FileAggregator,
60
+ LabelCountAggregator,
61
+ RelationshipAggregator,
62
+ SequenceAggregator,
63
+ TimeSeriesAggregator,
64
+ )
62
65
  from cognite_toolkit._cdf_tk.utils.http_client import (
63
66
  FailedRequestItems,
64
67
  FailedResponseItems,
@@ -107,6 +110,9 @@ class ToDelete(ABC):
107
110
  ) -> Callable[[CogniteResourceList], list[JsonVal]]:
108
111
  raise NotImplementedError()
109
112
 
113
+ def get_extra_fields(self) -> dict[str, JsonVal]:
114
+ return {}
115
+
110
116
 
111
117
  @dataclass
112
118
  class DataModelingToDelete(ToDelete):
@@ -165,6 +171,36 @@ class NodesToDelete(ToDelete):
165
171
  return check_for_data
166
172
 
167
173
 
174
+ @dataclass
175
+ class IdResourceToDelete(ToDelete):
176
+ def get_process_function(
177
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
178
+ ) -> Callable[[CogniteResourceList], list[JsonVal]]:
179
+ def as_id(chunk: CogniteResourceList) -> list[JsonVal]:
180
+ return [{"id": item.id} for item in chunk]
181
+
182
+ return as_id
183
+
184
+
185
+ @dataclass
186
+ class ExternalIdToDelete(ToDelete):
187
+ def get_process_function(
188
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
189
+ ) -> Callable[[CogniteResourceList], list[JsonVal]]:
190
+ def as_external_id(chunk: CogniteResourceList) -> list[JsonVal]:
191
+ return [{"externalId": item.external_id} for item in chunk]
192
+
193
+ return as_external_id
194
+
195
+
196
+ @dataclass
197
+ class AssetToDelete(IdResourceToDelete):
198
+ recursive: bool
199
+
200
+ def get_extra_fields(self) -> dict[str, JsonVal]:
201
+ return {"recursive": self.recursive}
202
+
203
+
168
204
  class PurgeCommand(ToolkitCommand):
169
205
  BATCH_SIZE_DM = 1000
170
206
 
@@ -218,8 +254,9 @@ class PurgeCommand(ToolkitCommand):
218
254
  print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
219
255
  return results
220
256
 
257
+ @staticmethod
221
258
  def _create_to_delete_list_purge_space(
222
- self, client: ToolkitClient, delete_datapoints: bool, delete_file_content: bool, stats: SpaceStatistics
259
+ client: ToolkitClient, delete_datapoints: bool, delete_file_content: bool, stats: SpaceStatistics
223
260
  ) -> list[ToDelete]:
224
261
  config = client.config
225
262
  to_delete = [
@@ -289,7 +326,7 @@ class PurgeCommand(ToolkitCommand):
289
326
  item.crud, space, data_set_external_id, batch_size=self.BATCH_SIZE_DM
290
327
  ),
291
328
  process=item.get_process_function(client, console, verbose, process_results),
292
- write=self._purge_batch(item.crud, item.delete_url, delete_client, write_results),
329
+ write=self._purge_batch(item, item.delete_url, delete_client, write_results),
293
330
  max_queue_size=10,
294
331
  iteration_count=iteration_count,
295
332
  download_description=f"Downloading {item.display_name}",
@@ -323,14 +360,24 @@ class PurgeCommand(ToolkitCommand):
323
360
 
324
361
  @staticmethod
325
362
  def _purge_batch(
326
- crud: ResourceCRUD, delete_url: str, delete_client: HTTPClient, result: ResourceDeployResult
363
+ delete_item: ToDelete, delete_url: str, delete_client: HTTPClient, result: ResourceDeployResult
327
364
  ) -> Callable[[list[JsonVal]], None]:
365
+ crud = delete_item.crud
366
+
367
+ def as_id(item: JsonVal) -> Hashable:
368
+ try:
369
+ return crud.get_id(item)
370
+ except KeyError:
371
+ # Fallback to internal ID
372
+ return crud.get_internal_id(item)
373
+
328
374
  def process(items: list[JsonVal]) -> None:
329
375
  responses = delete_client.request_with_retries(
330
376
  ItemsRequest(
331
377
  endpoint_url=delete_url,
332
378
  method="POST",
333
- items=[DeleteItem(item=item, as_id_fun=crud.get_id) for item in items],
379
+ items=[DeleteItem(item=item, as_id_fun=as_id) for item in items],
380
+ extra_body_fields=delete_item.get_extra_fields(),
334
381
  )
335
382
  )
336
383
  for response in responses:
@@ -341,27 +388,35 @@ class PurgeCommand(ToolkitCommand):
341
388
 
342
389
  return process
343
390
 
344
- @staticmethod
345
- def _get_dependencies(
346
- loader_cls: type[ResourceCRUD], exclude: set[type[ResourceCRUD]] | None = None
347
- ) -> dict[type[ResourceCRUD], frozenset[type[ResourceCRUD]]]:
348
- return {
349
- dep_cls: dep_cls.dependencies
350
- for dep_cls in RESOURCE_CRUD_LIST
351
- if loader_cls in dep_cls.dependencies and (exclude is None or dep_cls not in exclude)
352
- }
353
-
354
- def dataset_v2(
391
+ def dataset(
355
392
  self,
356
393
  client: ToolkitClient,
357
394
  selected_data_set_external_id: str,
358
- include_dataset: bool = False,
395
+ archive_dataset: bool = False,
359
396
  include_data: bool = True,
360
397
  include_configurations: bool = False,
398
+ asset_recursive: bool = False,
361
399
  dry_run: bool = False,
362
400
  auto_yes: bool = False,
363
401
  verbose: bool = False,
364
402
  ) -> DeployResults:
403
+ """Purge a dataset and all its content
404
+
405
+ Args:
406
+ client: The ToolkitClient to use
407
+ selected_data_set_external_id: The external ID of the dataset to purge
408
+ archive_dataset: Whether to archive the dataset itself after the purge
409
+ include_data: Whether to include data (assets, events, time series, files, sequences, 3D models, relationships, labels) in the purge
410
+ include_configurations: Whether to include configurations (workflows, transformations, extraction pipelines) in the purge
411
+ asset_recursive: Whether to recursively delete assets.
412
+ dry_run: Whether to perform a dry run
413
+ auto_yes: Whether to automatically confirm the purge
414
+ verbose: Whether to print verbose output
415
+
416
+ Returns:
417
+ DeployResults: The results of the purge operation
418
+
419
+ """
365
420
  # Warning Messages
366
421
  if not dry_run:
367
422
  self._print_panel("dataSet", selected_data_set_external_id)
@@ -389,6 +444,8 @@ class PurgeCommand(ToolkitCommand):
389
444
  client,
390
445
  include_data,
391
446
  include_configurations,
447
+ selected_data_set_external_id,
448
+ asset_recursive,
392
449
  )
393
450
  if dry_run:
394
451
  results = DeployResults([], "purge", dry_run=True)
@@ -397,11 +454,12 @@ class PurgeCommand(ToolkitCommand):
397
454
  else:
398
455
  results = self._delete_resources(to_delete, client, verbose, None, selected_data_set_external_id)
399
456
  print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
400
- if include_dataset and not dry_run:
457
+ if archive_dataset and not dry_run:
401
458
  self._archive_dataset(client, selected_data_set_external_id)
402
459
  return results
403
460
 
404
- def _archive_dataset(self, client: ToolkitClient, data_set: str) -> None:
461
+ @staticmethod
462
+ def _archive_dataset(client: ToolkitClient, data_set: str) -> None:
405
463
  archived = (
406
464
  DataSetUpdate(external_id=data_set)
407
465
  .external_id.set(str(uuid.uuid4()))
@@ -411,77 +469,92 @@ class PurgeCommand(ToolkitCommand):
411
469
  client.data_sets.update(archived)
412
470
  print(f"DataSet {data_set} archived")
413
471
 
472
+ @staticmethod
414
473
  def _create_to_delete_list_purge_dataset(
415
- self,
416
474
  client: ToolkitClient,
417
475
  include_data: bool,
418
476
  include_configurations: bool,
477
+ data_set_external_id: str,
478
+ asset_recursive: bool,
419
479
  ) -> list[ToDelete]:
420
- raise NotImplementedError()
480
+ config = client.config
481
+ to_delete: list[ToDelete] = []
421
482
 
422
- def dataset(
423
- self,
424
- client: ToolkitClient,
425
- external_id: str | None = None,
426
- include_dataset: bool = False,
427
- dry_run: bool = False,
428
- auto_yes: bool = False,
429
- verbose: bool = False,
430
- ) -> None:
431
- """Purge a dataset and all its content"""
432
- selected_dataset = self._get_selected_dataset(external_id, client)
433
- if external_id is None:
434
- # Interactive mode
435
- include_dataset = questionary.confirm(
436
- "Do you want to archive the dataset itself after the purge?", default=False
437
- ).ask()
438
- dry_run = questionary.confirm("Dry run?", default=True).ask()
439
- if not dry_run:
440
- self._print_panel("dataset", selected_dataset)
441
- if not auto_yes:
442
- confirm = questionary.confirm(
443
- f"Are you really sure you want to purge the {selected_dataset!r} dataset?", default=False
444
- ).ask()
445
- if not confirm:
446
- return
447
-
448
- loaders = self._get_dependencies(
449
- DataSetsCRUD,
450
- exclude={
451
- GroupCRUD,
452
- GroupResourceScopedCRUD,
453
- GroupAllScopedCRUD,
454
- StreamlitCRUD,
455
- HostedExtractorDestinationCRUD,
456
- FunctionCRUD,
457
- LocationFilterCRUD,
458
- InfieldV1CRUD,
459
- },
460
- )
461
- is_purged = self._purge(client, loaders, selected_data_set=selected_dataset, dry_run=dry_run, verbose=verbose)
462
- if include_dataset and is_purged:
463
- if dry_run:
464
- print(f"Would have archived {selected_dataset}")
465
- else:
466
- archived = (
467
- DataSetUpdate(external_id=selected_dataset)
468
- .external_id.set(str(uuid.uuid4()))
469
- .metadata.add({"archived": "true"})
470
- .write_protected.set(True)
471
- )
472
- client.data_sets.update(archived)
473
- print(f"DataSet {selected_dataset} archived")
474
- elif include_dataset:
475
- self.warn(
476
- HighSeverityWarning(f"DataSet {selected_dataset} was not archived due to errors during the purge")
483
+ if include_data:
484
+ three_d_crud = ThreeDModelCRUD.create_loader(client)
485
+ to_delete.extend(
486
+ [
487
+ ExternalIdToDelete(
488
+ RelationshipCRUD.create_loader(client),
489
+ RelationshipAggregator(client).count(data_set_external_id=data_set_external_id),
490
+ config.create_api_url("/relationships/delete"),
491
+ ),
492
+ IdResourceToDelete(
493
+ EventCRUD.create_loader(client),
494
+ EventAggregator(client).count(data_set_external_id=data_set_external_id),
495
+ config.create_api_url("/events/delete"),
496
+ ),
497
+ IdResourceToDelete(
498
+ FileMetadataCRUD.create_loader(client),
499
+ FileAggregator(client).count(data_set_external_id=data_set_external_id),
500
+ config.create_api_url("/files/delete"),
501
+ ),
502
+ IdResourceToDelete(
503
+ TimeSeriesCRUD.create_loader(client),
504
+ TimeSeriesAggregator(client).count(data_set_external_id=data_set_external_id),
505
+ config.create_api_url("/timeseries/delete"),
506
+ ),
507
+ IdResourceToDelete(
508
+ SequenceCRUD.create_loader(client),
509
+ SequenceAggregator(client).count(data_set_external_id=data_set_external_id),
510
+ config.create_api_url("/sequences/delete"),
511
+ ),
512
+ IdResourceToDelete(
513
+ three_d_crud,
514
+ sum(1 for _ in three_d_crud.iterate(data_set_external_id=data_set_external_id)),
515
+ config.create_api_url("/3d/models/delete"),
516
+ ),
517
+ AssetToDelete(
518
+ AssetCRUD.create_loader(client),
519
+ AssetAggregator(client).count(data_set_external_id=data_set_external_id),
520
+ config.create_api_url("/assets/delete"),
521
+ recursive=asset_recursive,
522
+ ),
523
+ ExternalIdToDelete(
524
+ LabelCRUD.create_loader(client),
525
+ LabelCountAggregator(client).count(data_set_external_id=data_set_external_id),
526
+ config.create_api_url("/labels/delete"),
527
+ ),
528
+ ]
477
529
  )
530
+ if include_configurations:
531
+ transformation_crud = TransformationCRUD.create_loader(client)
532
+ workflow_crud = WorkflowCRUD.create_loader(client)
533
+ extraction_pipeline_crud = ExtractionPipelineCRUD.create_loader(client)
534
+
535
+ to_delete.extend(
536
+ [
537
+ IdResourceToDelete(
538
+ transformation_crud,
539
+ sum(1 for _ in transformation_crud.iterate(data_set_external_id=data_set_external_id)),
540
+ config.create_api_url("/transformations/delete"),
541
+ ),
542
+ ExternalIdToDelete(
543
+ workflow_crud,
544
+ sum(1 for _ in workflow_crud.iterate(data_set_external_id=data_set_external_id)),
545
+ config.create_api_url("/workflows/delete"),
546
+ ),
547
+ IdResourceToDelete(
548
+ extraction_pipeline_crud,
549
+ sum(1 for _ in extraction_pipeline_crud.iterate(data_set_external_id=data_set_external_id)),
550
+ config.create_api_url("/extpipes/delete"),
551
+ ),
552
+ ]
553
+ )
554
+ return to_delete
478
555
 
479
- if not dry_run and is_purged:
480
- print(f"Purged dataset {selected_dataset!r} completed")
481
- elif not dry_run:
482
- print(f"Purged dataset {selected_dataset!r} partly completed. See warnings for details.")
483
-
484
- def _print_panel(self, resource_type: str, resource: str) -> None:
556
+ @staticmethod
557
+ def _print_panel(resource_type: str, resource: str) -> None:
485
558
  print(
486
559
  Panel(
487
560
  f"[red]WARNING:[/red] This operation [bold]cannot be undone[/bold]! "
@@ -494,256 +567,6 @@ class PurgeCommand(ToolkitCommand):
494
567
  )
495
568
  )
496
569
 
497
- @staticmethod
498
- def _get_selected_dataset(external_id: str | None, client: ToolkitClient) -> str:
499
- if external_id is None:
500
- datasets = client.data_sets.list(limit=-1)
501
- selected_dataset: str = questionary.select(
502
- "Which space are you going to purge (delete all resources in dataset)?",
503
- sorted([dataset.external_id for dataset in datasets if dataset.external_id]),
504
- ).ask()
505
- else:
506
- retrieved = client.data_sets.retrieve(external_id=external_id)
507
- if retrieved is None:
508
- raise ToolkitMissingResourceError(f"DataSet {external_id!r} does not exist")
509
- selected_dataset = external_id
510
-
511
- if selected_dataset is None:
512
- raise ToolkitValueError("No space selected")
513
- return selected_dataset
514
-
515
- def _purge(
516
- self,
517
- client: ToolkitClient,
518
- loaders: dict[type[ResourceCRUD], frozenset[type[ResourceCRUD]]],
519
- selected_space: str | None = None,
520
- selected_data_set: str | None = None,
521
- dry_run: bool = False,
522
- verbose: bool = False,
523
- batch_size: int = 1000,
524
- ) -> bool:
525
- is_purged = True
526
- results = DeployResults([], "purge", dry_run=dry_run)
527
- loader_cls: type[ResourceCRUD]
528
- has_purged_views = False
529
- with Console().status("...", spinner="aesthetic", speed=0.4) as status:
530
- for loader_cls in reversed(list(TopologicalSorter(loaders).static_order())):
531
- if loader_cls not in loaders:
532
- # Dependency that is included
533
- continue
534
- loader = loader_cls.create_loader(client, console=status.console)
535
- status_prefix = "Would have deleted" if dry_run else "Deleted"
536
- if isinstance(loader, ViewCRUD) and not dry_run:
537
- status_prefix = "Expected deleted" # Views are not always deleted immediately
538
- has_purged_views = True
539
-
540
- if not dry_run and isinstance(loader, AssetCRUD):
541
- # Special handling of assets as we must ensure all children are deleted before the parent.
542
- # In dry-run mode, we are not deleting the assets, so we can skip this.
543
- deleted_assets = self._purge_assets(loader, status, selected_data_set)
544
- results[loader.display_name] = ResourceDeployResult(
545
- loader.display_name, deleted=deleted_assets, total=deleted_assets
546
- )
547
- continue
548
-
549
- # Child loaders are, for example, WorkflowTriggerLoader, WorkflowVersionLoader for WorkflowLoader
550
- # These must delete all resources that are connected to the resource that the loader is deleting
551
- # Exclude loaders that we are already iterating over
552
- child_loader_classes = self._get_dependencies(loader_cls, exclude=set(loaders))
553
- child_loaders = [
554
- child_loader.create_loader(client)
555
- for child_loader in reversed(list(TopologicalSorter(child_loader_classes).static_order()))
556
- # Necessary as the topological sort includes dependencies that are not in the loaders
557
- if child_loader in child_loader_classes
558
- ]
559
- count = 0
560
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
561
- batch_ids: list[Hashable] = []
562
- for resource in loader.iterate(data_set_external_id=selected_data_set, space=selected_space):
563
- try:
564
- batch_ids.append(loader.get_id(resource))
565
- except (ToolkitRequiredValueError, KeyError) as e:
566
- try:
567
- batch_ids.append(loader.get_internal_id(resource))
568
- except (AttributeError, NotImplementedError):
569
- self.warn(
570
- HighSeverityWarning(
571
- f"Cannot delete {type(resource).__name__}. Failed to obtain ID: {e}"
572
- ),
573
- console=status.console,
574
- )
575
- is_purged = False
576
- continue
577
-
578
- if len(batch_ids) >= batch_size:
579
- child_deletion = self._delete_children(
580
- batch_ids, child_loaders, dry_run, status.console, verbose
581
- )
582
- batch_delete, batch_size = self._delete_batch(
583
- batch_ids, dry_run, loader, batch_size, status.console, verbose
584
- )
585
- count += batch_delete
586
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
587
- batch_ids = []
588
- # The DeployResults is overloaded such that the below accumulates the counts
589
- for name, child_count in child_deletion.items():
590
- results[name] = ResourceDeployResult(name, deleted=child_count, total=child_count)
591
-
592
- if batch_ids:
593
- child_deletion = self._delete_children(batch_ids, child_loaders, dry_run, status.console, verbose)
594
- batch_delete, batch_size = self._delete_batch(
595
- batch_ids, dry_run, loader, batch_size, status.console, verbose
596
- )
597
- count += batch_delete
598
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
599
- for name, child_count in child_deletion.items():
600
- results[name] = ResourceDeployResult(name, deleted=child_count, total=child_count)
601
- if count > 0:
602
- status.console.print(f"{status_prefix} {count:,} {loader.display_name}.")
603
- results[loader.display_name] = ResourceDeployResult(
604
- name=loader.display_name,
605
- deleted=count,
606
- total=count,
607
- )
608
- print(results.counts_table(exclude_columns={"Created", "Changed", "Untouched", "Total"}))
609
- if has_purged_views:
610
- print("You might need to run the purge command multiple times to delete all views.")
611
- return is_purged
612
-
613
- def _delete_batch(
614
- self,
615
- batch_ids: list[Hashable],
616
- dry_run: bool,
617
- loader: ResourceCRUD,
618
- batch_size: int,
619
- console: Console,
620
- verbose: bool,
621
- ) -> tuple[int, int]:
622
- if dry_run:
623
- deleted = len(batch_ids)
624
- else:
625
- try:
626
- deleted = loader.delete(batch_ids)
627
- except CogniteAPIError as delete_error:
628
- if (
629
- delete_error.code == 408
630
- and "timed out" in delete_error.message.casefold()
631
- and batch_size > 1
632
- and (len(batch_ids) > 1)
633
- ):
634
- self.warn(
635
- MediumSeverityWarning(
636
- f"Timed out deleting {loader.display_name}. Trying again with a smaller batch size."
637
- ),
638
- include_timestamp=True,
639
- console=console,
640
- )
641
- new_batch_size = len(batch_ids) // 2
642
- first = batch_ids[:new_batch_size]
643
- second = batch_ids[new_batch_size:]
644
- first_deleted, first_batch_size = self._delete_batch(
645
- first, dry_run, loader, new_batch_size, console, verbose
646
- )
647
- second_deleted, second_batch_size = self._delete_batch(
648
- second, dry_run, loader, new_batch_size, console, verbose
649
- )
650
- return first_deleted + second_deleted, min(first_batch_size, second_batch_size)
651
- else:
652
- raise delete_error
653
-
654
- if verbose:
655
- prefix = "Would delete" if dry_run else "Finished purging"
656
- console.print(f"{prefix} {deleted:,} {loader.display_name}")
657
- return deleted, batch_size
658
-
659
- @staticmethod
660
- def _delete_children(
661
- parent_ids: list[Hashable], child_loaders: list[ResourceCRUD], dry_run: bool, console: Console, verbose: bool
662
- ) -> dict[str, int]:
663
- child_deletion: dict[str, int] = {}
664
- for child_loader in child_loaders:
665
- child_ids = set()
666
- for child in child_loader.iterate(parent_ids=parent_ids):
667
- child_ids.add(child_loader.get_id(child))
668
- count = 0
669
- if child_ids:
670
- if dry_run:
671
- count = len(child_ids)
672
- else:
673
- count = child_loader.delete(list(child_ids))
674
-
675
- if verbose:
676
- prefix = "Would delete" if dry_run else "Deleted"
677
- console.print(f"{prefix} {count:,} {child_loader.display_name}")
678
- child_deletion[child_loader.display_name] = count
679
- return child_deletion
680
-
681
- def _purge_assets(
682
- self,
683
- loader: AssetCRUD,
684
- status: Status,
685
- selected_data_set: str | None = None,
686
- batch_size: int = 1000,
687
- ) -> int:
688
- # Using sets to avoid duplicates
689
- children_ids: set[int] = set()
690
- parent_ids: set[int] = set()
691
- is_first = True
692
- last_failed = False
693
- count = level = depth = last_parent_count = 0
694
- total_asset_count: int | None = None
695
- # Iterate through the asset hierarchy once per depth level. This is to delete all children before the parent.
696
- while is_first or level < depth:
697
- for asset in loader.iterate(data_set_external_id=selected_data_set):
698
- aggregates = cast(AggregateResultItem, asset.aggregates)
699
- if is_first and aggregates.depth is not None:
700
- depth = max(depth, aggregates.depth)
701
-
702
- if aggregates.child_count == 0:
703
- children_ids.add(asset.id)
704
- else:
705
- parent_ids.add(asset.id)
706
-
707
- if len(children_ids) >= batch_size:
708
- count += loader.delete(list(children_ids))
709
- if total_asset_count:
710
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
711
- else:
712
- status.update(f"Deleted {count:,} {loader.display_name}...")
713
- children_ids = set()
714
-
715
- if is_first:
716
- total_asset_count = count + len(parent_ids) + len(children_ids)
717
- if children_ids:
718
- count += loader.delete(list(children_ids))
719
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
720
- children_ids = set()
721
-
722
- if len(parent_ids) == last_parent_count and last_failed:
723
- try:
724
- # Just try to delete them all at once
725
- count += loader.delete(list(parent_ids))
726
- except CogniteAPIError as e:
727
- raise CDFAPIError(
728
- f"Failed to delete {len(parent_ids)} assets. This could be due to a parent-child cycle or an "
729
- "eventual consistency issue. Wait a few seconds and try again. An alternative is to use the "
730
- "Python-SDK to delete the asset hierarchy "
731
- "`client.assets.delete(external_id='my_root_asset', recursive=True)`"
732
- ) from e
733
- else:
734
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
735
- break
736
- elif len(parent_ids) == last_parent_count:
737
- last_failed = True
738
- else:
739
- last_failed = False
740
- level += 1
741
- last_parent_count = len(parent_ids)
742
- parent_ids.clear()
743
- is_first = False
744
- status.console.print(f"Finished purging {loader.display_name}.")
745
- return count
746
-
747
570
  def instances(
748
571
  self,
749
572
  client: ToolkitClient,