cognite-toolkit 0.6.83__py3-none-any.whl → 0.6.85__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-toolkit might be problematic. Click here for more details.

@@ -3,11 +3,10 @@ from abc import ABC, abstractmethod
3
3
  from collections.abc import Callable, Hashable, Iterable, Sequence
4
4
  from dataclasses import dataclass
5
5
  from functools import partial
6
- from graphlib import TopologicalSorter
7
6
  from typing import Literal, cast
8
7
 
9
8
  import questionary
10
- from cognite.client.data_classes import AggregateResultItem, DataSetUpdate
9
+ from cognite.client.data_classes import DataSetUpdate
11
10
  from cognite.client.data_classes._base import CogniteResourceList
12
11
  from cognite.client.data_classes.data_modeling import (
13
12
  EdgeList,
@@ -20,45 +19,49 @@ from cognite.client.utils._identifier import InstanceId
20
19
  from rich import print
21
20
  from rich.console import Console
22
21
  from rich.panel import Panel
23
- from rich.status import Status
24
22
 
25
23
  from cognite_toolkit._cdf_tk.client import ToolkitClient
26
24
  from cognite_toolkit._cdf_tk.cruds import (
27
- RESOURCE_CRUD_LIST,
28
25
  AssetCRUD,
29
26
  ContainerCRUD,
30
27
  DataModelCRUD,
31
- DataSetsCRUD,
32
28
  EdgeCRUD,
33
- FunctionCRUD,
34
- GroupAllScopedCRUD,
35
- GroupCRUD,
36
- GroupResourceScopedCRUD,
37
- HostedExtractorDestinationCRUD,
38
- InfieldV1CRUD,
39
- LocationFilterCRUD,
29
+ EventCRUD,
30
+ ExtractionPipelineCRUD,
31
+ FileMetadataCRUD,
32
+ LabelCRUD,
40
33
  NodeCRUD,
34
+ RelationshipCRUD,
41
35
  ResourceCRUD,
36
+ SequenceCRUD,
42
37
  SpaceCRUD,
43
- StreamlitCRUD,
38
+ ThreeDModelCRUD,
39
+ TimeSeriesCRUD,
40
+ TransformationCRUD,
44
41
  ViewCRUD,
42
+ WorkflowCRUD,
45
43
  )
46
44
  from cognite_toolkit._cdf_tk.data_classes import DeployResults, ResourceDeployResult
47
45
  from cognite_toolkit._cdf_tk.exceptions import (
48
46
  AuthorizationError,
49
- CDFAPIError,
50
47
  ToolkitMissingResourceError,
51
- ToolkitRequiredValueError,
52
- ToolkitValueError,
53
48
  )
54
49
  from cognite_toolkit._cdf_tk.storageio import InstanceIO
55
50
  from cognite_toolkit._cdf_tk.storageio.selectors import InstanceSelector
56
51
  from cognite_toolkit._cdf_tk.tk_warnings import (
57
52
  HighSeverityWarning,
58
53
  LimitedAccessWarning,
59
- MediumSeverityWarning,
60
54
  )
61
55
  from cognite_toolkit._cdf_tk.utils import humanize_collection
56
+ from cognite_toolkit._cdf_tk.utils.aggregators import (
57
+ AssetAggregator,
58
+ EventAggregator,
59
+ FileAggregator,
60
+ LabelCountAggregator,
61
+ RelationshipAggregator,
62
+ SequenceAggregator,
63
+ TimeSeriesAggregator,
64
+ )
62
65
  from cognite_toolkit._cdf_tk.utils.http_client import (
63
66
  FailedRequestItems,
64
67
  FailedResponseItems,
@@ -107,6 +110,9 @@ class ToDelete(ABC):
107
110
  ) -> Callable[[CogniteResourceList], list[JsonVal]]:
108
111
  raise NotImplementedError()
109
112
 
113
+ def get_extra_fields(self) -> dict[str, JsonVal]:
114
+ return {}
115
+
110
116
 
111
117
  @dataclass
112
118
  class DataModelingToDelete(ToDelete):
@@ -165,6 +171,36 @@ class NodesToDelete(ToDelete):
165
171
  return check_for_data
166
172
 
167
173
 
174
+ @dataclass
175
+ class IdResourceToDelete(ToDelete):
176
+ def get_process_function(
177
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
178
+ ) -> Callable[[CogniteResourceList], list[JsonVal]]:
179
+ def as_id(chunk: CogniteResourceList) -> list[JsonVal]:
180
+ return [{"id": item.id} for item in chunk]
181
+
182
+ return as_id
183
+
184
+
185
+ @dataclass
186
+ class ExternalIdToDelete(ToDelete):
187
+ def get_process_function(
188
+ self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
189
+ ) -> Callable[[CogniteResourceList], list[JsonVal]]:
190
+ def as_external_id(chunk: CogniteResourceList) -> list[JsonVal]:
191
+ return [{"externalId": item.external_id} for item in chunk]
192
+
193
+ return as_external_id
194
+
195
+
196
+ @dataclass
197
+ class AssetToDelete(IdResourceToDelete):
198
+ recursive: bool
199
+
200
+ def get_extra_fields(self) -> dict[str, JsonVal]:
201
+ return {"recursive": self.recursive}
202
+
203
+
168
204
  class PurgeCommand(ToolkitCommand):
169
205
  BATCH_SIZE_DM = 1000
170
206
 
@@ -218,8 +254,9 @@ class PurgeCommand(ToolkitCommand):
218
254
  print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
219
255
  return results
220
256
 
257
+ @staticmethod
221
258
  def _create_to_delete_list_purge_space(
222
- self, client: ToolkitClient, delete_datapoints: bool, delete_file_content: bool, stats: SpaceStatistics
259
+ client: ToolkitClient, delete_datapoints: bool, delete_file_content: bool, stats: SpaceStatistics
223
260
  ) -> list[ToDelete]:
224
261
  config = client.config
225
262
  to_delete = [
@@ -289,7 +326,7 @@ class PurgeCommand(ToolkitCommand):
289
326
  item.crud, space, data_set_external_id, batch_size=self.BATCH_SIZE_DM
290
327
  ),
291
328
  process=item.get_process_function(client, console, verbose, process_results),
292
- write=self._purge_batch(item.crud, item.delete_url, delete_client, write_results),
329
+ write=self._purge_batch(item, item.delete_url, delete_client, write_results),
293
330
  max_queue_size=10,
294
331
  iteration_count=iteration_count,
295
332
  download_description=f"Downloading {item.display_name}",
@@ -323,14 +360,24 @@ class PurgeCommand(ToolkitCommand):
323
360
 
324
361
  @staticmethod
325
362
  def _purge_batch(
326
- crud: ResourceCRUD, delete_url: str, delete_client: HTTPClient, result: ResourceDeployResult
363
+ delete_item: ToDelete, delete_url: str, delete_client: HTTPClient, result: ResourceDeployResult
327
364
  ) -> Callable[[list[JsonVal]], None]:
365
+ crud = delete_item.crud
366
+
367
+ def as_id(item: JsonVal) -> Hashable:
368
+ try:
369
+ return crud.get_id(item)
370
+ except KeyError:
371
+ # Fallback to internal ID
372
+ return crud.get_internal_id(item)
373
+
328
374
  def process(items: list[JsonVal]) -> None:
329
375
  responses = delete_client.request_with_retries(
330
376
  ItemsRequest(
331
377
  endpoint_url=delete_url,
332
378
  method="POST",
333
- items=[DeleteItem(item=item, as_id_fun=crud.get_id) for item in items],
379
+ items=[DeleteItem(item=item, as_id_fun=as_id) for item in items],
380
+ extra_body_fields=delete_item.get_extra_fields(),
334
381
  )
335
382
  )
336
383
  for response in responses:
@@ -341,27 +388,35 @@ class PurgeCommand(ToolkitCommand):
341
388
 
342
389
  return process
343
390
 
344
- @staticmethod
345
- def _get_dependencies(
346
- loader_cls: type[ResourceCRUD], exclude: set[type[ResourceCRUD]] | None = None
347
- ) -> dict[type[ResourceCRUD], frozenset[type[ResourceCRUD]]]:
348
- return {
349
- dep_cls: dep_cls.dependencies
350
- for dep_cls in RESOURCE_CRUD_LIST
351
- if loader_cls in dep_cls.dependencies and (exclude is None or dep_cls not in exclude)
352
- }
353
-
354
- def dataset_v2(
391
+ def dataset(
355
392
  self,
356
393
  client: ToolkitClient,
357
394
  selected_data_set_external_id: str,
358
- include_dataset: bool = False,
395
+ archive_dataset: bool = False,
359
396
  include_data: bool = True,
360
397
  include_configurations: bool = False,
398
+ asset_recursive: bool = False,
361
399
  dry_run: bool = False,
362
400
  auto_yes: bool = False,
363
401
  verbose: bool = False,
364
402
  ) -> DeployResults:
403
+ """Purge a dataset and all its content
404
+
405
+ Args:
406
+ client: The ToolkitClient to use
407
+ selected_data_set_external_id: The external ID of the dataset to purge
408
+ archive_dataset: Whether to archive the dataset itself after the purge
409
+ include_data: Whether to include data (assets, events, time series, files, sequences, 3D models, relationships, labels) in the purge
410
+ include_configurations: Whether to include configurations (workflows, transformations, extraction pipelines) in the purge
411
+ asset_recursive: Whether to recursively delete assets.
412
+ dry_run: Whether to perform a dry run
413
+ auto_yes: Whether to automatically confirm the purge
414
+ verbose: Whether to print verbose output
415
+
416
+ Returns:
417
+ DeployResults: The results of the purge operation
418
+
419
+ """
365
420
  # Warning Messages
366
421
  if not dry_run:
367
422
  self._print_panel("dataSet", selected_data_set_external_id)
@@ -375,6 +430,8 @@ class PurgeCommand(ToolkitCommand):
375
430
  # Validate Auth
376
431
  validator = ValidateAccess(client, "purge")
377
432
  data_set_id = client.lookup.data_sets.id(selected_data_set_external_id)
433
+ if data_set_id is None:
434
+ raise ToolkitMissingResourceError(f"DataSet {selected_data_set_external_id!r} does not exist")
378
435
  action = cast(Sequence[Literal["read", "write"]], ["read"] if dry_run else ["read", "write"])
379
436
  if include_data:
380
437
  # Check asset, events, time series, files, and sequences access, relationships, labels, 3D access.
@@ -387,6 +444,8 @@ class PurgeCommand(ToolkitCommand):
387
444
  client,
388
445
  include_data,
389
446
  include_configurations,
447
+ selected_data_set_external_id,
448
+ asset_recursive,
390
449
  )
391
450
  if dry_run:
392
451
  results = DeployResults([], "purge", dry_run=True)
@@ -395,11 +454,12 @@ class PurgeCommand(ToolkitCommand):
395
454
  else:
396
455
  results = self._delete_resources(to_delete, client, verbose, None, selected_data_set_external_id)
397
456
  print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
398
- if include_dataset and not dry_run:
457
+ if archive_dataset and not dry_run:
399
458
  self._archive_dataset(client, selected_data_set_external_id)
400
459
  return results
401
460
 
402
- def _archive_dataset(self, client: ToolkitClient, data_set: str) -> None:
461
+ @staticmethod
462
+ def _archive_dataset(client: ToolkitClient, data_set: str) -> None:
403
463
  archived = (
404
464
  DataSetUpdate(external_id=data_set)
405
465
  .external_id.set(str(uuid.uuid4()))
@@ -409,77 +469,92 @@ class PurgeCommand(ToolkitCommand):
409
469
  client.data_sets.update(archived)
410
470
  print(f"DataSet {data_set} archived")
411
471
 
472
+ @staticmethod
412
473
  def _create_to_delete_list_purge_dataset(
413
- self,
414
474
  client: ToolkitClient,
415
475
  include_data: bool,
416
476
  include_configurations: bool,
477
+ data_set_external_id: str,
478
+ asset_recursive: bool,
417
479
  ) -> list[ToDelete]:
418
- raise NotImplementedError()
480
+ config = client.config
481
+ to_delete: list[ToDelete] = []
419
482
 
420
- def dataset(
421
- self,
422
- client: ToolkitClient,
423
- external_id: str | None = None,
424
- include_dataset: bool = False,
425
- dry_run: bool = False,
426
- auto_yes: bool = False,
427
- verbose: bool = False,
428
- ) -> None:
429
- """Purge a dataset and all its content"""
430
- selected_dataset = self._get_selected_dataset(external_id, client)
431
- if external_id is None:
432
- # Interactive mode
433
- include_dataset = questionary.confirm(
434
- "Do you want to archive the dataset itself after the purge?", default=False
435
- ).ask()
436
- dry_run = questionary.confirm("Dry run?", default=True).ask()
437
- if not dry_run:
438
- self._print_panel("dataset", selected_dataset)
439
- if not auto_yes:
440
- confirm = questionary.confirm(
441
- f"Are you really sure you want to purge the {selected_dataset!r} dataset?", default=False
442
- ).ask()
443
- if not confirm:
444
- return
445
-
446
- loaders = self._get_dependencies(
447
- DataSetsCRUD,
448
- exclude={
449
- GroupCRUD,
450
- GroupResourceScopedCRUD,
451
- GroupAllScopedCRUD,
452
- StreamlitCRUD,
453
- HostedExtractorDestinationCRUD,
454
- FunctionCRUD,
455
- LocationFilterCRUD,
456
- InfieldV1CRUD,
457
- },
458
- )
459
- is_purged = self._purge(client, loaders, selected_data_set=selected_dataset, dry_run=dry_run, verbose=verbose)
460
- if include_dataset and is_purged:
461
- if dry_run:
462
- print(f"Would have archived {selected_dataset}")
463
- else:
464
- archived = (
465
- DataSetUpdate(external_id=selected_dataset)
466
- .external_id.set(str(uuid.uuid4()))
467
- .metadata.add({"archived": "true"})
468
- .write_protected.set(True)
469
- )
470
- client.data_sets.update(archived)
471
- print(f"DataSet {selected_dataset} archived")
472
- elif include_dataset:
473
- self.warn(
474
- HighSeverityWarning(f"DataSet {selected_dataset} was not archived due to errors during the purge")
483
+ if include_data:
484
+ three_d_crud = ThreeDModelCRUD.create_loader(client)
485
+ to_delete.extend(
486
+ [
487
+ ExternalIdToDelete(
488
+ RelationshipCRUD.create_loader(client),
489
+ RelationshipAggregator(client).count(data_set_external_id=data_set_external_id),
490
+ config.create_api_url("/relationships/delete"),
491
+ ),
492
+ IdResourceToDelete(
493
+ EventCRUD.create_loader(client),
494
+ EventAggregator(client).count(data_set_external_id=data_set_external_id),
495
+ config.create_api_url("/events/delete"),
496
+ ),
497
+ IdResourceToDelete(
498
+ FileMetadataCRUD.create_loader(client),
499
+ FileAggregator(client).count(data_set_external_id=data_set_external_id),
500
+ config.create_api_url("/files/delete"),
501
+ ),
502
+ IdResourceToDelete(
503
+ TimeSeriesCRUD.create_loader(client),
504
+ TimeSeriesAggregator(client).count(data_set_external_id=data_set_external_id),
505
+ config.create_api_url("/timeseries/delete"),
506
+ ),
507
+ IdResourceToDelete(
508
+ SequenceCRUD.create_loader(client),
509
+ SequenceAggregator(client).count(data_set_external_id=data_set_external_id),
510
+ config.create_api_url("/sequences/delete"),
511
+ ),
512
+ IdResourceToDelete(
513
+ three_d_crud,
514
+ sum(1 for _ in three_d_crud.iterate(data_set_external_id=data_set_external_id)),
515
+ config.create_api_url("/3d/models/delete"),
516
+ ),
517
+ AssetToDelete(
518
+ AssetCRUD.create_loader(client),
519
+ AssetAggregator(client).count(data_set_external_id=data_set_external_id),
520
+ config.create_api_url("/assets/delete"),
521
+ recursive=asset_recursive,
522
+ ),
523
+ ExternalIdToDelete(
524
+ LabelCRUD.create_loader(client),
525
+ LabelCountAggregator(client).count(data_set_external_id=data_set_external_id),
526
+ config.create_api_url("/labels/delete"),
527
+ ),
528
+ ]
475
529
  )
530
+ if include_configurations:
531
+ transformation_crud = TransformationCRUD.create_loader(client)
532
+ workflow_crud = WorkflowCRUD.create_loader(client)
533
+ extraction_pipeline_crud = ExtractionPipelineCRUD.create_loader(client)
534
+
535
+ to_delete.extend(
536
+ [
537
+ IdResourceToDelete(
538
+ transformation_crud,
539
+ sum(1 for _ in transformation_crud.iterate(data_set_external_id=data_set_external_id)),
540
+ config.create_api_url("/transformations/delete"),
541
+ ),
542
+ ExternalIdToDelete(
543
+ workflow_crud,
544
+ sum(1 for _ in workflow_crud.iterate(data_set_external_id=data_set_external_id)),
545
+ config.create_api_url("/workflows/delete"),
546
+ ),
547
+ IdResourceToDelete(
548
+ extraction_pipeline_crud,
549
+ sum(1 for _ in extraction_pipeline_crud.iterate(data_set_external_id=data_set_external_id)),
550
+ config.create_api_url("/extpipes/delete"),
551
+ ),
552
+ ]
553
+ )
554
+ return to_delete
476
555
 
477
- if not dry_run and is_purged:
478
- print(f"Purged dataset {selected_dataset!r} completed")
479
- elif not dry_run:
480
- print(f"Purged dataset {selected_dataset!r} partly completed. See warnings for details.")
481
-
482
- def _print_panel(self, resource_type: str, resource: str) -> None:
556
+ @staticmethod
557
+ def _print_panel(resource_type: str, resource: str) -> None:
483
558
  print(
484
559
  Panel(
485
560
  f"[red]WARNING:[/red] This operation [bold]cannot be undone[/bold]! "
@@ -492,256 +567,6 @@ class PurgeCommand(ToolkitCommand):
492
567
  )
493
568
  )
494
569
 
495
- @staticmethod
496
- def _get_selected_dataset(external_id: str | None, client: ToolkitClient) -> str:
497
- if external_id is None:
498
- datasets = client.data_sets.list(limit=-1)
499
- selected_dataset: str = questionary.select(
500
- "Which space are you going to purge (delete all resources in dataset)?",
501
- sorted([dataset.external_id for dataset in datasets if dataset.external_id]),
502
- ).ask()
503
- else:
504
- retrieved = client.data_sets.retrieve(external_id=external_id)
505
- if retrieved is None:
506
- raise ToolkitMissingResourceError(f"DataSet {external_id!r} does not exist")
507
- selected_dataset = external_id
508
-
509
- if selected_dataset is None:
510
- raise ToolkitValueError("No space selected")
511
- return selected_dataset
512
-
513
- def _purge(
514
- self,
515
- client: ToolkitClient,
516
- loaders: dict[type[ResourceCRUD], frozenset[type[ResourceCRUD]]],
517
- selected_space: str | None = None,
518
- selected_data_set: str | None = None,
519
- dry_run: bool = False,
520
- verbose: bool = False,
521
- batch_size: int = 1000,
522
- ) -> bool:
523
- is_purged = True
524
- results = DeployResults([], "purge", dry_run=dry_run)
525
- loader_cls: type[ResourceCRUD]
526
- has_purged_views = False
527
- with Console().status("...", spinner="aesthetic", speed=0.4) as status:
528
- for loader_cls in reversed(list(TopologicalSorter(loaders).static_order())):
529
- if loader_cls not in loaders:
530
- # Dependency that is included
531
- continue
532
- loader = loader_cls.create_loader(client, console=status.console)
533
- status_prefix = "Would have deleted" if dry_run else "Deleted"
534
- if isinstance(loader, ViewCRUD) and not dry_run:
535
- status_prefix = "Expected deleted" # Views are not always deleted immediately
536
- has_purged_views = True
537
-
538
- if not dry_run and isinstance(loader, AssetCRUD):
539
- # Special handling of assets as we must ensure all children are deleted before the parent.
540
- # In dry-run mode, we are not deleting the assets, so we can skip this.
541
- deleted_assets = self._purge_assets(loader, status, selected_data_set)
542
- results[loader.display_name] = ResourceDeployResult(
543
- loader.display_name, deleted=deleted_assets, total=deleted_assets
544
- )
545
- continue
546
-
547
- # Child loaders are, for example, WorkflowTriggerLoader, WorkflowVersionLoader for WorkflowLoader
548
- # These must delete all resources that are connected to the resource that the loader is deleting
549
- # Exclude loaders that we are already iterating over
550
- child_loader_classes = self._get_dependencies(loader_cls, exclude=set(loaders))
551
- child_loaders = [
552
- child_loader.create_loader(client)
553
- for child_loader in reversed(list(TopologicalSorter(child_loader_classes).static_order()))
554
- # Necessary as the topological sort includes dependencies that are not in the loaders
555
- if child_loader in child_loader_classes
556
- ]
557
- count = 0
558
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
559
- batch_ids: list[Hashable] = []
560
- for resource in loader.iterate(data_set_external_id=selected_data_set, space=selected_space):
561
- try:
562
- batch_ids.append(loader.get_id(resource))
563
- except (ToolkitRequiredValueError, KeyError) as e:
564
- try:
565
- batch_ids.append(loader.get_internal_id(resource))
566
- except (AttributeError, NotImplementedError):
567
- self.warn(
568
- HighSeverityWarning(
569
- f"Cannot delete {type(resource).__name__}. Failed to obtain ID: {e}"
570
- ),
571
- console=status.console,
572
- )
573
- is_purged = False
574
- continue
575
-
576
- if len(batch_ids) >= batch_size:
577
- child_deletion = self._delete_children(
578
- batch_ids, child_loaders, dry_run, status.console, verbose
579
- )
580
- batch_delete, batch_size = self._delete_batch(
581
- batch_ids, dry_run, loader, batch_size, status.console, verbose
582
- )
583
- count += batch_delete
584
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
585
- batch_ids = []
586
- # The DeployResults is overloaded such that the below accumulates the counts
587
- for name, child_count in child_deletion.items():
588
- results[name] = ResourceDeployResult(name, deleted=child_count, total=child_count)
589
-
590
- if batch_ids:
591
- child_deletion = self._delete_children(batch_ids, child_loaders, dry_run, status.console, verbose)
592
- batch_delete, batch_size = self._delete_batch(
593
- batch_ids, dry_run, loader, batch_size, status.console, verbose
594
- )
595
- count += batch_delete
596
- status.update(f"{status_prefix} {count:,} {loader.display_name}...")
597
- for name, child_count in child_deletion.items():
598
- results[name] = ResourceDeployResult(name, deleted=child_count, total=child_count)
599
- if count > 0:
600
- status.console.print(f"{status_prefix} {count:,} {loader.display_name}.")
601
- results[loader.display_name] = ResourceDeployResult(
602
- name=loader.display_name,
603
- deleted=count,
604
- total=count,
605
- )
606
- print(results.counts_table(exclude_columns={"Created", "Changed", "Untouched", "Total"}))
607
- if has_purged_views:
608
- print("You might need to run the purge command multiple times to delete all views.")
609
- return is_purged
610
-
611
- def _delete_batch(
612
- self,
613
- batch_ids: list[Hashable],
614
- dry_run: bool,
615
- loader: ResourceCRUD,
616
- batch_size: int,
617
- console: Console,
618
- verbose: bool,
619
- ) -> tuple[int, int]:
620
- if dry_run:
621
- deleted = len(batch_ids)
622
- else:
623
- try:
624
- deleted = loader.delete(batch_ids)
625
- except CogniteAPIError as delete_error:
626
- if (
627
- delete_error.code == 408
628
- and "timed out" in delete_error.message.casefold()
629
- and batch_size > 1
630
- and (len(batch_ids) > 1)
631
- ):
632
- self.warn(
633
- MediumSeverityWarning(
634
- f"Timed out deleting {loader.display_name}. Trying again with a smaller batch size."
635
- ),
636
- include_timestamp=True,
637
- console=console,
638
- )
639
- new_batch_size = len(batch_ids) // 2
640
- first = batch_ids[:new_batch_size]
641
- second = batch_ids[new_batch_size:]
642
- first_deleted, first_batch_size = self._delete_batch(
643
- first, dry_run, loader, new_batch_size, console, verbose
644
- )
645
- second_deleted, second_batch_size = self._delete_batch(
646
- second, dry_run, loader, new_batch_size, console, verbose
647
- )
648
- return first_deleted + second_deleted, min(first_batch_size, second_batch_size)
649
- else:
650
- raise delete_error
651
-
652
- if verbose:
653
- prefix = "Would delete" if dry_run else "Finished purging"
654
- console.print(f"{prefix} {deleted:,} {loader.display_name}")
655
- return deleted, batch_size
656
-
657
- @staticmethod
658
- def _delete_children(
659
- parent_ids: list[Hashable], child_loaders: list[ResourceCRUD], dry_run: bool, console: Console, verbose: bool
660
- ) -> dict[str, int]:
661
- child_deletion: dict[str, int] = {}
662
- for child_loader in child_loaders:
663
- child_ids = set()
664
- for child in child_loader.iterate(parent_ids=parent_ids):
665
- child_ids.add(child_loader.get_id(child))
666
- count = 0
667
- if child_ids:
668
- if dry_run:
669
- count = len(child_ids)
670
- else:
671
- count = child_loader.delete(list(child_ids))
672
-
673
- if verbose:
674
- prefix = "Would delete" if dry_run else "Deleted"
675
- console.print(f"{prefix} {count:,} {child_loader.display_name}")
676
- child_deletion[child_loader.display_name] = count
677
- return child_deletion
678
-
679
- def _purge_assets(
680
- self,
681
- loader: AssetCRUD,
682
- status: Status,
683
- selected_data_set: str | None = None,
684
- batch_size: int = 1000,
685
- ) -> int:
686
- # Using sets to avoid duplicates
687
- children_ids: set[int] = set()
688
- parent_ids: set[int] = set()
689
- is_first = True
690
- last_failed = False
691
- count = level = depth = last_parent_count = 0
692
- total_asset_count: int | None = None
693
- # Iterate through the asset hierarchy once per depth level. This is to delete all children before the parent.
694
- while is_first or level < depth:
695
- for asset in loader.iterate(data_set_external_id=selected_data_set):
696
- aggregates = cast(AggregateResultItem, asset.aggregates)
697
- if is_first and aggregates.depth is not None:
698
- depth = max(depth, aggregates.depth)
699
-
700
- if aggregates.child_count == 0:
701
- children_ids.add(asset.id)
702
- else:
703
- parent_ids.add(asset.id)
704
-
705
- if len(children_ids) >= batch_size:
706
- count += loader.delete(list(children_ids))
707
- if total_asset_count:
708
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
709
- else:
710
- status.update(f"Deleted {count:,} {loader.display_name}...")
711
- children_ids = set()
712
-
713
- if is_first:
714
- total_asset_count = count + len(parent_ids) + len(children_ids)
715
- if children_ids:
716
- count += loader.delete(list(children_ids))
717
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
718
- children_ids = set()
719
-
720
- if len(parent_ids) == last_parent_count and last_failed:
721
- try:
722
- # Just try to delete them all at once
723
- count += loader.delete(list(parent_ids))
724
- except CogniteAPIError as e:
725
- raise CDFAPIError(
726
- f"Failed to delete {len(parent_ids)} assets. This could be due to a parent-child cycle or an "
727
- "eventual consistency issue. Wait a few seconds and try again. An alternative is to use the "
728
- "Python-SDK to delete the asset hierarchy "
729
- "`client.assets.delete(external_id='my_root_asset', recursive=True)`"
730
- ) from e
731
- else:
732
- status.update(f"Deleted {count:,}/{total_asset_count:,} {loader.display_name}...")
733
- break
734
- elif len(parent_ids) == last_parent_count:
735
- last_failed = True
736
- else:
737
- last_failed = False
738
- level += 1
739
- last_parent_count = len(parent_ids)
740
- parent_ids.clear()
741
- is_first = False
742
- status.console.print(f"Finished purging {loader.display_name}.")
743
- return count
744
-
745
570
  def instances(
746
571
  self,
747
572
  client: ToolkitClient,
@@ -60,7 +60,7 @@ class _ReplaceMethod:
60
60
  """This is a small helper class used in the
61
61
  lookup and replace in the ACL scoped ids"""
62
62
 
63
- lookup_method: Callable[[str, bool], int]
63
+ lookup_method: Callable[[str, bool], int | None]
64
64
  reverse_lookup_method: Callable[[int], str | None]
65
65
  id_name: str
66
66