cognite-toolkit 0.6.81__py3-none-any.whl → 0.6.83__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_migrate/prepare.py

@@ -6,12 +6,14 @@ from cognite_toolkit._cdf_tk.commands.deploy import DeployCommand
 from cognite_toolkit._cdf_tk.cruds import (
     ContainerCRUD,
     DataModelCRUD,
+    ResourceCRUD,
     ResourceViewMappingCRUD,
     ResourceWorker,
     SpaceCRUD,
     ViewCRUD,
 )
 from cognite_toolkit._cdf_tk.data_classes import DeployResults
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning
 
 from .data_model import COGNITE_MIGRATION_MODEL, CONTAINERS, MODEL_ID, SPACE, VIEWS
 from .default_mappings import create_default_mappings
@@ -27,27 +29,30 @@ class MigrationPrepareCommand(ToolkitCommand):
         verb = "Would deploy" if dry_run else "Deploying"
         print(f"{verb} {MODEL_ID!r}")
         results = DeployResults([], "deploy", dry_run=dry_run)
-        for loader_cls, resource_list in [
+        crud_cls: type[ResourceCRUD]
+        for crud_cls, resource_list in [  # type: ignore[assignment]
             (SpaceCRUD, [SPACE]),
             (ContainerCRUD, CONTAINERS),
             (ViewCRUD, VIEWS),
             (DataModelCRUD, [COGNITE_MIGRATION_MODEL]),
             (ResourceViewMappingCRUD, create_default_mappings()),
         ]:
-            # MyPy does not understand that `loader_cls` has a `create_loader` method.
-            loader = loader_cls.create_loader(client)  # type: ignore[attr-defined]
-            worker = ResourceWorker(loader, "deploy")
+            crud = crud_cls.create_loader(client)
+            if warning := crud.prerequisite_warning():
+                self.warn(HighSeverityWarning(warning))
+                continue
+            worker = ResourceWorker(crud, "deploy")
             # MyPy does not understand that `loader` has a `get_id` method.
-            dump_arg = {"context": "local"} if loader_cls is ResourceViewMappingCRUD else {}
-            local_by_id = {loader.get_id(item): (item.dump(**dump_arg), item) for item in resource_list}  # type: ignore[attr-defined]
+            dump_arg = {"context": "local"} if crud_cls is ResourceViewMappingCRUD else {}
+            local_by_id = {crud.get_id(item): (item.dump(**dump_arg), item) for item in resource_list}  # type: ignore[attr-defined]
             worker.validate_access(local_by_id, is_dry_run=dry_run)
-            cdf_resources = loader.retrieve(list(local_by_id.keys()))
+            cdf_resources = crud.retrieve(list(local_by_id.keys()))
             resources = worker.categorize_resources(local_by_id, cdf_resources, False, verbose)
 
             if dry_run:
-                result = deploy_cmd.dry_run_deploy(resources, loader, False, False)
+                result = deploy_cmd.dry_run_deploy(resources, crud, False, False)
             else:
-                result = deploy_cmd.actual_deploy(resources, loader)
+                result = deploy_cmd.actual_deploy(resources, crud)
             if result:
                 results[result.name] = result
         if results.has_counts:

cognite_toolkit/_cdf_tk/commands/_purge.py

@@ -1,14 +1,20 @@
-import dataclasses
 import uuid
-from collections.abc import Callable, Hashable, Iterable
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Hashable, Iterable, Sequence
+from dataclasses import dataclass
 from functools import partial
 from graphlib import TopologicalSorter
-from typing import Any, cast
+from typing import Literal, cast
 
 import questionary
 from cognite.client.data_classes import AggregateResultItem, DataSetUpdate
 from cognite.client.data_classes._base import CogniteResourceList
-from cognite.client.data_classes.data_modeling import NodeId, NodeList
+from cognite.client.data_classes.data_modeling import (
+    EdgeList,
+    NodeId,
+    NodeList,
+)
+from cognite.client.data_classes.data_modeling.statistics import SpaceStatistics
 from cognite.client.exceptions import CogniteAPIError
 from cognite.client.utils._identifier import InstanceId
 from rich import print
@@ -67,13 +73,13 @@ from cognite_toolkit._cdf_tk.utils.validate_access import ValidateAccess
 from ._base import ToolkitCommand
 
 
-@dataclasses.dataclass
+@dataclass
 class DeleteResults:
     deleted: int = 0
     failed: int = 0
 
 
-@dataclasses.dataclass
+@dataclass
 class DeleteItem:
     item: JsonVal
     as_id_fun: Callable[[JsonVal], Hashable]
@@ -85,6 +91,80 @@ class DeleteItem:
         return self.as_id_fun(self.item)
 
 
+@dataclass
+class ToDelete(ABC):
+    crud: ResourceCRUD
+    total: int
+    delete_url: str
+
+    @property
+    def display_name(self) -> str:
+        return self.crud.display_name
+
+    @abstractmethod
+    def get_process_function(
+        self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
+    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
+        raise NotImplementedError()
+
+
+@dataclass
+class DataModelingToDelete(ToDelete):
+    def get_process_function(
+        self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
+    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
+        def as_id(chunk: CogniteResourceList) -> list[JsonVal]:
+            return [item.as_id().dump(include_type=False) for item in chunk]
+
+        return as_id
+
+
+@dataclass
+class EdgeToDelete(ToDelete):
+    def get_process_function(
+        self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
+    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
+        def as_id(chunk: CogniteResourceList) -> list[JsonVal]:
+            return [
+                {"space": item.space, "externalId": item.external_id, "instanceType": "edge"}
+                for item in cast(EdgeList, chunk)
+            ]
+
+        return as_id
+
+
+@dataclass
+class NodesToDelete(ToDelete):
+    delete_datapoints: bool
+    delete_file_content: bool
+
+    def get_process_function(
+        self, client: ToolkitClient, console: Console, verbose: bool, process_results: ResourceDeployResult
+    ) -> Callable[[CogniteResourceList], list[JsonVal]]:
+        def check_for_data(chunk: CogniteResourceList) -> list[JsonVal]:
+            node_ids = cast(NodeList, chunk).as_ids()
+            found_ids: set[InstanceId] = set()
+            if not self.delete_datapoints:
+                timeseries = client.time_series.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
+                found_ids |= {ts.instance_id for ts in timeseries if ts.instance_id is not None}
+            if not self.delete_file_content:
+                files = client.files.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
+                found_ids |= {f.instance_id for f in files if f.instance_id is not None}
+            if found_ids and verbose:
+                console.print(f"Skipping {found_ids} nodes as they have datapoints or file content")
+            process_results.unchanged += len(found_ids)
+            result: list[JsonVal] = []
+            for node_id in (n for n in node_ids if n not in found_ids):
+                dumped = node_id.dump(include_instance_type=True)
+                # The delete endpoint expects "instanceType" instead of "type"
+                dumped["instanceType"] = dumped.pop("type")
+                # MyPy think complains about invariant here, even though dict[str, str] is a type of JsonVal
+                result.append(dumped)  # type: ignore[arg-type]
+            return result
+
+        return check_for_data
+
+
 class PurgeCommand(ToolkitCommand):
     BATCH_SIZE_DM = 1000
 
@@ -99,17 +179,16 @@ class PurgeCommand(ToolkitCommand):
         auto_yes: bool = False,
         verbose: bool = False,
     ) -> DeployResults:
-        results = DeployResults([], "purge", dry_run=dry_run)
-
         # Warning Messages
         if not dry_run:
             self._print_panel("space", selected_space)
+
         if not dry_run and not auto_yes:
             confirm = questionary.confirm(
                 f"Are you really sure you want to purge the {selected_space!r} space?", default=False
             ).ask()
             if not confirm:
-                return results
+                return DeployResults([], "purge", dry_run=dry_run)
 
         stats = client.data_modeling.statistics.spaces.retrieve(selected_space)
         if stats is None:
@@ -118,91 +197,123 @@ class PurgeCommand(ToolkitCommand):
         # ValidateAuth
         validator = ValidateAccess(client, "purge")
         if include_space or (stats.containers + stats.views + stats.data_models) > 0:
+            # We check for write even in dry-run mode. This is because dry-run is expected to fail
+            # if the user cannot perform the purge.
             validator.data_model(["read", "write"], spaces={selected_space})
         if (stats.nodes + stats.edges) > 0:
             validator.instances(["read", "write"], spaces={selected_space})
 
-        config = client.config
-        total_by_crud_cls = {
-            EdgeCRUD: (stats.edges, config.create_api_url("/models/instances/delete")),
-            NodeCRUD: (stats.nodes, config.create_api_url("/models/instances/delete")),
-            DataModelCRUD: (stats.data_models, config.create_api_url("/models/datamodels/delete")),
-            ViewCRUD: (stats.views, config.create_api_url("/models/views/delete")),
-            ContainerCRUD: (stats.containers, config.create_api_url("/models/containers/delete")),
-        }
-        console = Console()
+        to_delete = self._create_to_delete_list_purge_space(client, delete_datapoints, delete_file_content, stats)
         if dry_run:
-            for crud_cls, (total, _) in total_by_crud_cls.items():
-                crud = crud_cls.create_loader(client)  # type: ignore[attr-defined]
-                results[crud.display_name] = ResourceDeployResult(crud.display_name, deleted=total)
+            results = DeployResults([], "purge", dry_run=True)
+            for item in to_delete:
+                results[item.display_name] = ResourceDeployResult(item.display_name, deleted=item.total)
             if include_space:
                 space_loader = SpaceCRUD.create_loader(client)
                 results[space_loader.display_name] = ResourceDeployResult(space_loader.display_name, deleted=1)
         else:
-            with HTTPClient(client.config, max_retries=10) as delete_client:
-                for crud_cls, (total, URL) in total_by_crud_cls.items():
-                    crud = crud_cls.create_loader(client)  # type: ignore[attr-defined]
-                    if total == 0:
-                        results[crud.display_name] = ResourceDeployResult(crud.display_name, deleted=0)
-                        continue
-                    # Two results objects since they are updated concurrently
-                    process_results = ResourceDeployResult(crud.display_name)
-                    write_results = ResourceDeployResult(crud.display_name)
-                    # Containers, DataModels and Views need special handling to avoid type info in the delete call
-                    # While for instance we need the type info in delete.
-                    if isinstance(crud, ContainerCRUD | DataModelCRUD | ViewCRUD):
-                        dump_args = {"include_type": False}
-                    elif isinstance(crud, EdgeCRUD):
-                        dump_args = {"include_instance_type": True}
-                    else:
-                        dump_args = {}
-                    process = partial(self._as_id_batch, dump_args=dump_args)
-                    if isinstance(crud, NodeCRUD):
-                        process = partial(
-                            self._check_data,
-                            client=client,
-                            delete_datapoints=delete_datapoints,
-                            delete_file_content=delete_file_content,
-                            process_results=process_results,
-                            console=console,
-                            verbose=verbose,
-                        )
-
-                    executor = ProducerWorkerExecutor[CogniteResourceList, list[JsonVal]](
-                        download_iterable=self._iterate_batch(crud, selected_space, batch_size=self.BATCH_SIZE_DM),
-                        process=process,
-                        write=self._purge_batch(crud, URL, delete_client, write_results),
-                        max_queue_size=10,
-                        iteration_count=total // self.BATCH_SIZE_DM + (1 if total % self.BATCH_SIZE_DM > 0 else 0),
-                        download_description=f"Downloading {crud.display_name}",
-                        process_description=f"Preparing {crud.display_name} for deletion",
-                        write_description=f"Deleting {crud.display_name}",
-                        console=console,
-                    )
-                    executor.run()
-                    write_results += process_results
-                    results[crud.display_name] = write_results
-
-                    if executor.error_occurred:
-                        self.warn(
-                            HighSeverityWarning(f"Failed to delete all {crud.display_name}. {executor.error_message}")
-                        )
+            results = self._delete_resources(to_delete, client, verbose, selected_space, None)
             if include_space:
-                space_loader = SpaceCRUD.create_loader(client)
-                try:
-                    space_loader.delete([selected_space])
-                    print(f"Space {selected_space} deleted")
-                except CogniteAPIError as e:
-                    self.warn(HighSeverityWarning(f"Failed to delete space {selected_space!r}: {e}"))
-                else:
-                    results[space_loader.display_name] = ResourceDeployResult(space_loader.display_name, deleted=1)
+                self._delete_space(client, selected_space, results)
         print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
         return results
 
+    def _create_to_delete_list_purge_space(
+        self, client: ToolkitClient, delete_datapoints: bool, delete_file_content: bool, stats: SpaceStatistics
+    ) -> list[ToDelete]:
+        config = client.config
+        to_delete = [
+            EdgeToDelete(
+                EdgeCRUD.create_loader(client), stats.edges, config.create_api_url("/models/instances/delete")
+            ),
+            NodesToDelete(
+                NodeCRUD.create_loader(client),
+                stats.nodes,
+                config.create_api_url(
+                    "/models/instances/delete",
+                ),
+                delete_datapoints=delete_datapoints,
+                delete_file_content=delete_file_content,
+            ),
+            DataModelingToDelete(
+                DataModelCRUD.create_loader(client),
+                stats.data_models,
+                config.create_api_url("/models/datamodels/delete"),
+            ),
+            DataModelingToDelete(
+                ViewCRUD.create_loader(client), stats.views, config.create_api_url("/models/views/delete")
+            ),
+            DataModelingToDelete(
+                ContainerCRUD.create_loader(client),
+                stats.containers,
+                config.create_api_url("/models/containers/delete"),
+            ),
+        ]
+        return to_delete
+
+    def _delete_space(self, client: ToolkitClient, selected_space: str, results: DeployResults) -> None:
+        space_loader = SpaceCRUD.create_loader(client)
+        try:
+            space_loader.delete([selected_space])
+            print(f"Space {selected_space} deleted")
+        except CogniteAPIError as e:
+            self.warn(HighSeverityWarning(f"Failed to delete space {selected_space!r}: {e}"))
+        else:
+            results[space_loader.display_name] = ResourceDeployResult(space_loader.display_name, deleted=1)
+
+    def _delete_resources(
+        self,
+        to_delete: list[ToDelete],
+        client: ToolkitClient,
+        verbose: bool,
+        space: str | None,
+        data_set_external_id: str | None,
+    ) -> DeployResults:
+        results = DeployResults([], "purge", dry_run=False)
+        console = Console()
+        with HTTPClient(client.config, max_retries=10) as delete_client:
+            for item in to_delete:
+                if item.total == 0:
+                    results[item.display_name] = ResourceDeployResult(item.display_name, deleted=0)
+                    continue
+                # Two results objects since they are updated concurrently
+                process_results = ResourceDeployResult(item.display_name)
+                write_results = ResourceDeployResult(item.display_name)
+                iteration_count: int | None = None
+                if item.total > 0:
+                    iteration_count = item.total // self.BATCH_SIZE_DM + (
+                        1 if item.total % self.BATCH_SIZE_DM > 0 else 0
+                    )
+                executor = ProducerWorkerExecutor[CogniteResourceList, list[JsonVal]](
+                    download_iterable=self._iterate_batch(
+                        item.crud, space, data_set_external_id, batch_size=self.BATCH_SIZE_DM
+                    ),
+                    process=item.get_process_function(client, console, verbose, process_results),
+                    write=self._purge_batch(item.crud, item.delete_url, delete_client, write_results),
+                    max_queue_size=10,
+                    iteration_count=iteration_count,
+                    download_description=f"Downloading {item.display_name}",
+                    process_description=f"Preparing {item.display_name} for deletion",
+                    write_description=f"Deleting {item.display_name}",
+                    console=console,
+                )
+                executor.run()
+                write_results += process_results
+                results[item.display_name] = write_results
+                if executor.error_occurred:
+                    if verbose and executor.error_traceback:
+                        executor.print_traceback()
+                    self.warn(
+                        HighSeverityWarning(f"Failed to delete all {item.display_name}. {executor.error_message}")
+                    )
+        return results
+
     @staticmethod
-    def _iterate_batch(crud: ResourceCRUD, selected_space: str, batch_size: int) -> Iterable[CogniteResourceList]:
+    def _iterate_batch(
+        crud: ResourceCRUD, selected_space: str | None, data_set_external_id: str | None, batch_size: int
+    ) -> Iterable[CogniteResourceList]:
         batch = crud.list_cls([])
-        for resource in crud.iterate(space=selected_space):
+        for resource in crud.iterate(space=selected_space, data_set_external_id=data_set_external_id):
             batch.append(resource)
             if len(batch) >= batch_size:
                 yield batch
@@ -210,41 +321,6 @@ class PurgeCommand(ToolkitCommand):
         if batch:
             yield batch
 
-    @staticmethod
-    def _as_id_batch(chunk: CogniteResourceList, dump_args: dict[str, Any]) -> list[JsonVal]:
-        return [item.as_id().dump(**dump_args) for item in chunk]
-
-    @staticmethod
-    def _check_data(
-        chunk: NodeList,
-        client: ToolkitClient,
-        delete_datapoints: bool,
-        delete_file_content: bool,
-        process_results: ResourceDeployResult,
-        console: Console,
-        verbose: bool,
-    ) -> list[JsonVal]:
-        """Check if the node has timeseries or files and delete the data if requested."""
-        node_ids = chunk.as_ids()
-        found_ids: set[InstanceId] = set()
-        if not delete_datapoints:
-            timeseries = client.time_series.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
-            found_ids |= {ts.instance_id for ts in timeseries if ts.instance_id is not None}
-        if not delete_file_content:
-            files = client.files.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
-            found_ids |= {f.instance_id for f in files if f.instance_id is not None}
-        if found_ids and verbose:
-            console.print(f"Skipping {found_ids} nodes as they have datapoints or file content")
-        process_results.unchanged += len(found_ids)
-        result: list[JsonVal] = []
-        for node_id in (n for n in node_ids if n not in found_ids):
-            dumped = node_id.dump(include_instance_type=True)
-            # The delete endpoint expects "instanceType" instead of "type"
-            dumped["instanceType"] = dumped.pop("type")
-            # MyPy think complains about invariant here, even though dict[str, str] is a type of JsonVal
-            result.append(dumped)  # type: ignore[arg-type]
-        return result
-
     @staticmethod
     def _purge_batch(
         crud: ResourceCRUD, delete_url: str, delete_client: HTTPClient, result: ResourceDeployResult
@@ -275,24 +351,71 @@ class PurgeCommand(ToolkitCommand):
             if loader_cls in dep_cls.dependencies and (exclude is None or dep_cls not in exclude)
         }
 
-    @staticmethod
-    def _get_selected_space(space: str | None, client: ToolkitClient) -> str:
-        if space is None:
-            spaces = client.data_modeling.spaces.list(limit=-1, include_global=False)
-            selected_space = questionary.select(
-                "Which space do you want to purge"
-                " (including all data models, views, containers, nodes and edges within that space)?",
-                sorted([space.space for space in spaces]),
+    def dataset_v2(
+        self,
+        client: ToolkitClient,
+        selected_data_set_external_id: str,
+        include_dataset: bool = False,
+        include_data: bool = True,
+        include_configurations: bool = False,
+        dry_run: bool = False,
+        auto_yes: bool = False,
+        verbose: bool = False,
+    ) -> DeployResults:
+        # Warning Messages
+        if not dry_run:
+            self._print_panel("dataSet", selected_data_set_external_id)
+        if not dry_run and not auto_yes:
+            confirm = questionary.confirm(
+                f"Are you really sure you want to purge the {selected_data_set_external_id!r} dataSet?", default=False
             ).ask()
+            if not confirm:
+                return DeployResults([], "purge", dry_run=dry_run)
+
+        # Validate Auth
+        validator = ValidateAccess(client, "purge")
+        data_set_id = client.lookup.data_sets.id(selected_data_set_external_id)
+        action = cast(Sequence[Literal["read", "write"]], ["read"] if dry_run else ["read", "write"])
+        if include_data:
+            # Check asset, events, time series, files, and sequences access, relationships, labels, 3D access.
+            validator.dataset_data(action, dataset_ids={data_set_id})
+        if include_configurations:
+            # Check workflow, transformations, extraction pipeline access
+            validator.dataset_configurations(action, dataset_ids={data_set_id})
+
+        to_delete: list[ToDelete] = self._create_to_delete_list_purge_dataset(
+            client,
+            include_data,
+            include_configurations,
+        )
+        if dry_run:
+            results = DeployResults([], "purge", dry_run=True)
+            for item in to_delete:
+                results[item.display_name] = ResourceDeployResult(item.display_name, deleted=item.total)
         else:
-            retrieved = client.data_modeling.spaces.retrieve(space)
-            if retrieved is None:
-                raise ToolkitMissingResourceError(f"Space {space} does not exist")
-            selected_space = space
+            results = self._delete_resources(to_delete, client, verbose, None, selected_data_set_external_id)
+        print(results.counts_table(exclude_columns={"Created", "Changed", "Total"}))
+        if include_dataset and not dry_run:
+            self._archive_dataset(client, selected_data_set_external_id)
+        return results
 
-        if selected_space is None:
-            raise ToolkitValueError("No space selected")
-        return selected_space
+    def _archive_dataset(self, client: ToolkitClient, data_set: str) -> None:
+        archived = (
+            DataSetUpdate(external_id=data_set)
+            .external_id.set(str(uuid.uuid4()))
+            .metadata.add({"archived": "true"})
+            .write_protected.set(True)
+        )
+        client.data_sets.update(archived)
+        print(f"DataSet {data_set} archived")
+
+    def _create_to_delete_list_purge_dataset(
+        self,
+        client: ToolkitClient,
+        include_data: bool,
+        include_configurations: bool,
+    ) -> list[ToDelete]:
+        raise NotImplementedError()
 
     def dataset(
         self,

cognite_toolkit/_cdf_tk/commands/auth.py

@@ -440,9 +440,11 @@ class AuthCommand(ToolkitCommand):
         loaders_by_capability_tuple: dict[tuple, list[str]] = defaultdict(list)
         capability_by_id: dict[frozenset[tuple], Capability] = {}
         project_type = client.project.status().this_project.data_modeling_status
-        for loader_cls in cruds.RESOURCE_CRUD_LIST:
-            loader = loader_cls.create_loader(client)
-            capability = loader_cls.get_required_capability(None, read_only=False)
+        for crud_cls in cruds.RESOURCE_CRUD_LIST:
+            crud = crud_cls.create_loader(client)
+            if crud.prerequisite_warning() is not None:
+                continue
+            capability = crud_cls.get_required_capability(None, read_only=False)
             capabilities = capability if isinstance(capability, list) else [capability]
             for cap in capabilities:
                 if project_type == "DATA_MODELING_ONLY" and isinstance(cap, AssetsAcl | RelationshipsAcl):
@@ -451,7 +453,7 @@ class AuthCommand(ToolkitCommand):
                 if id_ not in capability_by_id:
                     capability_by_id[id_] = cap
                 for cap_tuple in cap.as_tuples():
-                    loaders_by_capability_tuple[cap_tuple].append(loader.display_name)
+                    loaders_by_capability_tuple[cap_tuple].append(crud.display_name)
         return list(capability_by_id.values()), loaders_by_capability_tuple
 
     def check_has_any_access(self, client: ToolkitClient) -> TokenInspection:

cognite_toolkit/_cdf_tk/commands/modules.py

@@ -321,16 +321,22 @@ default_organization_dir = "{organization_dir.name}"''',
 
         modules_root_dir = organization_dir / MODULES
 
+        # Determine which library to use (if any)
+        library: Library | None = None
         if library_url:
             if not library_checksum:
                 raise ToolkitRequiredValueError(
                     "The '--library-checksum' is required when '--library-url' is provided."
                 )
+            library = Library(url=library_url, checksum=library_checksum)
+        elif not (organization_dir / CDFToml.file_name).exists():
+            # Load default library from resources when cdf.toml doesn't exist
+            default_cdf_toml = CDFToml.load(cwd=RESOURCES_PATH, use_singleton=False)
+            library = default_cdf_toml.libraries.get("toolkit-data")
+            if library is None:
+                raise ToolkitError("Default cdf.toml in resources is missing 'toolkit-data' library configuration.")
 
-            user_library = Library(url=library_url, checksum=library_checksum)
-            packages, modules_source_path = self._get_available_packages(user_library)
-        else:
-            packages, modules_source_path = self._get_available_packages()
+        packages, modules_source_path = self._get_available_packages(library)
 
         if select_all:
             print(Panel("Instantiating all available modules"))
@@ -767,7 +773,7 @@ default_organization_dir = "{organization_dir.name}"''',
         """
 
         cdf_toml = CDFToml.load()
-        if Flags.EXTERNAL_LIBRARIES.is_enabled() or user_library:
+        if (Flags.EXTERNAL_LIBRARIES.is_enabled() or user_library) and self._module_source_dir is None:
             libraries = {"userdefined": user_library} if user_library else cdf_toml.libraries
 
             for library_name, library in libraries.items():

cognite_toolkit/_cdf_tk/constants.py

@@ -30,7 +30,6 @@ CONFIG_FILE_SUFFIX = "config.yaml"
 # The global config file
 GLOBAL_CONFIG_FILE = "global.yaml"
 
-BUILTIN_MODULES = "_builtin_modules"
 COGNITE_MODULES = "cognite_modules"
 CUSTOM_MODULES = "custom_modules"
 EXTERNAL_PACKAGE = "external_package"
@@ -61,7 +60,6 @@ TABLE_FORMATS = frozenset([".csv", ".parquet"])
 ROOT_PATH = Path(__file__).parent.parent
 COGNITE_MODULES_PATH = ROOT_PATH / COGNITE_MODULES
 MODULES_PATH = ROOT_PATH / MODULES
-BUILTIN_MODULES_PATH = ROOT_PATH / BUILTIN_MODULES
 RESOURCES_PATH = ROOT_PATH / RESOURCES
 SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.2.0"
 # This is used in the build directory to keep track of order and flatten the

cognite_toolkit/_cdf_tk/cruds/_base_cruds.py

@@ -257,6 +257,15 @@ class ResourceCRUD(
     ) -> Iterable[T_WritableCogniteResource]:
         raise NotImplementedError
 
+    ### These methods can be optionally overwritten in the subclass ###
+    def prerequisite_warning(self) -> str | None:
+        """ "Returns a warning message if there are any prerequisites that must be met before using this CRUD.
+
+        This is used for special resource CRUDs that for example require data models/views to be deployed in CDF
+        to work. For example, the InfieldV1CRUD and the ResourceViewMappingCRUD.
+        """
+        return None
+
     @classmethod
     def get_dependent_items(cls, item: dict) -> "Iterable[tuple[type[ResourceCRUD], Hashable]]":
         """Returns all items that this item requires.

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py

@@ -75,6 +75,15 @@ class InfieldV1CRUD(ResourceCRUD[str, APMConfigWrite, APMConfig, APMConfigWriteL
 
         return DataModelInstancesAcl(actions, DataModelInstancesAcl.Scope.SpaceID([APMConfig.space]))
 
+    def prerequisite_warning(self) -> str | None:
+        views = self.client.data_modeling.views.retrieve(APMConfig.view_id)
+        if len(views) > 0:
+            return None
+        return (
+            f"{self.display_name} requires the {APMConfig.view_id!r} to be deployed. "
+            f"Install the infield options with cdf modules init/add to deploy it."
+        )
+
     def create(self, items: APMConfigWriteList) -> NodeApplyResultList:
         result = self.client.data_modeling.instances.apply(
             nodes=items.as_nodes(), auto_create_direct_relations=True, replace=False

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py

@@ -67,6 +67,13 @@ class ResourceViewMappingCRUD(
             actions=actions, scope=capabilities.DataModelInstancesAcl.Scope.SpaceID([COGNITE_MIGRATION_SPACE])
         )
 
+    def prerequisite_warning(self) -> str | None:
+        view_id = ResourceViewMapping.get_source()
+        views = self.client.data_modeling.views.retrieve(view_id)
+        if len(views) > 0:
+            return None
+        return f"{self.display_name} requires the {view_id!r} to be deployed. run `cdf migrate prepare` to deploy it."
+
     def create(self, items: NodeApplyList) -> Sized:
         return self.client.migration.resource_view_mapping.upsert(items)
 

cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py

@@ -15,7 +15,6 @@ from rich import print
 from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_ENVIRONMENT_FILE,
-    BUILTIN_MODULES,
     DEFAULT_CONFIG_FILE,
     DEFAULT_ENV,
     MODULES,
@@ -468,7 +467,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
         """Loads all default.config.yaml files in the cognite root module."""
 
         default_files_iterable: Iterable[Path]
-        if cognite_root_module.name in ROOT_MODULES or cognite_root_module.name == BUILTIN_MODULES:
+        if cognite_root_module.name in ROOT_MODULES:
             default_files_iterable = cognite_root_module.glob(f"**/{DEFAULT_CONFIG_FILE}")
         else:
             default_files_iterable = itertools.chain(