cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/modules.py

@@ -4,7 +4,6 @@ import tempfile
 import zipfile
 from collections import Counter
 from hashlib import sha256
-from importlib import resources
 from pathlib import Path
 from types import TracebackType
 from typing import Any, Literal
@@ -23,8 +22,7 @@ from rich.rule import Rule
 from rich.table import Table
 from rich.tree import Tree
 
-import cognite_toolkit
-from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml, Library
 from cognite_toolkit._cdf_tk.commands import _cli_commands as CLICommands
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
 from cognite_toolkit._cdf_tk.commands._changes import (
@@ -36,9 +34,10 @@ from cognite_toolkit._cdf_tk.commands._changes import (
     UpdateDockerImageVersion,
     UpdateModuleVersion,
 )
+from cognite_toolkit._cdf_tk.commands._questionary_style import custom_style_fancy
 from cognite_toolkit._cdf_tk.constants import (
-    BUILTIN_MODULES,
     MODULES,
+    RESOURCES_PATH,
     SUPPORT_MODULE_UPGRADE_FROM_VERSION,
     EnvType,
 )
@@ -52,7 +51,6 @@ from cognite_toolkit._cdf_tk.data_classes import (
     Packages,
 )
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError, ToolkitRequiredValueError, ToolkitValueError
-from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.hints import verify_module_directory
 from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
 from cognite_toolkit._cdf_tk.tk_warnings.other import HighSeverityWarning
@@ -66,20 +64,6 @@ if sys.version_info >= (3, 11):
     from typing import Self
 else:
     from typing_extensions import Self
-custom_style_fancy = questionary.Style(
-    [
-        ("qmark", "fg:#673ab7"),  # token in front of the question
-        ("question", "bold"),  # question text
-        ("answer", "fg:#f44336 bold"),  # submitted answer text behind the question
-        ("pointer", "fg:#673ab7 bold"),  # pointer used in select and checkbox prompts
-        ("highlighted", "fg:#673ab7 bold"),  # pointed-at choice in select and checkbox prompts
-        ("selected", "fg:#673ab7"),  # style for a selected item of a checkbox
-        ("separator", "fg:#cc5454"),  # separator in lists
-        ("instruction", ""),  # user instructions for select, rawselect, checkbox
-        ("text", ""),  # plain text
-        ("disabled", "fg:#858585 italic"),  # disabled choices for select and checkbox prompts
-    ]
-)
 
 INDENT = "  "
 POINTER = INDENT + "▶"
@@ -92,10 +76,19 @@ _FILE_DOWNLOADERS_BY_TYPE: dict[str, type[FileDownloader]] = {
 
 
 class ModulesCommand(ToolkitCommand):
-    def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False):
+    def __init__(
+        self,
+        print_warning: bool = True,
+        skip_tracking: bool = False,
+        silent: bool = False,
+        temp_dir_suffix: str | None = None,
+        module_source_dir: Path | None = None,
+    ):
         super().__init__(print_warning, skip_tracking, silent)
-        self._builtin_modules_path = Path(resources.files(cognite_toolkit.__name__)) / BUILTIN_MODULES  # type: ignore [arg-type]
-        self._temp_download_dir = Path(tempfile.gettempdir()) / MODULES
+        self._module_source_dir: Path | None = module_source_dir
+        # Use suffix to make temp directory unique (useful for parallel test execution)
+        modules_dir_name = f"{MODULES}.{temp_dir_suffix}" if temp_dir_suffix else MODULES
+        self._temp_download_dir = Path(tempfile.gettempdir()) / modules_dir_name
         if not self._temp_download_dir.exists():
             self._temp_download_dir.mkdir(parents=True, exist_ok=True)
 
@@ -166,6 +159,11 @@ class ModulesCommand(ToolkitCommand):
             print(f"{INDENT}[{'yellow' if mode == 'clean' else 'green'}]Creating {package_name}[/]")
 
             for module in package.modules:
+                if module.module_id:
+                    self._additional_tracking_info.installed_module_ids.add(module.module_id)
+                if module.package_id:
+                    self._additional_tracking_info.installed_package_ids.add(module.package_id)
+
                 if module.dir in seen_modules:
                     # A module can be part of multiple packages
                     continue
@@ -273,26 +271,7 @@ class ModulesCommand(ToolkitCommand):
             )
             safe_write(Path(organization_dir) / f"config.{environment}.yaml", config_init.dump_yaml_with_comments())
 
-        cdf_toml_content = self.create_cdf_toml(organization_dir, environments[0] if environments else "dev")
-
-        destination = Path.cwd() / CDFToml.file_name
-        if destination.exists():
-            print(f"{INDENT}[yellow]cdf.toml file already exists. Skipping creation.")
-        else:
-            destination.write_text(cdf_toml_content, encoding="utf-8")
-
-    def create_cdf_toml(self, organization_dir: Path, env: EnvType = "dev") -> str:
-        cdf_toml_content = safe_read(self._builtin_modules_path / CDFToml.file_name)
-        if organization_dir != Path.cwd():
-            cdf_toml_content = cdf_toml_content.replace(
-                "#<PLACEHOLDER>",
-                f'''
-default_organization_dir = "{organization_dir.name}"''',
-            )
-        else:
-            cdf_toml_content = cdf_toml_content.replace("#<PLACEHOLDER>", "")
-        cdf_toml_content = cdf_toml_content.replace("<DEFAULT_ENV_PLACEHOLDER>", env)
-        return cdf_toml_content
+        CDFToml.write(organization_dir, environments[0] if environments else "dev")
 
     def init(
         self,
@@ -302,20 +281,30 @@ default_organization_dir = "{organization_dir.name}"''',
         user_select: str | None = None,
         user_environments: list[str] | None = None,
         user_download_data: bool | None = None,
+        library_url: str | None = None,
+        library_checksum: str | None = None,
     ) -> None:
         if not organization_dir:
-            new_line = "\n    "
-            message = (
-                f"Which directory would you like to create templates in? (default: current directory){new_line}"
-                f"HINT Use an organization directory if you use the repository for more than Toolkit. "
-                f"If not, use the current (repository root) directory '.':"
-            )
-            organization_dir_raw = questionary.text(message=message, default="").ask()
-            organization_dir = Path(organization_dir_raw.strip())
+            organization_dir = ModulesCommand._prompt_organization_dir()
 
         modules_root_dir = organization_dir / MODULES
 
-        packages, modules_source_path = self._get_available_packages()
+        # Determine which library to use (if any)
+        library: Library | None = None
+        if library_url:
+            if not library_checksum:
+                raise ToolkitRequiredValueError(
+                    "The '--library-checksum' is required when '--library-url' is provided."
+                )
+            library = Library(url=library_url, checksum=library_checksum)
+        elif not (organization_dir / CDFToml.file_name).exists():
+            # Load default library from resources when cdf.toml doesn't exist
+            default_cdf_toml = CDFToml.load(cwd=RESOURCES_PATH, use_singleton=False)
+            library = default_cdf_toml.libraries.get("toolkit-data")
+            if library is None:
+                raise ToolkitError("Default cdf.toml in resources is missing 'toolkit-data' library configuration.")
+
+        packages, modules_source_path = self._get_available_packages(library)
 
         if select_all:
             print(Panel("Instantiating all available modules"))
@@ -424,6 +413,21 @@ default_organization_dir = "{organization_dir.name}"''',
         if not is_interactive:
             raise typer.Exit()
 
+    @staticmethod
+    def _prompt_organization_dir() -> Path:
+        print(
+            "\n".join(
+                [
+                    "Enter an optional subdirectory for Toolkit content.",
+                    "    [bold yellow]HINT[/bold yellow] Use an organization directory if you use the repository for more than Toolkit.",
+                    "    [dim]The current directory is the default.[/dim]",
+                ]
+            )
+        )
+
+        organization_dir_raw = questionary.text(message="", default="").ask()
+        return Path(organization_dir_raw.strip())
+
     @staticmethod
     def _get_download_data(selected: Packages) -> bool:
         example_data = {
@@ -745,16 +749,21 @@ default_organization_dir = "{organization_dir.name}"''',
             modules_source_path=modules_source_path,
         )
 
-    def _get_available_packages(self) -> tuple[Packages, Path]:
+    def _get_available_packages(self, user_library: Library | None = None) -> tuple[Packages, Path]:
         """
         Returns a list of available packages, either from the CDF TOML file or from external libraries if the feature flag is enabled.
-        If the feature flag is not enabled and no libraries are specified, it returns the built-in modules.
         """
 
         cdf_toml = CDFToml.load()
-        if Flags.EXTERNAL_LIBRARIES.is_enabled() and cdf_toml.libraries:
-            for library_name, library in cdf_toml.libraries.items():
+
+        if self._module_source_dir is None:
+            libraries = {"userdefined": user_library} if user_library else cdf_toml.libraries
+
+            for library_name, library in libraries.items():
                 try:
+                    if library_name:
+                        self._additional_tracking_info.downloaded_library_ids.add(library_name)
+
                     print(f"[green]Adding library {library_name} from {library.url}[/]")
                     # Extract filename from URL, fallback to library_name.zip if no filename found
                     from urllib.parse import urlparse
@@ -768,7 +777,20 @@ default_organization_dir = "{organization_dir.name}"''',
                     self._validate_checksum(library.checksum, file_path)
                     self._unpack(file_path)
                     packages = Packages().load(file_path.parent)
+                    if packages.warnings:
+                        for warning in packages.warnings:
+                            self.warn(warning)
                     self._validate_packages(packages, f"library {library_name}")
+
+                    # Track deployment pack download for each package and module
+                    for package in packages.values():
+                        if package.id:
+                            self._additional_tracking_info.downloaded_package_ids.add(package.id)
+
+                        for module in package.modules:
+                            if module.module_id:
+                                self._additional_tracking_info.downloaded_module_ids.add(module.module_id)
+
                     return packages, file_path.parent
                 except Exception as e:
                     if isinstance(e, ToolkitError):
@@ -779,12 +801,24 @@ default_organization_dir = "{organization_dir.name}"''',
                         ) from e
 
                     raise ToolkitError(f"Failed to add library {library_name}, {e}")
-            # If no libraries are specified or the flag is not enabled, load the built-in modules
+            # If no libraries are specified or the flag is not enabled, raise an error
             raise ValueError("No valid libraries found.")
         else:
-            packages = Packages.load(self._builtin_modules_path)
-            self._validate_packages(packages, "built-in modules")
-            return packages, self._builtin_modules_path
+            if user_library:
+                self.warn(
+                    MediumSeverityWarning(
+                        "External library provided but not enabled in cdf.toml. Please enable the feature flag and try again."
+                    )
+                )
+
+            if self._module_source_dir is None:
+                self.warn(
+                    MediumSeverityWarning("No external libraries and no local module source directory specified.")
+                )
+                return Packages(), Path(".")
+            packages = Packages.load(self._module_source_dir)
+            self._validate_packages(packages, self._module_source_dir.name)
+            return packages, self._module_source_dir
 
     def _validate_packages(self, packages: Packages, source_name: str) -> None:
         """

cognite_toolkit/_cdf_tk/commands/pull.py

@@ -11,7 +11,6 @@ from typing import Any, Union
 
 import questionary
 import yaml
-from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
 from questionary import Choice
 from rich import print
 from rich.markdown import Markdown
@@ -30,7 +29,6 @@ from cognite_toolkit._cdf_tk.cruds import (
     ResourceCRUD,
     StreamlitCRUD,
 )
-from cognite_toolkit._cdf_tk.cruds._base_cruds import T_ID, T_WritableCogniteResourceList
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildEnvironment,
     BuildVariable,
@@ -44,6 +42,7 @@ from cognite_toolkit._cdf_tk.data_classes import (
     YAMLComments,
 )
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError, ToolkitMissingResourceError, ToolkitValueError
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils import (
     YAMLComment,
@@ -59,6 +58,7 @@ from cognite_toolkit._cdf_tk.utils.modules import (
     module_directory_from_path,
     parse_user_selected_modules,
 )
+from cognite_toolkit._cdf_tk.utils.useful_types import T_ID
 
 from ._base import ToolkitCommand
 from .build_cmd import BuildCommand
@@ -542,15 +542,13 @@ class PullCommand(ToolkitCommand):
 
     def _pull_resources(
         self,
-        loader: ResourceCRUD[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         resources: BuiltFullResourceList[T_ID],
         dry_run: bool,
         environment_variables: dict[str, str | None],
     ) -> ResourceDeployResult:
         cdf_resources = loader.retrieve(resources.identifiers)
-        cdf_resource_by_id: dict[T_ID, T_WritableCogniteResource] = {loader.get_id(r): r for r in cdf_resources}
+        cdf_resource_by_id: dict[T_ID, T_ResourceResponse] = {loader.get_id(r): r for r in cdf_resources}
 
         resources_by_file = resources.by_file()
         file_results = ResourceDeployResult(loader.display_name)
@@ -561,7 +559,7 @@ class PullCommand(ToolkitCommand):
 
             if has_changes and not dry_run:
                 new_content, extra_files = self._to_write_content(
-                    safe_read(source_file), to_write, resources, environment_variables, loader
+                    safe_read(source_file), to_write, resources, environment_variables, loader, source_file
                 )
                 with source_file.open("w", encoding=ENCODING, newline=NEWLINE) as f:
                     f.write(new_content)
@@ -574,11 +572,9 @@ class PullCommand(ToolkitCommand):
     def _get_to_write(
         self,
         local_resource_by_id: dict[T_ID, dict[str, Any]],
-        cdf_resource_by_id: dict[T_ID, T_WritableCogniteResource],
+        cdf_resource_by_id: dict[T_ID, T_ResourceResponse],
         file_results: ResourceDeployResult,
-        loader: ResourceCRUD[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
     ) -> tuple[bool, dict[T_ID, dict[str, Any]]]:
         to_write: dict[T_ID, dict[str, Any]] = {}
         has_changes = False
@@ -607,9 +603,7 @@ class PullCommand(ToolkitCommand):
     @staticmethod
     def _get_local_resource_dict_by_id(
         resources: BuiltFullResourceList[T_ID],
-        loader: ResourceCRUD[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         environment_variables: dict[str, str | None],
     ) -> dict[T_ID, dict[str, Any]]:
         unique_destinations = {r.destination for r in resources if r.destination}
@@ -646,9 +640,8 @@ class PullCommand(ToolkitCommand):
         to_write: dict[T_ID, dict[str, Any]],
         resources: BuiltFullResourceList[T_ID],
         environment_variables: dict[str, str | None],
-        loader: ResourceCRUD[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
+        source_file: Path,
     ) -> tuple[str, dict[Path, str]]:
         # 1. Replace all variables with placeholders
         # 2. Load source and keep the comments
@@ -682,7 +675,7 @@ class PullCommand(ToolkitCommand):
                     variables_with_environment_list.append(variable)
             variables = BuildVariables(variables_with_environment_list)
 
-        content, value_by_placeholder = variables.replace(source, use_placeholder=True)
+        content, value_by_placeholder = variables.replace(source, source_file, use_placeholder=True)
         comments = YAMLComments.load(source)
         # If there is a variable in the identifier, we need to replace it with the value
         # such that we can look it up in the to_write dict.
@@ -690,10 +683,10 @@ class PullCommand(ToolkitCommand):
             # The safe read in ExtractionPipelineConfigLoader stringifies the config dict,
             # but we need to load it as a dict so we can write it back to the file maintaining
             # the order or the keys.
-            loaded = read_yaml_content(variables.replace(source))
+            loaded = read_yaml_content(variables.replace(source, source_file))
             loaded_with_placeholder = read_yaml_content(content)
         else:
-            loaded = read_yaml_content(loader.safe_read(variables.replace(source)))
+            loaded = read_yaml_content(loader.safe_read(variables.replace(source, source_file)))
             loaded_with_placeholder = read_yaml_content(loader.safe_read(content))
 
         built_by_identifier = {r.identifier: r for r in resources}
@@ -756,7 +749,7 @@ class PullCommand(ToolkitCommand):
             builder = create_builder(built.resource_dir, None)
             for extra in built.extra_sources:
                 extra_content, extra_placeholders = built.build_variables.replace(
-                    safe_read(extra.path), extra.path.suffix, use_placeholder=True
+                    safe_read(extra.path), extra.path, use_placeholder=True
                 )
                 key, _ = builder.load_extra_field(extra_content)
                 if key in item_write:

cognite_toolkit/_cdf_tk/commands/resources.py

@@ -0,0 +1,179 @@
+import difflib
+from pathlib import Path
+from typing import cast
+
+import questionary
+import typer
+from questionary import Choice
+from rich import print
+
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.constants import MODULES
+from cognite_toolkit._cdf_tk.cruds import RESOURCE_CRUD_LIST, ResourceCRUD
+from cognite_toolkit._cdf_tk.data_classes import ModuleDirectories
+from cognite_toolkit._cdf_tk.resource_classes import ToolkitResource
+from cognite_toolkit._cdf_tk.utils.collection import humanize_collection
+from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump
+
+
+class ResourcesCommand(ToolkitCommand):
+    def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
+        super().__init__(print_warning, skip_tracking, silent)
+
+    def _get_or_prompt_module_path(self, module: str | None, organization_dir: Path, verbose: bool) -> Path:
+        """
+        Check if the module exists in the organization directory and return the module path.
+        If module is not provided, ask the user to select or create a new module.
+        """
+        present_modules = ModuleDirectories.load(organization_dir, None)
+
+        if module:
+            for mod in present_modules:
+                if mod.name.casefold() == module.casefold():
+                    return mod.dir
+
+            if questionary.confirm(f"{module} module not found. Do you want to create a new one?").ask():
+                return organization_dir / MODULES / module
+
+            if verbose:
+                print(f"[red]Aborting as {module} module not found...[/red]")
+            else:
+                print("[red]Aborting...[/red]")
+            raise typer.Exit()
+
+        choices = [Choice(title=mod.name, value=mod.dir) for mod in present_modules]
+        choices.append(Choice(title="<Create new module>", value="NEW"))
+
+        selected = questionary.select("Select a module:", choices=choices).ask()
+
+        if selected == "NEW":
+            new_module_name = questionary.text("Enter name for new module:").ask()
+            if not new_module_name:
+                print("[red]No module name provided. Aborting...[/red]")
+                raise typer.Exit()
+            return organization_dir / MODULES / new_module_name
+
+        if not selected:
+            print("[red]No module selected. Aborting...[/red]")
+            raise typer.Exit()
+
+        return cast(Path, selected)
+
+    def _resolve_kinds(self, kinds: list[str] | None) -> list[type[ResourceCRUD]]:
+        """
+        Resolve kinds from list of strings or do an interactive selection.
+        """
+        all_cruds = {crud.kind.casefold(): crud for crud in RESOURCE_CRUD_LIST}
+
+        if not kinds:
+            sorted_cruds = sorted(RESOURCE_CRUD_LIST, key=lambda x: x.kind)
+            choices = [Choice(title=crud.kind, value=crud) for crud in sorted_cruds]
+
+            selected = questionary.select("Select resource type:", choices=choices).ask()
+            if not selected:
+                print("[red]No resource type selected. Aborting...[/red]")
+                raise typer.Exit()
+            return [selected]
+
+        resolved_cruds = []
+        for kind in kinds:
+            kind_lower = kind.casefold()
+            if kind_lower in all_cruds:
+                resolved_cruds.append(all_cruds[kind_lower])
+            else:
+                matches = difflib.get_close_matches(kind_lower, all_cruds.keys())
+                if matches:
+                    suggestion = all_cruds[matches[0]].kind
+                    print(f"[red]Unknown resource type '{kind}'. Did you mean '{suggestion}'?[/red]")
+                else:
+                    print(
+                        f"[red]Unknown resource type '{kind}'. "
+                        f"Available types: {humanize_collection(sorted([c.kind for c in RESOURCE_CRUD_LIST]))}[/red]"
+                    )
+                raise typer.Exit()
+
+        return resolved_cruds
+
+    def _create_resource_yaml_skeleton(self, yaml_cls: type[ToolkitResource]) -> dict[str, str]:
+        """
+        Build YAML skeleton from a Pydantic model class using JSON schema for better type information.
+        """
+        yaml_skeleton: dict[str, str] = {}
+        for field_name, field in yaml_cls.model_fields.items():
+            name = field.alias or field_name
+            description = field.description or name
+            if field.is_required():
+                yaml_skeleton[name] = f"(Required) {description}"
+            else:
+                yaml_skeleton[name] = description
+
+        return yaml_skeleton
+
+    def _get_resource_yaml_content(self, resource_crud: type[ResourceCRUD]) -> str:
+        """
+        Creates a new resource in the specified module using the resource_crud.yaml_cls.
+        """
+        yaml_header = (
+            f"# API docs: {resource_crud.doc_url()}\n"
+            f"# YAML reference: https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library"
+        )
+        yaml_skeleton = self._create_resource_yaml_skeleton(resource_crud.yaml_cls)
+        yaml_contents = yaml_safe_dump(yaml_skeleton)
+        return yaml_header + "\n\n" + yaml_contents
+
+    def _create_resource_yaml_file(
+        self,
+        resource_crud: type[ResourceCRUD],
+        module_path: Path,
+        prefix: str | None = None,
+        verbose: bool = False,
+    ) -> None:
+        """
+        Creates a new resource YAML file in the specified module using the resource_crud.yaml_cls.
+        """
+        resource_dir: Path = module_path / resource_crud.folder_name
+        if resource_crud.sub_folder_name:
+            resource_dir = resource_dir / resource_crud.sub_folder_name
+
+        if not resource_dir.exists():
+            resource_dir.mkdir(parents=True, exist_ok=True)
+
+        final_prefix = prefix if prefix is not None else f"my_{resource_crud.kind}"
+        file_name = f"{final_prefix}.{resource_crud.kind}.yaml"
+        file_path: Path = resource_dir / file_name
+
+        if file_path.exists() and not questionary.confirm(f"{file_path.name} file already exists. Overwrite?").ask():
+            print("[red]Skipping...[/red]")
+            return
+
+        yaml_content = self._get_resource_yaml_content(resource_crud)
+        file_path.write_text(yaml_content)
+        if verbose:
+            print(
+                f"[green]{resource_crud.kind} Resource YAML file created successfully at {file_path.as_posix()}[/green]"
+            )
+        else:
+            print(f"[green]Created {file_path.as_posix()}[/green]")
+
+    def create(
+        self,
+        organization_dir: Path,
+        module_name: str | None = None,
+        kind: list[str] | None = None,
+        prefix: str | None = None,
+        verbose: bool = False,
+    ) -> None:
+        """
+        create resource YAMLs.
+
+        Args:
+            organization_dir: The directory of the organization.
+            module_name: The name of the module.
+            kind: The kind(s) of resource to create.
+            prefix: The prefix for the resource file.
+            verbose: Whether to print verbose output.
+        """
+        module_path = self._get_or_prompt_module_path(module_name, organization_dir, verbose)
+        resource_cruds = self._resolve_kinds(kind)
+        for crud in resource_cruds:
+            self._create_resource_yaml_file(crud, module_path, prefix, verbose)

cognite_toolkit/_cdf_tk/constants.py

@@ -30,7 +30,6 @@ CONFIG_FILE_SUFFIX = "config.yaml"
 # The global config file
 GLOBAL_CONFIG_FILE = "global.yaml"
 
-BUILTIN_MODULES = "_builtin_modules"
 COGNITE_MODULES = "cognite_modules"
 CUSTOM_MODULES = "custom_modules"
 EXTERNAL_PACKAGE = "external_package"
@@ -38,6 +37,7 @@ MODULES = "modules"
 REPO_FILES_DIR = "_repo_files"
 DOCKER_IMAGE_NAME = "cognite/toolkit"
 BUILD_FOLDER_ENCODING = "utf-8"
+RESOURCES = "_resources"
 
 ROOT_MODULES = [MODULES, CUSTOM_MODULES, COGNITE_MODULES, EXTERNAL_PACKAGE]
 MODULE_PATH_SEP = "/"
@@ -60,8 +60,8 @@ TABLE_FORMATS = frozenset([".csv", ".parquet"])
 ROOT_PATH = Path(__file__).parent.parent
 COGNITE_MODULES_PATH = ROOT_PATH / COGNITE_MODULES
 MODULES_PATH = ROOT_PATH / MODULES
-BUILTIN_MODULES_PATH = ROOT_PATH / BUILTIN_MODULES
-SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.2.0"
+RESOURCES_PATH = ROOT_PATH / RESOURCES
+SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.4.0"
 # This is used in the build directory to keep track of order and flatten the
 # module directory structure with accounting for duplicated names.
 INDEX_PATTERN = re.compile("^[0-9]+\\.")
@@ -81,7 +81,7 @@ EnvType: TypeAlias = Literal["dev", "test", "staging", "qa", "prod"]
 USE_SENTRY = "pytest" not in sys.modules and os.environ.get("SENTRY_ENABLED", "true").lower() == "true"
 SPACE_FORMAT_PATTERN = r"^[a-zA-Z][a-zA-Z0-9_-]{0,41}[a-zA-Z0-9]?$"
 DM_EXTERNAL_ID_PATTERN = r"^[a-zA-Z]([a-zA-Z0-9_]{0,253}[a-zA-Z0-9])?$"
-INSTANCE_EXTERNAL_ID_PATTERN = r"^[^\\x00]{1,256}$"
+INSTANCE_EXTERNAL_ID_PATTERN = r"^[^\x00]{1,256}$"
 FORBIDDEN_SPACES = frozenset(["space", "cdf", "dms", "pg3", "shared", "system", "node", "edge"])
 FORBIDDEN_CONTAINER_AND_VIEW_EXTERNAL_IDS = frozenset(
     [
@@ -160,3 +160,31 @@ COGNITE_MIGRATION_SPACE = "cognite_migration"
 COGNITE_TIME_SERIES_CONTAINER = ContainerId("cdf_cdm", "CogniteTimeSeries")
 COGNITE_FILE_CONTAINER = ContainerId("cdf_cdm", "CogniteFile")
 CDF_UNIT_SPACE = "cdf_cdm_units"
+
+
+# Container properties that are read-only in DMS needs to be handled with extra care, as this aspect is not currently exposed by the API.
+READONLY_CONTAINER_PROPERTIES = {
+    ContainerId(space="cdf_cdm", external_id="CogniteAsset"): {
+        "assetHierarchy_path_last_updated_time",
+        "assetHierarchy_path",
+        "assetHierarchy_root",
+    },
+    ContainerId(space="cdf_cdm", external_id="CogniteFile"): {"isUploaded", "uploadedTime"},
+}
+
+# Data Plugin Constants
+DATA_DEFAULT_DIR = "data"
+DATA_RESOURCE_DIR = "resources"
+DATA_MANIFEST_STEM = "Manifest"
+DATA_MANIFEST_SUFFIX = f".{DATA_MANIFEST_STEM}.yaml"
+
+DATA_UPLOAD_URL = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/guides/plugins/data_plugin/index"
+# Migration Constants
+MISSING_INSTANCE_SPACE = "<InstanceSpaceMissing>"
+MISSING_EXTERNAL_ID = "INTERNAL_ID_project_{project}_id_{id}"
+
+# Stream Template names
+STREAM_IMMUTABLE_TEMPLATE_NAME = frozenset(("ImmutableTestStream", "BasicArchive"))
+STREAM_MUTABLE_TEMPLATE_NAME = frozenset(("BasicLiveData",))
+
+StreamTemplateName: TypeAlias = Literal["ImmutableTestStream", "BasicArchive", "BasicLiveData"]