cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/pull.py

@@ -11,7 +11,6 @@ from typing import Any, Union
 
 import questionary
 import yaml
-from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
 from questionary import Choice
 from rich import print
 from rich.markdown import Markdown
@@ -20,6 +19,16 @@ from rich.panel import Panel
 from cognite_toolkit._cdf_tk.builders import create_builder
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import BUILD_ENVIRONMENT_FILE, ENV_VAR_PATTERN
+from cognite_toolkit._cdf_tk.cruds import (
+    ExtractionPipelineConfigCRUD,
+    FunctionCRUD,
+    GraphQLCRUD,
+    GroupAllScopedCRUD,
+    HostedExtractorDestinationCRUD,
+    HostedExtractorSourceCRUD,
+    ResourceCRUD,
+    StreamlitCRUD,
+)
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildEnvironment,
     BuildVariable,
@@ -33,17 +42,7 @@ from cognite_toolkit._cdf_tk.data_classes import (
     YAMLComments,
 )
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError, ToolkitMissingResourceError, ToolkitValueError
-from cognite_toolkit._cdf_tk.loaders import (
-    ExtractionPipelineConfigLoader,
-    FunctionLoader,
-    GraphQLLoader,
-    GroupAllScopedLoader,
-    HostedExtractorDestinationLoader,
-    HostedExtractorSourceLoader,
-    ResourceLoader,
-    StreamlitLoader,
-)
-from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID, T_WritableCogniteResourceList
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils import (
     YAMLComment,
@@ -59,6 +58,7 @@ from cognite_toolkit._cdf_tk.utils.modules import (
     module_directory_from_path,
     parse_user_selected_modules,
 )
+from cognite_toolkit._cdf_tk.utils.useful_types import T_ID
 
 from ._base import ToolkitCommand
 from .build_cmd import BuildCommand
@@ -504,7 +504,7 @@ class PullCommand(ToolkitCommand):
 
         results = DeployResults([], action="pull", dry_run=dry_run)
         for loader_cls in selected_loaders:
-            if not issubclass(loader_cls, ResourceLoader):
+            if not issubclass(loader_cls, ResourceCRUD):
                 continue
             loader = loader_cls.create_loader(client, build_dir)
             resources: BuiltFullResourceList[T_ID] = built_modules.get_resources(  # type: ignore[valid-type]
@@ -516,12 +516,12 @@ class PullCommand(ToolkitCommand):
             )
             if not resources:
                 continue
-            if isinstance(loader, HostedExtractorSourceLoader | HostedExtractorDestinationLoader):
+            if isinstance(loader, HostedExtractorSourceCRUD | HostedExtractorDestinationCRUD):
                 self.warn(
                     LowSeverityWarning(f"Skipping {loader.display_name} as it is not supported by the pull command.")
                 )
                 continue
-            if isinstance(loader, GraphQLLoader | FunctionLoader | StreamlitLoader):
+            if isinstance(loader, GraphQLCRUD | FunctionCRUD | StreamlitCRUD):
                 self.warn(
                     LowSeverityWarning(
                         f"Skipping {loader.display_name} as it is not supported by the pull command due to"
@@ -529,7 +529,7 @@ class PullCommand(ToolkitCommand):
                     )
                 )
                 continue
-            if isinstance(loader, GroupAllScopedLoader):
+            if isinstance(loader, GroupAllScopedCRUD):
                 # We have two loaders for Groups. We skip this one and
                 # only use the GroupResourceScopedLoader
                 continue
@@ -542,15 +542,13 @@ class PullCommand(ToolkitCommand):
 
     def _pull_resources(
         self,
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         resources: BuiltFullResourceList[T_ID],
         dry_run: bool,
         environment_variables: dict[str, str | None],
     ) -> ResourceDeployResult:
-        cdf_resources = loader.retrieve(resources.identifiers)  # type: ignore[arg-type]
-        cdf_resource_by_id: dict[T_ID, T_WritableCogniteResource] = {loader.get_id(r): r for r in cdf_resources}
+        cdf_resources = loader.retrieve(resources.identifiers)
+        cdf_resource_by_id: dict[T_ID, T_ResourceResponse] = {loader.get_id(r): r for r in cdf_resources}
 
         resources_by_file = resources.by_file()
         file_results = ResourceDeployResult(loader.display_name)
@@ -560,8 +558,8 @@ class PullCommand(ToolkitCommand):
             has_changes, to_write = self._get_to_write(local_resource_by_id, cdf_resource_by_id, file_results, loader)
 
             if has_changes and not dry_run:
-                new_content, extra_files = self._to_write_content(  # type: ignore[arg-type]
-                    safe_read(source_file), to_write, resources, environment_variables, loader
+                new_content, extra_files = self._to_write_content(
+                    safe_read(source_file), to_write, resources, environment_variables, loader, source_file
                 )
                 with source_file.open("w", encoding=ENCODING, newline=NEWLINE) as f:
                     f.write(new_content)
@@ -574,11 +572,9 @@ class PullCommand(ToolkitCommand):
     def _get_to_write(
         self,
         local_resource_by_id: dict[T_ID, dict[str, Any]],
-        cdf_resource_by_id: dict[T_ID, T_WritableCogniteResource],
+        cdf_resource_by_id: dict[T_ID, T_ResourceResponse],
         file_results: ResourceDeployResult,
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
     ) -> tuple[bool, dict[T_ID, dict[str, Any]]]:
         to_write: dict[T_ID, dict[str, Any]] = {}
         has_changes = False
@@ -607,9 +603,7 @@ class PullCommand(ToolkitCommand):
     @staticmethod
     def _get_local_resource_dict_by_id(
         resources: BuiltFullResourceList[T_ID],
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         environment_variables: dict[str, str | None],
     ) -> dict[T_ID, dict[str, Any]]:
         unique_destinations = {r.destination for r in resources if r.destination}
@@ -625,7 +619,7 @@ class PullCommand(ToolkitCommand):
 
     @staticmethod
     def _select_resource_ids(
-        all_: bool, id_: T_ID, loader: ResourceLoader, local_resources: BuiltFullResourceList, organization_dir: Path
+        all_: bool, id_: T_ID, loader: ResourceCRUD, local_resources: BuiltFullResourceList, organization_dir: Path
     ) -> BuiltFullResourceList[T_ID]:
         if all_:
             return local_resources
@@ -646,9 +640,8 @@ class PullCommand(ToolkitCommand):
         to_write: dict[T_ID, dict[str, Any]],
         resources: BuiltFullResourceList[T_ID],
         environment_variables: dict[str, str | None],
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
+        source_file: Path,
     ) -> tuple[str, dict[Path, str]]:
         # 1. Replace all variables with placeholders
         # 2. Load source and keep the comments
@@ -682,18 +675,18 @@ class PullCommand(ToolkitCommand):
                     variables_with_environment_list.append(variable)
             variables = BuildVariables(variables_with_environment_list)
 
-        content, value_by_placeholder = variables.replace(source, use_placeholder=True)
+        content, value_by_placeholder = variables.replace(source, source_file, use_placeholder=True)
         comments = YAMLComments.load(source)
         # If there is a variable in the identifier, we need to replace it with the value
         # such that we can look it up in the to_write dict.
-        if isinstance(loader, ExtractionPipelineConfigLoader):
+        if isinstance(loader, ExtractionPipelineConfigCRUD):
             # The safe read in ExtractionPipelineConfigLoader stringifies the config dict,
             # but we need to load it as a dict so we can write it back to the file maintaining
             # the order or the keys.
-            loaded = read_yaml_content(variables.replace(source))
+            loaded = read_yaml_content(variables.replace(source, source_file))
             loaded_with_placeholder = read_yaml_content(content)
         else:
-            loaded = read_yaml_content(loader.safe_read(variables.replace(source)))
+            loaded = read_yaml_content(loader.safe_read(variables.replace(source, source_file)))
             loaded_with_placeholder = read_yaml_content(loader.safe_read(content))
 
         built_by_identifier = {r.identifier: r for r in resources}
@@ -756,7 +749,7 @@ class PullCommand(ToolkitCommand):
             builder = create_builder(built.resource_dir, None)
             for extra in built.extra_sources:
                 extra_content, extra_placeholders = built.build_variables.replace(
-                    safe_read(extra.path), extra.path.suffix, use_placeholder=True
+                    safe_read(extra.path), extra.path, use_placeholder=True
                 )
                 key, _ = builder.load_extra_field(extra_content)
                 if key in item_write:
@@ -774,7 +767,7 @@ class ResourceReplacer:
     The local resource dict order is maintained. In addition, placeholders are used for variables.
     """
 
-    def __init__(self, value_by_placeholder: dict[str, BuildVariable], loader: ResourceLoader) -> None:
+    def __init__(self, value_by_placeholder: dict[str, BuildVariable], loader: ResourceCRUD) -> None:
         self._value_by_placeholder = value_by_placeholder
         self._loader = loader
 

cognite_toolkit/_cdf_tk/commands/repo.py

@@ -44,7 +44,7 @@ class RepoCommand(ToolkitCommand):
                     elif cwd != git_root:
                         raise ToolkitValueError(
                             f"Current working directory is not the root of the git repository. "
-                            f"Please run this command from {git_root.as_posix()!r}."  # type: ignore [union-attr]
+                            f"Please run this command from {git_root.as_posix()!r}."
                         )
             else:
                 self.warn(

cognite_toolkit/_cdf_tk/commands/resources.py

@@ -0,0 +1,179 @@
+import difflib
+from pathlib import Path
+from typing import cast
+
+import questionary
+import typer
+from questionary import Choice
+from rich import print
+
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.constants import MODULES
+from cognite_toolkit._cdf_tk.cruds import RESOURCE_CRUD_LIST, ResourceCRUD
+from cognite_toolkit._cdf_tk.data_classes import ModuleDirectories
+from cognite_toolkit._cdf_tk.resource_classes import ToolkitResource
+from cognite_toolkit._cdf_tk.utils.collection import humanize_collection
+from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump
+
+
+class ResourcesCommand(ToolkitCommand):
+    def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
+        super().__init__(print_warning, skip_tracking, silent)
+
+    def _get_or_prompt_module_path(self, module: str | None, organization_dir: Path, verbose: bool) -> Path:
+        """
+        Check if the module exists in the organization directory and return the module path.
+        If module is not provided, ask the user to select or create a new module.
+        """
+        present_modules = ModuleDirectories.load(organization_dir, None)
+
+        if module:
+            for mod in present_modules:
+                if mod.name.casefold() == module.casefold():
+                    return mod.dir
+
+            if questionary.confirm(f"{module} module not found. Do you want to create a new one?").ask():
+                return organization_dir / MODULES / module
+
+            if verbose:
+                print(f"[red]Aborting as {module} module not found...[/red]")
+            else:
+                print("[red]Aborting...[/red]")
+            raise typer.Exit()
+
+        choices = [Choice(title=mod.name, value=mod.dir) for mod in present_modules]
+        choices.append(Choice(title="<Create new module>", value="NEW"))
+
+        selected = questionary.select("Select a module:", choices=choices).ask()
+
+        if selected == "NEW":
+            new_module_name = questionary.text("Enter name for new module:").ask()
+            if not new_module_name:
+                print("[red]No module name provided. Aborting...[/red]")
+                raise typer.Exit()
+            return organization_dir / MODULES / new_module_name
+
+        if not selected:
+            print("[red]No module selected. Aborting...[/red]")
+            raise typer.Exit()
+
+        return cast(Path, selected)
+
+    def _resolve_kinds(self, kinds: list[str] | None) -> list[type[ResourceCRUD]]:
+        """
+        Resolve kinds from list of strings or do an interactive selection.
+        """
+        all_cruds = {crud.kind.casefold(): crud for crud in RESOURCE_CRUD_LIST}
+
+        if not kinds:
+            sorted_cruds = sorted(RESOURCE_CRUD_LIST, key=lambda x: x.kind)
+            choices = [Choice(title=crud.kind, value=crud) for crud in sorted_cruds]
+
+            selected = questionary.select("Select resource type:", choices=choices).ask()
+            if not selected:
+                print("[red]No resource type selected. Aborting...[/red]")
+                raise typer.Exit()
+            return [selected]
+
+        resolved_cruds = []
+        for kind in kinds:
+            kind_lower = kind.casefold()
+            if kind_lower in all_cruds:
+                resolved_cruds.append(all_cruds[kind_lower])
+            else:
+                matches = difflib.get_close_matches(kind_lower, all_cruds.keys())
+                if matches:
+                    suggestion = all_cruds[matches[0]].kind
+                    print(f"[red]Unknown resource type '{kind}'. Did you mean '{suggestion}'?[/red]")
+                else:
+                    print(
+                        f"[red]Unknown resource type '{kind}'. "
+                        f"Available types: {humanize_collection(sorted([c.kind for c in RESOURCE_CRUD_LIST]))}[/red]"
+                    )
+                raise typer.Exit()
+
+        return resolved_cruds
+
+    def _create_resource_yaml_skeleton(self, yaml_cls: type[ToolkitResource]) -> dict[str, str]:
+        """
+        Build YAML skeleton from a Pydantic model class using JSON schema for better type information.
+        """
+        yaml_skeleton: dict[str, str] = {}
+        for field_name, field in yaml_cls.model_fields.items():
+            name = field.alias or field_name
+            description = field.description or name
+            if field.is_required():
+                yaml_skeleton[name] = f"(Required) {description}"
+            else:
+                yaml_skeleton[name] = description
+
+        return yaml_skeleton
+
+    def _get_resource_yaml_content(self, resource_crud: type[ResourceCRUD]) -> str:
+        """
+        Creates a new resource in the specified module using the resource_crud.yaml_cls.
+        """
+        yaml_header = (
+            f"# API docs: {resource_crud.doc_url()}\n"
+            f"# YAML reference: https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/resource_library"
+        )
+        yaml_skeleton = self._create_resource_yaml_skeleton(resource_crud.yaml_cls)
+        yaml_contents = yaml_safe_dump(yaml_skeleton)
+        return yaml_header + "\n\n" + yaml_contents
+
+    def _create_resource_yaml_file(
+        self,
+        resource_crud: type[ResourceCRUD],
+        module_path: Path,
+        prefix: str | None = None,
+        verbose: bool = False,
+    ) -> None:
+        """
+        Creates a new resource YAML file in the specified module using the resource_crud.yaml_cls.
+        """
+        resource_dir: Path = module_path / resource_crud.folder_name
+        if resource_crud.sub_folder_name:
+            resource_dir = resource_dir / resource_crud.sub_folder_name
+
+        if not resource_dir.exists():
+            resource_dir.mkdir(parents=True, exist_ok=True)
+
+        final_prefix = prefix if prefix is not None else f"my_{resource_crud.kind}"
+        file_name = f"{final_prefix}.{resource_crud.kind}.yaml"
+        file_path: Path = resource_dir / file_name
+
+        if file_path.exists() and not questionary.confirm(f"{file_path.name} file already exists. Overwrite?").ask():
+            print("[red]Skipping...[/red]")
+            return
+
+        yaml_content = self._get_resource_yaml_content(resource_crud)
+        file_path.write_text(yaml_content)
+        if verbose:
+            print(
+                f"[green]{resource_crud.kind} Resource YAML file created successfully at {file_path.as_posix()}[/green]"
+            )
+        else:
+            print(f"[green]Created {file_path.as_posix()}[/green]")
+
+    def create(
+        self,
+        organization_dir: Path,
+        module_name: str | None = None,
+        kind: list[str] | None = None,
+        prefix: str | None = None,
+        verbose: bool = False,
+    ) -> None:
+        """
+        create resource YAMLs.
+
+        Args:
+            organization_dir: The directory of the organization.
+            module_name: The name of the module.
+            kind: The kind(s) of resource to create.
+            prefix: The prefix for the resource file.
+            verbose: Whether to print verbose output.
+        """
+        module_path = self._get_or_prompt_module_path(module_name, organization_dir, verbose)
+        resource_cruds = self._resolve_kinds(kind)
+        for crud in resource_cruds:
+            self._create_resource_yaml_file(crud, module_path, prefix, verbose)

cognite_toolkit/_cdf_tk/commands/run.py

@@ -32,6 +32,8 @@ from rich.table import Table
 from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig
 from cognite_toolkit._cdf_tk.client.data_classes.functions import FunctionScheduleID
 from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER
+from cognite_toolkit._cdf_tk.cruds import FunctionCRUD, FunctionScheduleCRUD, WorkflowVersionCRUD
+from cognite_toolkit._cdf_tk.cruds._resource_cruds.workflow import WorkflowTriggerCRUD
 from cognite_toolkit._cdf_tk.data_classes import BuiltResourceFull, ModuleResources
 from cognite_toolkit._cdf_tk.exceptions import (
     AuthorizationError,
@@ -43,8 +45,6 @@ from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitValueError,
 )
 from cognite_toolkit._cdf_tk.hints import verify_module_directory
-from cognite_toolkit._cdf_tk.loaders import FunctionLoader, FunctionScheduleLoader, WorkflowVersionLoader
-from cognite_toolkit._cdf_tk.loaders._resource_loaders.workflow_loaders import WorkflowTriggerLoader
 from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils import in_dict
 from cognite_toolkit._cdf_tk.utils.auth import CLIENT_NAME, EnvironmentVariables
@@ -208,7 +208,7 @@ if __name__ == "__main__":
     @staticmethod
     def _get_function(external_id: str | None, resources: ModuleResources) -> BuiltResourceFull[str]:
         function_builds_by_identifier = {
-            build.identifier: build for build in resources.list_resources(str, "functions", FunctionLoader.kind)
+            build.identifier: build for build in resources.list_resources(str, "functions", FunctionCRUD.kind)
         }
 
         if len(function_builds_by_identifier) == 0:
@@ -282,15 +282,15 @@ if __name__ == "__main__":
     def _get_call_args_interactive(
         function_external_id: str, resources: ModuleResources
     ) -> tuple[dict[str, Any], ClientCredentials | None]:
-        schedules = resources.list_resources(FunctionScheduleID, "functions", FunctionScheduleLoader.kind)
+        schedules = resources.list_resources(FunctionScheduleID, "functions", FunctionScheduleCRUD.kind)
         options: dict[str, Any] = {
             f"FunctionSchedule: {schedule.identifier.name}": schedule
             for schedule in schedules
             if schedule.identifier.function_external_id == function_external_id
         }
-        workflows = resources.list_resources(WorkflowVersionId, "workflows", WorkflowVersionLoader.kind)
+        workflows = resources.list_resources(WorkflowVersionId, "workflows", WorkflowVersionCRUD.kind)
         raw_trigger_by_workflow_id: dict[WorkflowVersionId, dict[str, Any]] = {}
-        for trigger in resources.list_resources(str, "workflows", WorkflowTriggerLoader.kind):
+        for trigger in resources.list_resources(str, "workflows", WorkflowTriggerCRUD.kind):
             raw_trigger = trigger.load_resource_dict({}, validate=False)
             loaded_trigger = WorkflowTriggerUpsert.load(raw_trigger)
             raw_trigger_by_workflow_id[
@@ -341,7 +341,7 @@ if __name__ == "__main__":
     ) -> tuple[dict[str, Any], ClientCredentials | None]:
         data: dict[str, Any] | None = None
         credentials: ClientCredentials | None = None
-        workflows = resources.list_resources(WorkflowVersionId, "workflows", WorkflowVersionLoader.kind)
+        workflows = resources.list_resources(WorkflowVersionId, "workflows", WorkflowVersionCRUD.kind)
         found = False
         for workflow in workflows:
             if (isinstance(data_source, str) and workflow.identifier.workflow_external_id == data_source) or (
@@ -357,7 +357,7 @@ if __name__ == "__main__":
                         data = task.parameters.data if isinstance(task.parameters.data, dict) else {}
                         found = True
                         break
-            for trigger in resources.list_resources(str, "workflows", WorkflowTriggerLoader.kind):
+            for trigger in resources.list_resources(str, "workflows", WorkflowTriggerCRUD.kind):
                 raw_trigger = trigger.load_resource_dict({}, validate=False)
                 loaded_trigger = WorkflowTriggerUpsert.load(raw_trigger)
                 if (isinstance(data_source, str) and loaded_trigger.workflow_external_id == data_source) or (
@@ -379,7 +379,7 @@ if __name__ == "__main__":
         if not isinstance(data_source, str):
             raise ToolkitValueError(f"Data source {data_source} is not a valid workflow external id.")
 
-        for schedule in resources.list_resources(FunctionScheduleID, "functions", FunctionScheduleLoader.kind):
+        for schedule in resources.list_resources(FunctionScheduleID, "functions", FunctionScheduleCRUD.kind):
             if (
                 schedule.identifier.function_external_id == function_external_id
                 and schedule.identifier.name == data_source
@@ -683,7 +683,7 @@ class RunWorkflowCommand(ToolkitCommand):
         resources = ModuleResources(organization_dir, build_env_name)
         client = env_vars.get_client()
         is_interactive = external_id is None
-        workflows = resources.list_resources(WorkflowVersionId, "workflows", WorkflowVersionLoader.kind)
+        workflows = resources.list_resources(WorkflowVersionId, "workflows", WorkflowVersionCRUD.kind)
         if len(workflows) == 0:
             raise ToolkitMissingResourceError("No workflows found in modules.")
         if external_id is None:
@@ -704,7 +704,7 @@ class RunWorkflowCommand(ToolkitCommand):
                 raise ToolkitMissingResourceError(f"Could not find workflow with external id {external_id}")
             selected = selected_
         id_ = selected.identifier
-        triggers = resources.list_resources(str, "workflows", WorkflowTriggerLoader.kind)
+        triggers = resources.list_resources(str, "workflows", WorkflowTriggerCRUD.kind)
 
         credentials: ClientCredentials | None = None
         input_: dict | None = None

cognite_toolkit/_cdf_tk/constants.py

@@ -7,7 +7,7 @@ from typing import Literal, TypeAlias
 from cognite.client.data_classes.data_modeling import ContainerId
 
 try:
-    from pyodide.ffi import IN_BROWSER  # type: ignore [import-not-found]
+    from pyodide.ffi import IN_BROWSER
 except ModuleNotFoundError:
     IN_BROWSER = False
 # This is the default config located locally in each module.
@@ -30,15 +30,16 @@ CONFIG_FILE_SUFFIX = "config.yaml"
 # The global config file
 GLOBAL_CONFIG_FILE = "global.yaml"
 
-BUILTIN_MODULES = "_builtin_modules"
 COGNITE_MODULES = "cognite_modules"
 CUSTOM_MODULES = "custom_modules"
+EXTERNAL_PACKAGE = "external_package"
 MODULES = "modules"
 REPO_FILES_DIR = "_repo_files"
 DOCKER_IMAGE_NAME = "cognite/toolkit"
 BUILD_FOLDER_ENCODING = "utf-8"
+RESOURCES = "_resources"
 
-ROOT_MODULES = [MODULES, CUSTOM_MODULES, COGNITE_MODULES]
+ROOT_MODULES = [MODULES, CUSTOM_MODULES, COGNITE_MODULES, EXTERNAL_PACKAGE]
 MODULE_PATH_SEP = "/"
 
 MIN_TIMESTAMP_MS = -2208988800000  # 1900-01-01 00:00:00.000
@@ -59,8 +60,8 @@ TABLE_FORMATS = frozenset([".csv", ".parquet"])
 ROOT_PATH = Path(__file__).parent.parent
 COGNITE_MODULES_PATH = ROOT_PATH / COGNITE_MODULES
 MODULES_PATH = ROOT_PATH / MODULES
-BUILTIN_MODULES_PATH = ROOT_PATH / BUILTIN_MODULES
-SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.2.0"
+RESOURCES_PATH = ROOT_PATH / RESOURCES
+SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.4.0"
 # This is used in the build directory to keep track of order and flatten the
 # module directory structure with accounting for duplicated names.
 INDEX_PATTERN = re.compile("^[0-9]+\\.")
@@ -79,7 +80,8 @@ HINT_LEAD_TEXT_LEN = 5
 EnvType: TypeAlias = Literal["dev", "test", "staging", "qa", "prod"]
 USE_SENTRY = "pytest" not in sys.modules and os.environ.get("SENTRY_ENABLED", "true").lower() == "true"
 SPACE_FORMAT_PATTERN = r"^[a-zA-Z][a-zA-Z0-9_-]{0,41}[a-zA-Z0-9]?$"
-CONTAINER_AND_VIEW_EXTERNAL_ID_PATTERN = r"^[a-zA-Z]([a-zA-Z0-9_]{0,253}[a-zA-Z0-9])?$"
+DM_EXTERNAL_ID_PATTERN = r"^[a-zA-Z]([a-zA-Z0-9_]{0,253}[a-zA-Z0-9])?$"
+INSTANCE_EXTERNAL_ID_PATTERN = r"^[^\x00]{1,256}$"
 FORBIDDEN_SPACES = frozenset(["space", "cdf", "dms", "pg3", "shared", "system", "node", "edge"])
 FORBIDDEN_CONTAINER_AND_VIEW_EXTERNAL_IDS = frozenset(
     [
@@ -121,7 +123,7 @@ FORBIDDEN_CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER = frozenset(
     ]
 )
 CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN = r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0,253}[a-zA-Z0-9]?$"
-VIEW_VERSION_PATTERN = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
+DM_VERSION_PATTERN = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
 
 
 def clean_name(name: str) -> str:
@@ -157,3 +159,32 @@ COGNITE_MIGRATION_SPACE = "cognite_migration"
 
 COGNITE_TIME_SERIES_CONTAINER = ContainerId("cdf_cdm", "CogniteTimeSeries")
 COGNITE_FILE_CONTAINER = ContainerId("cdf_cdm", "CogniteFile")
+CDF_UNIT_SPACE = "cdf_cdm_units"
+
+
+# Container properties that are read-only in DMS needs to be handled with extra care, as this aspect is not currently exposed by the API.
+READONLY_CONTAINER_PROPERTIES = {
+    ContainerId(space="cdf_cdm", external_id="CogniteAsset"): {
+        "assetHierarchy_path_last_updated_time",
+        "assetHierarchy_path",
+        "assetHierarchy_root",
+    },
+    ContainerId(space="cdf_cdm", external_id="CogniteFile"): {"isUploaded", "uploadedTime"},
+}
+
+# Data Plugin Constants
+DATA_DEFAULT_DIR = "data"
+DATA_RESOURCE_DIR = "resources"
+DATA_MANIFEST_STEM = "Manifest"
+DATA_MANIFEST_SUFFIX = f".{DATA_MANIFEST_STEM}.yaml"
+
+DATA_UPLOAD_URL = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/guides/plugins/data_plugin/index"
+# Migration Constants
+MISSING_INSTANCE_SPACE = "<InstanceSpaceMissing>"
+MISSING_EXTERNAL_ID = "INTERNAL_ID_project_{project}_id_{id}"
+
+# Stream Template names
+STREAM_IMMUTABLE_TEMPLATE_NAME = frozenset(("ImmutableTestStream", "BasicArchive"))
+STREAM_MUTABLE_TEMPLATE_NAME = frozenset(("BasicLiveData",))
+
+StreamTemplateName: TypeAlias = Literal["ImmutableTestStream", "BasicArchive", "BasicLiveData"]