cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/data_classes/_built_resources.py

@@ -1,5 +1,4 @@
-from __future__ import annotations
-
+import sys
 from abc import abstractmethod
 from collections import defaultdict
 from collections.abc import Collection, Iterator, MutableSequence
@@ -8,18 +7,22 @@ from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generic, SupportsIndex, TypeVar, cast, overload
 
+from cognite_toolkit._cdf_tk.cruds import get_crud
+from cognite_toolkit._cdf_tk.cruds._base_cruds import T_ID, ResourceCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError
-from cognite_toolkit._cdf_tk.loaders import get_loader
-from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID, ResourceLoader
 from cognite_toolkit._cdf_tk.utils import (
     calculate_directory_hash,
-    calculate_str_or_file_hash,
+    calculate_hash,
     load_yaml_inject_variables,
     safe_read,
 )
 
 from ._build_variables import BuildVariables
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
 if TYPE_CHECKING:
     from ._built_modules import BuiltModule
 
@@ -47,7 +50,7 @@ class SourceLocation:
         }
 
     @classmethod
-    def load(cls, data: dict[str, Any]) -> SourceLocation:
+    def load(cls, data: dict[str, Any]) -> "SourceLocationEager":
         return SourceLocationEager(
             path=Path(data["path"]),
             _hash=str(data["hash"]),
@@ -63,7 +66,7 @@ class SourceLocationLazy(SourceLocation):
         if self.absolute_path.is_dir():
             return calculate_directory_hash(self.absolute_path, shorten=True)
         else:
-            return calculate_str_or_file_hash(self.absolute_path, shorten=True)
+            return calculate_hash(self.absolute_path, shorten=True)
 
 
 @dataclass
@@ -96,11 +99,11 @@ class BuiltResource(Generic[T_ID]):
     extra_sources: list[SourceLocation] | None
 
     @classmethod
-    def load(cls: type[T_BuiltResource], data: dict[str, Any], resource_folder: str) -> T_BuiltResource:
-        from cognite_toolkit._cdf_tk.loaders import ResourceLoader, get_loader
+    def load(cls, data: dict[str, Any], resource_folder: str) -> Self:
+        from cognite_toolkit._cdf_tk.cruds import ResourceCRUD, get_crud
 
         kind = data["kind"]
-        loader = cast(ResourceLoader, get_loader(resource_folder, kind))
+        loader = cast(ResourceCRUD, get_crud(resource_folder, kind))
         identifier = loader.get_id(data["identifier"])
 
         return cls(
@@ -112,9 +115,9 @@ class BuiltResource(Generic[T_ID]):
         )
 
     def dump(self, resource_folder: str, include_destination: bool = False) -> dict[str, Any]:
-        from cognite_toolkit._cdf_tk.loaders import ResourceLoader, get_loader
+        from cognite_toolkit._cdf_tk.cruds import ResourceCRUD, get_crud
 
-        loader = cast(ResourceLoader, get_loader(resource_folder, self.kind))
+        loader = cast(ResourceCRUD, get_crud(resource_folder, self.kind))
         dumped = loader.dump_id(self.identifier)
 
         output: dict[str, Any] = {
@@ -128,7 +131,7 @@ class BuiltResource(Generic[T_ID]):
             output["extra_sources"] = [source.dump() for source in self.extra_sources]
         return output
 
-    def create_full(self, module: BuiltModule, resource_dir: str) -> BuiltResourceFull[T_ID]:
+    def create_full(self, module: "BuiltModule", resource_dir: str) -> "BuiltResourceFull":
         return BuiltResourceFull(
             identifier=self.identifier,
             source=self.source,
@@ -155,8 +158,8 @@ class BuiltResourceFull(BuiltResource[T_ID]):
     def load_resource_dict(
         self, environment_variables: dict[str, str | None], validate: bool = False
     ) -> dict[str, Any]:
-        content = self.build_variables.replace(safe_read(self.source.path))
-        loader = cast(ResourceLoader, get_loader(self.resource_dir, self.kind))
+        content = self.build_variables.replace(safe_read(self.source.path), self.source.path)
+        loader = cast(ResourceCRUD, get_crud(self.resource_dir, self.kind))
         raw = load_yaml_inject_variables(
             content,
             environment_variables,
@@ -184,9 +187,9 @@ class BuiltResourceList(list, MutableSequence[BuiltResource[T_ID]], Generic[T_ID
     def __getitem__(self, index: SupportsIndex) -> BuiltResource[T_ID]: ...
 
     @overload
-    def __getitem__(self, index: slice) -> BuiltResourceList[T_ID]: ...
+    def __getitem__(self, index: slice) -> "BuiltResourceList[T_ID]": ...
 
-    def __getitem__(self, index: SupportsIndex | slice, /) -> BuiltResource[T_ID] | BuiltResourceList[T_ID]:
+    def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltResource[T_ID] | BuiltResourceList[T_ID]":
         if isinstance(index, slice):
             return BuiltResourceList[T_ID](super().__getitem__(index))
         return super().__getitem__(index)
@@ -196,7 +199,7 @@ class BuiltResourceList(list, MutableSequence[BuiltResource[T_ID]], Generic[T_ID
         return [resource.identifier for resource in self]
 
     @classmethod
-    def load(cls, data: list[dict[str, Any]], resource_folder: str) -> BuiltResourceList[T_ID]:
+    def load(cls, data: list[dict[str, Any]], resource_folder: str) -> "BuiltResourceList[T_ID]":
         return cls([BuiltResource.load(resource_data, resource_folder) for resource_data in data])
 
     def dump(self, resource_folder: str, include_destination: bool = False) -> list[dict[str, Any]]:
@@ -226,14 +229,14 @@ class BuiltFullResourceList(BuiltResourceList[T_ID]):
     def __getitem__(self, index: SupportsIndex) -> BuiltResourceFull[T_ID]: ...
 
     @overload
-    def __getitem__(self, index: slice) -> BuiltFullResourceList[T_ID]: ...
+    def __getitem__(self, index: slice) -> "BuiltFullResourceList[T_ID]": ...
 
-    def __getitem__(self, index: SupportsIndex | slice, /) -> BuiltResourceFull[T_ID] | BuiltFullResourceList[T_ID]:
+    def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltResourceFull[T_ID] | BuiltFullResourceList[T_ID]":
         if isinstance(index, slice):
             return BuiltFullResourceList[T_ID](super().__getitem__(index))
         return cast(BuiltResourceFull[T_ID], super().__getitem__(index))
 
-    def by_file(self) -> dict[Path, BuiltFullResourceList[T_ID]]:
+    def by_file(self) -> "dict[Path, BuiltFullResourceList[T_ID]]":
         resources_by_file: dict[Path, BuiltFullResourceList[T_ID]] = defaultdict(lambda: BuiltFullResourceList())
         for resource in self:
             resources_by_file[resource.source.path].append(resource)

cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py

@@ -1,8 +1,7 @@
-from __future__ import annotations
-
 import itertools
 import os
 import re
+import sys
 from abc import ABC
 from collections import UserDict, defaultdict
 from collections.abc import Hashable, Iterable, Sequence, Set
@@ -16,7 +15,6 @@ from rich import print
 from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_ENVIRONMENT_FILE,
-    BUILTIN_MODULES,
     DEFAULT_CONFIG_FILE,
     DEFAULT_ENV,
     MODULES,
@@ -24,9 +22,9 @@ from cognite_toolkit._cdf_tk.constants import (
     SEARCH_VARIABLES_SUFFIX,
     EnvType,
 )
+from cognite_toolkit._cdf_tk.cruds import CRUDS_BY_FOLDER_NAME, RawDatabaseCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitEnvError, ToolkitMissingModuleError
 from cognite_toolkit._cdf_tk.hints import ModuleDefinition
-from cognite_toolkit._cdf_tk.loaders import LOADER_BY_FOLDER_NAME, RawDatabaseLoader
 from cognite_toolkit._cdf_tk.tk_warnings import (
     FileReadWarning,
     MediumSeverityWarning,
@@ -38,7 +36,7 @@ from cognite_toolkit._cdf_tk.tk_warnings import (
 from cognite_toolkit._cdf_tk.utils import (
     YAMLComment,
     YAMLWithComments,
-    calculate_str_or_file_hash,
+    calculate_hash,
     flatten_dict,
     read_yaml_content,
     safe_read,
@@ -51,6 +49,10 @@ from ._base import ConfigCore, _load_version_variable
 from ._built_resources import BuiltResourceList
 from ._module_directories import ModuleDirectories, ReadModule
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
 _AVAILABLE_ENV_TYPES = tuple(get_args(EnvType))
 
 
@@ -66,7 +68,7 @@ class Environment:
         return self.validation_type.casefold() != "dev"
 
     @classmethod
-    def load(cls, data: dict[str, Any], build_name: str) -> Environment:
+    def load(cls, data: dict[str, Any], build_name: str) -> "Environment":
         _deprecation_selected(data)
         if "name" not in data:
             data["name"] = build_name
@@ -151,7 +153,7 @@ class BuildConfigYAML(ConfigYAMLCore, ConfigCore):
             return MediumSeverityWarning(mismatch_message)
 
     @classmethod
-    def load(cls, data: dict[str, Any], build_env_name: str, filepath: Path) -> BuildConfigYAML:
+    def load(cls, data: dict[str, Any], build_env_name: str, filepath: Path) -> Self:
         if "environment" not in data:
             err_msg = f"Expected 'environment' section in {filepath!s}."
             raise ToolkitEnvError(err_msg)
@@ -168,9 +170,9 @@ class BuildConfigYAML(ConfigYAMLCore, ConfigCore):
 
     def create_build_environment(
         self, built_modules: BuiltModuleList, selected_modules: ModuleDirectories
-    ) -> BuildEnvironment:
+    ) -> "BuildEnvironment":
         return BuildEnvironment(
-            name=self.environment.name,  # type: ignore[arg-type]
+            name=self.environment.name,
             project=self.environment.project,
             validation_type=self.environment.validation_type,
             selected=self.environment.selected,
@@ -223,7 +225,7 @@ class BuildConfigYAML(ConfigYAMLCore, ConfigCore):
         return selected_modules
 
     @classmethod
-    def load_default(cls, organization_dir: Path) -> BuildConfigYAML:
+    def load_default(cls, organization_dir: Path) -> Self:
         return cls(filepath=organization_dir / BuildConfigYAML.get_filename(DEFAULT_ENV))
 
     def dump(self) -> dict[str, Any]:
@@ -260,7 +262,7 @@ class BuildEnvironment(Environment):
     @classmethod
     def load(
         cls, data: dict[str, Any], build_name: str | None, action: Literal["build", "deploy", "clean", "pull"] = "build"
-    ) -> BuildEnvironment:
+    ) -> Self:
         if "name" in data and build_name is not None and data["name"] != build_name:
             raise ToolkitEnvError(
                 f"Expected to {action} for {build_name!r} environment, but the last "
@@ -304,7 +306,7 @@ class BuildEnvironment(Environment):
     def check_source_files_changed(self) -> WarningList[FileReadWarning]:
         warning_list = WarningList[FileReadWarning]()
         for resource_folder, resources in self.built_resources.items():
-            if resource_folder == RawDatabaseLoader.folder_name:
+            if resource_folder == RawDatabaseCRUD.folder_name:
                 # We modify the hash for RawDatabaseLoader, so we skip checking the hash for this folder.
                 continue
             for resource in resources:
@@ -320,7 +322,7 @@ class BuildEnvironment(Environment):
                         warning_list.append(
                             MissingFileWarning(source_filepath, attempted_check="source file has changed")
                         )
-                    elif source.hash != calculate_str_or_file_hash(source_filepath, shorten=True):
+                    elif source.hash != calculate_hash(source_filepath, shorten=True):
                         warning_list.append(SourceFileModifiedWarning(source_filepath))
         return warning_list
 
@@ -425,7 +427,7 @@ class WildcardSet(set, Set[_WildcardSequence]):
     """Sets that support wildcard matching."""
 
     @classmethod
-    def load(cls, items: Sequence[Sequence[str]]) -> WildcardSet:
+    def load(cls, items: Sequence[Sequence[str]]) -> Self:
         return cls(_WildcardSequence(item) for item in items)
 
     def __contains__(self, key: Any) -> bool:
@@ -461,11 +463,11 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
         cognite_root_module: Path,
         selected_paths: set[Path] | None = None,
         ignore_patterns: list[tuple[str, ...]] | None = None,
-    ) -> InitConfigYAML:
+    ) -> Self:
         """Loads all default.config.yaml files in the cognite root module."""
 
         default_files_iterable: Iterable[Path]
-        if cognite_root_module.name in ROOT_MODULES or cognite_root_module.name == BUILTIN_MODULES:
+        if cognite_root_module.name in ROOT_MODULES:
             default_files_iterable = cognite_root_module.glob(f"**/{DEFAULT_CONFIG_FILE}")
         else:
             default_files_iterable = itertools.chain(
@@ -487,14 +489,16 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
 
     def _load_defaults(
         self, cognite_root_module: Path, defaults_files: list[Path], ignore_patterns: list[tuple[str, ...]] | None
-    ) -> InitConfigYAML:
+    ) -> Self:
         """Loads all default.config.yaml files in the cognite root module.
 
         This extracts the default values from the default.config.yaml files and
         adds them to the config.yaml file.
 
         Args:
-            cognite_root_module: The root module for all cognite modules.
+            cognite_root_module: Path to the root directory containing all Cognite modules.
+            defaults_files: List of paths to default.config.yaml files to load.
+            ignore_patterns: Optional list of tuples containing patterns to ignore when loading defaults.
 
         Returns:
             self
@@ -507,6 +511,10 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
             raw_file = safe_read(default_config)
             file_comments = self._extract_comments(raw_file, key_prefix=tuple(parts))
             file_data = cast(dict, read_yaml_content(raw_file))
+
+            # a file may exist, but contain just comments, thus the file_data is None
+            if file_data is None:
+                continue
             for key, value in file_data.items():
                 if len(parts) >= 1 and parts[0] in ROOT_MODULES:
                     key_path = (self._variables, *parts, key)
@@ -528,7 +536,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
         return self
 
     @classmethod
-    def load_existing(cls, existing_config_yaml: str, build_env_name: str = "dev") -> InitConfigYAML:
+    def load_existing(cls, existing_config_yaml: str, build_env_name: str = "dev") -> Self:
         """Loads an existing config.yaml file.
 
         This does a yaml.safe_load, in addition to extracting comments from the file.
@@ -568,7 +576,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
             entries=entries,
         )
 
-    def load_variables(self, organization_dir: Path, propagate_reused_variables: bool = False) -> InitConfigYAML:
+    def load_variables(self, organization_dir: Path, propagate_reused_variables: bool = False) -> Self:
         """This scans the content the files in the given directories and finds the variables.
         The motivation is to find the variables that are used in the templates, as well
         as picking up variables that are used in custom modules.
@@ -590,7 +598,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
                 continue
             content = safe_read(filepath)
             key_parent = filepath.parent.relative_to(organization_dir).parts
-            if key_parent and key_parent[-1] in LOADER_BY_FOLDER_NAME:
+            if key_parent and key_parent[-1] in CRUDS_BY_FOLDER_NAME:
                 key_parent = key_parent[:-1]
 
             for match in re.findall(r"{{\s*([a-zA-Z0-9_]+)\s*}}", content):
@@ -600,7 +608,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
 
     def _load_variables(
         self, variable_by_parent_key: dict[str, set[tuple[str, ...]]], propagate_reused_variables: bool = False
-    ) -> InitConfigYAML:
+    ) -> Self:
         for variable, key_parents in variable_by_parent_key.items():
             if len(key_parents) > 1 and propagate_reused_variables:
                 key_parents = {self._find_common_parent(list(key_parents))}
@@ -609,7 +617,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
                 # Remove module subfolders.
                 key_parent_list = list(key_parent)
                 for i in range(len(key_parent_list)):
-                    if key_parent_list[i] in LOADER_BY_FOLDER_NAME:
+                    if key_parent_list[i] in CRUDS_BY_FOLDER_NAME:
                         key_parent_list = key_parent_list[:i]
                         break
                 key_parent = tuple(key_parent_list)
@@ -632,7 +640,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC
         ] = defaultdict(list)
         count_by_variable_keys: dict[str, set[float | int | str | bool | tuple[Hashable] | None]] = defaultdict(set)
         for key, entry in self.items():
-            value = tuple(entry.value) if isinstance(entry.value, list) else entry.value  # type: ignore[arg-type]
+            value = tuple(entry.value) if isinstance(entry.value, list) else entry.value
             variables_by_key_value[(key[-1], value)].append(entry)
             count_by_variable_keys[key[-1]].add(value)
 
@@ -725,7 +733,7 @@ class ConfigYAMLs(UserDict[str, InitConfigYAML]):
         super().__init__(entries or {})
 
     @classmethod
-    def load_default_environments(cls, default: dict[str, Any]) -> ConfigYAMLs:
+    def load_default_environments(cls, default: dict[str, Any]) -> Self:
         instance = cls()
         for environment_name, environment_config in default.items():
             environment = Environment.load(environment_config, environment_name)
@@ -733,7 +741,7 @@ class ConfigYAMLs(UserDict[str, InitConfigYAML]):
         return instance
 
     @classmethod
-    def load_existing_environments(cls, existing_config_yamls: Sequence[Path]) -> ConfigYAMLs:
+    def load_existing_environments(cls, existing_config_yamls: Sequence[Path]) -> Self:
         instance = cls()
         for config_yaml in existing_config_yamls:
             config = InitConfigYAML.load_existing(safe_read(config_yaml), config_yaml.name.split(".")[0])

cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py

@@ -1,5 +1,4 @@
-from __future__ import annotations
-
+import sys
 from abc import ABC
 from collections import UserDict
 from collections.abc import Iterable
@@ -9,6 +8,11 @@ from typing import Literal
 
 from rich.table import Table
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 
 @total_ordering
 @dataclass
@@ -40,7 +44,7 @@ class ResourceDeployResult(DeployResult):
     def calculated_total(self) -> int:
         return self.created + self.deleted + self.changed + self.unchanged
 
-    def __iadd__(self, other: ResourceDeployResult) -> ResourceDeployResult:
+    def __iadd__(self, other: "ResourceDeployResult") -> "ResourceDeployResult":
         if self.name != other.name:
             raise ValueError("Cannot add two DeployResult objects with different names")
         self.created += other.created
@@ -69,7 +73,7 @@ class ResourceContainerDeployResult(ResourceDeployResult):
     item_name: str = ""
     dropped_datapoints: int = 0
 
-    def __iadd__(self, other: ResourceDeployResult) -> ResourceContainerDeployResult:
+    def __iadd__(self, other: ResourceDeployResult) -> "ResourceContainerDeployResult":
         if self.name != other.name:
             raise ValueError("Cannot add two ResourceContainerDeployResult objects with different names")
         super().__iadd__(other)
@@ -77,13 +81,28 @@ class ResourceContainerDeployResult(ResourceDeployResult):
             self.dropped_datapoints += other.dropped_datapoints
         return self
 
+    @classmethod
+    def from_resource_deploy_result(
+        cls, result: ResourceDeployResult, item_name: str = "", dropped_datapoints: int = 0
+    ) -> Self:
+        return cls(
+            name=result.name,
+            created=result.created,
+            deleted=result.deleted,
+            changed=result.changed,
+            unchanged=result.unchanged,
+            total=result.total,
+            item_name=item_name,
+            dropped_datapoints=dropped_datapoints,
+        )
+
 
 @dataclass
 class UploadDeployResult(DeployResult):
     uploaded: int = 0
     item_name: str = ""
 
-    def __iadd__(self, other: UploadDeployResult) -> UploadDeployResult:
+    def __iadd__(self, other: "UploadDeployResult") -> "UploadDeployResult":
         if self.name != other.name:
             raise ValueError("Cannot add two DeployResult objects with different names")
         self.uploaded += other.uploaded
@@ -127,7 +146,7 @@ class DeployResults(UserDict):
     @property
     def has_uploads(self) -> bool:
         return any(
-            isinstance(entry, (UploadDeployResult, ResourceContainerDeployResult)) for entry in self.data.values()
+            isinstance(entry, UploadDeployResult | ResourceContainerDeployResult) for entry in self.data.values()
         )
 
     def counts_table(
@@ -185,7 +204,7 @@ class DeployResults(UserDict):
         for item in sorted(
             entry
             for entry in self.data.values()
-            if isinstance(entry, (UploadDeployResult, ResourceContainerDeployResult))
+            if isinstance(entry, UploadDeployResult | ResourceContainerDeployResult)
         ):
             if item.name == "raw.tables":
                 # We skip this as we cannot count the number of datapoints in a raw table

cognite_toolkit/_cdf_tk/data_classes/_module_directories.py

@@ -1,6 +1,5 @@
-from __future__ import annotations
-
 import shutil
+import sys
 from collections import defaultdict
 from collections.abc import Collection, Iterator, Sequence
 from dataclasses import dataclass
@@ -13,6 +12,11 @@ from cognite_toolkit._cdf_tk.utils import calculate_directory_hash, iterate_modu
 
 from ._module_toml import ModuleToml
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 
 @dataclass(frozen=True)
 class ModuleLocation:
@@ -46,6 +50,20 @@ class ModuleLocation:
             return self.definition.title
         return None
 
+    @property
+    def module_id(self) -> str | None:
+        """The ID of the module."""
+        if self.definition:
+            return self.definition.id
+        return None
+
+    @property
+    def package_id(self) -> str | None:
+        """The ID of the package."""
+        if self.definition:
+            return self.definition.package_id
+        return None
+
     @property
     def relative_path(self) -> Path:
         """The relative path to the module."""
@@ -142,10 +160,12 @@ class ModuleLocation:
     def __str__(self) -> str:
         return self.name
 
-    def as_read_module(self) -> ReadModule:
+    def as_read_module(self) -> "ReadModule":
         return ReadModule(
             dir=self.dir,
             resource_directories=tuple(self.resource_directories),
+            module_id=self.module_id,
+            package_id=self.package_id,
         )
 
 
@@ -160,6 +180,8 @@ class ReadModule:
 
     dir: Path
     resource_directories: tuple[str, ...]
+    module_id: str | None
+    package_id: str | None
 
     def resource_dir_path(self, resource_folder: str) -> Path | None:
         """Returns the path to a resource in the module.
@@ -176,16 +198,20 @@ class ReadModule:
         return None
 
     @classmethod
-    def load(cls, data: dict[str, Any]) -> ReadModule:
+    def load(cls, data: dict[str, Any]) -> Self:
         return cls(
             dir=Path(data["dir"]),
             resource_directories=tuple(data["resource_directories"]),
+            module_id=data.get("module_id"),
+            package_id=data.get("package_id"),
         )
 
     def dump(self) -> dict[str, Any]:
         return {
             "dir": self.dir.as_posix(),
             "resource_directories": list(self.resource_directories),
+            "module_id": self.module_id,
+            "package_id": self.package_id,
         }
 
 
@@ -197,7 +223,7 @@ class ModuleDirectories(tuple, Sequence[ModuleLocation]):
 
     # Subclassing tuple to make the class immutable. ModuleDirectories is expected to be initialized and
     # then used as a read-only object.
-    def __new__(cls, collection: Collection[ModuleLocation] | None) -> ModuleDirectories:
+    def __new__(cls, collection: Collection[ModuleLocation] | None) -> Self:
         # Need to override __new__ to as we are subclassing a tuple:
         #   https://stackoverflow.com/questions/1565374/subclassing-tuple-with-multiple-init-arguments
         return super().__new__(cls, tuple(collection or []))
@@ -209,7 +235,7 @@ class ModuleDirectories(tuple, Sequence[ModuleLocation]):
         return {selection for module_location in self for selection in module_location.module_selections}
 
     @cached_property
-    def selected(self) -> ModuleDirectories:
+    def selected(self) -> "ModuleDirectories":
         return ModuleDirectories([module for module in self if module.is_selected])
 
     @cached_property
@@ -225,7 +251,7 @@ class ModuleDirectories(tuple, Sequence[ModuleLocation]):
         cls,
         organization_dir: Path,
         user_selected_modules: set[str | Path] | None = None,
-    ) -> ModuleDirectories:
+    ) -> Self:
         """Loads the modules in the source directory.
 
         Args:
@@ -240,17 +266,15 @@ class ModuleDirectories(tuple, Sequence[ModuleLocation]):
         for module, source_paths in iterate_modules(organization_dir):
             relative_module_dir = module.relative_to(organization_dir)
             module_toml: ModuleToml | None = None
-            tags: set[str] = set()
             if (module / ModuleToml.filename).exists():
                 module_toml = ModuleToml.load(module / ModuleToml.filename)
-                tags = set(module_toml.tags)
 
             module_locations.append(
                 ModuleLocation(
                     module,
                     organization_dir,
                     source_paths,
-                    cls._is_selected_module(relative_module_dir, user_selected_modules, tags),
+                    cls._is_selected_module(relative_module_dir, user_selected_modules),
                     module_toml,
                 )
             )
@@ -273,17 +297,12 @@ class ModuleDirectories(tuple, Sequence[ModuleLocation]):
                 shutil.copy(source_file, absolute_file_path)
 
     @classmethod
-    def _is_selected_module(
-        cls, relative_module_dir: Path, user_selected: set[str | Path], module_tags: set[str]
-    ) -> bool:
+    def _is_selected_module(cls, relative_module_dir: Path, user_selected: set[str | Path]) -> bool:
         """Checks whether a module is selected by the user."""
         return (
             relative_module_dir.name in user_selected
             or relative_module_dir in user_selected
             or any(parent in user_selected for parent in relative_module_dir.parents)
-            # Check if the module has any tags that the user has selected,
-            # i.e., that the intersection of the module tags and the user selected tags is not empty.
-            or bool(module_tags & user_selected)
         )
 
     def as_path_by_name(self) -> dict[str, list[Path]]:
@@ -300,9 +319,9 @@ class ModuleDirectories(tuple, Sequence[ModuleLocation]):
     def __getitem__(self, index: SupportsIndex) -> ModuleLocation: ...
 
     @overload
-    def __getitem__(self, index: slice) -> ModuleDirectories: ...
+    def __getitem__(self, index: slice) -> "ModuleDirectories": ...
 
-    def __getitem__(self, index: SupportsIndex | slice, /) -> ModuleLocation | ModuleDirectories:
+    def __getitem__(self, index: SupportsIndex | slice, /) -> "ModuleLocation | ModuleDirectories":
         if isinstance(index, slice):
             return ModuleDirectories(super().__getitem__(index))
         return super().__getitem__(index)

cognite_toolkit/_cdf_tk/data_classes/_module_resources.py

@@ -1,5 +1,4 @@
-from __future__ import annotations
-
+import sys
 from dataclasses import dataclass
 from functools import cached_property
 from pathlib import Path
@@ -10,8 +9,8 @@ import yaml
 from cognite_toolkit import _version
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.constants import DEFAULT_ENV
-from cognite_toolkit._cdf_tk.loaders import ResourceTypes
-from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID
+from cognite_toolkit._cdf_tk.cruds import ResourceTypes
+from cognite_toolkit._cdf_tk.cruds._base_cruds import T_ID
 from cognite_toolkit._cdf_tk.utils import (
     safe_write,
     tmp_build_directory,
@@ -25,6 +24,11 @@ from ._built_resources import BuiltFullResourceList
 from ._config_yaml import BuildConfigYAML
 from ._module_directories import ModuleDirectories
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 
 @dataclass
 class ModulesInfo:
@@ -32,7 +36,7 @@ class ModulesInfo:
     modules: BuiltModuleList
 
     @classmethod
-    def load(cls, data: dict[str, Any]) -> ModulesInfo:
+    def load(cls, data: dict[str, Any]) -> Self:
         return cls(
             version=data["version"],
             modules=BuiltModuleList([BuiltModule.load(module_data) for module_data in data["modules"]]),
@@ -52,13 +56,11 @@ class BuildInfo(ConfigCore):
     modules: ModulesInfo
 
     @classmethod
-    def load(cls, data: dict[str, Any], build_env: str, filepath: Path) -> BuildInfo:
+    def load(cls, data: dict[str, Any], build_env: str, filepath: Path) -> Self:
         return cls(filepath, ModulesInfo.load(data["modules"]))
 
     @classmethod
-    def rebuild(
-        cls, organization_dir: Path, build_env: str | None, needs_rebuild: set[Path] | None = None
-    ) -> BuildInfo:
+    def rebuild(cls, organization_dir: Path, build_env: str | None, needs_rebuild: set[Path] | None = None) -> Self:
         # To avoid circular imports
         # Ideally, this class should be in a separate module
         from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand
@@ -110,7 +112,7 @@ class BuildInfo(ConfigCore):
         return new_build
 
     @classmethod
-    def _get_existing(cls, organization_dir: Path, build_env: str) -> BuildInfo | None:
+    def _get_existing(cls, organization_dir: Path, build_env: str) -> Self | None:
         try:
             existing = cls.load_from_directory(organization_dir, build_env)
         except FileNotFoundError: