cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/{loaders → cruds}/_worker.py

@@ -1,54 +1,63 @@
-from __future__ import annotations
-
 import re
 import warnings
-from collections.abc import Hashable
+from collections.abc import Hashable, Sequence
 from copy import deepcopy
+from dataclasses import dataclass, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generic, Literal, overload
+from typing import TYPE_CHECKING, Any, Generic, cast
 
-from cognite.client.data_classes._base import (
-    T_CogniteResourceList,
-    T_WritableCogniteResource,
-    T_WriteClass,
-)
+from cognite.client.data_classes import FunctionWrite
+from cognite.client.data_classes.capabilities import Capability
 from rich import print
 from rich.panel import Panel
 from yaml import YAMLError
 
 from cognite_toolkit._cdf_tk.constants import TABLE_FORMATS
 from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError, ToolkitYAMLFormatError
+from cognite_toolkit._cdf_tk.feature_flags import Flags
 from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, catch_warnings
 from cognite_toolkit._cdf_tk.utils import to_diff
 
-from ._base_loaders import T_ID, ResourceLoader, T_WritableCogniteResourceList
+from . import FunctionCRUD
+from ._base_cruds import T_ID, ResourceCRUD, T_ResourceRequest, T_ResourceResponse
 
 if TYPE_CHECKING:
     from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
 
 
-class ResourceWorker(
-    Generic[T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList]
-):
+@dataclass
+class CategorizedResources(Generic[T_ID, T_ResourceRequest]):
+    to_create: list[T_ResourceRequest] = field(default_factory=list)
+    to_update: list[T_ResourceRequest] = field(default_factory=list)
+    to_delete: list[T_ID] = field(default_factory=list)
+    unchanged: list[T_ResourceRequest] = field(default_factory=list)
+
+
+class ResourceWorker(Generic[T_ID, T_ResourceRequest, T_ResourceResponse]):
     def __init__(
         self,
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
+        action: str,
     ):
         self.loader = loader
+        self.duplicates: list[T_ID] = []
+        self.action = action
 
     def load_files(
-        self, sort: bool = True, directory: Path | None = None, read_modules: list[ReadModule] | None = None
+        self, sort: bool = True, directory: Path | None = None, read_modules: "list[ReadModule] | None" = None
     ) -> list[Path]:
         filepaths = self.loader.find_files(directory)
 
-        for read_module in read_modules or []:
-            if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
-                # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
-                # These table formats are not built, i.e., no variable replacement is done,
-                # so we load them directly from the source module.
-                filepaths.extend(self.loader.find_files(resource_dir, include_formats=TABLE_FORMATS))
+        if not Flags.v08.is_enabled():
+            for read_module in read_modules or []:
+                if resource_dir := read_module.resource_dir_path(self.loader.folder_name):
+                    # As of 05/11/24, Asset support csv and parquet files in addition to YAML.
+                    # These table formats are not built, i.e., no variable replacement is done,
+                    # so we load them directly from the source module.
+                    table_files = [
+                        file for file in self.loader.find_files(resource_dir) if file.suffix in TABLE_FORMATS
+                    ]
+                    filepaths.extend(table_files)
 
         if not sort:
             return filepaths
@@ -64,45 +73,40 @@ class ResourceWorker(
         # sort.
         return sorted(filepaths, key=sort_key)
 
-    @overload
-    def load_resources(
+    def prepare_resources(
         self,
         filepaths: list[Path],
-        return_existing: Literal[True],
         environment_variables: dict[str, str | None] | None = None,
         is_dry_run: bool = False,
         force_update: bool = False,
         verbose: bool = False,
-    ) -> tuple[T_WritableCogniteResourceList, list[T_ID]]: ...
+    ) -> CategorizedResources:
+        """Prepare resources for deployment by loading them from files, validating access, and categorizing them into create, update, delete, and unchanged lists.
 
-    @overload
-    def load_resources(
-        self,
-        filepaths: list[Path],
-        return_existing: Literal[False] = False,
-        environment_variables: dict[str, str | None] | None = None,
-        is_dry_run: bool = False,
-        force_update: bool = False,
-        verbose: bool = False,
-    ) -> tuple[T_CogniteResourceList, T_CogniteResourceList, list[T_ID], T_CogniteResourceList, list[T_ID]]: ...
+        Args:
+            filepaths: The list of file paths to load resources from.
+            environment_variables: Environment variables to use for variable replacement in the resource files.
+            is_dry_run: Whether to perform a dry run (no actual changes made).
+            force_update: Whether to force update existing resources even if they are unchanged.
+            verbose: Whether to print detailed information about the resources being processed.
+
+        Returns:
+            CategorizedResources: A categorized list of resources to create, update, delete, and unchanged.
+        """
+        local_by_id = self.load_resources(filepaths, environment_variables, is_dry_run)
+
+        self.validate_access(local_by_id, is_dry_run)
+
+        # Lookup the existing resources in CDF
+        cdf_resources = self.loader.retrieve(list(local_by_id.keys()))
+        return self.categorize_resources(local_by_id, cdf_resources, force_update, verbose)
 
     def load_resources(
-        self,
-        filepaths: list[Path],
-        return_existing: bool = False,
-        environment_variables: dict[str, str | None] | None = None,
-        is_dry_run: bool = False,
-        force_update: bool = False,
-        verbose: bool = False,
-    ) -> (
-        tuple[T_CogniteResourceList, T_CogniteResourceList, list[T_ID], T_CogniteResourceList, list[T_ID]]
-        | tuple[T_WritableCogniteResourceList, list[T_ID]]
-    ):
-        duplicates: list[T_ID] = []
-        local_by_id: dict[T_ID, tuple[dict[str, Any], T_WriteClass]] = {}  # type: ignore[assignment]
+        self, filepaths: list[Path], environment_variables: dict[str, str | None] | None, is_dry_run: bool
+    ) -> dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]]:
+        local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]] = {}
         # Load all resources from files, get ids, and remove duplicates.
         environment_variables = environment_variables or {}
-
         for filepath in filepaths:
             with catch_warnings(EnvironmentVariableMissingWarning) as warning_list:
                 try:
@@ -122,7 +126,7 @@ class ResourceWorker(
                     # should be handled by the other loader.
                     continue
                 if identifier in local_by_id:
-                    duplicates.append(identifier)
+                    self.duplicates.append(identifier)
                 else:
                     local_by_id[identifier] = resource_dict, loaded
 
@@ -134,43 +138,46 @@ class ResourceWorker(
                     warnings.warn(warning, stacklevel=2)
                 else:
                     warning.print_warning()
-
-        capabilities = self.loader.get_required_capability(
-            [item for _, item in local_by_id.values()], read_only=is_dry_run
-        )
+        return local_by_id
+
+    def validate_access(
+        self, local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]], is_dry_run: bool
+    ) -> None:
+        capabilities: Capability | list[Capability]
+        if isinstance(self.loader, FunctionCRUD):
+            function_loader: FunctionCRUD = self.loader
+            function_items = cast(list[FunctionWrite], [item for _, item in local_by_id.values()])
+            capabilities = function_loader.get_function_required_capabilities(function_items, read_only=is_dry_run)
+        else:
+            capabilities = self.loader.get_required_capability(
+                [item for _, item in local_by_id.values()], read_only=is_dry_run
+            )
         if capabilities and (missing := self.loader.client.verify.authorization(capabilities)):
-            raise self.loader.client.verify.create_error(missing, action=f"clean {self.loader.display_name}")
+            raise self.loader.client.verify.create_error(missing, action=f"{self.action} {self.loader.display_name}")
 
-        # Lookup the existing resources in CDF
-        cdf_resources: T_WritableCogniteResourceList
-        cdf_resources = self.loader.retrieve(list(local_by_id.keys()))
-        if return_existing:
-            return cdf_resources, duplicates
-
-        to_create: T_CogniteResourceList
-        to_update: T_CogniteResourceList
-        to_delete: list[T_ID] = []
-        unchanged: T_CogniteResourceList
-        to_create, to_update, unchanged = (
-            self.loader.list_write_cls([]),
-            self.loader.list_write_cls([]),
-            self.loader.list_write_cls([]),
-        )
+    def categorize_resources(
+        self,
+        local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]],
+        cdf_resources: Sequence[T_ResourceResponse],
+        force_update: bool,
+        verbose: bool,
+    ) -> CategorizedResources[T_ID, T_ResourceRequest]:
+        resources: CategorizedResources[T_ID, T_ResourceRequest] = CategorizedResources()
         cdf_resource_by_id = {self.loader.get_id(resource): resource for resource in cdf_resources}
         for identifier, (local_dict, local_resource) in local_by_id.items():
             cdf_resource = cdf_resource_by_id.get(identifier)
             if cdf_resource is None:
-                to_create.append(local_resource)
+                resources.to_create.append(local_resource)
                 continue
             cdf_dict = self.loader.dump_resource(cdf_resource, local_dict)
             if not force_update and cdf_dict == local_dict:
-                unchanged.append(local_resource)
+                resources.unchanged.append(local_resource)
                 continue
             if self.loader.support_update:
-                to_update.append(local_resource)
+                resources.to_update.append(local_resource)
             else:
-                to_delete.append(identifier)
-                to_create.append(local_resource)
+                resources.to_delete.append(identifier)
+                resources.to_create.append(local_resource)
             if verbose:
                 diff_str = "\n".join(to_diff(cdf_dict, local_dict))
                 for sensitive in self.loader.sensitive_strings(local_resource):
@@ -182,4 +189,4 @@ class ResourceWorker(
                         expand=False,
                     )
                 )
-        return to_create, to_update, to_delete, unchanged, duplicates
+        return resources

cognite_toolkit/_cdf_tk/data_classes/__init__.py

@@ -32,6 +32,7 @@ from ._deploy_results import (
 from ._module_directories import ModuleDirectories, ModuleLocation
 from ._module_resources import ModuleResources
 from ._packages import Package, Packages
+from ._tracking_info import CommandTrackingInfo
 from ._yaml_comments import YAMLComments
 
 __all__ = [
@@ -47,6 +48,7 @@ __all__ = [
     "BuiltResource",
     "BuiltResourceFull",
     "BuiltResourceList",
+    "CommandTrackingInfo",
     "ConfigEntry",
     "ConfigYAMLs",
     "DatapointDeployResult",

cognite_toolkit/_cdf_tk/data_classes/_base.py

@@ -1,5 +1,4 @@
-from __future__ import annotations
-
+import sys
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from pathlib import Path
@@ -10,6 +9,11 @@ from cognite_toolkit._cdf_tk.constants import BUILD_ENVIRONMENT_FILE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitRequiredValueError, ToolkitVersionError
 from cognite_toolkit._cdf_tk.utils import read_yaml_file
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 
 @dataclass
 class ConfigCore(ABC):
@@ -23,7 +27,7 @@ class ConfigCore(ABC):
         return cls.filename.format(build_env=build_env)
 
     @classmethod
-    def load_from_directory(cls: type[T_BuildConfig], organization_dir: Path, build_env: str) -> T_BuildConfig:
+    def load_from_directory(cls, organization_dir: Path, build_env: str) -> Self:
         filename = cls.get_filename(build_env)
         filepath = organization_dir / filename
         filepath = filepath if filepath.is_file() else Path.cwd() / filename
@@ -37,7 +41,7 @@ class ConfigCore(ABC):
 
     @classmethod
     @abstractmethod
-    def load(cls: type[T_BuildConfig], data: dict[str, Any], build_env: str, filepath: Path) -> T_BuildConfig:
+    def load(cls, data: dict[str, Any], build_env: str, filepath: Path) -> Self:
         raise NotImplementedError
 
 

cognite_toolkit/_cdf_tk/data_classes/_build_files.py

@@ -2,8 +2,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
-from cognite_toolkit._cdf_tk.loaders import (
-    ResourceLoader,
+from cognite_toolkit._cdf_tk.cruds import (
+    ResourceCRUD,
 )
 from cognite_toolkit._cdf_tk.tk_warnings import (
     WarningList,
@@ -27,7 +27,7 @@ class BuildSourceFile:
 class BuildDestinationFile:
     path: Path
     loaded: list[dict[str, Any]] | dict[str, Any]
-    loader: type[ResourceLoader]
+    loader: type[ResourceCRUD]
     source: SourceLocation
     extra_sources: list[SourceLocation] | None
     warnings: WarningList[FileReadWarning] = field(default_factory=WarningList[FileReadWarning])

cognite_toolkit/_cdf_tk/data_classes/_build_variables.py

@@ -1,6 +1,5 @@
-from __future__ import annotations
-
 import re
+import sys
 import uuid
 from collections import defaultdict
 from collections.abc import Collection, Iterator, Sequence
@@ -9,10 +8,15 @@ from functools import cached_property
 from pathlib import Path
 from typing import Any, Literal, SupportsIndex, overload
 
+from cognite_toolkit._cdf_tk.cruds._resource_cruds.transformation import TransformationCRUD
+from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.feature_flags import Flags
 
-from ._module_directories import ModuleLocation
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
 
 
 @dataclass(frozen=True)
@@ -51,7 +55,7 @@ class BuildVariable:
         }
 
     @classmethod
-    def load(cls, data: dict[str, Any]) -> BuildVariable:
+    def load(cls, data: dict[str, Any]) -> Self:
         if isinstance(data["value"], list):
             # Convert the list to a tuple to make it hashable
             value = tuple(data["value"])
@@ -68,7 +72,7 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
 
     # Subclassing tuple to make the class immutable. BuildVariables is expected to be initialized and
     # then used as a read-only object.
-    def __new__(cls, collection: Collection[BuildVariable], source_path: Path | None = None) -> BuildVariables:
+    def __new__(cls, collection: Collection[BuildVariable], source_path: Path | None = None) -> Self:
         # Need to override __new__ to as we are subclassing a tuple:
         #   https://stackoverflow.com/questions/1565374/subclassing-tuple-with-multiple-init-arguments
         return super().__new__(cls, tuple(collection))
@@ -78,7 +82,7 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
         self.source_path = source_path
 
     @cached_property
-    def selected(self) -> BuildVariables:
+    def selected(self) -> "BuildVariables":
         return BuildVariables([variable for variable in self if variable.is_selected])
 
     @classmethod
@@ -88,7 +92,7 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
         available_modules: set[Path],
         selected_modules: set[Path] | None = None,
         source_path: Path | None = None,
-    ) -> BuildVariables:
+    ) -> Self:
         """Loads the variables from the user input."""
         variables = []
         to_check: list[tuple[Path, int | None, dict[str, Any]]] = [(Path(""), None, raw_variable)]
@@ -119,11 +123,11 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
         return cls(variables, source_path=source_path)
 
     @classmethod
-    def load(cls, data: list[dict[str, Any]]) -> BuildVariables:
+    def load(cls, data: list[dict[str, Any]]) -> Self:
         """Loads the variables from a dictionary."""
         return cls([BuildVariable.load(variable) for variable in data])
 
-    def get_module_variables(self, module: ModuleLocation) -> list[BuildVariables]:
+    def get_module_variables(self, module: ModuleLocation) -> "list[BuildVariables]":
         """Gets the variables for a specific module."""
         variables_by_key_by_iteration: dict[int | None, dict[str, list[BuildVariable]]] = defaultdict(
             lambda: defaultdict(list)
@@ -157,16 +161,19 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
         ]
 
     @overload
-    def replace(self, content: str, file_suffix: str = ".yaml", use_placeholder: Literal[False] = False) -> str: ...
+    def replace(self, content: str, file_path: Path | None = None, use_placeholder: Literal[False] = False) -> str: ...
 
     @overload
     def replace(
-        self, content: str, file_suffix: str = ".yaml", use_placeholder: Literal[True] = True
+        self, content: str, file_path: Path | None = None, use_placeholder: Literal[True] = True
     ) -> tuple[str, dict[str, BuildVariable]]: ...
 
     def replace(
-        self, content: str, file_suffix: str = ".yaml", use_placeholder: bool = False
+        self, content: str, file_path: Path | None = None, use_placeholder: bool = False
     ) -> str | tuple[str, dict[str, BuildVariable]]:
+        # Extract file suffix from path, default to .yaml if not provided
+        file_suffix = file_path.suffix if file_path and file_path.suffix else ".yaml"
+
         variable_by_placeholder: dict[str, BuildVariable] = {}
         for variable in self:
             if not use_placeholder:
@@ -176,22 +183,125 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
                 variable_by_placeholder[replace] = variable
 
             _core_pattern = rf"{{{{\s*{variable.key}\s*}}}}"
-            if file_suffix in {".yaml", ".yml", ".json"}:
-                # Preserve data types
-                pattern = _core_pattern
-                if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
-                    replace = f'"{replace}"'
-                    pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
-                elif replace is None:
-                    replace = "null"
-                content = re.sub(pattern, str(replace), content)
+            if file_suffix == ".sql":
+                # For SQL files, convert lists to SQL-style tuples
+                if isinstance(replace, list):
+                    replace = self._format_list_as_sql_tuple(replace)
+                content = re.sub(_core_pattern, str(replace), content)
+            elif file_suffix in {".yaml", ".yml", ".json"}:
+                # Check if this is a transformation file (ends with Transformation.yaml/yml)
+                is_transformation_file = file_path is not None and f".{TransformationCRUD.kind}." in file_path.name
+                # Check if variable is within a query field (SQL context)
+                is_in_query_field = self._is_in_query_field(content, variable.key)
+
+                # For lists in query fields, use SQL-style tuples
+                # For transformation files, ensure SQL conversion is applied to query property variables
+                if is_transformation_file and is_in_query_field and isinstance(replace, list):
+                    replace = self._format_list_as_sql_tuple(replace)
+                    # Use simple pattern for SQL context (no YAML quoting needed)
+                    content = re.sub(_core_pattern, str(replace), content)
+                else:
+                    # Preserve data types for YAML
+                    pattern = _core_pattern
+                    if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
+                        replace = f'"{replace}"'
+                        pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
+                    elif replace is None:
+                        replace = "null"
+                    content = re.sub(pattern, str(replace), content)
             else:
+                # For other file types, use simple string replacement
                 content = re.sub(_core_pattern, str(replace), content)
         if use_placeholder:
             return content, variable_by_placeholder
         else:
             return content
 
+    @staticmethod
+    def _is_transformation_file(file_path: Path) -> bool:
+        """Check if the file path indicates a transformation YAML file.
+
+        Transformation files are YAML files in the "transformations" folder.
+
+        Args:
+            file_path: The file path to check
+
+        Returns:
+            True if the file is a transformation YAML file
+        """
+        # Check if path contains "transformations" folder and ends with .yaml/.yml
+        path_str = file_path.as_posix().lower()
+        return "transformations" in path_str and file_path.suffix.lower() in {".yaml", ".yml"}
+
+    @staticmethod
+    def _format_list_as_sql_tuple(replace: list[Any]) -> str:
+        """Format a list as a SQL-style tuple string.
+
+        Args:
+            replace: The list to format
+
+        Returns:
+            SQL tuple string, e.g., "('A', 'B', 'C')" or "()" for empty lists
+        """
+        if not replace:
+            # Empty list becomes empty SQL tuple
+            return "()"
+        else:
+            # Format list as SQL tuple: ('A', 'B', 'C')
+            formatted_items = []
+            for item in replace:
+                if item is None:
+                    formatted_items.append("NULL")
+                elif isinstance(item, str):
+                    formatted_items.append(f"'{item}'")
+                else:
+                    formatted_items.append(str(item))
+            return f"({', '.join(formatted_items)})"
+
+    @staticmethod
+    def _is_in_query_field(content: str, variable_key: str) -> bool:
+        """Check if a variable is within a query field in YAML.
+
+        Assumes query is a top-level property. This detects various YAML formats:
+        - query: >-
+        - query: |
+        - query: "..."
+        - query: ...
+        """
+        lines = content.split("\n")
+        variable_pattern = rf"{{{{\s*{re.escape(variable_key)}\s*}}}}"
+        in_query_field = False
+
+        for line in lines:
+            # Check if this line starts a top-level query field
+            query_match = re.match(r"^query\s*:\s*(.*)$", line)
+            if query_match:
+                in_query_field = True
+                query_content_start = query_match.group(1).strip()
+
+                # Check if variable is on the same line as query: declaration
+                if re.search(variable_pattern, line):
+                    return True
+
+                # If query content starts on same line (not a block scalar), check it
+                if query_content_start and not query_content_start.startswith(("|", ">", "|-", ">-", "|+", ">+")):
+                    if re.search(variable_pattern, query_content_start):
+                        return True
+                continue
+
+            # Check if we're still in the query field
+            if in_query_field:
+                # If we hit another top-level property, we've exited the query field
+                if re.match(r"^\w+\s*:", line):
+                    in_query_field = False
+                    continue
+
+                # We're still in the query field, check for variable
+                if re.search(variable_pattern, line):
+                    return True
+
+        return False
+
     # Implemented to get correct type hints
     def __iter__(self) -> Iterator[BuildVariable]:
         return super().__iter__()
@@ -200,9 +310,9 @@ class BuildVariables(tuple, Sequence[BuildVariable]):
     def __getitem__(self, index: SupportsIndex) -> BuildVariable: ...
 
     @overload
-    def __getitem__(self, index: slice) -> BuildVariables: ...
+    def __getitem__(self, index: slice) -> Self: ...
 
-    def __getitem__(self, index: SupportsIndex | slice, /) -> BuildVariable | BuildVariables:
+    def __getitem__(self, index: SupportsIndex | slice, /) -> "BuildVariable | BuildVariables":
         if isinstance(index, slice):
             return BuildVariables(super().__getitem__(index))
         return super().__getitem__(index)

cognite_toolkit/_cdf_tk/data_classes/_built_modules.py

@@ -1,12 +1,11 @@
-from __future__ import annotations
-
+import sys
 from collections.abc import Callable, Collection, Iterator, MutableSequence
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, SupportsIndex, overload
 
-from cognite_toolkit._cdf_tk.loaders import ResourceTypes
-from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID
+from cognite_toolkit._cdf_tk.cruds import ResourceTypes
+from cognite_toolkit._cdf_tk.cruds._base_cruds import T_ID
 
 from ._build_variables import BuildVariables
 from ._built_resources import (
@@ -16,6 +15,11 @@ from ._built_resources import (
     SourceLocation,
 )
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 
 @dataclass
 class BuiltModule:
@@ -26,9 +30,11 @@ class BuiltModule:
     warning_count: int
     status: str
     iteration: int
+    package_id: str | None = None
+    module_id: str | None = None
 
     @classmethod
-    def load(cls, data: dict[str, Any]) -> BuiltModule:
+    def load(cls, data: dict[str, Any]) -> Self:
         return cls(
             name=data["name"],
             location=SourceLocation.load(data["location"]),
@@ -40,6 +46,8 @@ class BuiltModule:
             warning_count=data.get("warning_count", 0),
             status=data.get("status", "Success"),
             iteration=data.get("iteration", 1),
+            package_id=data.get("package_id"),
+            module_id=data.get("module_id"),
         )
 
     def dump(self) -> dict[str, Any]:
@@ -53,6 +61,8 @@ class BuiltModule:
             "warning_count": self.warning_count,
             "status": self.status,
             "iteration": self.iteration,
+            "package_id": self.package_id,
+            "module_id": self.module_id,
         }
 
 
@@ -69,9 +79,9 @@ class BuiltModuleList(list, MutableSequence[BuiltModule]):
     def __getitem__(self, index: SupportsIndex) -> BuiltModule: ...
 
     @overload
-    def __getitem__(self, index: slice) -> BuiltModuleList: ...
+    def __getitem__(self, index: slice) -> Self: ...
 
-    def __getitem__(self, index: SupportsIndex | slice, /) -> BuiltModule | BuiltModuleList:
+    def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltModule | BuiltModuleList":
         if isinstance(index, slice):
             return BuiltModuleList(super().__getitem__(index))
         return super().__getitem__(index)
@@ -83,7 +93,7 @@ class BuiltModuleList(list, MutableSequence[BuiltModule]):
         kind: str | None = None,
         selected: Path | str | None = None,
         is_supported_file: Callable[[Path], bool] | None = None,
-    ) -> BuiltFullResourceList[T_ID]:
+    ) -> "BuiltFullResourceList[T_ID]":
         resources = (
             resource.create_full(module, resource_dir)
             for module in self
@@ -96,7 +106,7 @@ class BuiltModuleList(list, MutableSequence[BuiltModule]):
             resources = (
                 resource
                 for resource in resources
-                if (resource.source.path == selected or resource.source.path.is_relative_to(selected))
+                if (resource.source.path == selected or self._are_relative(resource.source.path, selected))
             )
         if is_supported_file:
             # This is necessary as the destination file can be created from a source file that is not supported.
@@ -111,3 +121,7 @@ class BuiltModuleList(list, MutableSequence[BuiltModule]):
             for resource_dir, resources in module.resources.items():
                 resources_by_folder.setdefault(resource_dir, BuiltResourceList()).extend(resources)
         return resources_by_folder
+
+    @staticmethod
+    def _are_relative(filepath: Path, select_path: Path) -> bool:
+        return filepath.resolve().is_relative_to(select_path.resolve())