cognite-toolkit 0.7.47__py3-none-any.whl → 0.7.48__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py

@@ -1,5 +1,7 @@
+import re
+from itertools import groupby
 from pathlib import Path
-from typing import Any, Literal, TypedDict
+from typing import Any, Literal
 
 from rich import print
 from rich.panel import Panel
@@ -7,39 +9,29 @@ from rich.panel import Panel
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
 from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand as OldBuildCommand
-from cognite_toolkit._cdf_tk.commands.build_v2.build_input import BuildInput
-from cognite_toolkit._cdf_tk.commands.build_v2.build_issues import BuildIssue, BuildIssueList
-from cognite_toolkit._cdf_tk.data_classes import (
-    BuildConfigYAML,
-    BuildVariables,
-    BuiltModuleList,
-    ModuleDirectories,
-)
+from cognite_toolkit._cdf_tk.commands.build_v2._modules_parser import ModulesParser
+from cognite_toolkit._cdf_tk.commands.build_v2.build_parameters import BuildParameters
+from cognite_toolkit._cdf_tk.commands.build_v2.data_classes._modules import Module
+from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, BuildVariables, BuiltModuleList
+from cognite_toolkit._cdf_tk.data_classes._issues import Issue, IssueList
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError
-from cognite_toolkit._cdf_tk.hints import verify_module_directory
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
 from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
-from cognite_toolkit._cdf_tk.validation import validate_module_selection, validate_modules_variables
 from cognite_toolkit._version import __version__
 
 
-class BuildWarnings(TypedDict):
-    warning: ToolkitWarning
-    location: list[Path]
-
-
 class BuildCommand(ToolkitCommand):
     def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
         super().__init__(print_warning, skip_tracking, silent)
-        self.issues = BuildIssueList()
+        self.issues = IssueList()
 
     def execute(
         self,
         verbose: bool,
-        organization_dir: Path,
+        base_dir: Path,
         build_dir: Path,
         selected: list[str | Path] | None,
-        build_env_name: str | None,
+        build_env: str | None,
         no_clean: bool,
         client: ToolkitClient | None = None,
         on_error: Literal["continue", "raise"] = "continue",
@@ -51,30 +43,48 @@ class BuildCommand(ToolkitCommand):
         self.verbose = verbose
         self.on_error = on_error
 
-        # Tracking the project and cluster for the build.
-        if client:
-            self._additional_tracking_info.project = client.config.project
-            self._additional_tracking_info.cluster = client.config.cdf_cluster
-
-        # Setting the parameters for the build.
-        input = BuildInput.load(organization_dir, build_dir, build_env_name, client, selected)
+        build_parameters = BuildParameters.load(
+            organization_dir=base_dir,
+            build_dir=build_dir,
+            build_env_name=build_env,
+            client=client,
+            user_selected=selected,
+        )
 
         # Print the build input.
         if self.verbose:
-            self._print_build_input(input)
+            self._print_build_input(build_parameters)
 
-        # Capture warnings from module structure integrity
-        if module_selection_issues := self._validate_modules(input):
-            self.issues.extend(module_selection_issues)
+        # Tracking the project and cluster for the build.
+        if build_parameters.client:
+            self._additional_tracking_info.project = build_parameters.client.config.project
+            self._additional_tracking_info.cluster = build_parameters.client.config.cdf_cluster
+
+        # Load modules
+        modules_parser = ModulesParser(organization_dir=base_dir, selected=selected)
+        module_paths = modules_parser.parse()
+        module_loading_issues = modules_parser.issues
+        if module_loading_issues:
+            self.issues.extend(module_loading_issues)
+            self._print_or_log_issues_by_category(self.issues)
+            raise ToolkitError("Module loading issues encountered. Cannot continue. See above for details.")
+
+        # Load modules
+        if module_paths:
+            pass
+
+        # modules = [Module.load(path) for path in module_paths]
+        # for module in modules:
+        #    continue
 
         # Logistics: clean and create build directory
-        if prepare_issues := self._prepare_target_directory(input, not no_clean):
+        if prepare_issues := self._prepare_target_directory(build_dir, not no_clean):
             self.issues.extend(prepare_issues)
 
         # Compile the configuration and variables,
         # check syntax on module and resource level
         # for any "compilation errors and warnings"
-        built_modules, build_integrity_issues = self._build_configuration(input)
+        built_modules, build_integrity_issues = self._build_configuration(build_parameters)
         if build_integrity_issues:
             self.issues.extend(build_integrity_issues)
 
@@ -83,84 +93,48 @@ class BuildCommand(ToolkitCommand):
             self.issues.extend(build_quality_issues)
 
         # Finally, print warnings grouped by category/code and location.
-        self._print_or_log_warnings_by_category(self.issues)
+        self._print_or_log_issues_by_category(self.issues)
 
         return built_modules
 
-    def _print_build_input(self, input: BuildInput) -> None:
+    def _print_build_input(self, build_input: BuildParameters) -> None:
         print(
             Panel(
-                f"Building {input.organization_dir!s}:\n  - Toolkit Version '{__version__!s}'\n"
-                f"  - Environment name {input.build_env_name!r}, validation-type {input.config.environment.validation_type!r}.\n"
-                f"  - Config '{input.config.filepath!s}'",
+                f"Building {build_input.organization_dir!s}:\n  - Toolkit Version '{__version__!s}'\n"
+                f"  - Environment name {build_input.build_env_name!r}, validation-type {build_input.config.environment.validation_type!r}.\n"
+                f"  - Config '{build_input.config.filepath!s}'",
                 expand=False,
             )
         )
 
-    def _prepare_target_directory(self, input: BuildInput, clean: bool = False) -> BuildIssueList:
+    def _prepare_target_directory(self, build_dir: Path, clean: bool = False) -> IssueList:
         """
         Directory logistics
         """
-        issues = BuildIssueList()
-        if input.build_dir.exists() and any(input.build_dir.iterdir()):
+        issues = IssueList()
+        if build_dir.exists() and any(build_dir.iterdir()):
             if not clean:
                 raise ToolkitError("Build directory is not empty. Run without --no-clean to remove existing files.")
 
             if self.verbose:
-                issues.append(BuildIssue(description=f"Build directory {input.build_dir!s} is not empty. Clearing."))
-            safe_rmtree(input.build_dir)
-        input.build_dir.mkdir(parents=True, exist_ok=True)
+                issues.append(Issue(code="BUILD_001"))
+            safe_rmtree(build_dir)
+        build_dir.mkdir(parents=True, exist_ok=True)
         return issues
 
-    def _validate_modules(self, input: BuildInput) -> BuildIssueList:
-        issues = BuildIssueList()
-        # Verify that the modules exists, are not duplicates,
-        # and at least one is selected
-        verify_module_directory(input.organization_dir, input.build_env_name)
-
-        # Validate module selection
-        user_selected_modules = input.config.environment.get_selected_modules({})
-        module_warnings = validate_module_selection(
-            modules=input.modules,
-            config=input.config,
-            packages={},
-            selected_modules=user_selected_modules,
-            organization_dir=input.organization_dir,
-        )
-        if module_warnings:
-            issues.extend(BuildIssueList.from_warning_list(module_warnings))
-
-        # Validate variables. Note: this looks for non-replaced template
-        # variables <.*?> and can be improved in the future.
-        # Keeping for reference.
-        variables_warnings = validate_modules_variables(input.variables, input.config.filepath)
-        if variables_warnings:
-            issues.extend(BuildIssueList.from_warning_list(variables_warnings))
-
-        # Track LOC of managed configuration
-        # Note: _track is not implemented yet, so we skip it for now
-        # self._track(input)
-
-        return issues
-
-    def _build_configuration(self, input: BuildInput) -> tuple[BuiltModuleList, BuildIssueList]:
-        issues = BuildIssueList()
-        # Use input.modules.selected directly (it's already a ModuleDirectories)
-        if not input.modules.selected:
-            return BuiltModuleList(), issues
-
-        # first collect variables into practical lookup
-        # TODO: parallelism is not implemented yet. I'm sure there are optimizations to be had here, but we'll focus on process parallelism since we believe loading yaml and file i/O are the biggest bottlenecks.
+    def _build_configuration(self, build_input: BuildParameters) -> tuple[BuiltModuleList, IssueList]:
+        issues = IssueList()
+        # Use build_input.modules directly (it is already filtered by selection)
 
         old_build_command = OldBuildCommand(print_warning=False, skip_tracking=False)
         built_modules = old_build_command.build_config(
-            build_dir=input.build_dir,
-            organization_dir=input.organization_dir,
-            config=input.config,
+            build_dir=build_input.build_dir,
+            organization_dir=build_input.organization_dir,
+            config=build_input.config,
             packages={},
             clean=False,
             verbose=self.verbose,
-            client=input.client,
+            client=build_input.client,
             progress_bar=False,
             on_error=self.on_error,
         )
@@ -172,29 +146,42 @@ class BuildCommand(ToolkitCommand):
         # Always convert warnings to issues, even if the list appears empty
         # (WarningList might have custom __bool__ behavior)
         if old_build_command.warning_list:
-            converted_issues = BuildIssueList.from_warning_list(old_build_command.warning_list)
+            converted_issues = IssueList.from_warning_list(old_build_command.warning_list)
             issues.extend(converted_issues)
         return built_modules, issues
 
-    def _verify_build_quality(self, built_modules: BuiltModuleList) -> BuildIssueList:
-        issues = BuildIssueList()
+    def _verify_build_quality(self, built_modules: BuiltModuleList) -> IssueList:
+        issues = IssueList()
         return issues
 
-    def _write(self, input: BuildInput) -> None:
+    def _write(self, build_input: BuildParameters) -> None:
         # Write the build to the build directory.
         # Track lines of code built.
         raise NotImplementedError()
 
-    def _track(self, input: BuildInput) -> None:
+    def _track(self, build_input: BuildParameters) -> None:
         raise NotImplementedError()
 
-    def _print_or_log_warnings_by_category(self, issues: BuildIssueList) -> None:
-        pass
+    def _print_or_log_issues_by_category(self, issues: IssueList) -> None:
+        issues_sorted = sorted(issues, key=self._issue_sort_key)
+        for code, grouped_issues in groupby(issues_sorted, key=lambda issue: issue.code or ""):
+            print(f"[bold]{code}[/]")
+            for issue in grouped_issues:
+                message = issue.message or ""
+                print(f"  - {message}")
+
+    def _issue_sort_key(self, issue: Issue) -> tuple[str, str]:
+        code = issue.code or ""
+        if not issue.message:
+            return code, ""
+        match = re.search(r"'([^']+)'", issue.message)
+        path = match.group(1) if match else issue.message
+        return code, path
 
     # Delegate to old BuildCommand for backward compatibility with tests
     def build_modules(
         self,
-        modules: ModuleDirectories,
+        modules: list[Module],
         build_dir: Path,
         variables: BuildVariables,
         verbose: bool = False,
@@ -204,10 +191,10 @@ class BuildCommand(ToolkitCommand):
         """Delegate to old BuildCommand for backward compatibility."""
         old_cmd = OldBuildCommand()
 
-        built_modules = old_cmd.build_modules(modules, build_dir, variables, verbose, progress_bar, on_error)
+        built_modules = old_cmd.build_modules(modules, build_dir, variables, verbose, progress_bar, on_error)  # type: ignore[arg-type]
         self._additional_tracking_info.package_ids.update(old_cmd._additional_tracking_info.package_ids)
         self._additional_tracking_info.module_ids.update(old_cmd._additional_tracking_info.module_ids)
-        self.issues.extend(BuildIssueList.from_warning_list(old_cmd.warning_list or WarningList[ToolkitWarning]()))
+        self.issues.extend(IssueList.from_warning_list(old_cmd.warning_list or WarningList[ToolkitWarning]()))
         return built_modules
 
     def build_config(

cognite_toolkit/_cdf_tk/commands/build_v2/{build_input.py → build_parameters.py}

@@ -1,5 +1,4 @@
 import sys
-from functools import cached_property
 from pathlib import Path
 
 if sys.version_info >= (3, 11):
@@ -13,14 +12,12 @@ from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import DEFAULT_ENV
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildConfigYAML,
-    BuildVariables,
-    ModuleDirectories,
 )
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 
 
-class BuildInput(BaseModel):
+class BuildParameters(BaseModel):
     """Input to the build process."""
 
     # need this until we turn BuildConfigYaml and ToolkitClient into Pydantic models
@@ -31,8 +28,8 @@ class BuildInput(BaseModel):
     build_env_name: str
     config: BuildConfigYAML
     client: ToolkitClient | None = None
-    selected: list[str | Path] | None = None
     warnings: WarningList[ToolkitWarning] | None = None
+    user_selected: list[str | Path] | None = None
 
     @classmethod
     def load(
@@ -41,24 +38,24 @@ class BuildInput(BaseModel):
         build_dir: Path,
         build_env_name: str | None,
         client: ToolkitClient | None,
-        selected: list[str | Path] | None = None,
+        user_selected: list[str | Path] | None = None,
     ) -> Self:
         resolved_org_dir = Path.cwd() if organization_dir in {Path("."), Path("./")} else organization_dir
         resolved_env = build_env_name or DEFAULT_ENV
-        config, warnings = cls._load_config(resolved_org_dir, resolved_env, selected)
+        config, warnings = cls._load_config(resolved_org_dir, resolved_env, user_selected)
         return cls(
             organization_dir=resolved_org_dir,
             build_dir=build_dir,
             build_env_name=resolved_env,
             config=config,
             client=client,
-            selected=selected,
             warnings=warnings,
+            user_selected=user_selected,
         )
 
     @classmethod
     def _load_config(
-        cls, organization_dir: Path, build_env_name: str, selected: list[str | Path] | None
+        cls, organization_dir: Path, build_env_name: str, user_selected: list[str | Path] | None
     ) -> tuple[BuildConfigYAML, WarningList[ToolkitWarning]]:
         warnings: WarningList[ToolkitWarning] = WarningList[ToolkitWarning]()
         if (organization_dir / BuildConfigYAML.get_filename(build_env_name or DEFAULT_ENV)).exists():
@@ -66,20 +63,9 @@ class BuildInput(BaseModel):
         else:
             # Loads the default environment
             config = BuildConfigYAML.load_default(organization_dir)
-        if selected:
-            config.environment.selected = parse_user_selected_modules(selected, organization_dir)
+        if user_selected:
+            config.environment.selected = list(set(parse_user_selected_modules(list(user_selected), organization_dir)))
         config.set_environment_variables()
         if environment_warning := config.validate_environment():
             warnings.append(environment_warning)
         return config, warnings
-
-    @cached_property
-    def modules(self) -> ModuleDirectories:
-        user_selected_modules = self.config.environment.get_selected_modules({})
-        return ModuleDirectories.load(self.organization_dir, user_selected_modules)
-
-    @cached_property
-    def variables(self) -> BuildVariables:
-        return BuildVariables.load_raw(
-            self.config.variables, self.modules.available_paths, self.modules.selected.available_paths
-        )

cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_modules.py

@@ -0,0 +1,27 @@
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from cognite_toolkit._cdf_tk.data_classes._module_toml import ModuleToml
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class Module(BaseModel):
+    model_config = ConfigDict(
+        frozen=True,
+        validate_assignment=True,
+        arbitrary_types_allowed=True,
+    )
+
+    path: Path
+    definition: ModuleToml | None = None
+
+    @classmethod
+    def load(cls, path: Path) -> Self:
+        definition = ModuleToml.load(path / ModuleToml.filename) if (path / ModuleToml.filename).exists() else None
+        return cls(path=path, definition=definition)

cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_resource.py

@@ -0,0 +1,22 @@
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class Resource(BaseModel):
+    model_config = ConfigDict(
+        frozen=True,
+        validate_assignment=True,
+    )
+
+    path: Path
+
+    @classmethod
+    def load(cls, path: Path) -> Self:
+        return cls(path=path)

cognite_toolkit/_cdf_tk/cruds/__init__.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import itertools
+from collections import defaultdict
 from typing import Literal, TypeAlias
 
 from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags
@@ -90,24 +91,28 @@ if not FeatureFlag.is_enabled(Flags.STREAMS):
 if not FeatureFlag.is_enabled(Flags.SIMULATORS):
     _EXCLUDED_CRUDS.add(SimulatorModelCRUD)
 
-CRUDS_BY_FOLDER_NAME: dict[str, list[type[Loader]]] = {}
+CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA: defaultdict[str, list[type[Loader]]] = defaultdict(list)
+CRUDS_BY_FOLDER_NAME: defaultdict[str, list[type[Loader]]] = defaultdict(list)
 for _loader in itertools.chain(
     ResourceCRUD.__subclasses__(),
     ResourceContainerCRUD.__subclasses__(),
     DataCRUD.__subclasses__(),
     GroupCRUD.__subclasses__(),
 ):
-    if _loader in [ResourceCRUD, ResourceContainerCRUD, DataCRUD, GroupCRUD] or _loader in _EXCLUDED_CRUDS:
+    if _loader in [ResourceCRUD, ResourceContainerCRUD, DataCRUD, GroupCRUD]:
         # Skipping base classes
         continue
-    if _loader.folder_name not in CRUDS_BY_FOLDER_NAME:  # type: ignore[attr-defined]
-        CRUDS_BY_FOLDER_NAME[_loader.folder_name] = []  # type: ignore[attr-defined]
     # MyPy bug: https://github.com/python/mypy/issues/4717
-    CRUDS_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
+    CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
+
+    if _loader not in _EXCLUDED_CRUDS:
+        CRUDS_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
 del _loader  # cleanup module namespace
 
+
 # For backwards compatibility
 CRUDS_BY_FOLDER_NAME["data_models"] = CRUDS_BY_FOLDER_NAME["data_modeling"]  # Todo: Remove in v1.0
+CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA["data_models"] = CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA["data_modeling"]
 RESOURCE_CRUD_BY_FOLDER_NAME = {
     folder_name: cruds
     for folder_name, loaders in CRUDS_BY_FOLDER_NAME.items()
@@ -166,6 +171,7 @@ __all__ = [
     "RESOURCE_CRUD_CONTAINER_LIST",
     "RESOURCE_CRUD_LIST",
     "RESOURCE_DATA_CRUD_LIST",
+    "_EXCLUDED_CRUDS",
     "AgentCRUD",
     "AssetCRUD",
     "CogniteFileCRUD",

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py

@@ -4,11 +4,10 @@ from typing import Any, final
 from cognite.client.data_classes.capabilities import Capability, StreamsAcl
 from cognite.client.utils.useful_types import SequenceNotStr
 
-from cognite_toolkit._cdf_tk.client.http_client import ToolkitAPIError
+from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId
 from cognite_toolkit._cdf_tk.client.resource_classes.streams import (
     StreamRequest,
     StreamResponse,
-    StreamResponseList,
 )
 from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
 from cognite_toolkit._cdf_tk.resource_classes import StreamYAML
@@ -17,7 +16,7 @@ from .datamodel import ContainerCRUD
 
 
 @final
-class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
+class StreamCRUD(ResourceCRUD[ExternalId, StreamRequest, StreamResponse]):
     folder_name = "streams"
     resource_cls = StreamResponse
     resource_write_cls = StreamRequest
@@ -32,14 +31,14 @@ class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
         return "streams"
 
     @classmethod
-    def get_id(cls, item: StreamRequest | StreamResponse | dict) -> str:
+    def get_id(cls, item: StreamRequest | StreamResponse | dict) -> ExternalId:
         if isinstance(item, dict):
-            return item["externalId"]
-        return item.external_id
+            return ExternalId(external_id=item["externalId"])
+        return ExternalId(external_id=item.external_id)
 
     @classmethod
-    def dump_id(cls, id: str) -> dict[str, Any]:
-        return {"externalId": id}
+    def dump_id(cls, id: ExternalId) -> dict[str, Any]:
+        return id.dump()
 
     @classmethod
     def get_required_capability(
@@ -55,30 +54,15 @@ class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
         )
         return StreamsAcl(actions, StreamsAcl.Scope.All())
 
-    def create(self, items: Sequence[StreamRequest]) -> StreamResponseList:
-        created = self.client.streams.create(list(items))
-        return StreamResponseList(created)
+    def create(self, items: Sequence[StreamRequest]) -> list[StreamResponse]:
+        return self.client.streams.create(items)
 
-    def retrieve(self, ids: SequenceNotStr[str]) -> StreamResponseList:
-        retrieved: list[StreamResponse] = []
-        for _id in ids:
-            try:
-                _resp = self.client.streams.retrieve(_id)
-            except ToolkitAPIError:
-                continue
-            if _resp is not None:
-                retrieved.append(_resp)
-        return StreamResponseList(retrieved)
+    def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[StreamResponse]:
+        return self.client.streams.retrieve(list(ids), ignore_unknown_ids=True)
 
-    def delete(self, ids: SequenceNotStr[str]) -> int:
-        count = 0
-        for _id in ids:
-            try:
-                self.client.streams.delete(_id)
-            except ToolkitAPIError:
-                continue
-            count += 1
-        return count
+    def delete(self, ids: SequenceNotStr[ExternalId]) -> int:
+        self.client.streams.delete(list(ids), ignore_unknown_ids=True)
+        return len(ids)
 
     def _iterate(
         self,

cognite_toolkit/_cdf_tk/data_classes/__init__.py

@@ -29,6 +29,7 @@ from ._deploy_results import (
     ResourceDeployResult,
     UploadDeployResult,
 )
+from ._issues import Issue, IssueList
 from ._module_directories import ModuleDirectories, ModuleLocation
 from ._module_resources import ModuleResources
 from ._packages import Package, Packages
@@ -56,6 +57,8 @@ __all__ = [
     "DeployResults",
     "Environment",
     "InitConfigYAML",
+    "Issue",
+    "IssueList",
     "ModuleDirectories",
     "ModuleLocation",
     "ModuleResources",

cognite_toolkit/_cdf_tk/data_classes/_issues.py

@@ -0,0 +1,36 @@
+import sys
+from collections import UserList
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from pydantic import BaseModel, Field
+
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+
+MODULE_ISSUE_CODE = "MOD"
+
+
+class Issue(BaseModel):
+    """Base class for all issues"""
+
+    code: str
+    message: str | None = Field(default=None)
+
+
+# temporary adapter to manage existing warnings
+class IssueList(UserList[Issue]):
+    """List of build issues."""
+
+    @classmethod
+    def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self:
+        """Create a IssueList from a WarningList."""
+        return cls([Issue(code="WARN", message=warning.get_message()) for warning in warning_list])
+
+
+class ModuleLoadingIssue(Issue):
+    """Issue with the loading of a module folder."""
+
+    code: str = "MOD_001"

cognite_toolkit/_cdf_tk/data_classes/_module_directories.py

@@ -8,7 +8,8 @@ from pathlib import Path
 from typing import Any, SupportsIndex, overload
 
 from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN
-from cognite_toolkit._cdf_tk.utils import calculate_directory_hash, iterate_modules, resource_folder_from_path
+from cognite_toolkit._cdf_tk.utils.hashing import calculate_directory_hash
+from cognite_toolkit._cdf_tk.utils.modules import iterate_modules, resource_folder_from_path
 
 from ._module_toml import ModuleToml
 

cognite_toolkit/_cdf_tk/storageio/_base.py

@@ -11,6 +11,7 @@ from cognite_toolkit._cdf_tk.utils.collection import chunker
 from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader, SchemaColumn
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
 
+from .logger import DataLogger, NoOpLogger
 from .selectors import DataSelector
 
 
@@ -76,6 +77,7 @@ class StorageIO(ABC, Generic[T_Selector, T_ResourceResponse]):
 
     def __init__(self, client: ToolkitClient) -> None:
         self.client = client
+        self.logger: DataLogger = NoOpLogger()
 
     @abstractmethod
     def as_id(self, item: T_ResourceResponse) -> str: