cognite-toolkit 0.5.44__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py

@@ -0,0 +1,241 @@
+from pathlib import Path
+from typing import Any, Literal, TypedDict
+
+from rich import print
+from rich.panel import Panel
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand as OldBuildCommand
+from cognite_toolkit._cdf_tk.commands.build_v2.build_input import BuildInput
+from cognite_toolkit._cdf_tk.commands.build_v2.build_issues import BuildIssue, BuildIssueList
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildConfigYAML,
+    BuildVariables,
+    BuiltModuleList,
+    ModuleDirectories,
+)
+from cognite_toolkit._cdf_tk.exceptions import ToolkitError
+from cognite_toolkit._cdf_tk.hints import verify_module_directory
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
+from cognite_toolkit._cdf_tk.validation import validate_module_selection, validate_modules_variables
+from cognite_toolkit._version import __version__
+
+
+class BuildWarnings(TypedDict):
+    warning: ToolkitWarning
+    location: list[Path]
+
+
+class BuildCommand(ToolkitCommand):
+    def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
+        super().__init__(print_warning, skip_tracking, silent)
+        self.issues = BuildIssueList()
+
+    def execute(
+        self,
+        verbose: bool,
+        organization_dir: Path,
+        build_dir: Path,
+        selected: list[str | Path] | None,
+        build_env_name: str | None,
+        no_clean: bool,
+        client: ToolkitClient | None = None,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """
+        Build the resources into deployable artifacts in the build directory.
+        """
+
+        self.verbose = verbose
+        self.on_error = on_error
+
+        # Tracking the project and cluster for the build.
+        if client:
+            self._additional_tracking_info.project = client.config.project
+            self._additional_tracking_info.cluster = client.config.cdf_cluster
+
+        # Setting the parameters for the build.
+        input = BuildInput.load(organization_dir, build_dir, build_env_name, client, selected)
+
+        # Print the build input.
+        if self.verbose:
+            self._print_build_input(input)
+
+        # Capture warnings from module structure integrity
+        if module_selection_issues := self._validate_modules(input):
+            self.issues.extend(module_selection_issues)
+
+        # Logistics: clean and create build directory
+        if prepare_issues := self._prepare_target_directory(input, not no_clean):
+            self.issues.extend(prepare_issues)
+
+        # Compile the configuration and variables,
+        # check syntax on module and resource level
+        # for any "compilation errors and warnings"
+        built_modules, build_integrity_issues = self._build_configuration(input)
+        if build_integrity_issues:
+            self.issues.extend(build_integrity_issues)
+
+        # This is where we would add any recommendations for the user to improve the build.
+        if build_quality_issues := self._verify_build_quality(built_modules):
+            self.issues.extend(build_quality_issues)
+
+        # Finally, print warnings grouped by category/code and location.
+        self._print_or_log_warnings_by_category(self.issues)
+
+        return built_modules
+
+    def _print_build_input(self, input: BuildInput) -> None:
+        print(
+            Panel(
+                f"Building {input.organization_dir!s}:\n  - Toolkit Version '{__version__!s}'\n"
+                f"  - Environment name {input.build_env_name!r}, validation-type {input.config.environment.validation_type!r}.\n"
+                f"  - Config '{input.config.filepath!s}'",
+                expand=False,
+            )
+        )
+
+    def _prepare_target_directory(self, input: BuildInput, clean: bool = False) -> BuildIssueList:
+        """
+        Directory logistics
+        """
+        issues = BuildIssueList()
+        if input.build_dir.exists() and any(input.build_dir.iterdir()):
+            if not clean:
+                raise ToolkitError("Build directory is not empty. Run without --no-clean to remove existing files.")
+
+            if self.verbose:
+                issues.append(BuildIssue(description=f"Build directory {input.build_dir!s} is not empty. Clearing."))
+            safe_rmtree(input.build_dir)
+        input.build_dir.mkdir(parents=True, exist_ok=True)
+        return issues
+
+    def _validate_modules(self, input: BuildInput) -> BuildIssueList:
+        issues = BuildIssueList()
+        # Verify that the modules exists, are not duplicates,
+        # and at least one is selected
+        verify_module_directory(input.organization_dir, input.build_env_name)
+
+        # Validate module selection
+        user_selected_modules = input.config.environment.get_selected_modules({})
+        module_warnings = validate_module_selection(
+            modules=input.modules,
+            config=input.config,
+            packages={},
+            selected_modules=user_selected_modules,
+            organization_dir=input.organization_dir,
+        )
+        if module_warnings:
+            issues.extend(BuildIssueList.from_warning_list(module_warnings))
+
+        # Validate variables. Note: this looks for non-replaced template
+        # variables <.*?> and can be improved in the future.
+        # Keeping for reference.
+        variables_warnings = validate_modules_variables(input.variables, input.config.filepath)
+        if variables_warnings:
+            issues.extend(BuildIssueList.from_warning_list(variables_warnings))
+
+        # Track LOC of managed configuration
+        # Note: _track is not implemented yet, so we skip it for now
+        # self._track(input)
+
+        return issues
+
+    def _build_configuration(self, input: BuildInput) -> tuple[BuiltModuleList, BuildIssueList]:
+        issues = BuildIssueList()
+        # Use input.modules.selected directly (it's already a ModuleDirectories)
+        if not input.modules.selected:
+            return BuiltModuleList(), issues
+
+        # first collect variables into practical lookup
+        # TODO: parallelism is not implemented yet. I'm sure there are optimizations to be had here, but we'll focus on process parallelism since we believe loading yaml and file i/O are the biggest bottlenecks.
+
+        old_build_command = OldBuildCommand(print_warning=False, skip_tracking=False)
+        built_modules = old_build_command.build_config(
+            build_dir=input.build_dir,
+            organization_dir=input.organization_dir,
+            config=input.config,
+            packages={},
+            clean=False,
+            verbose=self.verbose,
+            client=input.client,
+            progress_bar=False,
+            on_error=self.on_error,
+        )
+        # Copy tracking info from old command to self
+        self._additional_tracking_info.package_ids.update(old_build_command._additional_tracking_info.package_ids)
+        self._additional_tracking_info.module_ids.update(old_build_command._additional_tracking_info.module_ids)
+
+        # Collect warnings from the old build command and convert to issues
+        # Always convert warnings to issues, even if the list appears empty
+        # (WarningList might have custom __bool__ behavior)
+        if old_build_command.warning_list:
+            converted_issues = BuildIssueList.from_warning_list(old_build_command.warning_list)
+            issues.extend(converted_issues)
+        return built_modules, issues
+
+    def _verify_build_quality(self, built_modules: BuiltModuleList) -> BuildIssueList:
+        issues = BuildIssueList()
+        return issues
+
+    def _write(self, input: BuildInput) -> None:
+        # Write the build to the build directory.
+        # Track lines of code built.
+        raise NotImplementedError()
+
+    def _track(self, input: BuildInput) -> None:
+        raise NotImplementedError()
+
+    def _print_or_log_warnings_by_category(self, issues: BuildIssueList) -> None:
+        pass
+
+    # Delegate to old BuildCommand for backward compatibility with tests
+    def build_modules(
+        self,
+        modules: ModuleDirectories,
+        build_dir: Path,
+        variables: BuildVariables,
+        verbose: bool = False,
+        progress_bar: bool = False,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+
+        built_modules = old_cmd.build_modules(modules, build_dir, variables, verbose, progress_bar, on_error)
+        self._additional_tracking_info.package_ids.update(old_cmd._additional_tracking_info.package_ids)
+        self._additional_tracking_info.module_ids.update(old_cmd._additional_tracking_info.module_ids)
+        self.issues.extend(BuildIssueList.from_warning_list(old_cmd.warning_list or WarningList[ToolkitWarning]()))
+        return built_modules
+
+    def build_config(
+        self,
+        build_dir: Path,
+        organization_dir: Path,
+        config: BuildConfigYAML,
+        packages: dict[str, list[str]],
+        clean: bool = False,
+        verbose: bool = False,
+        client: ToolkitClient | None = None,
+        progress_bar: bool = False,
+        on_error: Literal["continue", "raise"] = "continue",
+    ) -> BuiltModuleList:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+        return old_cmd.build_config(
+            build_dir, organization_dir, config, packages, clean, verbose, client, progress_bar, on_error
+        )
+
+    def _replace_variables(
+        self,
+        resource_files: list[Path],
+        variables: BuildVariables,
+        resource_name: str,
+        module_dir: Path,
+        verbose: bool = False,
+    ) -> list[Any]:
+        """Delegate to old BuildCommand for backward compatibility."""
+        old_cmd = OldBuildCommand()
+        return old_cmd._replace_variables(resource_files, variables, resource_name, module_dir, verbose)

cognite_toolkit/_cdf_tk/commands/build_v2/build_input.py

@@ -0,0 +1,85 @@
+import sys
+from functools import cached_property
+from pathlib import Path
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from pydantic import BaseModel, ConfigDict
+
+from cognite_toolkit._cdf_tk.client import ToolkitClient
+from cognite_toolkit._cdf_tk.constants import DEFAULT_ENV
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildConfigYAML,
+    BuildVariables,
+    ModuleDirectories,
+)
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
+
+
+class BuildInput(BaseModel):
+    """Input to the build process."""
+
+    # need this until we turn BuildConfigYaml and ToolkitClient into Pydantic models
+    model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
+
+    organization_dir: Path
+    build_dir: Path
+    build_env_name: str
+    config: BuildConfigYAML
+    client: ToolkitClient | None = None
+    selected: list[str | Path] | None = None
+    warnings: WarningList[ToolkitWarning] | None = None
+
+    @classmethod
+    def load(
+        cls,
+        organization_dir: Path,
+        build_dir: Path,
+        build_env_name: str | None,
+        client: ToolkitClient | None,
+        selected: list[str | Path] | None = None,
+    ) -> Self:
+        resolved_org_dir = Path.cwd() if organization_dir in {Path("."), Path("./")} else organization_dir
+        resolved_env = build_env_name or DEFAULT_ENV
+        config, warnings = cls._load_config(resolved_org_dir, resolved_env, selected)
+        return cls(
+            organization_dir=resolved_org_dir,
+            build_dir=build_dir,
+            build_env_name=resolved_env,
+            config=config,
+            client=client,
+            selected=selected,
+            warnings=warnings,
+        )
+
+    @classmethod
+    def _load_config(
+        cls, organization_dir: Path, build_env_name: str, selected: list[str | Path] | None
+    ) -> tuple[BuildConfigYAML, WarningList[ToolkitWarning]]:
+        warnings: WarningList[ToolkitWarning] = WarningList[ToolkitWarning]()
+        if (organization_dir / BuildConfigYAML.get_filename(build_env_name or DEFAULT_ENV)).exists():
+            config = BuildConfigYAML.load_from_directory(organization_dir, build_env_name or DEFAULT_ENV)
+        else:
+            # Loads the default environment
+            config = BuildConfigYAML.load_default(organization_dir)
+        if selected:
+            config.environment.selected = parse_user_selected_modules(selected, organization_dir)
+        config.set_environment_variables()
+        if environment_warning := config.validate_environment():
+            warnings.append(environment_warning)
+        return config, warnings
+
+    @cached_property
+    def modules(self) -> ModuleDirectories:
+        user_selected_modules = self.config.environment.get_selected_modules({})
+        return ModuleDirectories.load(self.organization_dir, user_selected_modules)
+
+    @cached_property
+    def variables(self) -> BuildVariables:
+        return BuildVariables.load_raw(
+            self.config.variables, self.modules.available_paths, self.modules.selected.available_paths
+        )

cognite_toolkit/_cdf_tk/commands/build_v2/build_issues.py

@@ -0,0 +1,27 @@
+import sys
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from collections import UserList
+
+from pydantic import BaseModel
+
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
+
+
+class BuildIssue(BaseModel):
+    """Issue with the build. Can have a recommendation for the user to improve the build."""
+
+    description: str
+
+
+class BuildIssueList(UserList[BuildIssue]):
+    """List of build issues."""
+
+    @classmethod
+    def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self:
+        """Create a BuildIssueList from a WarningList."""
+        return cls([BuildIssue(description=warning.get_message()) for warning in warning_list])

cognite_toolkit/_cdf_tk/commands/clean.py

@@ -1,10 +1,9 @@
-from __future__ import annotations
-
 import traceback
+from collections.abc import Sequence
 from graphlib import TopologicalSorter
 from pathlib import Path
 
-from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
+import questionary
 from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
 from cognite.client.utils.useful_types import SequenceNotStr
 from rich import print
@@ -17,6 +16,16 @@ from cognite_toolkit._cdf_tk.constants import (
     HINT_LEAD_TEXT,
     HINT_LEAD_TEXT_LEN,
 )
+from cognite_toolkit._cdf_tk.cruds import (
+    CRUDS_BY_FOLDER_NAME,
+    DataCRUD,
+    DataSetsCRUD,
+    RawDatabaseCRUD,
+    ResourceContainerCRUD,
+    ResourceCRUD,
+    ResourceWorker,
+)
+from cognite_toolkit._cdf_tk.cruds._base_cruds import Loader
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildEnvironment,
     DeployResults,
@@ -26,19 +35,12 @@ from cognite_toolkit._cdf_tk.data_classes import (
 from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitCleanResourceError,
+    ToolkitMissingModuleError,
     ToolkitNotADirectoryError,
     ToolkitValidationError,
+    ToolkitValueError,
 )
-from cognite_toolkit._cdf_tk.loaders import (
-    LOADER_BY_FOLDER_NAME,
-    DataLoader,
-    DataSetsLoader,
-    RawDatabaseLoader,
-    ResourceContainerLoader,
-    ResourceLoader,
-    ResourceWorker,
-)
-from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID, Loader, T_WritableCogniteResourceList
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.tk_warnings import (
     LowSeverityWarning,
     MediumSeverityWarning,
@@ -50,18 +52,17 @@ from cognite_toolkit._cdf_tk.utils import (
     read_yaml_file,
 )
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
+from cognite_toolkit._cdf_tk.utils.useful_types import T_ID
 
 from ._utils import _print_ids_or_length
 
-AVAILABLE_DATA_TYPES: tuple[str, ...] = tuple(LOADER_BY_FOLDER_NAME)
+AVAILABLE_DATA_TYPES: tuple[str, ...] = tuple(CRUDS_BY_FOLDER_NAME)
 
 
 class CleanCommand(ToolkitCommand):
     def clean_resources(
         self,
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         env_vars: EnvironmentVariables,
         read_modules: list[ReadModule],
         dry_run: bool = False,
@@ -69,13 +70,13 @@ class CleanCommand(ToolkitCommand):
         drop_data: bool = False,
         verbose: bool = False,
     ) -> ResourceDeployResult | None:
-        if not isinstance(loader, ResourceContainerLoader) and not drop:
+        if not isinstance(loader, ResourceContainerCRUD) and not drop:
             # Skipping silently as this, we will not drop data or delete this resource
             return ResourceDeployResult(name=loader.display_name)
         if not loader.support_drop:
             print(f"  [bold green]INFO:[/] {loader.display_name!r} cleaning is not supported, skipping...")
             return ResourceDeployResult(name=loader.display_name)
-        elif isinstance(loader, ResourceContainerLoader) and not drop_data:
+        elif isinstance(loader, ResourceContainerCRUD) and not drop_data:
             print(
                 f"  [bold]INFO:[/] Skipping cleaning of {loader.display_name!r}. This is a data resource (it contains "
                 f"data and is not only configuration/metadata) and therefore "
@@ -83,33 +84,32 @@ class CleanCommand(ToolkitCommand):
             )
             return ResourceContainerDeployResult(name=loader.display_name, item_name=loader.item_name)
 
-        worker = ResourceWorker(loader)
+        worker = ResourceWorker(loader, "clean")
         files = worker.load_files(read_modules=read_modules)
         if not files:
             return None
         # Since we do a clean, we do not want to verify that everything exists wrt data sets, spaces etc.
-        existing_resources, duplicated = worker.load_resources(
+        local_resources = worker.load_resources(
             filepaths=files,
-            return_existing=True,
             environment_variables=env_vars.dump(include_os=True),
             is_dry_run=True,
-            verbose=verbose,
         )
+        existing_resources = loader.retrieve(list(local_resources.keys()))
         nr_of_existing = len(existing_resources)
 
         if drop:
             prefix = "Would clean" if dry_run else "Cleaning"
-            with_data = "with data " if isinstance(loader, ResourceContainerLoader) else ""
+            with_data = "with data " if isinstance(loader, ResourceContainerCRUD) else ""
         else:
             prefix = "Would drop data from" if dry_run else "Dropping data from"
             with_data = ""
         print(f"[bold]{prefix} {nr_of_existing} {loader.display_name} {with_data}from CDF...[/]")
-        if not isinstance(loader, RawDatabaseLoader):
-            for duplicate in duplicated:
+        if not isinstance(loader, RawDatabaseCRUD):
+            for duplicate in worker.duplicates:
                 self.warn(LowSeverityWarning(f"Duplicate {loader.display_name} {duplicate}."))
 
         # Deleting resources.
-        if isinstance(loader, ResourceContainerLoader) and drop_data:
+        if isinstance(loader, ResourceContainerCRUD) and drop_data:
             nr_of_dropped_datapoints = self._drop_data(existing_resources, loader, dry_run, verbose)
             if drop:
                 nr_of_deleted = self._delete_resources(existing_resources, loader, dry_run, verbose)
@@ -124,7 +124,7 @@ class CleanCommand(ToolkitCommand):
                 dropped_datapoints=nr_of_dropped_datapoints,
                 item_name=loader.item_name,
             )
-        elif not isinstance(self, ResourceContainerLoader) and drop:
+        elif not isinstance(self, ResourceContainerCRUD) and drop:
             nr_of_deleted = self._delete_resources(existing_resources, loader, dry_run, verbose)
             if verbose:
                 print("")
@@ -133,7 +133,7 @@ class CleanCommand(ToolkitCommand):
             return ResourceDeployResult(name=loader.display_name)
 
     def _delete_resources(
-        self, loaded_resources: T_CogniteResourceList, loader: ResourceLoader, dry_run: bool, verbose: bool
+        self, loaded_resources: Sequence[T_ResourceResponse], loader: ResourceCRUD, dry_run: bool, verbose: bool
     ) -> int:
         nr_of_deleted = 0
         resource_ids = loader.get_ids(loaded_resources)
@@ -158,7 +158,11 @@ class CleanCommand(ToolkitCommand):
         return nr_of_deleted
 
     def _drop_data(
-        self, loaded_resources: T_CogniteResourceList, loader: ResourceContainerLoader, dry_run: bool, verbose: bool
+        self,
+        loaded_resources: Sequence[T_ResourceResponse],
+        loader: ResourceContainerCRUD,
+        dry_run: bool,
+        verbose: bool,
     ) -> int:
         nr_of_dropped = 0
         resource_ids = loader.get_ids(loaded_resources)
@@ -182,8 +186,25 @@ class CleanCommand(ToolkitCommand):
                 self._verbose_print_drop(resource_drop_count, resource_ids, loader, dry_run)
         return nr_of_dropped
 
+    def _interactive_module_selection(self, built_modules: list[ReadModule] | None) -> list[ReadModule] | None:
+        if not built_modules:
+            return None
+        choices = [
+            questionary.Choice(title=built_module.dir.name, value=built_module) for built_module in built_modules
+        ]
+
+        selected_modules = questionary.checkbox(
+            "Which modules would you like to clean?",
+            instruction="Use arrow up/down, press space to select item(s) and enter to save",
+            choices=choices,
+        ).ask()
+
+        if not selected_modules:
+            return None
+        return selected_modules
+
     def _verbose_print_drop(
-        self, drop_count: int, resource_ids: SequenceNotStr[T_ID], loader: ResourceContainerLoader, dry_run: bool
+        self, drop_count: int, resource_ids: SequenceNotStr[T_ID], loader: ResourceContainerCRUD, dry_run: bool
     ) -> None:
         prefix = "Would have dropped" if dry_run else "Dropped"
         if drop_count > 0:
@@ -201,6 +222,11 @@ class CleanCommand(ToolkitCommand):
             # Count is not supported
             print(f" {prefix} all {loader.item_name} from {loader.display_name}: {_print_ids_or_length(resource_ids)}.")
 
+    def _select_modules(self, clean_state: BuildEnvironment, module_str: str | None) -> list[ReadModule] | None:
+        if module_str:
+            return [module for module in clean_state.read_modules if module.dir.name == module_str]
+        return self._interactive_module_selection(clean_state.read_modules)
+
     def execute(
         self,
         env_vars: EnvironmentVariables,
@@ -208,7 +234,9 @@ class CleanCommand(ToolkitCommand):
         build_env_name: str | None,
         dry_run: bool,
         include: list[str] | None,
+        module_str: str | None,
         verbose: bool,
+        all_modules: bool = False,
     ) -> None:
         if not build_dir.exists():
             raise ToolkitNotADirectoryError(
@@ -242,9 +270,31 @@ class CleanCommand(ToolkitCommand):
         )
 
         if not build_dir.is_dir():
-            raise ToolkitNotADirectoryError(f"'{build_dir}'. Did you forget to run `cdf-tk build` first?")
+            raise ToolkitNotADirectoryError(f"'{build_dir}'. Did you forget to run `cdf build` first?")
+
+        selected_modules: list[ReadModule]
+        if all_modules:
+            selected_modules = clean_state.read_modules or []
+            if not selected_modules:
+                raise ToolkitValueError("No modules available to clean.")
+        elif module_str:
+            selected_modules = [module for module in clean_state.read_modules if module.dir.name == module_str]
+            if not selected_modules:
+                available_module_names = {module.dir.name for module in clean_state.read_modules}
+                raise ToolkitMissingModuleError(
+                    f"No modules matched the selection: {module_str}. Available modules: {sorted(available_module_names)}"
+                )
+        else:
+            selected_modules = self._interactive_module_selection(clean_state.read_modules) or []
+            if not selected_modules:
+                raise ToolkitValueError(
+                    "No module specified with the --module option and no modules selected interactively."
+                )
 
-        selected_loaders = self.get_selected_loaders(build_dir, clean_state.read_resource_folders, include)
+        selected_resource_folders = {
+            resource_folder for module in selected_modules for resource_folder in module.resource_directories
+        }
+        selected_loaders = self.get_selected_loaders(build_dir, selected_resource_folders, include)
 
         results = DeployResults([], "clean", dry_run=dry_run)
 
@@ -262,16 +312,16 @@ class CleanCommand(ToolkitCommand):
             self.warn(ToolkitDependenciesIncludedWarning([item.folder_name for item in should_include]))
 
         for loader_cls in reversed(resolved_list):
-            if not issubclass(loader_cls, ResourceLoader):
+            if not issubclass(loader_cls, ResourceCRUD):
                 continue
             loader = loader_cls.create_loader(client, build_dir)
-            if type(loader) is DataSetsLoader:
+            if type(loader) is DataSetsCRUD:
                 self.warn(ToolkitNotSupportedWarning(feature="Dataset clean."))
                 continue
             result = self.clean_resources(
                 loader,
                 env_vars=env_vars,
-                read_modules=clean_state.read_modules,
+                read_modules=selected_modules,
                 drop=True,
                 dry_run=dry_run,
                 drop_data=True,
@@ -288,7 +338,7 @@ class CleanCommand(ToolkitCommand):
         self, build_dir: Path, read_resource_folders: set[str], include: list[str] | None
     ) -> dict[type[Loader], frozenset[type[Loader]]]:
         selected_loaders: dict[type[Loader], frozenset[type[Loader]]] = {}
-        for folder_name, loader_classes in LOADER_BY_FOLDER_NAME.items():
+        for folder_name, loader_classes in CRUDS_BY_FOLDER_NAME.items():
             if include is not None and folder_name not in include:
                 continue
             if folder_name in read_resource_folders:
@@ -301,7 +351,7 @@ class CleanCommand(ToolkitCommand):
                 if loader_cls.any_supported_files(build_dir / folder_name):
                     folder_has_supported_files = True
                     selected_loaders[loader_cls] = loader_cls.dependencies
-                elif issubclass(loader_cls, DataLoader):
+                elif issubclass(loader_cls, DataCRUD):
                     # Data Loaders are always included, as they will have
                     # the files in the module folder and not the build folder.
                     selected_loaders[loader_cls] = loader_cls.dependencies
@@ -322,7 +372,7 @@ class CleanCommand(ToolkitCommand):
         return selected_loaders
 
     @staticmethod
-    def _process_include(include: list[str] | None) -> list[str]:
+    def validate_include(include: list[str] | None) -> list[str]:
         if include and (invalid_types := set(include).difference(AVAILABLE_DATA_TYPES)):
             raise ToolkitValidationError(
                 f"Invalid resource types specified: {invalid_types}, available types: {AVAILABLE_DATA_TYPES}"