cognite-toolkit 0.7.39__py3-none-any.whl → 0.7.51__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py

@@ -1,5 +1,7 @@
+import re
+from itertools import groupby
 from pathlib import Path
-from typing import Any, Literal, TypedDict
+from typing import Any, Literal
 
 from rich import print
 from rich.panel import Panel
@@ -7,39 +9,29 @@ from rich.panel import Panel
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
 from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand as OldBuildCommand
-from cognite_toolkit._cdf_tk.commands.build_v2.build_input import BuildInput
-from cognite_toolkit._cdf_tk.commands.build_v2.build_issues import BuildIssue, BuildIssueList
-from cognite_toolkit._cdf_tk.data_classes import (
-    BuildConfigYAML,
-    BuildVariables,
-    BuiltModuleList,
-    ModuleDirectories,
-)
+from cognite_toolkit._cdf_tk.commands.build_v2._modules_parser import ModulesParser
+from cognite_toolkit._cdf_tk.commands.build_v2.build_parameters import BuildParameters
+from cognite_toolkit._cdf_tk.commands.build_v2.data_classes._modules import Module
+from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, BuildVariables, BuiltModuleList
+from cognite_toolkit._cdf_tk.data_classes._issues import Issue, IssueList
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError
-from cognite_toolkit._cdf_tk.hints import verify_module_directory
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
 from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
-from cognite_toolkit._cdf_tk.validation import validate_module_selection, validate_modules_variables
 from cognite_toolkit._version import __version__
 
 
-class BuildWarnings(TypedDict):
-    warning: ToolkitWarning
-    location: list[Path]
-
-
 class BuildCommand(ToolkitCommand):
     def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
         super().__init__(print_warning, skip_tracking, silent)
-        self.issues = BuildIssueList()
+        self.issues = IssueList()
 
     def execute(
         self,
         verbose: bool,
-        organization_dir: Path,
+        base_dir: Path,
         build_dir: Path,
         selected: list[str | Path] | None,
-        build_env_name: str | None,
+        build_env: str | None,
         no_clean: bool,
         client: ToolkitClient | None = None,
         on_error: Literal["continue", "raise"] = "continue",
@@ -51,30 +43,48 @@ class BuildCommand(ToolkitCommand):
         self.verbose = verbose
         self.on_error = on_error
 
-        # Tracking the project and cluster for the build.
-        if client:
-            self._additional_tracking_info.project = client.config.project
-            self._additional_tracking_info.cluster = client.config.cdf_cluster
-
-        # Setting the parameters for the build.
-        input = BuildInput.load(organization_dir, build_dir, build_env_name, client, selected)
+        build_parameters = BuildParameters.load(
+            organization_dir=base_dir,
+            build_dir=build_dir,
+            build_env_name=build_env,
+            client=client,
+            user_selected=selected,
+        )
 
         # Print the build input.
         if self.verbose:
-            self._print_build_input(input)
+            self._print_build_input(build_parameters)
 
-        # Capture warnings from module structure integrity
-        if module_selection_issues := self._validate_modules(input):
-            self.issues.extend(module_selection_issues)
+        # Tracking the project and cluster for the build.
+        if build_parameters.client:
+            self._additional_tracking_info.project = build_parameters.client.config.project
+            self._additional_tracking_info.cluster = build_parameters.client.config.cdf_cluster
+
+        # Load modules
+        modules_parser = ModulesParser(organization_dir=base_dir, selected=selected)
+        module_paths = modules_parser.parse()
+        module_loading_issues = modules_parser.issues
+        if module_loading_issues:
+            self.issues.extend(module_loading_issues)
+            self._print_or_log_issues_by_category(self.issues)
+            raise ToolkitError("Module loading issues encountered. Cannot continue. See above for details.")
+
+        # Load modules
+        if module_paths:
+            pass
+
+        # modules = [Module.load(path) for path in module_paths]
+        # for module in modules:
+        #    continue
 
         # Logistics: clean and create build directory
-        if prepare_issues := self._prepare_target_directory(input, not no_clean):
+        if prepare_issues := self._prepare_target_directory(build_dir, not no_clean):
             self.issues.extend(prepare_issues)
 
         # Compile the configuration and variables,
         # check syntax on module and resource level
         # for any "compilation errors and warnings"
-        built_modules, build_integrity_issues = self._build_configuration(input)
+        built_modules, build_integrity_issues = self._build_configuration(build_parameters)
         if build_integrity_issues:
             self.issues.extend(build_integrity_issues)
 
@@ -83,84 +93,48 @@ class BuildCommand(ToolkitCommand):
             self.issues.extend(build_quality_issues)
 
         # Finally, print warnings grouped by category/code and location.
-        self._print_or_log_warnings_by_category(self.issues)
+        self._print_or_log_issues_by_category(self.issues)
 
         return built_modules
 
-    def _print_build_input(self, input: BuildInput) -> None:
+    def _print_build_input(self, build_input: BuildParameters) -> None:
         print(
             Panel(
-                f"Building {input.organization_dir!s}:\n  - Toolkit Version '{__version__!s}'\n"
-                f"  - Environment name {input.build_env_name!r}, validation-type {input.config.environment.validation_type!r}.\n"
-                f"  - Config '{input.config.filepath!s}'",
+                f"Building {build_input.organization_dir!s}:\n  - Toolkit Version '{__version__!s}'\n"
+                f"  - Environment name {build_input.build_env_name!r}, validation-type {build_input.config.environment.validation_type!r}.\n"
+                f"  - Config '{build_input.config.filepath!s}'",
                 expand=False,
             )
         )
 
-    def _prepare_target_directory(self, input: BuildInput, clean: bool = False) -> BuildIssueList:
+    def _prepare_target_directory(self, build_dir: Path, clean: bool = False) -> IssueList:
         """
         Directory logistics
         """
-        issues = BuildIssueList()
-        if input.build_dir.exists() and any(input.build_dir.iterdir()):
+        issues = IssueList()
+        if build_dir.exists() and any(build_dir.iterdir()):
             if not clean:
                 raise ToolkitError("Build directory is not empty. Run without --no-clean to remove existing files.")
 
             if self.verbose:
-                issues.append(BuildIssue(description=f"Build directory {input.build_dir!s} is not empty. Clearing."))
-            safe_rmtree(input.build_dir)
-        input.build_dir.mkdir(parents=True, exist_ok=True)
+                issues.append(Issue(code="BUILD_001"))
+            safe_rmtree(build_dir)
+        build_dir.mkdir(parents=True, exist_ok=True)
         return issues
 
-    def _validate_modules(self, input: BuildInput) -> BuildIssueList:
-        issues = BuildIssueList()
-        # Verify that the modules exists, are not duplicates,
-        # and at least one is selected
-        verify_module_directory(input.organization_dir, input.build_env_name)
-
-        # Validate module selection
-        user_selected_modules = input.config.environment.get_selected_modules({})
-        module_warnings = validate_module_selection(
-            modules=input.modules,
-            config=input.config,
-            packages={},
-            selected_modules=user_selected_modules,
-            organization_dir=input.organization_dir,
-        )
-        if module_warnings:
-            issues.extend(BuildIssueList.from_warning_list(module_warnings))
-
-        # Validate variables. Note: this looks for non-replaced template
-        # variables <.*?> and can be improved in the future.
-        # Keeping for reference.
-        variables_warnings = validate_modules_variables(input.variables, input.config.filepath)
-        if variables_warnings:
-            issues.extend(BuildIssueList.from_warning_list(variables_warnings))
-
-        # Track LOC of managed configuration
-        # Note: _track is not implemented yet, so we skip it for now
-        # self._track(input)
-
-        return issues
-
-    def _build_configuration(self, input: BuildInput) -> tuple[BuiltModuleList, BuildIssueList]:
-        issues = BuildIssueList()
-        # Use input.modules.selected directly (it's already a ModuleDirectories)
-        if not input.modules.selected:
-            return BuiltModuleList(), issues
-
-        # first collect variables into practical lookup
-        # TODO: parallelism is not implemented yet. I'm sure there are optimizations to be had here, but we'll focus on process parallelism since we believe loading yaml and file i/O are the biggest bottlenecks.
+    def _build_configuration(self, build_input: BuildParameters) -> tuple[BuiltModuleList, IssueList]:
+        issues = IssueList()
+        # Use build_input.modules directly (it is already filtered by selection)
 
         old_build_command = OldBuildCommand(print_warning=False, skip_tracking=False)
         built_modules = old_build_command.build_config(
-            build_dir=input.build_dir,
-            organization_dir=input.organization_dir,
-            config=input.config,
+            build_dir=build_input.build_dir,
+            organization_dir=build_input.organization_dir,
+            config=build_input.config,
             packages={},
             clean=False,
             verbose=self.verbose,
-            client=input.client,
+            client=build_input.client,
             progress_bar=False,
             on_error=self.on_error,
         )
@@ -172,29 +146,42 @@ class BuildCommand(ToolkitCommand):
         # Always convert warnings to issues, even if the list appears empty
         # (WarningList might have custom __bool__ behavior)
         if old_build_command.warning_list:
-            converted_issues = BuildIssueList.from_warning_list(old_build_command.warning_list)
+            converted_issues = IssueList.from_warning_list(old_build_command.warning_list)
             issues.extend(converted_issues)
         return built_modules, issues
 
-    def _verify_build_quality(self, built_modules: BuiltModuleList) -> BuildIssueList:
-        issues = BuildIssueList()
+    def _verify_build_quality(self, built_modules: BuiltModuleList) -> IssueList:
+        issues = IssueList()
         return issues
 
-    def _write(self, input: BuildInput) -> None:
+    def _write(self, build_input: BuildParameters) -> None:
         # Write the build to the build directory.
         # Track lines of code built.
         raise NotImplementedError()
 
-    def _track(self, input: BuildInput) -> None:
+    def _track(self, build_input: BuildParameters) -> None:
         raise NotImplementedError()
 
-    def _print_or_log_warnings_by_category(self, issues: BuildIssueList) -> None:
-        pass
+    def _print_or_log_issues_by_category(self, issues: IssueList) -> None:
+        issues_sorted = sorted(issues, key=self._issue_sort_key)
+        for code, grouped_issues in groupby(issues_sorted, key=lambda issue: issue.code or ""):
+            print(f"[bold]{code}[/]")
+            for issue in grouped_issues:
+                message = issue.message or ""
+                print(f"  - {message}")
+
+    def _issue_sort_key(self, issue: Issue) -> tuple[str, str]:
+        code = issue.code or ""
+        if not issue.message:
+            return code, ""
+        match = re.search(r"'([^']+)'", issue.message)
+        path = match.group(1) if match else issue.message
+        return code, path
 
     # Delegate to old BuildCommand for backward compatibility with tests
     def build_modules(
         self,
-        modules: ModuleDirectories,
+        modules: list[Module],
         build_dir: Path,
         variables: BuildVariables,
         verbose: bool = False,
@@ -204,10 +191,10 @@ class BuildCommand(ToolkitCommand):
         """Delegate to old BuildCommand for backward compatibility."""
         old_cmd = OldBuildCommand()
 
-        built_modules = old_cmd.build_modules(modules, build_dir, variables, verbose, progress_bar, on_error)
+        built_modules = old_cmd.build_modules(modules, build_dir, variables, verbose, progress_bar, on_error)  # type: ignore[arg-type]
         self._additional_tracking_info.package_ids.update(old_cmd._additional_tracking_info.package_ids)
         self._additional_tracking_info.module_ids.update(old_cmd._additional_tracking_info.module_ids)
-        self.issues.extend(BuildIssueList.from_warning_list(old_cmd.warning_list or WarningList[ToolkitWarning]()))
+        self.issues.extend(IssueList.from_warning_list(old_cmd.warning_list or WarningList[ToolkitWarning]()))
         return built_modules
 
     def build_config(

cognite_toolkit/_cdf_tk/commands/build_v2/{build_input.py → build_parameters.py}

@@ -1,5 +1,4 @@
 import sys
-from functools import cached_property
 from pathlib import Path
 
 if sys.version_info >= (3, 11):
@@ -13,14 +12,12 @@ from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.constants import DEFAULT_ENV
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildConfigYAML,
-    BuildVariables,
-    ModuleDirectories,
 )
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 
 
-class BuildInput(BaseModel):
+class BuildParameters(BaseModel):
     """Input to the build process."""
 
     # need this until we turn BuildConfigYaml and ToolkitClient into Pydantic models
@@ -31,8 +28,8 @@ class BuildInput(BaseModel):
     build_env_name: str
     config: BuildConfigYAML
     client: ToolkitClient | None = None
-    selected: list[str | Path] | None = None
     warnings: WarningList[ToolkitWarning] | None = None
+    user_selected: list[str | Path] | None = None
 
     @classmethod
     def load(
@@ -41,24 +38,24 @@ class BuildInput(BaseModel):
         build_dir: Path,
         build_env_name: str | None,
         client: ToolkitClient | None,
-        selected: list[str | Path] | None = None,
+        user_selected: list[str | Path] | None = None,
     ) -> Self:
         resolved_org_dir = Path.cwd() if organization_dir in {Path("."), Path("./")} else organization_dir
         resolved_env = build_env_name or DEFAULT_ENV
-        config, warnings = cls._load_config(resolved_org_dir, resolved_env, selected)
+        config, warnings = cls._load_config(resolved_org_dir, resolved_env, user_selected)
         return cls(
             organization_dir=resolved_org_dir,
             build_dir=build_dir,
             build_env_name=resolved_env,
             config=config,
             client=client,
-            selected=selected,
             warnings=warnings,
+            user_selected=user_selected,
         )
 
     @classmethod
     def _load_config(
-        cls, organization_dir: Path, build_env_name: str, selected: list[str | Path] | None
+        cls, organization_dir: Path, build_env_name: str, user_selected: list[str | Path] | None
     ) -> tuple[BuildConfigYAML, WarningList[ToolkitWarning]]:
         warnings: WarningList[ToolkitWarning] = WarningList[ToolkitWarning]()
         if (organization_dir / BuildConfigYAML.get_filename(build_env_name or DEFAULT_ENV)).exists():
@@ -66,20 +63,9 @@ class BuildInput(BaseModel):
         else:
             # Loads the default environment
             config = BuildConfigYAML.load_default(organization_dir)
-        if selected:
-            config.environment.selected = parse_user_selected_modules(selected, organization_dir)
+        if user_selected:
+            config.environment.selected = list(set(parse_user_selected_modules(list(user_selected), organization_dir)))
         config.set_environment_variables()
         if environment_warning := config.validate_environment():
             warnings.append(environment_warning)
         return config, warnings
-
-    @cached_property
-    def modules(self) -> ModuleDirectories:
-        user_selected_modules = self.config.environment.get_selected_modules({})
-        return ModuleDirectories.load(self.organization_dir, user_selected_modules)
-
-    @cached_property
-    def variables(self) -> BuildVariables:
-        return BuildVariables.load_raw(
-            self.config.variables, self.modules.available_paths, self.modules.selected.available_paths
-        )

cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_modules.py

@@ -0,0 +1,27 @@
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from cognite_toolkit._cdf_tk.data_classes._module_toml import ModuleToml
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class Module(BaseModel):
+    model_config = ConfigDict(
+        frozen=True,
+        validate_assignment=True,
+        arbitrary_types_allowed=True,
+    )
+
+    path: Path
+    definition: ModuleToml | None = None
+
+    @classmethod
+    def load(cls, path: Path) -> Self:
+        definition = ModuleToml.load(path / ModuleToml.filename) if (path / ModuleToml.filename).exists() else None
+        return cls(path=path, definition=definition)

cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_resource.py

@@ -0,0 +1,22 @@
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class Resource(BaseModel):
+    model_config = ConfigDict(
+        frozen=True,
+        validate_assignment=True,
+    )
+
+    path: Path
+
+    @classmethod
+    def load(cls, path: Path) -> Self:
+        return cls(path=path)

cognite_toolkit/_cdf_tk/commands/dump_resource.py

@@ -52,10 +52,10 @@ from rich.console import Console
 from rich.panel import Panel
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.location_filters import LocationFilterList
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.search_config import SearchConfigList
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.search_config import ViewId as SearchConfigViewId
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.streamlit_ import Streamlit, StreamlitList
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.location_filters import LocationFilterList
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.search_config import SearchConfigList
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.search_config import ViewId as SearchConfigViewId
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.streamlit_ import Streamlit, StreamlitList
 from cognite_toolkit._cdf_tk.cruds import (
     AgentCRUD,
     ContainerCRUD,

cognite_toolkit/_cdf_tk/commands/pull.py

@@ -1,5 +1,6 @@
 import dataclasses
 import itertools
+import json
 import re
 import sys
 import tempfile
@@ -28,6 +29,7 @@ from cognite_toolkit._cdf_tk.cruds import (
     HostedExtractorSourceCRUD,
     ResourceCRUD,
     StreamlitCRUD,
+    ViewCRUD,
 )
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildEnvironment,
@@ -643,6 +645,44 @@ class PullCommand(ToolkitCommand):
         loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         source_file: Path,
     ) -> tuple[str, dict[Path, str]]:
+        """Convert resource data from CDF into YAML file content ready to be written to disk.
+
+        This method takes the raw CDF resource data and transforms it back into a properly
+        formatted YAML file that preserves:
+        - Template variables (e.g., {{ variable_name }}) instead of their resolved values
+        - YAML comments from the original source file
+        - The original key ordering in dictionaries
+
+        The transformation process:
+        1. Replace all template variables with unique placeholders
+        2. Load source YAML content while preserving comments
+        3. Update the resource data with placeholder values where variables were used
+        4. Dump the updated data back to YAML format
+        5. Replace placeholders with the original template variable syntax
+        6. Restore the YAML comments
+
+        Args:
+            source: The original YAML file content as a string.
+            to_write: A mapping from resource identifiers to their updated data dictionaries
+                pulled from CDF.
+            resources: The list of built resources containing build variables and metadata.
+            environment_variables: A mapping of environment variable names to their values,
+                used to resolve variables like ${VAR_NAME} in template values.
+            loader: The ResourceCRUD loader instance for this resource type.
+            source_file: The path to the source file being processed.
+
+        Returns:
+            A tuple containing:
+            - The final YAML content string ready to be written to disk.
+            - A dictionary mapping extra file paths to their content (for resources
+              that have additional files, like SQL queries for transformations).
+
+        Raises:
+            ValueError: If the loaded YAML structure doesn't match between the original
+                and placeholder versions.
+            ToolkitMissingResourceError: If a resource identifier is not found in the
+                to_write or resources mappings.
+        """
         # 1. Replace all variables with placeholders
         # 2. Load source and keep the comments
         # 3. Update the to_write dict with the placeholders
@@ -762,9 +802,22 @@ class PullCommand(ToolkitCommand):
 
 
 class ResourceReplacer:
-    """Replaces values in a local resource directory with the updated values from CDF.
+    """Replaces values in a local resource dictionary with the updated values from CDF.
 
     The local resource dict order is maintained. In addition, placeholders are used for variables.
+
+    This class is responsible for merging CDF resource values back into local configuration files
+    while preserving:
+    - The original key ordering in dictionaries
+    - Template variable placeholders (e.g., {{ variable_name }})
+    - Comments and formatting where possible
+
+    Args:
+        value_by_placeholder: A mapping from placeholder strings to their corresponding
+            BuildVariable objects. Placeholders are temporary substitutes for template
+            variables during processing.
+        loader: The ResourceCRUD loader instance used to determine how to diff lists
+            and handle resource-specific logic.
     """
 
     def __init__(self, value_by_placeholder: dict[str, BuildVariable], loader: ResourceCRUD) -> None:
@@ -777,7 +830,49 @@ class ResourceReplacer:
         placeholder: dict[str, Any],
         to_write: dict[str, Any],
     ) -> dict[str, Any]:
-        return self._replace_dict(current, placeholder, to_write, tuple())
+        """Replace values in a local resource dict with updated values from CDF.
+
+        Merges the CDF resource values into the local configuration while maintaining
+        the original dictionary key ordering and preserving template variable placeholders.
+
+        Args:
+            current: The current local resource dictionary with resolved variable values.
+                This represents the source file content after template variables have
+                been substituted with their actual values.
+            placeholder: The local resource dictionary with placeholder strings instead
+                of resolved values. Used to identify which values contain template
+                variables that should be preserved.
+            to_write: The resource dictionary from CDF containing the updated values
+                to merge into the local configuration.
+
+        Returns:
+            A new dictionary with CDF values merged in, maintaining the original key
+            order from `current`. Template variables are preserved as placeholders
+            (to be converted back to {{ variable }} syntax by the caller). New keys
+            from CDF are appended at the end, and removed keys are omitted.
+
+        Raises:
+            ToolkitValueError: If a list variable has changed and cannot be updated,
+                or if there's a type mismatch between local and CDF values.
+        """
+        has_stringified_view_filter = False
+        if isinstance(self._loader, ViewCRUD):
+            # view.filter are recursive nested dicts that are complex. To avoid issues with comparing
+            # lists inside the filters, we stringify them before processing such that they are compared
+            # as strings.
+            processed = []
+            for d in (current, placeholder, to_write):
+                if isinstance(d.get("filter"), dict):
+                    d = d.copy()
+                    d["filter"] = json.dumps(d["filter"])
+                    has_stringified_view_filter = True
+                processed.append(d)
+            current, placeholder, to_write = processed
+        output = self._replace_dict(current, placeholder, to_write, tuple())
+        if has_stringified_view_filter and "filter" in output:
+            # Special case for ViewCRUD where the filter is stringified in CDF
+            output["filter"] = json.loads(output["filter"])
+        return output
 
     def _replace_dict(
         self,

cognite_toolkit/_cdf_tk/commands/run.py

@@ -30,7 +30,7 @@ from rich.progress import Progress
 from rich.table import Table
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig
-from cognite_toolkit._cdf_tk.client.data_classes.legacy.functions import FunctionScheduleID
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.functions import FunctionScheduleID
 from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER
 from cognite_toolkit._cdf_tk.cruds import FunctionCRUD, FunctionScheduleCRUD, WorkflowVersionCRUD
 from cognite_toolkit._cdf_tk.cruds._resource_cruds.workflow import WorkflowTriggerCRUD

cognite_toolkit/_cdf_tk/cruds/__init__.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import itertools
+from collections import defaultdict
 from typing import Literal, TypeAlias
 
 from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags
@@ -60,6 +61,7 @@ from ._resource_cruds import (
     SecurityCategoryCRUD,
     SequenceCRUD,
     SequenceRowCRUD,
+    SimulatorModelCRUD,
     SpaceCRUD,
     StreamCRUD,
     StreamlitCRUD,
@@ -86,26 +88,31 @@ if not FeatureFlag.is_enabled(Flags.MIGRATE):
     _EXCLUDED_CRUDS.add(ResourceViewMappingCRUD)
 if not FeatureFlag.is_enabled(Flags.STREAMS):
     _EXCLUDED_CRUDS.add(StreamCRUD)
+if not FeatureFlag.is_enabled(Flags.SIMULATORS):
+    _EXCLUDED_CRUDS.add(SimulatorModelCRUD)
 
-
-CRUDS_BY_FOLDER_NAME: dict[str, list[type[Loader]]] = {}
+CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA: defaultdict[str, list[type[Loader]]] = defaultdict(list)
+CRUDS_BY_FOLDER_NAME: defaultdict[str, list[type[Loader]]] = defaultdict(list)
 for _loader in itertools.chain(
     ResourceCRUD.__subclasses__(),
     ResourceContainerCRUD.__subclasses__(),
     DataCRUD.__subclasses__(),
     GroupCRUD.__subclasses__(),
 ):
-    if _loader in [ResourceCRUD, ResourceContainerCRUD, DataCRUD, GroupCRUD] or _loader in _EXCLUDED_CRUDS:
+    if _loader in [ResourceCRUD, ResourceContainerCRUD, DataCRUD, GroupCRUD]:
         # Skipping base classes
         continue
-    if _loader.folder_name not in CRUDS_BY_FOLDER_NAME:  # type: ignore[attr-defined]
-        CRUDS_BY_FOLDER_NAME[_loader.folder_name] = []  # type: ignore[attr-defined]
     # MyPy bug: https://github.com/python/mypy/issues/4717
-    CRUDS_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
+    CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
+
+    if _loader not in _EXCLUDED_CRUDS:
+        CRUDS_BY_FOLDER_NAME[_loader.folder_name].append(_loader)  # type: ignore[arg-type, attr-defined]
 del _loader  # cleanup module namespace
 
+
 # For backwards compatibility
 CRUDS_BY_FOLDER_NAME["data_models"] = CRUDS_BY_FOLDER_NAME["data_modeling"]  # Todo: Remove in v1.0
+CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA["data_models"] = CRUDS_BY_FOLDER_NAME_INCLUDE_ALPHA["data_modeling"]
 RESOURCE_CRUD_BY_FOLDER_NAME = {
     folder_name: cruds
     for folder_name, loaders in CRUDS_BY_FOLDER_NAME.items()
@@ -142,6 +149,7 @@ ResourceTypes: TypeAlias = Literal[
     "functions",
     "raw",
     "robotics",
+    "simulators",
     "streams",
     "streamlit",
     "workflows",
@@ -163,6 +171,7 @@ __all__ = [
     "RESOURCE_CRUD_CONTAINER_LIST",
     "RESOURCE_CRUD_LIST",
     "RESOURCE_DATA_CRUD_LIST",
+    "_EXCLUDED_CRUDS",
     "AgentCRUD",
     "AssetCRUD",
     "CogniteFileCRUD",
@@ -209,6 +218,7 @@ __all__ = [
     "SecurityCategoryCRUD",
     "SequenceCRUD",
     "SequenceRowCRUD",
+    "SimulatorModelCRUD",
     "SpaceCRUD",
     "StreamlitCRUD",
     "ThreeDModelCRUD",