cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.39__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -1,32 +1,33 @@
+from collections import Counter
 from collections.abc import Sequence
 from functools import partial
 from pathlib import Path
 
-from cognite.client.data_classes._base import T_CogniteResource
-from pydantic import ValidationError
+from cognite.client.data_classes.data_modeling import (
+    ViewId,
+)
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_SUFFIX, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.cruds import ViewCRUD
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio import (
     T_Selector,
     UploadableStorageIO,
-    are_same_kind,
     get_upload_io,
 )
-from cognite_toolkit._cdf_tk.storageio._base import T_WriteCogniteResource, TableUploadableStorageIO, UploadItem
-from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.storageio._base import TableUploadableStorageIO, UploadItem
+from cognite_toolkit._cdf_tk.storageio.selectors import Selector, load_selector
+from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning, ToolkitWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
-from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
-from cognite_toolkit._cdf_tk.utils.fileio import TABLE_READ_CLS_BY_FORMAT, FileReader
+from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
-from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._base import ToolkitCommand
 from .deploy import DeployCommand
@@ -76,53 +77,76 @@ class UploadCommand(ToolkitCommand):
         │   │   └── table2.Table.yaml
         │   └── ...
         ├── datafile1.kind.ndjson # Data file of a specific kind
-        ├── datafile1.Metadata.yaml       # Metadata file for datafile1
+        ├── datafile1.Manifest.yaml       # Manifest for datafile1
         ├── datafile2.kind2.ndjson # Another data file of the same or different kind
-        ├── datafile2.Metadata.yaml       # Metadata file for datafile2
+        ├── datafile2.Manifest.yaml       # Manifest file for datafile2
         └── ...
         """
-        console = Console()
-        data_files_by_selector = self._find_data_files(input_dir, kind)
+        console = client.console
+        data_files_by_selector = self._find_data_files(input_dir)
 
         self._deploy_resource_folder(input_dir / DATA_RESOURCE_DIR, deploy_resources, client, console, dry_run, verbose)
 
+        data_files_by_selector = self._topological_sort_if_instance_selector(data_files_by_selector, client)
+
         self._upload_data(data_files_by_selector, client, dry_run, input_dir, console, verbose)
 
+    def _topological_sort_if_instance_selector(
+        self, data_files_by_selector: dict[Selector, list[Path]], client: ToolkitClient
+    ) -> dict[Selector, list[Path]]:
+        """Topologically sorts InstanceSpaceSelectors (if they are present) to determine the order of upload based on container dependencies from the views.
+
+        Args:
+            data_files_by_selector: A dictionary mapping selectors to their data files.
+            client: The cognite client to use for the upload.
+
+        Returns:
+            A dictionary mapping selectors to their data files with necessary preprocessing.
+        """
+        counts = Counter(type(selector) for selector in data_files_by_selector.keys())
+        if counts[InstanceSpaceSelector] <= 1:
+            return data_files_by_selector
+
+        selector_by_view_id: dict[ViewId, Selector] = {}
+        for selector in data_files_by_selector:
+            if isinstance(selector, InstanceSpaceSelector) and selector.view is not None:
+                selector_by_view_id[selector.view.as_id()] = selector
+
+        view_dependencies = ViewCRUD.create_loader(client).topological_sort_container_constraints(
+            list(selector_by_view_id.keys())
+        )
+        prepared_selectors: dict[Selector, list[Path]] = {}
+
+        # Reorder selectors according to the dependency-sorted view list
+        for view_id in view_dependencies:
+            selector = selector_by_view_id[view_id]
+            prepared_selectors[selector] = data_files_by_selector[selector]
+
+        # Preserve selectors that aren't affected by view dependencies
+        # (e.g., raw tables, time series, non-view instance data)
+        for selector in data_files_by_selector.keys():
+            if selector not in prepared_selectors:
+                prepared_selectors[selector] = data_files_by_selector[selector]
+
+        return prepared_selectors
+
     def _find_data_files(
         self,
         input_dir: Path,
-        kind: str | None = None,
     ) -> dict[Selector, list[Path]]:
         """Finds data files and their corresponding metadata files in the input directory."""
-        manifest_file_endswith = f".{DATA_MANIFEST_STEM}.yaml"
         data_files_by_metadata: dict[Selector, list[Path]] = {}
-        for metadata_file in input_dir.glob(f"*{manifest_file_endswith}"):
-            data_file_prefix = metadata_file.name.removesuffix(manifest_file_endswith)
-            data_files = [
-                file
-                for file in input_dir.glob(f"{data_file_prefix}*")
-                if not file.name.endswith(manifest_file_endswith)
-            ]
-            if kind is not None and data_files:
-                data_files = [data_file for data_file in data_files if are_same_kind(kind, data_file)]
-                if not data_files:
-                    continue
+        for manifest_file in input_dir.glob(f"*{DATA_MANIFEST_SUFFIX}"):
+            selector_or_warning = load_selector(manifest_file)
+            if isinstance(selector_or_warning, ToolkitWarning):
+                self.warn(selector_or_warning)
+                continue
+            selector: Selector = selector_or_warning
+            data_files = selector.find_data_files(input_dir, manifest_file)
             if not data_files:
                 self.warn(
                     MediumSeverityWarning(
-                        f"Metadata file {metadata_file.as_posix()!r} has no corresponding data files, skipping.",
-                    )
-                )
-                continue
-
-            selector_dict = read_yaml_file(metadata_file, expected_output="dict")
-            try:
-                selector = SelectorAdapter.validate_python(selector_dict)
-            except ValidationError as e:
-                errors = humanize_validation_error(e)
-                self.warn(
-                    ResourceFormatWarning(
-                        metadata_file, tuple(errors), text="Invalid selector in metadata file, skipping."
+                        f"Metadata file {manifest_file.as_posix()!r} has no corresponding data files, skipping.",
                     )
                 )
                 continue
@@ -178,49 +202,48 @@ class UploadCommand(ToolkitCommand):
                 io = self._create_selected_io(selector, datafiles[0], client)
                 if io is None:
                     continue
-                for data_file in datafiles:
-                    file_display = self._path_as_display_name(data_file)
-                    if verbose:
-                        console.print(f"{action} {selector.display_name} from {file_display.as_posix()!r}")
-                    reader = FileReader.from_filepath(data_file)
-                    is_table = reader.format in TABLE_READ_CLS_BY_FORMAT
-                    if is_table and not isinstance(io, TableUploadableStorageIO):
-                        raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
-                    tracker = ProgressTracker[str]([self._UPLOAD])
-                    executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
-                        download_iterable=io.read_chunks(reader),
-                        process=partial(io.rows_to_data, selector=selector)
-                        if is_table and isinstance(io, TableUploadableStorageIO)
-                        else io.json_chunk_to_data,
-                        write=partial(
-                            self._upload_items,
-                            upload_client=upload_client,
-                            io=io,
-                            dry_run=dry_run,
-                            selector=selector,
-                            tracker=tracker,
-                            console=console,
-                        ),
-                        iteration_count=None,
-                        max_queue_size=self._MAX_QUEUE_SIZE,
-                        download_description=f"Reading {file_count:,}/{total_file_count + 1:,}: {file_display.as_posix()!s}",
-                        process_description="Processing",
-                        write_description=f"{action} {selector.display_name!r}",
+                reader = MultiFileReader(datafiles)
+                if reader.is_table and not isinstance(io, TableUploadableStorageIO):
+                    raise ToolkitValueError(f"{selector.display_name} does not support {reader.format!r} files.")
+
+                chunk_count = io.count_chunks(reader)
+                iteration_count = chunk_count // io.CHUNK_SIZE + (1 if chunk_count % io.CHUNK_SIZE > 0 else 0)
+
+                tracker = ProgressTracker[str]([self._UPLOAD])
+                executor = ProducerWorkerExecutor[list[tuple[str, dict[str, JsonVal]]], Sequence[UploadItem]](
+                    download_iterable=io.read_chunks(reader, selector),
+                    process=partial(io.rows_to_data, selector=selector)
+                    if reader.is_table and isinstance(io, TableUploadableStorageIO)
+                    else io.json_chunk_to_data,
+                    write=partial(
+                        self._upload_items,
+                        upload_client=upload_client,
+                        io=io,
+                        dry_run=dry_run,
+                        selector=selector,
+                        tracker=tracker,
                         console=console,
-                    )
-                    executor.run()
-                    file_count += 1
-                    executor.raise_on_error()
-                    final_action = "Uploaded" if not dry_run else "Would upload"
-                    suffix = " successfully" if not dry_run else ""
-                    results = tracker.aggregate()
-                    success = results.get((self._UPLOAD, "success"), 0)
-                    failed = results.get((self._UPLOAD, "failed"), 0)
-                    if failed > 0:
-                        suffix += f", {failed:,} failed"
-                    console.print(
-                        f"{final_action} {success:,} {selector.display_name} from {file_display.as_posix()!r}{suffix}."
-                    )
+                    ),
+                    iteration_count=iteration_count,
+                    max_queue_size=self._MAX_QUEUE_SIZE,
+                    download_description=f"Reading {selector.display_name!r} files",
+                    process_description="Processing",
+                    write_description=f"{action} {selector.display_name!r}",
+                    console=console,
+                )
+                executor.run()
+                file_count += len(datafiles)
+                executor.raise_on_error()
+                final_action = "Uploaded" if not dry_run else "Would upload"
+                suffix = " successfully" if not dry_run else ""
+                results = tracker.aggregate()
+                success = results.get((self._UPLOAD, "success"), 0)
+                failed = results.get((self._UPLOAD, "failed"), 0)
+                if failed > 0:
+                    suffix += f", {failed:,} failed"
+                console.print(
+                    f"{final_action} {success:,} {selector.display_name} from {len(datafiles)} files{suffix}."
+                )
 
     @staticmethod
     def _path_as_display_name(input_path: Path, cwd: Path = Path.cwd()) -> Path:
@@ -233,7 +256,7 @@ class UploadCommand(ToolkitCommand):
         self, selector: Selector, data_file: Path, client: ToolkitClient
     ) -> UploadableStorageIO | None:
         try:
-            io_cls = get_upload_io(type(selector), kind=data_file)
+            io_cls = get_upload_io(selector)
         except ValueError as e:
             self.warn(HighSeverityWarning(f"Could not find StorageIO for selector {selector}: {e}"))
             return None
@@ -242,9 +265,9 @@ class UploadCommand(ToolkitCommand):
     @classmethod
     def _upload_items(
         cls,
-        data_chunk: Sequence[UploadItem],
+        data_chunk: Sequence[UploadItem[T_ResourceRequest]],
         upload_client: HTTPClient,
-        io: UploadableStorageIO[T_Selector, T_CogniteResource, T_WriteCogniteResource],
+        io: UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest],
         selector: T_Selector,
         dry_run: bool,
         tracker: ProgressTracker[str],

cognite_toolkit/_cdf_tk/commands/about.py

@@ -0,0 +1,221 @@
+import platform
+import sys
+from pathlib import Path
+
+from rich import print
+from rich.table import Table
+
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml, _read_toml
+from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.constants import clean_name
+from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.plugins import Plugins
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, MediumSeverityWarning
+from cognite_toolkit._version import __version__
+
+
+class AboutCommand(ToolkitCommand):
+    def execute(self, cwd: Path) -> None:
+        # Version information
+        print(f"\n[bold cyan]Cognite Toolkit[/bold cyan] version: [yellow]{__version__}[/yellow]")
+        print(f"Python version: {sys.version.split()[0]}")
+        print(f"Platform: {platform.system()} {platform.release()}")
+
+        # Check for cdf.toml in the current directory
+        cdf_toml_path = cwd / CDFToml.file_name
+
+        if cdf_toml_path.exists():
+            print(f"\n[bold green]Configuration file found:[/bold green] {cdf_toml_path}")
+
+            cdf_toml = CDFToml.load(cwd)
+
+            # We need to read the raw TOML to get original key names for plugins and alpha flags
+            raw_toml = _read_toml(cdf_toml_path)
+
+            self._check_unrecognized_sections(raw_toml)
+            self._display_plugins(cdf_toml, raw_toml)
+            self._display_alpha_flags(cdf_toml, raw_toml)
+            self._display_additional_config(cdf_toml)
+
+        else:
+            # Search for cdf.toml in subdirectories
+            found_files = self._search_cdf_toml(cwd)
+
+            if found_files:
+                print(f"\n[bold yellow]No cdf.toml found in current directory:[/bold yellow] {cwd}")
+                print("\n[bold]Found cdf.toml files in subdirectories:[/bold]")
+                for file in found_files:
+                    rel_path = file.relative_to(cwd)
+                    print(f"  • {rel_path}")
+                print(f"\n[bold cyan]Hint:[/bold cyan] Move one of these files to {cwd} or navigate to its directory.")
+            else:
+                print("\n[bold yellow]No cdf.toml found[/bold yellow] in current directory or subdirectories.")
+                print(f"Current directory: {cwd}")
+                print("\n[bold cyan]Hint:[/bold cyan] Run [yellow]cdf init[/yellow] to create a new project.")
+
+    def _check_unrecognized_sections(self, raw_toml: dict) -> None:
+        """Check for unrecognized tables in cdf.toml and warn about them."""
+        # Valid top-level tables in cdf.toml
+        valid_tables = {"cdf", "modules", "alpha_flags", "feature_flags", "plugins", "library"}
+
+        # Filter out empty keys, whitespace-only keys, and check for unrecognized tables
+        unrecognized_tables = [key for key in raw_toml.keys() if key and key.strip() and key not in valid_tables]
+
+        if unrecognized_tables:
+            print()
+
+        for table in unrecognized_tables:
+            # Try to find a matching valid table by stripping non-alphabetical characters
+            suggestion = self._find_similar_table(table, valid_tables)
+
+            message = f"Table '{table}' in cdf.toml is not recognized and will have no effect."
+            if suggestion:
+                message += f" Did you mean '{suggestion}'?"
+
+            self.warn(MediumSeverityWarning(message))
+
+    @staticmethod
+    def _find_similar_table(unrecognized: str, valid_tables: set[str]) -> str | None:
+        """Find a similar valid table by comparing alphabetical characters only.
+
+        Returns None if the unrecognized table is already valid or if no similar match is found.
+        """
+        # If it's already a valid table, return None (no suggestion needed)
+        if unrecognized in valid_tables:
+            return None
+
+        # Keep only alphabetical characters and lowercase
+        normalized_unrecognized = "".join(c for c in unrecognized if c.isalpha()).lower()
+
+        # First, try exact match (after normalization)
+        for valid in valid_tables:
+            normalized_valid = "".join(c for c in valid if c.isalpha()).lower()
+            if normalized_unrecognized == normalized_valid:
+                return valid
+
+        # If no match, check for singular/plural variations (missing 's')
+        for valid in valid_tables:
+            normalized_valid = "".join(c for c in valid if c.isalpha()).lower()
+
+            # Check if adding 's' to unrecognized matches valid (e.g., "plugin" -> "plugins")
+            if normalized_unrecognized + "s" == normalized_valid:
+                return valid
+
+        return None
+
+    def _display_plugins(self, cdf_toml: CDFToml, raw_toml: dict) -> None:
+        """Display all available plugins and their status."""
+        table = Table(title="Plugins", show_header=True)
+        table.add_column("Plugin", justify="left", style="cyan")
+        table.add_column("Status", justify="center")
+        table.add_column("Description", justify="left")
+
+        # Track which plugins we've seen
+        seen_plugins = set()
+
+        # Show all plugins from the enum
+        for plugin in Plugins:
+            plugin_name = plugin.value.name
+            cleaned_key = clean_name(plugin_name)
+            seen_plugins.add(cleaned_key)
+
+            is_enabled = cdf_toml.plugins.get(cleaned_key, False)
+            if is_enabled:
+                status = "[green]✓ enabled[/green]"
+            else:
+                status = "[dim]○ disabled[/dim]"
+
+            table.add_row(plugin_name, status, plugin.value.description)
+
+        print()
+        print(table)
+
+        # Show any unrecognized plugins from cdf.toml using original key names
+        raw_plugins = raw_toml.get("plugins", {})
+        unrecognized = []
+        for original_key, value in raw_plugins.items():
+            cleaned_key = clean_name(original_key)
+            if cleaned_key not in seen_plugins:
+                unrecognized.append((original_key, value))
+
+        for original_key, is_enabled in unrecognized:
+            status = "enabled" if is_enabled else "disabled"
+            self.warn(
+                LowSeverityWarning(f"Plugin '{original_key}' in cdf.toml is not recognized and will have no effect.")
+            )
+
+    def _display_alpha_flags(self, cdf_toml: CDFToml, raw_toml: dict) -> None:
+        """Display available alpha flags and their status."""
+        table = Table(title="Alpha Flags", show_header=True)
+        table.add_column("Flag", justify="left", style="yellow")
+        table.add_column("Status", justify="center")
+        table.add_column("Description", justify="left")
+
+        # Track which flags we've seen
+        seen_flags = set()
+
+        # Show flags from the enum that are either enabled or visible
+        for flag in Flags:
+            cleaned_key = clean_name(flag.name)
+            seen_flags.add(cleaned_key)
+
+            is_enabled = cdf_toml.alpha_flags.get(cleaned_key, False)
+
+            # Only show if enabled or visible
+            if is_enabled or flag.value.visible:
+                # Convert enum name to kebab-case for display
+                display_name = flag.name.lower().replace("_", "-")
+
+                if is_enabled:
+                    status = "[green]✓ enabled[/green]"
+                else:
+                    status = "[dim]○ disabled[/dim]"
+
+                table.add_row(display_name, status, flag.value.description)
+
+        print()
+        print(table)
+
+        # Show any unrecognized flags from cdf.toml using original key names
+        raw_flags = raw_toml.get("alpha_flags", {})
+        unrecognized = []
+        for original_key, value in raw_flags.items():
+            cleaned_key = clean_name(original_key)
+            if cleaned_key not in seen_flags:
+                unrecognized.append((original_key, value))
+
+        for original_key, is_enabled in unrecognized:
+            status = "enabled" if is_enabled else "disabled"
+            self.warn(
+                LowSeverityWarning(
+                    f"Alpha flag '{original_key}' in cdf.toml is not recognized and will have no effect."
+                )
+            )
+
+    def _display_additional_config(self, cdf_toml: CDFToml) -> None:
+        """Display additional configuration information."""
+        print("\n[bold]Additional Configuration:[/bold]")
+
+        print(f"  Default environment: [cyan]{cdf_toml.cdf.default_env}[/cyan]")
+
+        if cdf_toml.cdf.has_user_set_default_org:
+            print(f"  Default organization dir: [cyan]{cdf_toml.cdf.default_organization_dir}[/cyan]")
+
+        if cdf_toml.cdf.file_encoding:
+            print(f"  File encoding: [cyan]{cdf_toml.cdf.file_encoding}[/cyan]")
+
+        print(f"  Modules version: [cyan]{cdf_toml.modules.version}[/cyan]")
+
+        if cdf_toml.libraries:
+            print(f"  Configured libraries: [cyan]{len(cdf_toml.libraries)}[/cyan]")
+            for lib_name, lib_config in cdf_toml.libraries.items():
+                print(f"    • {lib_name}: [dim]{lib_config.url}[/dim]")
+
+    def _search_cdf_toml(self, cwd: Path) -> list[Path]:
+        """Search for cdf.toml files in immediate subdirectories (one level down)."""
+        try:
+            return sorted(
+                [potential_file for potential_file in cwd.glob(f"*/{CDFToml.file_name}") if potential_file.is_file()]
+            )
+        except PermissionError:
+            return []

cognite_toolkit/_cdf_tk/commands/auth.py

@@ -26,6 +26,7 @@ import questionary
 from cognite.client.data_classes.capabilities import (
     AssetsAcl,
     Capability,
+    ExtractionConfigsAcl,
     FunctionsAcl,
     GroupsAcl,
     ProjectsAcl,
@@ -46,6 +47,7 @@ from cognite_toolkit._cdf_tk.constants import (
     TOOLKIT_DEMO_GROUP_NAME,
     TOOLKIT_SERVICE_PRINCIPAL_GROUP_NAME,
 )
+from cognite_toolkit._cdf_tk.cruds import ExtractionPipelineConfigCRUD
 from cognite_toolkit._cdf_tk.exceptions import (
     AuthenticationError,
     AuthorizationError,
@@ -95,16 +97,6 @@ class AuthCommand(ToolkitCommand):
             raise AuthenticationError(f"Unable to verify the credentials.\n{e}")
 
         print("[green]The credentials are valid.[/green]")
-        if no_verify:
-            return
-        print(
-            Panel(
-                "Running verification, 'cdf auth verify'...",
-                title="",
-                expand=False,
-            )
-        )
-        self.verify(client, dry_run)
 
     def _store_dotenv(self, env_vars: EnvironmentVariables) -> None:
         new_env_file = env_vars.create_dotenv_file()
@@ -444,8 +436,23 @@ class AuthCommand(ToolkitCommand):
             crud = crud_cls.create_loader(client)
             if crud.prerequisite_warning() is not None:
                 continue
-            capability = crud_cls.get_required_capability(None, read_only=False)
-            capabilities = capability if isinstance(capability, list) else [capability]
+            if isinstance(crud, ExtractionPipelineConfigCRUD):
+                # The Extraction Pipeline Config CRUD requires special handling.
+                # The .get_required_capability is used in the DeployCommand as well. Since, there is no way to no
+                # the extraction pipeline ID or dataSetId from an ExtractionPipelineConfigWrite object, we do not
+                # check those there. If we returned the full capability, it would always have to be all scoped.
+                # That is too restrictive in the deploy command, so we return an empty list, essentially not checking
+                # anything there. Here, we want to add the all scoped capability, so that the Toolkit group gets the
+                # correct capability.
+                capabilities: list[Capability] = [
+                    ExtractionConfigsAcl(
+                        [ExtractionConfigsAcl.Action.Read, ExtractionConfigsAcl.Action.Write],
+                        ExtractionConfigsAcl.Scope.All(),
+                    )
+                ]
+            else:
+                capability = crud_cls.get_required_capability(None, read_only=False)
+                capabilities = capability if isinstance(capability, list) else [capability]
             for cap in capabilities:
                 if project_type == "DATA_MODELING_ONLY" and isinstance(cap, AssetsAcl | RelationshipsAcl):
                     continue

cognite_toolkit/_cdf_tk/commands/build_cmd.py

@@ -14,13 +14,12 @@ from rich.progress import track
 from cognite_toolkit._cdf_tk.builders import Builder, create_builder
 from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.raw import RawDatabase
+from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawDatabase
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
 from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_FOLDER_ENCODING,
     DEFAULT_ENV,
-    DEV_ONLY_MODULES,
     HINT_LEAD_TEXT,
     ROOT_MODULES,
     TEMPLATE_VARS_FILE_SUFFIXES,
@@ -59,10 +58,7 @@ from cognite_toolkit._cdf_tk.data_classes import (
     SourceLocationLazy,
 )
 from cognite_toolkit._cdf_tk.exceptions import (
-    ToolkitDuplicatedModuleError,
-    ToolkitEnvError,
     ToolkitError,
-    ToolkitMissingModuleError,
     ToolkitYAMLFormatError,
 )
 from cognite_toolkit._cdf_tk.hints import Hint, ModuleDefinition, verify_module_directory
@@ -89,6 +85,7 @@ from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 from cognite_toolkit._cdf_tk.validation import (
     validate_data_set_is_set,
+    validate_module_selection,
     validate_modules_variables,
     validate_resource_yaml_pydantic,
 )
@@ -207,7 +204,11 @@ class BuildCommand(ToolkitCommand):
 
         user_selected_modules = config.environment.get_selected_modules(packages)
         modules = ModuleDirectories.load(organization_dir, user_selected_modules)
-        self._validate_modules(modules, config, packages, user_selected_modules, organization_dir)
+        module_warnings = validate_module_selection(modules, config, packages, user_selected_modules, organization_dir)
+        if module_warnings:
+            self.warning_list.extend(module_warnings)
+            if self.print_warning:
+                print(str(module_warnings))
 
         if verbose:
             self.console("Selected packages:")
@@ -358,6 +359,15 @@ class BuildCommand(ToolkitCommand):
         for resource_name, resource_files in module.source_paths_by_resource_folder.items():
             source_files = self._replace_variables(resource_files, module_variables, resource_name, module.dir, verbose)
 
+            if resource_name == "data_models":
+                resource_name = "data_modeling"
+                self.warn(
+                    MediumSeverityWarning(
+                        "The resource folder 'data_models' is deprecated and will be removed in v1.0. "
+                        "Please rename the folder to 'data_modeling'."
+                    )
+                )
+
             builder = self._get_builder(build_dir, resource_name)
 
             built_resources = BuiltResourceList[Hashable]()
@@ -413,62 +423,6 @@ class BuildCommand(ToolkitCommand):
         builder = self._builder_by_resource_folder[resource_name]
         return builder
 
-    def _validate_modules(
-        self,
-        modules: ModuleDirectories,
-        config: BuildConfigYAML,
-        packages: dict[str, list[str]],
-        selected_modules: set[str | Path],
-        organization_dir: Path,
-    ) -> None:
-        # Validations: Ambiguous selection.
-        selected_names = {s for s in config.environment.selected if isinstance(s, str)}
-        if duplicate_modules := {
-            module_name: paths
-            for module_name, paths in modules.as_path_by_name().items()
-            if len(paths) > 1 and module_name in selected_names
-        }:
-            # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
-            # Note, if the user uses a path to select a module, this error will not be raised.
-            raise ToolkitDuplicatedModuleError(
-                f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
-            )
-        # Package Referenced Modules Exists
-        for package, package_modules in packages.items():
-            if package not in selected_names:
-                # We do not check packages that are not selected.
-                # Typically, the user will delete the modules that are irrelevant for them;
-                # thus we only check the selected packages.
-                continue
-            if missing_packages := set(package_modules) - modules.available_names:
-                ToolkitMissingModuleError(
-                    f"Package {package} defined in {CDFToml.file_name!s} is referring "
-                    f"the following missing modules {missing_packages}."
-                )
-
-        # Selected modules does not exists
-        if missing_modules := set(selected_modules) - modules.available:
-            hint = ModuleDefinition.long(missing_modules, organization_dir)
-            raise ToolkitMissingModuleError(
-                f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
-            )
-
-        # Nothing is Selected
-        if not modules.selected:
-            raise ToolkitEnvError(
-                f"No selected modules specified in {config.filepath!s}, have you configured "
-                f"the environment ({config.environment.name})?"
-            )
-
-        dev_modules = modules.available_names & DEV_ONLY_MODULES
-        if dev_modules and config.environment.validation_type != "dev":
-            self.warn(
-                MediumSeverityWarning(
-                    "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
-                    f"{humanize_collection(dev_modules)!r}",
-                )
-            )
-
     def _replace_variables(
         self,
         resource_files: Sequence[Path],