cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/deploy.py

@@ -1,8 +1,8 @@
-from collections.abc import Hashable
+from collections.abc import Hashable, Sequence
 from graphlib import TopologicalSorter
 from pathlib import Path
+from typing import overload
 
-from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
 from cognite.client.exceptions import CogniteAPIError, CogniteDuplicatedError
 from cognite.client.utils._identifier import T_ID
 from rich import print
@@ -15,8 +15,18 @@ from cognite_toolkit._cdf_tk.commands.clean import CleanCommand
 from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_ENVIRONMENT_FILE,
+    DATA_UPLOAD_URL,
     HINT_LEAD_TEXT,
 )
+from cognite_toolkit._cdf_tk.cruds import (
+    DataCRUD,
+    Loader,
+    RawDatabaseCRUD,
+    ResourceContainerCRUD,
+    ResourceCRUD,
+    ResourceWorker,
+)
+from cognite_toolkit._cdf_tk.cruds._worker import CategorizedResources
 from cognite_toolkit._cdf_tk.data_classes import (
     BuildEnvironment,
     DatapointDeployResult,
@@ -34,19 +44,14 @@ from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitFileNotFoundError,
     ToolkitNotADirectoryError,
 )
-from cognite_toolkit._cdf_tk.loaders import (
-    DataLoader,
-    Loader,
-    RawDatabaseLoader,
-    ResourceContainerLoader,
-    ResourceLoader,
-    ResourceWorker,
+from cognite_toolkit._cdf_tk.protocols import (
+    T_ResourceRequest,
+    T_ResourceResponse,
 )
-from cognite_toolkit._cdf_tk.loaders._base_loaders import T_WritableCogniteResourceList
-from cognite_toolkit._cdf_tk.loaders._worker import CategorizedResources
 from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning
 from cognite_toolkit._cdf_tk.tk_warnings.base import WarningList, catch_warnings
 from cognite_toolkit._cdf_tk.tk_warnings.other import (
+    HighSeverityWarning,
     LowSeverityWarning,
     ToolkitDependenciesIncludedWarning,
 )
@@ -76,7 +81,19 @@ class DeployCommand(ToolkitCommand):
         include = self._clean_command.validate_include(include)
 
         build = self._load_build(build_dir, build_env_name)
+
         client = env_vars.get_client(build.is_strict_validation)
+
+        self._additional_tracking_info.project = client.config.project
+        self._additional_tracking_info.cluster = client.config.cdf_cluster
+
+        if not dry_run:
+            for module in build.read_modules:
+                if module.module_id:
+                    self._additional_tracking_info.module_ids.add(module.module_id)
+                if module.package_id:
+                    self._additional_tracking_info.package_ids.add(module.package_id)
+
         selected_loaders = self._clean_command.get_selected_loaders(build_dir, build.read_resource_folders, include)
         ordered_loaders = self._order_loaders(selected_loaders, build_dir)
         self._start_message(build_dir, dry_run, env_vars)
@@ -87,17 +104,17 @@ class DeployCommand(ToolkitCommand):
             )
 
         self.deploy_all_resources(
-            client,
-            results,
-            build,
-            build_dir,
-            drop,
-            drop_data,
-            dry_run,
-            env_vars,
-            force_update,
-            ordered_loaders,
-            verbose,
+            client=client,
+            build_dir=build_dir,
+            env_vars=env_vars,
+            results=results,
+            dry_run=dry_run,
+            has_dropped=drop,
+            has_dropped_data=drop_data,
+            force_update=force_update,
+            ordered_loaders=ordered_loaders,
+            build=build,
+            verbose=verbose,
         )
 
         if results.has_counts:
@@ -182,9 +199,9 @@ class DeployCommand(ToolkitCommand):
             return None
 
         for loader_cls in reversed(ordered_loaders):
-            if not issubclass(loader_cls, ResourceLoader):
+            if not issubclass(loader_cls, ResourceCRUD):
                 continue
-            loader: ResourceLoader = loader_cls.create_loader(client, build_dir)
+            loader: ResourceCRUD = loader_cls.create_loader(client, build_dir)
             result = self._clean_command.clean_resources(
                 loader,
                 env_vars=env_vars,
@@ -199,55 +216,129 @@ class DeployCommand(ToolkitCommand):
         print("[bold]...cleaning complete![/]")
         return None
 
+    @overload
     def deploy_all_resources(
         self,
         client: ToolkitClient,
+        build_dir: Path,
+        env_vars: EnvironmentVariables,
+        results: None = None,
+        dry_run: bool = False,
+        has_dropped: bool = False,
+        has_dropped_data: bool = False,
+        force_update: bool = False,
+        ordered_loaders: list[type[Loader]] | None = None,
+        build: BuildEnvironment | None = None,
+        verbose: bool = False,
+    ) -> DeployResults: ...
+
+    @overload
+    def deploy_all_resources(
+        self,
+        client: ToolkitClient,
+        build_dir: Path,
+        env_vars: EnvironmentVariables,
         results: DeployResults,
-        build: BuildEnvironment,
+        dry_run: bool = False,
+        has_dropped: bool = False,
+        has_dropped_data: bool = False,
+        force_update: bool = False,
+        ordered_loaders: list[type[Loader]] | None = None,
+        build: BuildEnvironment | None = None,
+        verbose: bool = False,
+    ) -> None: ...
+
+    def deploy_all_resources(
+        self,
+        client: ToolkitClient,
         build_dir: Path,
-        drop: bool,
-        drop_data: bool,
-        dry_run: bool,
         env_vars: EnvironmentVariables,
-        force_update: bool,
-        ordered_loaders: list[type[Loader]],
-        verbose: bool,
-    ) -> None:
+        results: DeployResults | None = None,
+        dry_run: bool = False,
+        has_dropped: bool = False,
+        has_dropped_data: bool = False,
+        force_update: bool = False,
+        ordered_loaders: list[type[Loader]] | None = None,
+        build: BuildEnvironment | None = None,
+        verbose: bool = False,
+    ) -> DeployResults | None:
+        """Deploy all resources in the build directory.
+
+        Args:
+            client: The Cognite client to use for deployment.
+            build_dir: The directory containing the built resource files.
+            env_vars: The environment variables to use for variable replacement in the resource files.
+            results: An optional DeployResults object to store the results of the deployment. If None, a new object will be created and returned.
+            dry_run: If True, performs a dry run without actually deploying the resources.
+            has_dropped: If True, indicates that resources have been dropped prior to deployment.
+            has_dropped_data: If True, indicates that data resources have been dropped prior to deployment.
+            force_update: If True, forces an update of resources even if they are unchanged.
+            ordered_loaders: An optional list of loader classes to use for deployment. If None, all loaders will be used in the order determined by dependencies.
+            build: An optional BuildEnvironment object containing information about the build. If None, data uploads will be skipped.
+            verbose: If True, prints detailed information about the deployment process.
+
+        Returns:
+            A DeployResults object containing the results of the deployment if results is None, otherwise None.
+
+        """
         if verbose:
             print(Panel("[bold]DEPLOYING resources...[/]"))
 
+        if ordered_loaders is None:
+            selected_loaders = self._clean_command.get_selected_loaders(build_dir, set(), None)
+            ordered_loaders = self._order_loaders(selected_loaders, build_dir)
+        read_modules: list[ReadModule] | None = None
+        if build is not None:
+            read_modules = build.read_modules
+        output_results = DeployResults([], "deploy", dry_run=dry_run) if results is None else results
         for loader_cls in ordered_loaders:
+            if issubclass(loader_cls, DataCRUD):
+                self.warn(
+                    HighSeverityWarning(
+                        f"Uploading {loader_cls.kind} data is deprecated and will be removed in v0.8. "
+                        f"Use the `cdf data upload dir` command instead. See [{DATA_UPLOAD_URL}]({DATA_UPLOAD_URL}) for more information about "
+                        "the upload command."
+                    )
+                )
+
             loader = loader_cls.create_loader(client, build_dir)
             resource_result: DeployResult | None
-            if isinstance(loader, ResourceLoader):
+            if isinstance(loader, ResourceCRUD):
                 resource_result = self.deploy_resource_type(
                     loader,
                     env_vars,
-                    build.read_modules,
+                    read_modules,
                     dry_run,
-                    drop,
-                    drop_data,
+                    has_dropped,
+                    has_dropped_data,
                     force_update,
                     verbose,
                 )
-            elif isinstance(loader, DataLoader):
+            elif isinstance(loader, DataCRUD) and build is not None:
                 resource_result = self.upload_data(loader, build, dry_run, verbose)
+            elif isinstance(loader, DataCRUD) and build is None:
+                self.warn(
+                    LowSeverityWarning(
+                        f"Skipping data upload for {loader.display_name} as it is not supported in this context."
+                    )
+                )
+                continue
             else:
                 raise ValueError(f"Unsupported loader type {type(loader)}.")
 
             if resource_result:
-                results[resource_result.name] = resource_result
+                output_results[resource_result.name] = resource_result
             if verbose:
                 print("")  # Extra newline
+        if results is None:
+            return output_results
         return None
 
     def deploy_resource_type(
         self,
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         env_vars: EnvironmentVariables,
-        read_modules: list[ReadModule],
+        read_modules: list[ReadModule] | None = None,
         dry_run: bool = False,
         has_done_drop: bool = False,
         has_dropped_data: bool = False,
@@ -281,7 +372,7 @@ class DeployCommand(ToolkitCommand):
         print(f"[bold]{prefix} {nr_of_items} {loader.display_name} to CDF...[/]")
 
         # Moved here to avoid printing before the above message.
-        if not isinstance(loader, RawDatabaseLoader):
+        if not isinstance(loader, RawDatabaseCRUD):
             for duplicate in worker.duplicates:
                 self.warn(LowSeverityWarning(f"Skipping duplicate {loader.display_name} {duplicate}."))
 
@@ -298,7 +389,7 @@ class DeployCommand(ToolkitCommand):
         if verbose:
             self._verbose_print(resources, loader, dry_run)
 
-        if isinstance(loader, ResourceContainerLoader):
+        if isinstance(loader, ResourceContainerCRUD):
             return ResourceContainerDeployResult.from_resource_deploy_result(
                 result,
                 item_name=loader.item_name,
@@ -307,10 +398,8 @@ class DeployCommand(ToolkitCommand):
 
     def actual_deploy(
         self,
-        resources: CategorizedResources[T_ID, T_CogniteResourceList],
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        resources: CategorizedResources[T_ID, T_ResourceRequest],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         env_var_warnings: WarningList | None = None,
     ) -> ResourceDeployResult:
         environment_variable_warning_by_id = {
@@ -341,17 +430,15 @@ class DeployCommand(ToolkitCommand):
 
     @staticmethod
     def dry_run_deploy(
-        resources: CategorizedResources[T_ID, T_CogniteResourceList],
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
+        resources: CategorizedResources[T_ID, T_ResourceRequest],
+        loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
         has_done_drop: bool,
         has_dropped_data: bool,
     ) -> ResourceDeployResult:
         if (
             loader.support_drop
             and has_done_drop
-            and (not isinstance(loader, ResourceContainerLoader) or has_dropped_data)
+            and (not isinstance(loader, ResourceContainerCRUD) or has_dropped_data)
         ):
             # Means the resources will be deleted and not left unchanged or changed
             for item in resources.unchanged:
@@ -375,8 +462,8 @@ class DeployCommand(ToolkitCommand):
 
     @staticmethod
     def _verbose_print(
-        resources: CategorizedResources[T_ID, T_CogniteResourceList],
-        loader: ResourceLoader,
+        resources: CategorizedResources[T_ID, T_ResourceRequest],
+        loader: ResourceCRUD,
         dry_run: bool,
     ) -> None:
         print_outs = []
@@ -399,8 +486,8 @@ class DeployCommand(ToolkitCommand):
 
     def _create_resources(
         self,
-        resources: T_CogniteResourceList,
-        loader: ResourceLoader,
+        resources: Sequence[T_ResourceRequest],
+        loader: ResourceCRUD,
         environment_variable_warning_by_id: dict[Hashable, EnvironmentVariableMissingWarning],
     ) -> int:
         try:
@@ -422,8 +509,8 @@ class DeployCommand(ToolkitCommand):
 
     def _update_resources(
         self,
-        resources: T_CogniteResourceList,
-        loader: ResourceLoader,
+        resources: Sequence[T_ResourceRequest],
+        loader: ResourceCRUD,
         environment_variable_warning_by_id: dict[Hashable, EnvironmentVariableMissingWarning],
     ) -> int:
         try:
@@ -455,7 +542,7 @@ class DeployCommand(ToolkitCommand):
 
     @staticmethod
     def upload_data(
-        loader: DataLoader,
+        loader: DataCRUD,
         state: BuildEnvironment,
         dry_run: bool = False,
         verbose: bool = False,