cognite-toolkit 0.7.28__tar.gz → 0.7.30__tar.gz

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.28
+Version: 0.7.30
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/client/_toolkit_client.py

@@ -35,19 +35,26 @@ class ToolAPI:
 
 
 class ToolkitClient(CogniteClient):
-    def __init__(self, config: ToolkitClientConfig | None = None, enable_set_pending_ids: bool = False) -> None:
+    def __init__(
+        self,
+        config: ToolkitClientConfig | None = None,
+        enable_set_pending_ids: bool = False,
+        console: Console | None = None,
+    ) -> None:
         super().__init__(config=config)
         http_client = HTTPClient(self.config)
         self.http_client = http_client
         toolkit_config = ToolkitClientConfig.from_client_config(self.config)
-        self.console = Console()
+        self.console = console or Console()
         self.tool = ToolAPI(http_client, self.console)
         self.search = SearchAPI(self._config, self._API_VERSION, self)
         self.robotics = RoboticsAPI(self._config, self._API_VERSION, self)
         self.dml = DMLAPI(self._config, self._API_VERSION, self)
         self.verify = VerifyAPI(self._config, self._API_VERSION, self)
         self.lookup = LookUpGroup(self._config, self._API_VERSION, self, self.console)
-        self.functions: ExtendedFunctionsAPI = ExtendedFunctionsAPI(toolkit_config, self._API_VERSION, self)
+        self.functions: ExtendedFunctionsAPI = ExtendedFunctionsAPI(
+            toolkit_config, self._API_VERSION, self, self.console
+        )
         self.data_modeling: ExtendedDataModelingAPI = ExtendedDataModelingAPI(self._config, self._API_VERSION, self)
         if enable_set_pending_ids:
             self.time_series: ExtendedTimeSeriesAPI = ExtendedTimeSeriesAPI(self._config, self._API_VERSION, self)

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/client/api/extended_functions.py

@@ -17,15 +17,16 @@ class ExtendedFunctionsAPI(FunctionsAPI):
         config: ToolkitClientConfig,
         api_version: str | None,
         cognite_client: CogniteClient,
+        console: Console | None = None,
     ) -> None:
         """
         Extended Functions API to include custom headers and payload preparation.
         """
         super().__init__(config, api_version, cognite_client)
         self._toolkit_config = config
-        self._toolkit_http_client = HTTPClient(config, max_retries=global_config.max_retries)
+        self._toolkit_http_client = HTTPClient(config, max_retries=global_config.max_retries, console=console)
 
-    def create_with_429_retry(self, function: FunctionWrite, console: Console | None = None) -> Function:
+    def create_with_429_retry(self, function: FunctionWrite) -> Function:
         """Create a function with manual retry handling for 429 Too Many Requests responses.
 
         This method is a workaround for scenarios where the function creation API is temporarily unavailable
@@ -43,16 +44,13 @@ class ExtendedFunctionsAPI(FunctionsAPI):
                 endpoint_url=self._toolkit_config.create_api_url("/functions"),
                 method="POST",
                 body_content={"items": [function.dump(camel_case=True)]},
-            ),
-            console=console,
+            )
         )
         result.raise_for_status()
         # We assume the API response is one item on a successful creation
         return Function._load(result.get_first_body()["items"][0], cognite_client=self._cognite_client)  # type: ignore[arg-type,index]
 
-    def delete_with_429_retry(
-        self, external_id: SequenceNotStr[str], ignore_unknown_ids: bool = False, console: Console | None = None
-    ) -> None:
+    def delete_with_429_retry(self, external_id: SequenceNotStr[str], ignore_unknown_ids: bool = False) -> None:
         """Delete one or more functions with retry handling for 429 Too Many Requests responses.
 
         This method is an improvement over the standard delete method in the FunctionsAPI.
@@ -77,7 +75,6 @@ class ExtendedFunctionsAPI(FunctionsAPI):
                     endpoint_url=self._toolkit_config.create_api_url("/functions/delete"),
                     method="POST",
                     body_content=body_content,
-                ),
-                console=console,
+                )
             ).raise_for_status()
         return None

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py

@@ -231,6 +231,8 @@ class AnnotationMigrationIO(
     CHUNK_SIZE = 1000
     UPLOAD_ENDPOINT = InstanceIO.UPLOAD_ENDPOINT
 
+    SUPPORTED_ANNOTATION_TYPES = frozenset({"diagrams.AssetLink", "diagrams.FileLink"})
+
     def __init__(
         self,
         client: ToolkitClient,
@@ -272,6 +274,10 @@ class AnnotationMigrationIO(
         for data_chunk in self.annotation_io.stream_data(asset_centric_selector, limit):
             mapping_list = AssetCentricMappingList[Annotation]([])
             for resource in data_chunk.items:
+                if resource.annotation_type not in self.SUPPORTED_ANNOTATION_TYPES:
+                    # This should not happen, as the annotation_io should already filter these out.
+                    # This is just in case.
+                    continue
                 mapping = AnnotationMapping(
                     instance_id=EdgeId(space=self.instance_space, external_id=f"annotation_{resource.id!r}"),
                     id=resource.id,
@@ -294,11 +300,15 @@ class AnnotationMigrationIO(
             resources = self.client.annotations.retrieve_multiple(current_batch.get_ids())
             resources_by_id = {resource.id: resource for resource in resources}
             not_found = 0
+            incorrect_type_count = 0
             for mapping in current_batch:
                 resource = resources_by_id.get(mapping.id)
                 if resource is None:
                     not_found += 1
                     continue
+                if resource.annotation_type not in self.SUPPORTED_ANNOTATION_TYPES:
+                    incorrect_type_count += 1
+                    continue
                 mapping.ingestion_view = self._get_mapping(mapping.ingestion_view, resource)
                 chunk.append(AssetCentricMapping(mapping=mapping, resource=resource))
             if chunk:
@@ -308,6 +318,11 @@ class AnnotationMigrationIO(
                 MediumSeverityWarning(
                     f"Could not find {not_found} annotations referenced in the CSV file. They will be skipped during migration."
                 ).print_warning(include_timestamp=True, console=self.client.console)
+            if incorrect_type_count:
+                MediumSeverityWarning(
+                    f"Found {incorrect_type_count} annotations with unsupported types. Only 'diagrams.AssetLink' and "
+                    "'diagrams.FileLink' are supported. These annotations will be skipped during migration."
+                ).print_warning(include_timestamp=True, console=self.client.console)
 
     def _get_mapping(self, current_mapping: str | None, resource: Annotation) -> str:
         try:

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/commands/_upload.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from cognite.client.data_classes.data_modeling import (
     ViewId,
 )
-from pydantic import ValidationError
 from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
@@ -20,18 +19,15 @@ from cognite_toolkit._cdf_tk.storageio import (
     get_upload_io,
 )
 from cognite_toolkit._cdf_tk.storageio._base import TableUploadableStorageIO, UploadItem
-from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
+from cognite_toolkit._cdf_tk.storageio.selectors import Selector, load_selector
 from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning, ToolkitWarning
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
-from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
 from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
 from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
 from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
 from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
 from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
-from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._base import ToolkitCommand
 from .deploy import DeployCommand
@@ -141,17 +137,11 @@ class UploadCommand(ToolkitCommand):
         """Finds data files and their corresponding metadata files in the input directory."""
         data_files_by_metadata: dict[Selector, list[Path]] = {}
         for manifest_file in input_dir.glob(f"*{DATA_MANIFEST_SUFFIX}"):
-            selector_dict = read_yaml_file(manifest_file, expected_output="dict")
-            try:
-                selector = SelectorAdapter.validate_python(selector_dict)
-            except ValidationError as e:
-                errors = humanize_validation_error(e)
-                self.warn(
-                    ResourceFormatWarning(
-                        manifest_file, tuple(errors), text="Invalid selector in metadata file, skipping."
-                    )
-                )
+            selector_or_warning = load_selector(manifest_file)
+            if isinstance(selector_or_warning, ToolkitWarning):
+                self.warn(selector_or_warning)
                 continue
+            selector: Selector = selector_or_warning
             data_files = selector.find_data_files(input_dir, manifest_file)
             if not data_files:
                 self.warn(

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/commands/build_cmd.py

@@ -20,7 +20,6 @@ from cognite_toolkit._cdf_tk.constants import (
     _RUNNING_IN_BROWSER,
     BUILD_FOLDER_ENCODING,
     DEFAULT_ENV,
-    DEV_ONLY_MODULES,
     HINT_LEAD_TEXT,
     ROOT_MODULES,
     TEMPLATE_VARS_FILE_SUFFIXES,
@@ -59,10 +58,7 @@ from cognite_toolkit._cdf_tk.data_classes import (
     SourceLocationLazy,
 )
 from cognite_toolkit._cdf_tk.exceptions import (
-    ToolkitDuplicatedModuleError,
-    ToolkitEnvError,
     ToolkitError,
-    ToolkitMissingModuleError,
     ToolkitYAMLFormatError,
 )
 from cognite_toolkit._cdf_tk.hints import Hint, ModuleDefinition, verify_module_directory
@@ -89,6 +85,7 @@ from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
 from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules
 from cognite_toolkit._cdf_tk.validation import (
     validate_data_set_is_set,
+    validate_module_selection,
     validate_modules_variables,
     validate_resource_yaml_pydantic,
 )
@@ -207,7 +204,11 @@ class BuildCommand(ToolkitCommand):
 
         user_selected_modules = config.environment.get_selected_modules(packages)
         modules = ModuleDirectories.load(organization_dir, user_selected_modules)
-        self._validate_modules(modules, config, packages, user_selected_modules, organization_dir)
+        module_warnings = validate_module_selection(modules, config, packages, user_selected_modules, organization_dir)
+        if module_warnings:
+            self.warning_list.extend(module_warnings)
+            if self.print_warning:
+                print(str(module_warnings))
 
         if verbose:
             self.console("Selected packages:")
@@ -422,62 +423,6 @@ class BuildCommand(ToolkitCommand):
         builder = self._builder_by_resource_folder[resource_name]
         return builder
 
-    def _validate_modules(
-        self,
-        modules: ModuleDirectories,
-        config: BuildConfigYAML,
-        packages: dict[str, list[str]],
-        selected_modules: set[str | Path],
-        organization_dir: Path,
-    ) -> None:
-        # Validations: Ambiguous selection.
-        selected_names = {s for s in config.environment.selected if isinstance(s, str)}
-        if duplicate_modules := {
-            module_name: paths
-            for module_name, paths in modules.as_path_by_name().items()
-            if len(paths) > 1 and module_name in selected_names
-        }:
-            # If the user has selected a module by name, and there are multiple modules with that name, raise an error.
-            # Note, if the user uses a path to select a module, this error will not be raised.
-            raise ToolkitDuplicatedModuleError(
-                f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules
-            )
-        # Package Referenced Modules Exists
-        for package, package_modules in packages.items():
-            if package not in selected_names:
-                # We do not check packages that are not selected.
-                # Typically, the user will delete the modules that are irrelevant for them;
-                # thus we only check the selected packages.
-                continue
-            if missing_packages := set(package_modules) - modules.available_names:
-                ToolkitMissingModuleError(
-                    f"Package {package} defined in {CDFToml.file_name!s} is referring "
-                    f"the following missing modules {missing_packages}."
-                )
-
-        # Selected modules does not exists
-        if missing_modules := set(selected_modules) - modules.available:
-            hint = ModuleDefinition.long(missing_modules, organization_dir)
-            raise ToolkitMissingModuleError(
-                f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}"
-            )
-
-        # Nothing is Selected
-        if not modules.selected:
-            raise ToolkitEnvError(
-                f"No selected modules specified in {config.filepath!s}, have you configured "
-                f"the environment ({config.environment.name})?"
-            )
-
-        dev_modules = modules.available_names & DEV_ONLY_MODULES
-        if dev_modules and config.environment.validation_type != "dev":
-            self.warn(
-                MediumSeverityWarning(
-                    "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): "
-                    f"{humanize_collection(dev_modules)!r}",
-                )
-            )
-
     def _replace_variables(
         self,
         resource_files: Sequence[Path],

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py

@@ -320,7 +320,7 @@ class FunctionCRUD(ResourceCRUD[str, FunctionWrite, Function]):
                     " problem persists, please contact Cognite support."
                 )
             item.file_id = file_id
-            created_item = self.client.functions.create_with_429_retry(item, console=self.console)
+            created_item = self.client.functions.create_with_429_retry(item)
             self._warn_if_cpu_or_memory_changed(created_item, item)
             created.append(created_item)
         return created
@@ -404,7 +404,7 @@ class FunctionCRUD(ResourceCRUD[str, FunctionWrite, Function]):
     def delete(self, ids: SequenceNotStr[str]) -> int:
         functions = self.retrieve(ids)
 
-        self.client.functions.delete_with_429_retry(external_id=ids, ignore_unknown_ids=True, console=self.console)
+        self.client.functions.delete_with_429_retry(external_id=ids, ignore_unknown_ids=True)
         file_ids = {func.file_id for func in functions if func.file_id}
         self.client.files.delete(id=list(file_ids), ignore_unknown_ids=True)
         return len(ids)

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/feature_flags.py

@@ -57,6 +57,10 @@ class Flags(Enum):
         visible=True,
         description="Enables extended download to support downloading file content and datapoints",
     )
+    EXTEND_UPLOAD = FlagMetadata(
+        visible=True,
+        description="Enables extended upload to support uploading individual files",
+    )
 
     def is_enabled(self) -> bool:
         return FeatureFlag.is_enabled(self)

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/storageio/_annotations.py

@@ -33,7 +33,8 @@ class AnnotationIO(StorageIO[AssetCentricSelector, Annotation]):
                         annotated_resource_type="file",
                         annotated_resource_ids=[{"id": file_metadata.id} for file_metadata in file_chunk.items],
                         annotation_type=annotation_type,
-                    )
+                    ),
+                    limit=-1,
                 )
                 if limit is not None and total + len(results) > limit:
                     results = results[: limit - total]

{cognite_toolkit-0.7.28 → cognite_toolkit-0.7.30}/cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py

@@ -1,6 +1,13 @@
+from pathlib import Path
 from typing import Annotated
 
-from pydantic import Field, TypeAdapter
+from pydantic import Field, TypeAdapter, ValidationError
+
+from cognite_toolkit._cdf_tk.feature_flags import Flags
+from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning, ToolkitWarning
+from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
+from cognite_toolkit._cdf_tk.utils.file import read_yaml_file
+from cognite_toolkit._cdf_tk.validation import humanize_validation_error
 
 from ._asset_centric import AssetCentricFileSelector, AssetCentricSelector, AssetSubtreeSelector, DataSetSelector
 from ._base import DataSelector
@@ -52,9 +59,33 @@ Selector = Annotated[
     Field(discriminator="type"),
 ]
 
+ALPHA_SELECTORS = {FileIdentifierSelector}
+
 SelectorAdapter: TypeAdapter[Selector] = TypeAdapter(Selector)
 
 
+def load_selector(manifest_file: Path) -> Selector | ToolkitWarning:
+    """Loads a selector from a manifest file.
+
+    Args:
+        manifest_file: Path to the manifest file.
+
+    Returns:
+        A selector object or a toolkit warning if loading fails or the selector is an alpha feature that is not enabled.
+    """
+    selector_dict = read_yaml_file(manifest_file, expected_output="dict")
+    try:
+        selector = SelectorAdapter.validate_python(selector_dict)
+    except ValidationError as e:
+        errors = humanize_validation_error(e)
+        return ResourceFormatWarning(manifest_file, tuple(errors), text="Invalid selector in metadata file, skipping.")
+    if not Flags.EXTEND_UPLOAD.is_enabled() and type(selector) in ALPHA_SELECTORS:
+        return MediumSeverityWarning(
+            f"Selector type '{type(selector).__name__}' in file '{manifest_file}' is in alpha. To enable it set the alpha flag 'extend-upload = true' in your CDF.toml file."
+        )
+    return selector
+
+
 __all__ = [
     "AllChartsSelector",
     "AssetCentricFileSelector",
@@ -89,4 +120,5 @@ __all__ = [
     "Selector",
     "SelectorAdapter",
     "TimeSeriesColumn",
+    "load_selector",
 ]