cognite-toolkit 0.7.60__tar.gz → 0.7.62__tar.gz

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.60
+Version: 0.7.62
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -313,6 +313,14 @@ class MigrateApp(typer.Typer):
                 help="If set, the migration will not be executed, but only a report of what would be done is printed.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="(Only used with mapping-file) If set, no confirmation prompt will be shown before proceeding with the migration.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -331,6 +339,7 @@ class MigrateApp(typer.Typer):
             consumption_view=consumption_view,
             ingestion_mapping=ingestion_mapping,
             dry_run=dry_run,
+            auto_yes=auto_yes,
             verbose=verbose,
             kind="Assets",
             resource_type="asset",
@@ -356,6 +365,7 @@ class MigrateApp(typer.Typer):
         data_set_id: str | None,
         consumption_view: str | None,
         ingestion_mapping: str | None,
+        auto_yes: bool,
         dry_run: bool,
         verbose: bool,
         kind: AssetCentricKind,
@@ -365,7 +375,19 @@ class MigrateApp(typer.Typer):
         if data_set_id is not None and mapping_file is not None:
             raise typer.BadParameter("Cannot specify both data_set_id and mapping_file")
         elif mapping_file is not None:
-            selected: AssetCentricMigrationSelector = MigrationCSVFileSelector(datafile=mapping_file, kind=kind)
+            file_selector = MigrationCSVFileSelector(datafile=mapping_file, kind=kind)
+            selected: AssetCentricMigrationSelector = file_selector
+
+            panel = file_selector.items.print_status()
+            if panel is not None:
+                client.console.print(panel)
+                if not auto_yes:
+                    proceed = questionary.confirm(
+                        "Do you want to proceed with the migration?", default=False
+                    ).unsafe_ask()
+                    if not proceed:
+                        client.console.print("Migration aborted by user.")
+                        raise typer.Abort()
         elif data_set_id is not None:
             parsed_view = parse_view_str(consumption_view) if consumption_view is not None else None
             selected = MigrateDataSetSelector(
@@ -460,6 +482,14 @@ class MigrateApp(typer.Typer):
                 help="If set, the migration will not be executed, but only a report of what would be done is printed.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="(Only used with mapping-file) If set, no confirmation prompt will be shown before proceeding with the migration.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -478,6 +508,7 @@ class MigrateApp(typer.Typer):
             consumption_view=consumption_view,
             ingestion_mapping=ingestion_mapping,
             dry_run=dry_run,
+            auto_yes=auto_yes,
             verbose=verbose,
             kind="Events",
             resource_type="event",
@@ -563,6 +594,14 @@ class MigrateApp(typer.Typer):
                 help="If set, the migration will not be executed, but only a report of what would be done is printed.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="(Only used with mapping-file) If set, no confirmation prompt will be shown before proceeding with the migration.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -582,6 +621,7 @@ class MigrateApp(typer.Typer):
             consumption_view=consumption_view,
             ingestion_mapping=ingestion_mapping,
             dry_run=dry_run,
+            auto_yes=auto_yes,
             verbose=verbose,
             kind="TimeSeries",
             resource_type="timeseries",
@@ -670,6 +710,14 @@ class MigrateApp(typer.Typer):
                 help="If set, the migration will not be executed, but only a report of what would be done is printed.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="(Only used with mapping-file) If set, no confirmation prompt will be shown before proceeding with the migration.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -689,6 +737,7 @@ class MigrateApp(typer.Typer):
             consumption_view=consumption_view,
             ingestion_mapping=ingestion_mapping,
             dry_run=dry_run,
+            auto_yes=auto_yes,
             verbose=verbose,
             kind="FileMetadata",
             resource_type="file",
@@ -696,7 +745,7 @@ class MigrateApp(typer.Typer):
         )
         cmd = MigrationCommand(client=client)
 
-        if data_set_id is None:
+        if data_set_id is None and mapping_file is None:
             skip_linking = not questionary.confirm(
                 "Do you want to link old and new Files?", default=not skip_linking
             ).unsafe_ask()

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py

@@ -137,11 +137,19 @@ class DirectRelationCache:
         if source_ids:
             # SourceSystems are not cached in the client, so we have to handle the caching ourselves.
             cache = cast(dict[str, DirectRelationReference], self._cache_map[self.TableName.SOURCE_NAME])
-            missing = set(source_ids) - set(cache.keys())
+            missing: dict[str, str] = {}
+            for source_id in source_ids:
+                if source_id.casefold() not in cache:
+                    missing[source_id.casefold()] = source_id
+                elif source_id not in cache:
+                    missing[source_id] = source_id
             if missing:
                 source_systems = self._client.migration.created_source_system.retrieve(list(missing))
                 for source_system in source_systems:
-                    cache[source_system.source] = source_system.as_direct_relation_reference()
+                    source_reference = source_system.as_direct_relation_reference()
+                    cache[source_system.source] = source_reference
+                    if original_str := missing.get(source_system.source):
+                        cache[original_str] = source_reference
         if file_ids:
             self._update_cache(self._client.migration.lookup.files(id=list(file_ids)), self.TableName.FILE_ID)
         if asset_external_ids:

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py

@@ -10,6 +10,8 @@ from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, Nod
 from cognite.client.utils._identifier import InstanceId
 from cognite.client.utils._text import to_camel_case
 from pydantic import BaseModel, BeforeValidator, Field, field_validator, model_validator
+from rich.panel import Panel
+from rich.text import Text
 
 from cognite_toolkit._cdf_tk.client._resource_base import BaseModelObject, RequestResource
 from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import InternalId
@@ -24,6 +26,7 @@ from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import (
 )
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
+from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types import (
     AssetCentricKindExtended,
     JsonVal,
@@ -155,10 +158,46 @@ class MigrationMappingList(ModelList[MigrationMapping]):
         }
         if resource_type not in cls_by_resource_type:
             raise ToolkitValueError(
-                f"Invalid resource type '{resource_type}'. Must be one of 'asset', 'timeseries', or 'file'."
+                f"Invalid resource type '{resource_type}'. Must be one of {humanize_collection(cls_by_resource_type.keys())}."
             )
         return cls_by_resource_type[resource_type].read_csv_file(filepath, resource_type=None)
 
+    def print_status(self) -> Panel | None:
+        if not self:
+            return None
+        first = self[0]
+        resource_type = first.resource_type
+
+        text = Text()
+        text.append(f"Migrating {len(self)} {resource_type}", style="bold")
+        if "ingestionView" in self.columns:
+            text.append("\n[green]Mapping column set[/green]")
+        else:
+            text.append(
+                "\n[WARNING] 'ingestionView' column not set in CSV file. This is NOT recommended. "
+                f"All {resource_type}s will be ingested into CogniteCore. If you want to ingest the {resource_type}s "
+                f"into your own data modeling views, please add an 'ingestionView' column to the CSV file.",
+                style="red",
+            )
+        if "consumerViewSpace" in self.columns and "consumerViewExternalId" in self.columns:
+            consumer_columns = ["consumerViewSpace", "consumerViewExternalId"]
+            if "consumerViewVersion" in self.columns:
+                consumer_columns.append("consumerViewVersion")
+            text.append(
+                "\nPreferred consumer views specified "
+                f"for the mappings using the {humanize_collection(consumer_columns)} columns.",
+                style="green",
+            )
+        else:
+            text.append(
+                "\n[WARNING] Consumer views have not been specified for the instances. "
+                f"This is NOT recommended as this is used to determine which view to use when migrating the {resource_type}s in applications like Canvas. "
+                "To specify preferred consumer views, add 'consumerViewSpace', 'consumerViewExternalId', and optionally 'consumerViewVersion' columns to the CSV file.",
+                style="red",
+            )
+
+        return Panel(text, title="Ready for migration", expand=False)
+
 
 def _validate_node_id(value: Any) -> Any:
     if isinstance(value, dict):

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/cognite_toolkit/_cdf_tk/storageio/_data_classes.py

@@ -26,8 +26,10 @@ class ModelList(UserList[T_BaseModel], ABC):
         collection: Collection[T_BaseModel] | None = None,
         invalid_rows: dict[int, ResourceFormatWarning] | None = None,
         unexpected_columns: set[str] | None = None,
+        columns: set[str] | None = None,
     ) -> None:
         super().__init__(collection or [])
+        self.columns = columns or set()
         self.invalid_rows = invalid_rows or {}
         self.unexpected_columns = unexpected_columns or set()
 
@@ -53,7 +55,8 @@ class ModelList(UserList[T_BaseModel], ABC):
         # We only need to read one row to get the header
         schema = CSVReader.sniff_schema(filepath, sniff_rows=1)
         cls._validate_schema(schema)
-        unexpected_columns = {col.name for col in schema} - cls._required_header_names() - cls._optional_header_names()
+        csv_file_columns = {col.name for col in schema}
+        unexpected_columns = csv_file_columns - cls._required_header_names() - cls._optional_header_names()
         reader = CSVReader(input_file=filepath)
         items: list[T_BaseModel] = []
         invalid_rows: dict[int, ResourceFormatWarning] = {}
@@ -67,7 +70,7 @@ class ModelList(UserList[T_BaseModel], ABC):
             else:
                 raise TypeError(f"Unexpected result type: {type(result)}")
 
-        return cls(items, invalid_rows, unexpected_columns)
+        return cls(items, invalid_rows, unexpected_columns, columns=csv_file_columns)
 
     @classmethod
     def _validate_schema(cls, schema: list[SchemaColumn]) -> None:

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.60
+      image: cognite/toolkit:0.7.62
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.60
+      image: cognite/toolkit:0.7.62
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.60"
+version = "0.7.62"
 
 
 [plugins]

cognite_toolkit-0.7.62/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.7.62"

{cognite_toolkit-0.7.60 → cognite_toolkit-0.7.62}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.7.60"
+version = "0.7.62"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

cognite_toolkit-0.7.60/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.7.60"