cognite-toolkit 0.7.59__tar.gz → 0.7.61__tar.gz

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.59
+Version: 0.7.61
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/cognite_toolkit/_cdf_tk/apps/_dump_app.py

@@ -17,6 +17,7 @@ from cognite_toolkit._cdf_tk.commands.dump_resource import (
     GroupFinder,
     LocationFilterFinder,
     NodeFinder,
+    ResourceViewMappingFinder,
     SearchConfigFinder,
     SpaceFinder,
     StreamlitFinder,
@@ -52,6 +53,8 @@ class DumpApp(typer.Typer):
         self.command("agents")(DumpConfigApp.dump_agents)
 
         self.command("search-config")(DumpConfigApp.dump_search_config)
+        if Flags.MIGRATE.is_enabled():
+            self.command("resource-view-mapping")(DumpConfigApp.dump_resource_view_mapping)
 
     @staticmethod
     def dump_main(ctx: typer.Context) -> None:
@@ -79,6 +82,8 @@ class DumpConfigApp(typer.Typer):
         self.command("streamlit")(DumpConfigApp.dump_streamlit)
         self.command("agents")(self.dump_agents)
         self.command("search-config")(self.dump_search_config)
+        if Flags.MIGRATE.is_enabled():
+            self.command("resource-view-mapping")(self.dump_resource_view_mapping)
 
     @staticmethod
     def dump_config_main(ctx: typer.Context) -> None:
@@ -758,6 +763,54 @@ class DumpConfigApp(typer.Typer):
             )
         )
 
+    @staticmethod
+    def dump_resource_view_mapping(
+        ctx: typer.Context,
+        external_id: Annotated[
+            list[str] | None,
+            typer.Argument(
+                help="The external ID(s) of the resource view mapping(s) you want to dump. "
+                "If nothing is provided, an interactive prompt will be shown to select the resource view mappings.",
+            ),
+        ] = None,
+        output_dir: Annotated[
+            Path,
+            typer.Option(
+                "--output-dir",
+                "-o",
+                help="Where to dump the resource view mapping files.",
+                allow_dash=True,
+            ),
+        ] = Path("tmp"),
+        clean: Annotated[
+            bool,
+            typer.Option(
+                "--clean",
+                "-c",
+                help="Delete the output directory before dumping the resource view mappings.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """[MIGRATION] This command will dump the selected resource view mapping(s) as yaml to the folder specified, defaults to /tmp."""
+        client = EnvironmentVariables.create_from_environment().get_client()
+        cmd = DumpResourceCommand(client=client)
+        cmd.run(
+            lambda: cmd.dump_to_yamls(
+                ResourceViewMappingFinder(client, tuple(external_id) if external_id else None),
+                output_dir=output_dir,
+                clean=clean,
+                verbose=verbose,
+            )
+        )
+
 
 class DumpDataApp(typer.Typer):
     def __init__(self, *args: Any, **kwargs: Any) -> None:

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -313,6 +313,14 @@ class MigrateApp(typer.Typer):
                 help="If set, the migration will not be executed, but only a report of what would be done is printed.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="(Only used with mapping-file) If set, no confirmation prompt will be shown before proceeding with the migration.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -331,6 +339,7 @@ class MigrateApp(typer.Typer):
             consumption_view=consumption_view,
             ingestion_mapping=ingestion_mapping,
             dry_run=dry_run,
+            auto_yes=auto_yes,
             verbose=verbose,
             kind="Assets",
             resource_type="asset",
@@ -356,6 +365,7 @@ class MigrateApp(typer.Typer):
         data_set_id: str | None,
         consumption_view: str | None,
         ingestion_mapping: str | None,
+        auto_yes: bool,
         dry_run: bool,
         verbose: bool,
         kind: AssetCentricKind,
@@ -365,7 +375,19 @@ class MigrateApp(typer.Typer):
         if data_set_id is not None and mapping_file is not None:
             raise typer.BadParameter("Cannot specify both data_set_id and mapping_file")
         elif mapping_file is not None:
-            selected: AssetCentricMigrationSelector = MigrationCSVFileSelector(datafile=mapping_file, kind=kind)
+            file_selector = MigrationCSVFileSelector(datafile=mapping_file, kind=kind)
+            selected: AssetCentricMigrationSelector = file_selector
+
+            panel = file_selector.items.print_status()
+            if panel is not None:
+                client.console.print(panel)
+                if not auto_yes:
+                    proceed = questionary.confirm(
+                        "Do you want to proceed with the migration?", default=False
+                    ).unsafe_ask()
+                    if not proceed:
+                        client.console.print("Migration aborted by user.")
+                        raise typer.Abort()
         elif data_set_id is not None:
             parsed_view = parse_view_str(consumption_view) if consumption_view is not None else None
             selected = MigrateDataSetSelector(
@@ -460,6 +482,14 @@ class MigrateApp(typer.Typer):
                 help="If set, the migration will not be executed, but only a report of what would be done is printed.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="(Only used with mapping-file) If set, no confirmation prompt will be shown before proceeding with the migration.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -478,6 +508,7 @@ class MigrateApp(typer.Typer):
             consumption_view=consumption_view,
             ingestion_mapping=ingestion_mapping,
             dry_run=dry_run,
+            auto_yes=auto_yes,
             verbose=verbose,
             kind="Events",
             resource_type="event",
@@ -563,6 +594,14 @@ class MigrateApp(typer.Typer):
                 help="If set, the migration will not be executed, but only a report of what would be done is printed.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="(Only used with mapping-file) If set, no confirmation prompt will be shown before proceeding with the migration.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -582,6 +621,7 @@ class MigrateApp(typer.Typer):
             consumption_view=consumption_view,
             ingestion_mapping=ingestion_mapping,
             dry_run=dry_run,
+            auto_yes=auto_yes,
             verbose=verbose,
             kind="TimeSeries",
             resource_type="timeseries",
@@ -670,6 +710,14 @@ class MigrateApp(typer.Typer):
                 help="If set, the migration will not be executed, but only a report of what would be done is printed.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="(Only used with mapping-file) If set, no confirmation prompt will be shown before proceeding with the migration.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -689,6 +737,7 @@ class MigrateApp(typer.Typer):
             consumption_view=consumption_view,
             ingestion_mapping=ingestion_mapping,
             dry_run=dry_run,
+            auto_yes=auto_yes,
             verbose=verbose,
             kind="FileMetadata",
             resource_type="file",

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py

@@ -10,6 +10,8 @@ from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, Nod
 from cognite.client.utils._identifier import InstanceId
 from cognite.client.utils._text import to_camel_case
 from pydantic import BaseModel, BeforeValidator, Field, field_validator, model_validator
+from rich.panel import Panel
+from rich.text import Text
 
 from cognite_toolkit._cdf_tk.client._resource_base import BaseModelObject, RequestResource
 from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import InternalId
@@ -24,6 +26,7 @@ from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import (
 )
 from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
 from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
+from cognite_toolkit._cdf_tk.utils import humanize_collection
 from cognite_toolkit._cdf_tk.utils.useful_types import (
     AssetCentricKindExtended,
     JsonVal,
@@ -155,10 +158,46 @@ class MigrationMappingList(ModelList[MigrationMapping]):
         }
         if resource_type not in cls_by_resource_type:
             raise ToolkitValueError(
-                f"Invalid resource type '{resource_type}'. Must be one of 'asset', 'timeseries', or 'file'."
+                f"Invalid resource type '{resource_type}'. Must be one of {humanize_collection(cls_by_resource_type.keys())}."
             )
         return cls_by_resource_type[resource_type].read_csv_file(filepath, resource_type=None)
 
+    def print_status(self) -> Panel | None:
+        if not self:
+            return None
+        first = self[0]
+        resource_type = first.resource_type
+
+        text = Text()
+        text.append(f"Migrating {len(self)} {resource_type}", style="bold")
+        if "ingestionView" in self.columns:
+            text.append("\n[green]Mapping column set[/green]")
+        else:
+            text.append(
+                "\n[WARNING] 'ingestionView' column not set in CSV file. This is NOT recommended. "
+                f"All {resource_type}s will be ingested into CogniteCore. If you want to ingest the {resource_type}s "
+                f"into your own data modeling views, please add an 'ingestionView' column to the CSV file.",
+                style="red",
+            )
+        if "consumerViewSpace" in self.columns and "consumerViewExternalId" in self.columns:
+            consumer_columns = ["consumerViewSpace", "consumerViewExternalId"]
+            if "consumerViewVersion" in self.columns:
+                consumer_columns.append("consumerViewVersion")
+            text.append(
+                "\nPreferred consumer views specified "
+                f"for the mappings using the {humanize_collection(consumer_columns)} columns.",
+                style="green",
+            )
+        else:
+            text.append(
+                "\n[WARNING] Consumer views have not been specified for the instances. "
+                f"This is NOT recommended as this is used to determine which view to use when migrating the {resource_type}s in applications like Canvas. "
+                "To specify preferred consumer views, add 'consumerViewSpace', 'consumerViewExternalId', and optionally 'consumerViewVersion' columns to the CSV file.",
+                style="red",
+            )
+
+        return Panel(text, title="Ready for migration", expand=False)
+
 
 def _validate_node_id(value: Any) -> Any:
     if isinstance(value, dict):

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/cognite_toolkit/_cdf_tk/commands/dump_resource.py

@@ -28,7 +28,7 @@ from cognite.client.data_classes._base import (
 from cognite.client.data_classes.agents import (
     AgentList,
 )
-from cognite.client.data_classes.data_modeling import DataModelId
+from cognite.client.data_classes.data_modeling import DataModelId, NodeList
 from cognite.client.data_classes.documents import SourceFileProperty
 from cognite.client.data_classes.extractionpipelines import ExtractionPipelineConfigList
 from cognite.client.data_classes.functions import (
@@ -53,6 +53,7 @@ from rich.panel import Panel
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient
 from cognite_toolkit._cdf_tk.client.resource_classes.legacy.location_filters import LocationFilterList
+from cognite_toolkit._cdf_tk.client.resource_classes.legacy.migration import ResourceViewMapping
 from cognite_toolkit._cdf_tk.client.resource_classes.legacy.search_config import SearchConfigList
 from cognite_toolkit._cdf_tk.client.resource_classes.legacy.search_config import ViewId as SearchConfigViewId
 from cognite_toolkit._cdf_tk.client.resource_classes.legacy.streamlit_ import Streamlit, StreamlitList
@@ -69,6 +70,7 @@ from cognite_toolkit._cdf_tk.cruds import (
     LocationFilterCRUD,
     NodeCRUD,
     ResourceCRUD,
+    ResourceViewMappingCRUD,
     SearchConfigCRUD,
     SpaceCRUD,
     StreamlitCRUD,
@@ -797,6 +799,44 @@ class SearchConfigFinder(ResourceFinder[tuple[SearchConfigViewId, ...]]):
             yield list(self.identifier), None, loader, None
 
 
+class ResourceViewMappingFinder(ResourceFinder[tuple[str, ...]]):
+    def __init__(self, client: ToolkitClient, identifier: tuple[str, ...] | None = None):
+        super().__init__(client, identifier)
+        self.resource_view_mappings: list[ResourceViewMapping] | None = None
+
+    def _interactive_select(self) -> tuple[str, ...]:
+        mappings = self.client.migration.resource_view_mapping.list(limit=-1)
+        if not mappings:
+            raise ToolkitMissingResourceError("No resource view mappings found")
+        self.resource_view_mappings = list(mappings)
+        choices = [
+            Choice(
+                f"{mapping.external_id} ({mapping.resource_type} -> {mapping.view_id.external_id})",
+                value=mapping.external_id,
+            )
+            for mapping in sorted(mappings, key=lambda m: m.external_id)
+        ]
+        selected_ids: list[str] | None = questionary.checkbox(
+            "Which resource view mapping(s) would you like to dump?",
+            choices=choices,
+            validate=lambda choices: True if choices else "You must select at least one resource view mapping.",
+        ).unsafe_ask()
+        if not selected_ids:
+            raise ToolkitValueError(f"No resource view mappings selected for dumping.{_INTERACTIVE_SELECT_HELPER_TEXT}")
+        return tuple(selected_ids)
+
+    def __iter__(self) -> Iterator[tuple[list[Hashable], CogniteResourceList | None, ResourceCRUD, None | str]]:
+        self.identifier = self._selected()
+        loader = ResourceViewMappingCRUD.create_loader(self.client)
+        if self.resource_view_mappings:
+            selected_mappings = NodeList[ResourceViewMapping](
+                [m for m in self.resource_view_mappings if m.external_id in self.identifier]
+            )
+            yield [], selected_mappings, loader, None
+        else:
+            yield list(self.identifier), None, loader, None
+
+
 class DumpResourceCommand(ToolkitCommand):
     def dump_to_yamls(
         self,

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/cognite_toolkit/_cdf_tk/storageio/_data_classes.py

@@ -26,8 +26,10 @@ class ModelList(UserList[T_BaseModel], ABC):
         collection: Collection[T_BaseModel] | None = None,
         invalid_rows: dict[int, ResourceFormatWarning] | None = None,
         unexpected_columns: set[str] | None = None,
+        columns: set[str] | None = None,
     ) -> None:
         super().__init__(collection or [])
+        self.columns = columns or set()
         self.invalid_rows = invalid_rows or {}
         self.unexpected_columns = unexpected_columns or set()
 
@@ -53,7 +55,8 @@ class ModelList(UserList[T_BaseModel], ABC):
         # We only need to read one row to get the header
         schema = CSVReader.sniff_schema(filepath, sniff_rows=1)
         cls._validate_schema(schema)
-        unexpected_columns = {col.name for col in schema} - cls._required_header_names() - cls._optional_header_names()
+        csv_file_columns = {col.name for col in schema}
+        unexpected_columns = csv_file_columns - cls._required_header_names() - cls._optional_header_names()
         reader = CSVReader(input_file=filepath)
         items: list[T_BaseModel] = []
         invalid_rows: dict[int, ResourceFormatWarning] = {}
@@ -67,7 +70,7 @@ class ModelList(UserList[T_BaseModel], ABC):
             else:
                 raise TypeError(f"Unexpected result type: {type(result)}")
 
-        return cls(items, invalid_rows, unexpected_columns)
+        return cls(items, invalid_rows, unexpected_columns, columns=csv_file_columns)
 
     @classmethod
     def _validate_schema(cls, schema: list[SchemaColumn]) -> None:

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.7.59
+      image: cognite/toolkit:0.7.61
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.7.59
+      image: cognite/toolkit:0.7.61
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/cognite_toolkit/_resources/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf modules upgrade' command.
-version = "0.7.59"
+version = "0.7.61"
 
 
 [plugins]

cognite_toolkit-0.7.61/cognite_toolkit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.7.61"

{cognite_toolkit-0.7.59 → cognite_toolkit-0.7.61}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.7.59"
+version = "0.7.61"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

cognite_toolkit-0.7.59/cognite_toolkit/_version.py

@@ -1 +0,0 @@
-__version__ = "0.7.59"