cognite-toolkit 0.7.32__tar.gz → 0.7.34__tar.gz

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite_toolkit
-Version: 0.7.32
+Version: 0.7.34
 Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
 Author: Cognite AS
 Author-email: Cognite AS <support@cognite.com>

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/cognite_toolkit/_cdf_tk/apps/_migrate_app.py

@@ -15,10 +15,16 @@ from cognite_toolkit._cdf_tk.commands._migrate.creators import (
     InstanceSpaceCreator,
     SourceSystemCreator,
 )
-from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper, CanvasMapper, ChartMapper
+from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import (
+    AssetCentricMapper,
+    CanvasMapper,
+    ChartMapper,
+    ThreeDMapper,
+)
 from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
     AnnotationMigrationIO,
     AssetCentricMigrationIO,
+    ThreeDMigrationIO,
 )
 from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
     AssetCentricMigrationSelector,
@@ -26,7 +32,11 @@ from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
     MigrationCSVFileSelector,
 )
 from cognite_toolkit._cdf_tk.storageio import CanvasIO, ChartIO
-from cognite_toolkit._cdf_tk.storageio.selectors import CanvasExternalIdSelector, ChartExternalIdSelector
+from cognite_toolkit._cdf_tk.storageio.selectors import (
+    CanvasExternalIdSelector,
+    ChartExternalIdSelector,
+    ThreeDModelIdSelector,
+)
 from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
 from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
 from cognite_toolkit._cdf_tk.utils.interactive_select import (
@@ -36,6 +46,7 @@ from cognite_toolkit._cdf_tk.utils.interactive_select import (
     InteractiveCanvasSelect,
     InteractiveChartSelect,
     ResourceViewMappingInteractiveSelect,
+    ThreeDInteractiveSelect,
 )
 from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind
 
@@ -56,6 +67,7 @@ class MigrateApp(typer.Typer):
         self.command("annotations")(self.annotations)
         self.command("canvas")(self.canvas)
         self.command("charts")(self.charts)
+        self.command("3d")(self.three_d)
         # Uncomment when infield v2 config migration is ready
         # self.command("infield-configs")(self.infield_configs)
 
@@ -980,6 +992,68 @@ class MigrateApp(typer.Typer):
             )
         )
 
+    @staticmethod
+    def three_d(
+        ctx: typer.Context,
+        id: Annotated[
+            list[int] | None,
+            typer.Argument(
+                help="The ID of the 3D Model to migrate. If not provided, an interactive selection will be "
+                "performed to select the 3D Models to migrate."
+            ),
+        ] = None,
+        log_dir: Annotated[
+            Path,
+            typer.Option(
+                "--log-dir",
+                "-l",
+                help="Path to the directory where migration logs will be stored.",
+            ),
+        ] = Path(f"migration_logs_{TODAY}"),
+        dry_run: Annotated[
+            bool,
+            typer.Option(
+                "--dry-run",
+                "-d",
+                help="If set, the migration will not be executed, but only a report of "
+                "what would be done is printed. This is useful for checking that all resources referenced by the 3D Models "
+                "have been migrated to the new data modeling resources in CDF.",
+            ),
+        ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
+    ) -> None:
+        """Migrate 3D Models from Asset-Centric to data modeling in CDF.
+
+        This command expects that the CogniteMigration data model is already deployed, and that the Mapping view
+        is populated with the mapping from Asset-Centric resources to the new data modeling resources.
+        """
+        client = EnvironmentVariables.create_from_environment().get_client()
+        selected_ids: list[int]
+        if id:
+            selected_ids = id
+        else:
+            selected_models = ThreeDInteractiveSelect(client, "migrate").select_three_d_models("classic")
+            selected_ids = [model.id for model in selected_models]
+
+        cmd = MigrationCommand()
+        cmd.run(
+            lambda: cmd.migrate(
+                selected=ThreeDModelIdSelector(ids=tuple(selected_ids)),
+                data=ThreeDMigrationIO(client),
+                mapper=ThreeDMapper(client),
+                log_dir=log_dir,
+                dry_run=dry_run,
+                verbose=verbose,
+            )
+        )
+
     @staticmethod
     def infield_configs(
         ctx: typer.Context,

cognite_toolkit-0.7.34/cognite_toolkit/_cdf_tk/client/api/three_d.py

@@ -0,0 +1,131 @@
+from collections.abc import Sequence
+
+from rich.console import Console
+
+from cognite_toolkit._cdf_tk.client.data_classes.api_classes import PagedResponse
+from cognite_toolkit._cdf_tk.client.data_classes.three_d import ThreeDModelClassicRequest, ThreeDModelResponse
+from cognite_toolkit._cdf_tk.utils.http_client import (
+    HTTPClient,
+    ItemsRequest,
+    RequestMessage2,
+    SimpleBodyRequest,
+)
+from cognite_toolkit._cdf_tk.utils.useful_types import PrimitiveType
+
+
+class ThreeDModelAPI:
+    ENDPOINT = "/3d/models"
+    MAX_CLASSIC_MODELS_PER_CREATE_REQUEST = 1000
+    MAX_MODELS_PER_DELETE_REQUEST = 1000
+    _LIST_REQUEST_MAX_LIMIT = 1000
+
+    def __init__(self, http_client: HTTPClient, console: Console) -> None:
+        self._http_client = http_client
+        self._console = console
+        self._config = http_client.config
+
+    def create(self, models: Sequence[ThreeDModelClassicRequest]) -> list[ThreeDModelResponse]:
+        """Create 3D models in classic format.
+
+        Args:
+            models (Sequence[ThreeDModelClassicRequest]): The 3D model(s) to create.
+
+        Returns:
+            list[ThreeDModelResponse]: The created 3D model(s).
+        """
+        if not models:
+            return []
+        if len(models) > self.MAX_CLASSIC_MODELS_PER_CREATE_REQUEST:
+            raise ValueError("Cannot create more than 1000 3D models in a single request.")
+        responses = self._http_client.request_with_retries(
+            ItemsRequest(
+                endpoint_url=self._config.create_api_url(self.ENDPOINT),
+                method="POST",
+                items=list(models),
+            )
+        )
+        responses.raise_for_status()
+        body = responses.get_first_body()
+        return PagedResponse[ThreeDModelResponse].model_validate(body).items
+
+    def delete(self, ids: Sequence[int]) -> None:
+        """Delete 3D models by their IDs.
+
+        Args:
+            ids (Sequence[int]): The IDs of the 3D models to delete.
+        """
+        if not ids:
+            return None
+        if len(ids) > self.MAX_MODELS_PER_DELETE_REQUEST:
+            raise ValueError("Cannot delete more than 1000 3D models in a single request.")
+        responses = self._http_client.request_with_retries(
+            SimpleBodyRequest(
+                endpoint_url=self._config.create_api_url(self.ENDPOINT + "/delete"),
+                method="POST",
+                body_content={"items": [{"id": id_} for id_ in ids]},
+            )
+        )
+        responses.raise_for_status()
+
+    def iterate(
+        self,
+        published: bool | None = None,
+        include_revision_info: bool = False,
+        limit: int = 100,
+        cursor: str | None = None,
+    ) -> PagedResponse[ThreeDModelResponse]:
+        if not (0 < limit <= self._LIST_REQUEST_MAX_LIMIT):
+            raise ValueError(f"Limit must be between 1 and {self._LIST_REQUEST_MAX_LIMIT}, got {limit}.")
+        parameters: dict[str, PrimitiveType] = {
+            # There is a bug in the API. The parameter includeRevisionInfo is expected to be lower case and not
+            # camel case as documented. You get error message: Unrecognized query parameter includeRevisionInfo,
+            # did you mean includerevisioninfo?
+            "includerevisioninfo": include_revision_info,
+            "limit": limit,
+        }
+        if published is not None:
+            parameters["published"] = published
+        if cursor is not None:
+            parameters["cursor"] = cursor
+        responses = self._http_client.request_single_retries(
+            RequestMessage2(
+                endpoint_url=self._config.create_api_url(self.ENDPOINT),
+                method="GET",
+                parameters=parameters,
+            )
+        )
+        success_response = responses.get_success_or_raise()
+        return PagedResponse[ThreeDModelResponse].model_validate(success_response.body_json)
+
+    def list(
+        self,
+        published: bool | None = None,
+        include_revision_info: bool = False,
+        limit: int | None = 100,
+        cursor: str | None = None,
+    ) -> list[ThreeDModelResponse]:
+        results: list[ThreeDModelResponse] = []
+        while True:
+            request_limit = (
+                self._LIST_REQUEST_MAX_LIMIT
+                if limit is None
+                else min(limit - len(results), self._LIST_REQUEST_MAX_LIMIT)
+            )
+            if request_limit <= 0:
+                break
+            page = self.iterate(
+                published=published,
+                include_revision_info=include_revision_info,
+                limit=request_limit,
+                cursor=cursor,
+            )
+            results.extend(page.items)
+            if page.next_cursor is None:
+                break
+            cursor = page.next_cursor
+        return results
+
+
+class ThreeDAPI:
+    def __init__(self, http_client: HTTPClient, console: Console) -> None:
+        self.models = ThreeDModelAPI(http_client, console)

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/cognite_toolkit/_cdf_tk/client/data_classes/base.py

@@ -26,6 +26,10 @@ class ResponseResource(BaseModelObject, Generic[T_RequestResource], ABC):
         """Convert the response resource to a request resource."""
         ...
 
+    def as_write(self) -> T_RequestResource:
+        """Alias for as_request_resource to match protocol signature."""
+        return self.as_request_resource()
+
 
 class Identifier(BaseModel):
     """Base class for all identifier classes."""

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/cognite_toolkit/_cdf_tk/client/data_classes/instance_api.py

@@ -7,7 +7,7 @@ from .base import BaseModelObject, Identifier, RequestResource
 InstanceType: TypeAlias = Literal["node", "edge"]
 
 
-class InstanceIdentifier(Identifier):
+class TypedInstanceIdentifier(Identifier):
     """Identifier for an Instance instance."""
 
     instance_type: InstanceType
@@ -15,11 +15,16 @@ class InstanceIdentifier(Identifier):
     external_id: str
 
 
-class NodeIdentifier(InstanceIdentifier):
+class InstanceIdentifier(Identifier):
+    space: str
+    external_id: str
+
+
+class NodeIdentifier(TypedInstanceIdentifier):
     instance_type: Literal["node"] = "node"
 
 
-class EdgeIdentifier(InstanceIdentifier):
+class EdgeIdentifier(TypedInstanceIdentifier):
     instance_type: Literal["edge"] = "edge"
 
 
@@ -32,8 +37,8 @@ class InstanceResult(BaseModelObject):
     created_time: int
     last_updated_time: int
 
-    def as_id(self) -> InstanceIdentifier:
-        return InstanceIdentifier(
+    def as_id(self) -> TypedInstanceIdentifier:
+        return TypedInstanceIdentifier(
             instance_type=self.instance_type,
             space=self.space,
             external_id=self.external_id,
@@ -62,8 +67,8 @@ class InstanceRequestResource(RequestResource):
     space: str
     external_id: str
 
-    def as_id(self) -> InstanceIdentifier:
-        return InstanceIdentifier(
+    def as_id(self) -> TypedInstanceIdentifier:
+        return TypedInstanceIdentifier(
             instance_type=self.instance_type,
             space=self.space,
             external_id=self.external_id,
@@ -115,8 +120,8 @@ class InstanceRequestItem(RequestResource):
     existing_version: int | None = None
     sources: list[InstanceSource] | None = None
 
-    def as_id(self) -> InstanceIdentifier:
-        return InstanceIdentifier(
+    def as_id(self) -> TypedInstanceIdentifier:
+        return TypedInstanceIdentifier(
             instance_type=self.instance_type,
             space=self.space,
             external_id=self.external_id,
@@ -128,7 +133,7 @@ class InstanceResponseItem(BaseModelObject):
     space: str
     external_id: str
     version: int
-    type: InstanceIdentifier | None = None
+    type: TypedInstanceIdentifier | None = None
     created_time: int
     last_updated_time: int
     deleted_time: int | None = None
@@ -149,8 +154,8 @@ class InstanceResponseItem(BaseModelObject):
         output.update(space_properties.get(view_version, {}))
         return output
 
-    def as_id(self) -> InstanceIdentifier:
-        return InstanceIdentifier(
+    def as_id(self) -> TypedInstanceIdentifier:
+        return TypedInstanceIdentifier(
             instance_type=self.instance_type,
             space=self.space,
             external_id=self.external_id,

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/cognite_toolkit/_cdf_tk/client/data_classes/three_d.py

@@ -27,6 +27,9 @@ class ThreeDModelClassicRequest(ThreeDModelRequest):
     data_set_id: int | None = None
     metadata: dict[str, str] | None = None
 
+    def as_id(self) -> str:
+        return self.name
+
 
 class ThreeDModelDMSRequest(ThreeDModelRequest):
     space: str

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/cognite_toolkit/_cdf_tk/client/testing.py

@@ -13,6 +13,7 @@ from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client._toolkit_client import ToolkitClient
 
+from ._toolkit_client import ToolAPI
 from .api.canvas import CanvasAPI, IndustrialCanvasAPI
 from .api.charts import ChartsAPI
 from .api.dml import DMLAPI
@@ -53,6 +54,7 @@ from .api.robotics.maps import MapsAPI
 from .api.search import SearchAPI
 from .api.search_config import SearchConfigurationsAPI
 from .api.streams import StreamsAPI
+from .api.three_d import ThreeDAPI, ThreeDModelAPI
 from .api.token import TokenAPI
 from .api.verify import VerifyAPI
 
@@ -131,6 +133,10 @@ class ToolkitClientMock(CogniteClientMock):
         self.time_series.data.synthetic = MagicMock(spec_set=SyntheticDatapointsAPI)
         self.time_series.subscriptions = MagicMock(spec_set=DatapointsSubscriptionAPI)
 
+        self.tool = MagicMock(spec=ToolAPI)
+        self.tool.three_d = MagicMock(spec=ThreeDAPI)
+        self.tool.three_d.models = MagicMock(spec_set=ThreeDModelAPI)
+
         self.streams = MagicMock(spec=StreamsAPI)
 
         # This is a helper API, not a real API.

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py

@@ -9,8 +9,10 @@ from cognite.client.data_classes._base import (
 from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, NodeId, ViewId
 from cognite.client.utils._identifier import InstanceId
 from cognite.client.utils._text import to_camel_case
-from pydantic import BaseModel, BeforeValidator, field_validator, model_validator
+from pydantic import BaseModel, BeforeValidator, Field, field_validator, model_validator
 
+from cognite_toolkit._cdf_tk.client.data_classes.base import BaseModelObject, RequestResource
+from cognite_toolkit._cdf_tk.client.data_classes.instance_api import InstanceIdentifier
 from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
 from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
 from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
@@ -261,3 +263,32 @@ class AssetCentricMappingList(
 
     def as_write(self) -> InstanceApplyList:
         return InstanceApplyList([item.as_write() for item in self])
+
+
+class Model(BaseModelObject):
+    instance_id: InstanceIdentifier
+
+
+class Thumbnail(BaseModelObject):
+    instance_id: InstanceIdentifier
+
+
+class ThreeDRevisionMigrationRequest(RequestResource):
+    space: str
+    type: Literal["CAD", "PointCloud", "Image360"]
+    revision_id: int
+    model: Model
+
+    def as_id(self) -> int:
+        return self.revision_id
+
+
+class ThreeDMigrationRequest(RequestResource):
+    model_id: int
+    type: Literal["CAD", "PointCloud", "Image360"]
+    space: str
+    thumbnail: Thumbnail | None = None
+    revision: ThreeDRevisionMigrationRequest = Field(exclude=True)
+
+    def as_id(self) -> int:
+        return self.model_id

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from collections.abc import Callable, Sequence
-from typing import Generic, cast
+from typing import Generic, Literal, cast
 from uuid import uuid4
 
 from cognite.client.data_classes.data_modeling import (
@@ -27,26 +27,33 @@ from cognite_toolkit._cdf_tk.client.data_classes.charts_data import (
     ChartSource,
     ChartTimeseries,
 )
+from cognite_toolkit._cdf_tk.client.data_classes.instance_api import InstanceIdentifier
 from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMappingApply
+from cognite_toolkit._cdf_tk.client.data_classes.three_d import RevisionStatus, ThreeDModelResponse
 from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
-from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
+from cognite_toolkit._cdf_tk.commands._migrate.data_classes import (
+    Model,
+    ThreeDMigrationRequest,
+    ThreeDRevisionMigrationRequest,
+)
 from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
 from cognite_toolkit._cdf_tk.commands._migrate.issues import (
     CanvasMigrationIssue,
     ChartMigrationIssue,
     ConversionIssue,
     MigrationIssue,
+    ThreeDModelMigrationIssue,
 )
-from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
 from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitMigrationError, ToolkitValueError
 from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
 from cognite_toolkit._cdf_tk.storageio._base import T_Selector
-from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector
+from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector, ThreeDSelector
 from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.useful_types import (
-    T_AssetCentricResourceExtended,
-)
+from cognite_toolkit._cdf_tk.utils.useful_types import T_AssetCentricResourceExtended
+
+from .data_classes import AssetCentricMapping
+from .selectors import AssetCentricMigrationSelector
 
 
 class DataMapper(Generic[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC):
@@ -383,3 +390,82 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
             max_width=reference.max_width,
             max_height=reference.max_height,
         )
+
+
+class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrationRequest]):
+    def __init__(self, client: ToolkitClient) -> None:
+        self.client = client
+
+    def map(
+        self, source: Sequence[ThreeDModelResponse]
+    ) -> Sequence[tuple[ThreeDMigrationRequest | None, MigrationIssue]]:
+        self._populate_cache(source)
+        output: list[tuple[ThreeDMigrationRequest | None, MigrationIssue]] = []
+        for item in source:
+            mapped_item, issue = self._map_single_item(item)
+            output.append((mapped_item, issue))
+        return output
+
+    def _populate_cache(self, source: Sequence[ThreeDModelResponse]) -> None:
+        dataset_ids: set[int] = set()
+        for model in source:
+            if model.data_set_id is not None:
+                dataset_ids.add(model.data_set_id)
+        self.client.migration.space_source.retrieve(list(dataset_ids))
+
+    def _map_single_item(
+        self, item: ThreeDModelResponse
+    ) -> tuple[ThreeDMigrationRequest | None, ThreeDModelMigrationIssue]:
+        issue = ThreeDModelMigrationIssue(model_name=item.name, model_id=item.id)
+        instance_space: str | None = None
+        last_revision_id: int | None = None
+        model_type: Literal["CAD", "PointCloud", "Image360"] | None = None
+        if item.data_set_id is None:
+            issue.error_message.append("3D model is not associated with any dataset.")
+        else:
+            space_source = self.client.migration.space_source.retrieve(item.data_set_id)
+            if space_source is not None:
+                instance_space = space_source.instance_space
+        if instance_space is None and item.data_set_id is not None:
+            issue.error_message.append(f"Missing instance space for dataset ID {item.data_set_id!r}")
+        if item.last_revision_info is None:
+            issue.error_message.append("3D model has no revisions.")
+        else:
+            model_type = self._get_type(item.last_revision_info)
+            last_revision_id = item.last_revision_info.revision_id
+            if last_revision_id is None:
+                issue.error_message.append("3D model's last revision has no revision ID.")
+
+        if model_type is None:
+            issue.error_message.append("3D model's last revision has no recognized type.")
+
+        if instance_space is None or last_revision_id is None or model_type is None or issue.has_issues:
+            return None, issue
+
+        mapped_request = ThreeDMigrationRequest(
+            model_id=item.id,
+            type=model_type,
+            space=instance_space,
+            revision=ThreeDRevisionMigrationRequest(
+                space=instance_space,
+                type=model_type,
+                revision_id=last_revision_id,
+                model=Model(
+                    instance_id=InstanceIdentifier(
+                        space=instance_space,
+                        external_id=f"cog_3d_model_{item.id!s}",
+                    )
+                ),
+            ),
+        )
+        return mapped_request, issue
+
+    @staticmethod
+    def _get_type(revision: RevisionStatus) -> Literal["CAD", "PointCloud", "Image360"] | None:
+        types = revision.types or []
+        if any("gltf-directory" in t for t in types):
+            return "CAD"
+        elif any("ept-pointcloud" in t for t in types):
+            return "PointCloud"
+        else:
+            return None

{cognite_toolkit-0.7.32 → cognite_toolkit-0.7.34}/cognite_toolkit/_cdf_tk/commands/_migrate/issues.py

@@ -30,6 +30,24 @@ class MigrationIssue(MigrationObject):
         return True
 
 
+class ThreeDModelMigrationIssue(MigrationIssue):
+    """Represents a 3D model migration issue encountered during migration.
+
+    Attributes:
+        model_external_id (str): The external ID of the 3D model that could not be migrated.
+    """
+
+    type: ClassVar[str] = "threeDModelMigration"
+    model_name: str
+    model_id: int
+    error_message: list[str] = Field(default_factory=list)
+
+    @property
+    def has_issues(self) -> bool:
+        """Check if there are any issues recorded in this ThreeDModelMigrationIssue."""
+        return bool(self.error_message)
+
+
 class ChartMigrationIssue(MigrationIssue):
     """Represents a chart migration issue encountered during migration.