cognite-toolkit 0.7.31__py3-none-any.whl → 0.7.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf.py +5 -6
- cognite_toolkit/_cdf_tk/apps/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/apps/_import_app.py +41 -0
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +76 -2
- cognite_toolkit/_cdf_tk/client/api/extended_functions.py +9 -9
- cognite_toolkit/_cdf_tk/client/api/infield.py +23 -17
- cognite_toolkit/_cdf_tk/client/api/project.py +8 -7
- cognite_toolkit/_cdf_tk/client/api/streams.py +19 -14
- cognite_toolkit/_cdf_tk/client/api/three_d.py +89 -8
- cognite_toolkit/_cdf_tk/client/data_classes/base.py +6 -22
- cognite_toolkit/_cdf_tk/client/data_classes/instance_api.py +18 -13
- cognite_toolkit/_cdf_tk/client/data_classes/three_d.py +6 -0
- cognite_toolkit/_cdf_tk/client/testing.py +6 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py +32 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +93 -7
- cognite_toolkit/_cdf_tk/commands/_migrate/issues.py +18 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +107 -1
- cognite_toolkit/_cdf_tk/resource_classes/workflow_version.py +164 -5
- cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py +10 -1
- cognite_toolkit/_cdf_tk/storageio/selectors/_three_d.py +34 -0
- cognite_toolkit/_cdf_tk/utils/http_client/__init__.py +28 -0
- cognite_toolkit/_cdf_tk/utils/http_client/_client.py +3 -2
- cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py +69 -7
- cognite_toolkit/_cdf_tk/utils/interactive_select.py +47 -0
- cognite_toolkit/_cdf_tk/validation.py +4 -0
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.31.dist-info → cognite_toolkit-0.7.33.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.7.31.dist-info → cognite_toolkit-0.7.33.dist-info}/RECORD +34 -34
- {cognite_toolkit-0.7.31.dist-info → cognite_toolkit-0.7.33.dist-info}/WHEEL +1 -1
- cognite_toolkit/_cdf_tk/prototypes/commands/__init__.py +0 -0
- cognite_toolkit/_cdf_tk/prototypes/import_app.py +0 -41
- /cognite_toolkit/_cdf_tk/{prototypes/commands/import_.py → commands/_import_cmd.py} +0 -0
- {cognite_toolkit-0.7.31.dist-info → cognite_toolkit-0.7.33.dist-info}/entry_points.txt +0 -0
|
@@ -7,7 +7,7 @@ from .base import BaseModelObject, Identifier, RequestResource
|
|
|
7
7
|
InstanceType: TypeAlias = Literal["node", "edge"]
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
class
|
|
10
|
+
class TypedInstanceIdentifier(Identifier):
|
|
11
11
|
"""Identifier for an Instance instance."""
|
|
12
12
|
|
|
13
13
|
instance_type: InstanceType
|
|
@@ -15,11 +15,16 @@ class InstanceIdentifier(Identifier):
|
|
|
15
15
|
external_id: str
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
class
|
|
18
|
+
class InstanceIdentifier(Identifier):
|
|
19
|
+
space: str
|
|
20
|
+
external_id: str
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class NodeIdentifier(TypedInstanceIdentifier):
|
|
19
24
|
instance_type: Literal["node"] = "node"
|
|
20
25
|
|
|
21
26
|
|
|
22
|
-
class EdgeIdentifier(
|
|
27
|
+
class EdgeIdentifier(TypedInstanceIdentifier):
|
|
23
28
|
instance_type: Literal["edge"] = "edge"
|
|
24
29
|
|
|
25
30
|
|
|
@@ -32,8 +37,8 @@ class InstanceResult(BaseModelObject):
|
|
|
32
37
|
created_time: int
|
|
33
38
|
last_updated_time: int
|
|
34
39
|
|
|
35
|
-
def as_id(self) ->
|
|
36
|
-
return
|
|
40
|
+
def as_id(self) -> TypedInstanceIdentifier:
|
|
41
|
+
return TypedInstanceIdentifier(
|
|
37
42
|
instance_type=self.instance_type,
|
|
38
43
|
space=self.space,
|
|
39
44
|
external_id=self.external_id,
|
|
@@ -62,8 +67,8 @@ class InstanceRequestResource(RequestResource):
|
|
|
62
67
|
space: str
|
|
63
68
|
external_id: str
|
|
64
69
|
|
|
65
|
-
def as_id(self) ->
|
|
66
|
-
return
|
|
70
|
+
def as_id(self) -> TypedInstanceIdentifier:
|
|
71
|
+
return TypedInstanceIdentifier(
|
|
67
72
|
instance_type=self.instance_type,
|
|
68
73
|
space=self.space,
|
|
69
74
|
external_id=self.external_id,
|
|
@@ -107,7 +112,7 @@ class InstanceSource(BaseModelObject):
|
|
|
107
112
|
return value
|
|
108
113
|
|
|
109
114
|
|
|
110
|
-
class InstanceRequestItem(
|
|
115
|
+
class InstanceRequestItem(RequestResource):
|
|
111
116
|
model_config = ConfigDict(populate_by_name=True)
|
|
112
117
|
instance_type: InstanceType
|
|
113
118
|
space: str
|
|
@@ -115,8 +120,8 @@ class InstanceRequestItem(BaseModelObject):
|
|
|
115
120
|
existing_version: int | None = None
|
|
116
121
|
sources: list[InstanceSource] | None = None
|
|
117
122
|
|
|
118
|
-
def as_id(self) ->
|
|
119
|
-
return
|
|
123
|
+
def as_id(self) -> TypedInstanceIdentifier:
|
|
124
|
+
return TypedInstanceIdentifier(
|
|
120
125
|
instance_type=self.instance_type,
|
|
121
126
|
space=self.space,
|
|
122
127
|
external_id=self.external_id,
|
|
@@ -128,7 +133,7 @@ class InstanceResponseItem(BaseModelObject):
|
|
|
128
133
|
space: str
|
|
129
134
|
external_id: str
|
|
130
135
|
version: int
|
|
131
|
-
type:
|
|
136
|
+
type: TypedInstanceIdentifier | None = None
|
|
132
137
|
created_time: int
|
|
133
138
|
last_updated_time: int
|
|
134
139
|
deleted_time: int | None = None
|
|
@@ -149,8 +154,8 @@ class InstanceResponseItem(BaseModelObject):
|
|
|
149
154
|
output.update(space_properties.get(view_version, {}))
|
|
150
155
|
return output
|
|
151
156
|
|
|
152
|
-
def as_id(self) ->
|
|
153
|
-
return
|
|
157
|
+
def as_id(self) -> TypedInstanceIdentifier:
|
|
158
|
+
return TypedInstanceIdentifier(
|
|
154
159
|
instance_type=self.instance_type,
|
|
155
160
|
space=self.space,
|
|
156
161
|
external_id=self.external_id,
|
|
@@ -19,11 +19,17 @@ class RevisionStatus(BaseModelObject):
|
|
|
19
19
|
class ThreeDModelRequest(RequestResource):
|
|
20
20
|
name: str
|
|
21
21
|
|
|
22
|
+
def as_id(self) -> str:
|
|
23
|
+
return self.name
|
|
24
|
+
|
|
22
25
|
|
|
23
26
|
class ThreeDModelClassicRequest(ThreeDModelRequest):
|
|
24
27
|
data_set_id: int | None = None
|
|
25
28
|
metadata: dict[str, str] | None = None
|
|
26
29
|
|
|
30
|
+
def as_id(self) -> str:
|
|
31
|
+
return self.name
|
|
32
|
+
|
|
27
33
|
|
|
28
34
|
class ThreeDModelDMSRequest(ThreeDModelRequest):
|
|
29
35
|
space: str
|
|
@@ -13,6 +13,7 @@ from rich.console import Console
|
|
|
13
13
|
|
|
14
14
|
from cognite_toolkit._cdf_tk.client._toolkit_client import ToolkitClient
|
|
15
15
|
|
|
16
|
+
from ._toolkit_client import ToolAPI
|
|
16
17
|
from .api.canvas import CanvasAPI, IndustrialCanvasAPI
|
|
17
18
|
from .api.charts import ChartsAPI
|
|
18
19
|
from .api.dml import DMLAPI
|
|
@@ -53,6 +54,7 @@ from .api.robotics.maps import MapsAPI
|
|
|
53
54
|
from .api.search import SearchAPI
|
|
54
55
|
from .api.search_config import SearchConfigurationsAPI
|
|
55
56
|
from .api.streams import StreamsAPI
|
|
57
|
+
from .api.three_d import ThreeDAPI, ThreeDModelAPI
|
|
56
58
|
from .api.token import TokenAPI
|
|
57
59
|
from .api.verify import VerifyAPI
|
|
58
60
|
|
|
@@ -131,6 +133,10 @@ class ToolkitClientMock(CogniteClientMock):
|
|
|
131
133
|
self.time_series.data.synthetic = MagicMock(spec_set=SyntheticDatapointsAPI)
|
|
132
134
|
self.time_series.subscriptions = MagicMock(spec_set=DatapointsSubscriptionAPI)
|
|
133
135
|
|
|
136
|
+
self.tool = MagicMock(spec=ToolAPI)
|
|
137
|
+
self.tool.three_d = MagicMock(spec=ThreeDAPI)
|
|
138
|
+
self.tool.three_d.models = MagicMock(spec_set=ThreeDModelAPI)
|
|
139
|
+
|
|
134
140
|
self.streams = MagicMock(spec=StreamsAPI)
|
|
135
141
|
|
|
136
142
|
# This is a helper API, not a real API.
|
|
@@ -9,8 +9,10 @@ from cognite.client.data_classes._base import (
|
|
|
9
9
|
from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, NodeId, ViewId
|
|
10
10
|
from cognite.client.utils._identifier import InstanceId
|
|
11
11
|
from cognite.client.utils._text import to_camel_case
|
|
12
|
-
from pydantic import BaseModel, BeforeValidator, field_validator, model_validator
|
|
12
|
+
from pydantic import BaseModel, BeforeValidator, Field, field_validator, model_validator
|
|
13
13
|
|
|
14
|
+
from cognite_toolkit._cdf_tk.client.data_classes.base import BaseModelObject, RequestResource
|
|
15
|
+
from cognite_toolkit._cdf_tk.client.data_classes.instance_api import InstanceIdentifier
|
|
14
16
|
from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
|
|
15
17
|
from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
|
|
16
18
|
from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
|
|
@@ -261,3 +263,32 @@ class AssetCentricMappingList(
|
|
|
261
263
|
|
|
262
264
|
def as_write(self) -> InstanceApplyList:
|
|
263
265
|
return InstanceApplyList([item.as_write() for item in self])
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class Model(BaseModelObject):
|
|
269
|
+
instance_id: InstanceIdentifier
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
class Thumbnail(BaseModelObject):
|
|
273
|
+
instance_id: InstanceIdentifier
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
class ThreeDRevisionMigrationRequest(RequestResource):
|
|
277
|
+
space: str
|
|
278
|
+
type: Literal["CAD", "PointCloud", "Image360"]
|
|
279
|
+
revision_id: int
|
|
280
|
+
model: Model
|
|
281
|
+
|
|
282
|
+
def as_id(self) -> int:
|
|
283
|
+
return self.revision_id
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class ThreeDMigrationRequest(RequestResource):
|
|
287
|
+
model_id: int
|
|
288
|
+
type: Literal["CAD", "PointCloud", "Image360"]
|
|
289
|
+
space: str
|
|
290
|
+
thumbnail: Thumbnail | None = None
|
|
291
|
+
revision: ThreeDRevisionMigrationRequest = Field(exclude=True)
|
|
292
|
+
|
|
293
|
+
def as_id(self) -> int:
|
|
294
|
+
return self.model_id
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
from collections import defaultdict
|
|
3
3
|
from collections.abc import Callable, Sequence
|
|
4
|
-
from typing import Generic, cast
|
|
4
|
+
from typing import Generic, Literal, cast
|
|
5
5
|
from uuid import uuid4
|
|
6
6
|
|
|
7
7
|
from cognite.client.data_classes.data_modeling import (
|
|
@@ -27,26 +27,33 @@ from cognite_toolkit._cdf_tk.client.data_classes.charts_data import (
|
|
|
27
27
|
ChartSource,
|
|
28
28
|
ChartTimeseries,
|
|
29
29
|
)
|
|
30
|
+
from cognite_toolkit._cdf_tk.client.data_classes.instance_api import InstanceIdentifier
|
|
30
31
|
from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMappingApply
|
|
32
|
+
from cognite_toolkit._cdf_tk.client.data_classes.three_d import RevisionStatus, ThreeDModelResponse
|
|
31
33
|
from cognite_toolkit._cdf_tk.commands._migrate.conversion import DirectRelationCache, asset_centric_to_dm
|
|
32
|
-
from cognite_toolkit._cdf_tk.commands._migrate.data_classes import
|
|
34
|
+
from cognite_toolkit._cdf_tk.commands._migrate.data_classes import (
|
|
35
|
+
Model,
|
|
36
|
+
ThreeDMigrationRequest,
|
|
37
|
+
ThreeDRevisionMigrationRequest,
|
|
38
|
+
)
|
|
33
39
|
from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
|
|
34
40
|
from cognite_toolkit._cdf_tk.commands._migrate.issues import (
|
|
35
41
|
CanvasMigrationIssue,
|
|
36
42
|
ChartMigrationIssue,
|
|
37
43
|
ConversionIssue,
|
|
38
44
|
MigrationIssue,
|
|
45
|
+
ThreeDModelMigrationIssue,
|
|
39
46
|
)
|
|
40
|
-
from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
|
|
41
47
|
from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
|
|
42
48
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitMigrationError, ToolkitValueError
|
|
43
49
|
from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
|
|
44
50
|
from cognite_toolkit._cdf_tk.storageio._base import T_Selector
|
|
45
|
-
from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector
|
|
51
|
+
from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector, ThreeDSelector
|
|
46
52
|
from cognite_toolkit._cdf_tk.utils import humanize_collection
|
|
47
|
-
from cognite_toolkit._cdf_tk.utils.useful_types import
|
|
48
|
-
|
|
49
|
-
|
|
53
|
+
from cognite_toolkit._cdf_tk.utils.useful_types import T_AssetCentricResourceExtended
|
|
54
|
+
|
|
55
|
+
from .data_classes import AssetCentricMapping
|
|
56
|
+
from .selectors import AssetCentricMigrationSelector
|
|
50
57
|
|
|
51
58
|
|
|
52
59
|
class DataMapper(Generic[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC):
|
|
@@ -383,3 +390,82 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
|
|
|
383
390
|
max_width=reference.max_width,
|
|
384
391
|
max_height=reference.max_height,
|
|
385
392
|
)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrationRequest]):
|
|
396
|
+
def __init__(self, client: ToolkitClient) -> None:
|
|
397
|
+
self.client = client
|
|
398
|
+
|
|
399
|
+
def map(
|
|
400
|
+
self, source: Sequence[ThreeDModelResponse]
|
|
401
|
+
) -> Sequence[tuple[ThreeDMigrationRequest | None, MigrationIssue]]:
|
|
402
|
+
self._populate_cache(source)
|
|
403
|
+
output: list[tuple[ThreeDMigrationRequest | None, MigrationIssue]] = []
|
|
404
|
+
for item in source:
|
|
405
|
+
mapped_item, issue = self._map_single_item(item)
|
|
406
|
+
output.append((mapped_item, issue))
|
|
407
|
+
return output
|
|
408
|
+
|
|
409
|
+
def _populate_cache(self, source: Sequence[ThreeDModelResponse]) -> None:
|
|
410
|
+
dataset_ids: set[int] = set()
|
|
411
|
+
for model in source:
|
|
412
|
+
if model.data_set_id is not None:
|
|
413
|
+
dataset_ids.add(model.data_set_id)
|
|
414
|
+
self.client.migration.space_source.retrieve(list(dataset_ids))
|
|
415
|
+
|
|
416
|
+
def _map_single_item(
|
|
417
|
+
self, item: ThreeDModelResponse
|
|
418
|
+
) -> tuple[ThreeDMigrationRequest | None, ThreeDModelMigrationIssue]:
|
|
419
|
+
issue = ThreeDModelMigrationIssue(model_name=item.name, model_id=item.id)
|
|
420
|
+
instance_space: str | None = None
|
|
421
|
+
last_revision_id: int | None = None
|
|
422
|
+
model_type: Literal["CAD", "PointCloud", "Image360"] | None = None
|
|
423
|
+
if item.data_set_id is None:
|
|
424
|
+
issue.error_message.append("3D model is not associated with any dataset.")
|
|
425
|
+
else:
|
|
426
|
+
space_source = self.client.migration.space_source.retrieve(item.data_set_id)
|
|
427
|
+
if space_source is not None:
|
|
428
|
+
instance_space = space_source.instance_space
|
|
429
|
+
if instance_space is None and item.data_set_id is not None:
|
|
430
|
+
issue.error_message.append(f"Missing instance space for dataset ID {item.data_set_id!r}")
|
|
431
|
+
if item.last_revision_info is None:
|
|
432
|
+
issue.error_message.append("3D model has no revisions.")
|
|
433
|
+
else:
|
|
434
|
+
model_type = self._get_type(item.last_revision_info)
|
|
435
|
+
last_revision_id = item.last_revision_info.revision_id
|
|
436
|
+
if last_revision_id is None:
|
|
437
|
+
issue.error_message.append("3D model's last revision has no revision ID.")
|
|
438
|
+
|
|
439
|
+
if model_type is None:
|
|
440
|
+
issue.error_message.append("3D model's last revision has no recognized type.")
|
|
441
|
+
|
|
442
|
+
if instance_space is None or last_revision_id is None or model_type is None or issue.has_issues:
|
|
443
|
+
return None, issue
|
|
444
|
+
|
|
445
|
+
mapped_request = ThreeDMigrationRequest(
|
|
446
|
+
model_id=item.id,
|
|
447
|
+
type=model_type,
|
|
448
|
+
space=instance_space,
|
|
449
|
+
revision=ThreeDRevisionMigrationRequest(
|
|
450
|
+
space=instance_space,
|
|
451
|
+
type=model_type,
|
|
452
|
+
revision_id=last_revision_id,
|
|
453
|
+
model=Model(
|
|
454
|
+
instance_id=InstanceIdentifier(
|
|
455
|
+
space=instance_space,
|
|
456
|
+
external_id=f"cog_3d_model_{item.id!s}",
|
|
457
|
+
)
|
|
458
|
+
),
|
|
459
|
+
),
|
|
460
|
+
)
|
|
461
|
+
return mapped_request, issue
|
|
462
|
+
|
|
463
|
+
@staticmethod
|
|
464
|
+
def _get_type(revision: RevisionStatus) -> Literal["CAD", "PointCloud", "Image360"] | None:
|
|
465
|
+
types = revision.types or []
|
|
466
|
+
if any("gltf-directory" in t for t in types):
|
|
467
|
+
return "CAD"
|
|
468
|
+
elif any("ept-pointcloud" in t for t in types):
|
|
469
|
+
return "PointCloud"
|
|
470
|
+
else:
|
|
471
|
+
return None
|
|
@@ -30,6 +30,24 @@ class MigrationIssue(MigrationObject):
|
|
|
30
30
|
return True
|
|
31
31
|
|
|
32
32
|
|
|
33
|
+
class ThreeDModelMigrationIssue(MigrationIssue):
|
|
34
|
+
"""Represents a 3D model migration issue encountered during migration.
|
|
35
|
+
|
|
36
|
+
Attributes:
|
|
37
|
+
model_external_id (str): The external ID of the 3D model that could not be migrated.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
type: ClassVar[str] = "threeDModelMigration"
|
|
41
|
+
model_name: str
|
|
42
|
+
model_id: int
|
|
43
|
+
error_message: list[str] = Field(default_factory=list)
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def has_issues(self) -> bool:
|
|
47
|
+
"""Check if there are any issues recorded in this ThreeDModelMigrationIssue."""
|
|
48
|
+
return bool(self.error_message)
|
|
49
|
+
|
|
50
|
+
|
|
33
51
|
class ChartMigrationIssue(MigrationIssue):
|
|
34
52
|
"""Represents a chart migration issue encountered during migration.
|
|
35
53
|
|
|
@@ -6,6 +6,8 @@ from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, Nod
|
|
|
6
6
|
|
|
7
7
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
8
8
|
from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
|
|
9
|
+
from cognite_toolkit._cdf_tk.client.data_classes.three_d import ThreeDModelResponse
|
|
10
|
+
from cognite_toolkit._cdf_tk.commands._migrate.data_classes import ThreeDMigrationRequest
|
|
9
11
|
from cognite_toolkit._cdf_tk.constants import MISSING_EXTERNAL_ID, MISSING_INSTANCE_SPACE
|
|
10
12
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError, ToolkitValueError
|
|
11
13
|
from cognite_toolkit._cdf_tk.storageio import (
|
|
@@ -15,9 +17,22 @@ from cognite_toolkit._cdf_tk.storageio import (
|
|
|
15
17
|
UploadableStorageIO,
|
|
16
18
|
)
|
|
17
19
|
from cognite_toolkit._cdf_tk.storageio._base import Page, UploadItem
|
|
20
|
+
from cognite_toolkit._cdf_tk.storageio.selectors import (
|
|
21
|
+
ThreeDModelFilteredSelector,
|
|
22
|
+
ThreeDModelIdSelector,
|
|
23
|
+
ThreeDSelector,
|
|
24
|
+
)
|
|
18
25
|
from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
|
|
19
26
|
from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
|
|
20
|
-
from cognite_toolkit._cdf_tk.utils.http_client import
|
|
27
|
+
from cognite_toolkit._cdf_tk.utils.http_client import (
|
|
28
|
+
FailedResponse,
|
|
29
|
+
HTTPClient,
|
|
30
|
+
HTTPMessage,
|
|
31
|
+
ItemsRequest,
|
|
32
|
+
SimpleBodyRequest,
|
|
33
|
+
SuccessResponseItems,
|
|
34
|
+
ToolkitAPIError,
|
|
35
|
+
)
|
|
21
36
|
from cognite_toolkit._cdf_tk.utils.useful_types import (
|
|
22
37
|
AssetCentricKindExtended,
|
|
23
38
|
AssetCentricType,
|
|
@@ -348,3 +363,94 @@ class AnnotationMigrationIO(
|
|
|
348
363
|
selector: AssetCentricMigrationSelector | None = None,
|
|
349
364
|
) -> list[dict[str, JsonVal]]:
|
|
350
365
|
raise NotImplementedError("Serializing Annotation Migrations to JSON is not supported.")
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
class ThreeDMigrationIO(UploadableStorageIO[ThreeDSelector, ThreeDModelResponse, ThreeDMigrationRequest]):
|
|
369
|
+
KIND = "3DMigration"
|
|
370
|
+
SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
|
|
371
|
+
SUPPORTED_COMPRESSIONS = frozenset({".gz"})
|
|
372
|
+
SUPPORTED_READ_FORMATS = frozenset({".ndjson"})
|
|
373
|
+
DOWNLOAD_LIMIT = 1000
|
|
374
|
+
CHUNK_SIZE = 1
|
|
375
|
+
UPLOAD_ENDPOINT = "/3d/migrate/models"
|
|
376
|
+
REVISION_ENDPOINT = "/3d/migrate/revisions"
|
|
377
|
+
|
|
378
|
+
def as_id(self, item: ThreeDModelResponse) -> str:
|
|
379
|
+
return f"{item.name}_{item.id!s}"
|
|
380
|
+
|
|
381
|
+
def stream_data(self, selector: ThreeDSelector, limit: int | None = None) -> Iterable[Page[ThreeDModelResponse]]:
|
|
382
|
+
published: bool | None = None
|
|
383
|
+
if isinstance(selector, ThreeDModelFilteredSelector):
|
|
384
|
+
published = selector.published
|
|
385
|
+
included_models: set[int] | None = None
|
|
386
|
+
if isinstance(selector, ThreeDModelIdSelector):
|
|
387
|
+
included_models = set(selector.ids)
|
|
388
|
+
cursor: str | None = None
|
|
389
|
+
total = 0
|
|
390
|
+
while True:
|
|
391
|
+
request_limit = min(self.DOWNLOAD_LIMIT, limit - total) if limit is not None else self.DOWNLOAD_LIMIT
|
|
392
|
+
response = self.client.tool.three_d.models.iterate(
|
|
393
|
+
published=published, include_revision_info=True, limit=request_limit, cursor=cursor
|
|
394
|
+
)
|
|
395
|
+
# Only include asset-centric 3D models
|
|
396
|
+
items = [
|
|
397
|
+
item
|
|
398
|
+
for item in response.items
|
|
399
|
+
if item.space is None and (included_models is None or item.id in included_models)
|
|
400
|
+
]
|
|
401
|
+
total += len(items)
|
|
402
|
+
if items:
|
|
403
|
+
yield Page(worker_id="main", items=items, next_cursor=response.next_cursor)
|
|
404
|
+
if response.next_cursor is None:
|
|
405
|
+
break
|
|
406
|
+
cursor = response.next_cursor
|
|
407
|
+
|
|
408
|
+
def count(self, selector: ThreeDSelector) -> int | None:
|
|
409
|
+
# There is no efficient way to count 3D models in CDF.
|
|
410
|
+
return None
|
|
411
|
+
|
|
412
|
+
def data_to_json_chunk(
|
|
413
|
+
self, data_chunk: Sequence[ThreeDModelResponse], selector: ThreeDSelector | None = None
|
|
414
|
+
) -> list[dict[str, JsonVal]]:
|
|
415
|
+
raise NotImplementedError("Deserializing Annotation Migrations from JSON is not supported.")
|
|
416
|
+
|
|
417
|
+
def json_to_resource(self, item_json: dict[str, JsonVal]) -> ThreeDMigrationRequest:
|
|
418
|
+
raise NotImplementedError("Deserializing ThreeD Migrations from JSON is not supported.")
|
|
419
|
+
|
|
420
|
+
def upload_items(
|
|
421
|
+
self,
|
|
422
|
+
data_chunk: Sequence[UploadItem[ThreeDMigrationRequest]],
|
|
423
|
+
http_client: HTTPClient,
|
|
424
|
+
selector: ThreeDSelector | None = None,
|
|
425
|
+
) -> Sequence[HTTPMessage]:
|
|
426
|
+
"""Migrate 3D models by uploading them to the migrate/models endpoint."""
|
|
427
|
+
if len(data_chunk) > self.CHUNK_SIZE:
|
|
428
|
+
raise RuntimeError(f"Uploading more than {self.CHUNK_SIZE} 3D models at a time is not supported.")
|
|
429
|
+
|
|
430
|
+
results: list[HTTPMessage] = []
|
|
431
|
+
responses = http_client.request_with_retries(
|
|
432
|
+
message=ItemsRequest(
|
|
433
|
+
endpoint_url=self.client.config.create_api_url(self.UPLOAD_ENDPOINT),
|
|
434
|
+
method="POST",
|
|
435
|
+
items=list(data_chunk),
|
|
436
|
+
)
|
|
437
|
+
)
|
|
438
|
+
if (
|
|
439
|
+
failed_response := next((res for res in responses if isinstance(res, FailedResponse)), None)
|
|
440
|
+
) and failed_response.status_code == 400:
|
|
441
|
+
raise ToolkitAPIError("3D model migration failed. You need to enable the 3D migration alpha feature flag.")
|
|
442
|
+
|
|
443
|
+
results.extend(responses)
|
|
444
|
+
success_ids = {id for res in responses if isinstance(res, SuccessResponseItems) for id in res.ids}
|
|
445
|
+
for data in data_chunk:
|
|
446
|
+
if data.source_id not in success_ids:
|
|
447
|
+
continue
|
|
448
|
+
revision = http_client.request_with_retries(
|
|
449
|
+
message=SimpleBodyRequest(
|
|
450
|
+
endpoint_url=self.client.config.create_api_url(self.REVISION_ENDPOINT),
|
|
451
|
+
method="POST",
|
|
452
|
+
body_content={"items": [data.item.revision.dump(camel_case=True)]},
|
|
453
|
+
)
|
|
454
|
+
)
|
|
455
|
+
results.extend(revision.as_item_responses(data.source_id))
|
|
456
|
+
return results
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
from
|
|
1
|
+
from abc import ABC
|
|
2
|
+
from typing import Annotated, Literal
|
|
2
3
|
|
|
3
4
|
from cognite.client.data_classes import WorkflowVersionUpsert
|
|
4
5
|
from pydantic import Field, JsonValue
|
|
@@ -6,18 +7,141 @@ from pydantic import Field, JsonValue
|
|
|
6
7
|
from .base import BaseModelResource, ToolkitResource
|
|
7
8
|
|
|
8
9
|
|
|
10
|
+
class WorkflowVersionId(BaseModelResource):
|
|
11
|
+
workflow_external_id: str = Field(
|
|
12
|
+
max_length=255,
|
|
13
|
+
description="Identifier for a workflow. Must be unique for the project. No trailing or leading whitespace and no null characters allowed.",
|
|
14
|
+
)
|
|
15
|
+
version: str = Field(
|
|
16
|
+
max_length=255,
|
|
17
|
+
description="Identifier for a version. Must be unique for the workflow. No trailing or leading whitespace and no null characters allowed.",
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class CogniteFunctionRef(BaseModelResource):
|
|
22
|
+
external_id: str = Field(
|
|
23
|
+
description="The external id of the Cognite Function in the project. This can be either a function external ID or a reference like ${myTaskExternalId.output.someKey}"
|
|
24
|
+
)
|
|
25
|
+
data: str | JsonValue | None = Field(
|
|
26
|
+
None, description="Input data that will be passed to the Cognite Function. Limited to 100KB in size."
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class FunctionTaskParameters(BaseModelResource):
|
|
31
|
+
function: CogniteFunctionRef
|
|
32
|
+
is_async_complete: bool = Field(
|
|
33
|
+
False, description="Defines if the execution of the task should be completed asynchronously."
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class TransformationRef(BaseModelResource):
|
|
38
|
+
external_id: str = Field(
|
|
39
|
+
description="The external id of the Transformation in the project. This can be either a transformation external ID or a reference like ${myTaskExternalId.output.someKey}"
|
|
40
|
+
)
|
|
41
|
+
concurrency_policy: Literal["fail", "waitForCurrent", "restartAfterCurrent"] = Field(
|
|
42
|
+
"fail",
|
|
43
|
+
description="""Determines the behavior of the task if the Transformation is already running.
|
|
44
|
+
|
|
45
|
+
fail: The task fails if another instance of the Transformation is currently running.
|
|
46
|
+
waitForCurrent: The task will pause and wait for the already running Transformation to complete. Once completed, the task is completed. This mode is useful for preventing redundant Transformation runs.
|
|
47
|
+
restartAfterCurrent: The task waits for the ongoing Transformation to finish. After completion, the task restarts the Transformation. This mode ensures that the most recent data can be used by following tasks.""",
|
|
48
|
+
)
|
|
49
|
+
use_transformation_credentials: bool = Field(
|
|
50
|
+
False,
|
|
51
|
+
description="If set to true, the transformation will run using the client credentials configured on the transformation. If set to false, the transformation will run using the client credentials used to trigger the workflow.",
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class TransformationTaskParameters(BaseModelResource):
|
|
56
|
+
transformation: TransformationRef
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class CDFRequest(BaseModelResource):
|
|
60
|
+
resource_path: str = Field(
|
|
61
|
+
description="The path of the request. The path should be prefixed by {cluster}.cognitedata.com/api/v1/project/{project} based on the relevant cluster and project. It can also contain references like ${myTaskExternalId.output.someKey}"
|
|
62
|
+
)
|
|
63
|
+
method: Literal["POST", "GET", "PUT"] | str = Field(
|
|
64
|
+
description="The HTTP method of the request. It can also be a reference like ${myTaskExternalId.output.someKey}"
|
|
65
|
+
)
|
|
66
|
+
query_parameters: dict[str, JsonValue] | str | None = Field(
|
|
67
|
+
None,
|
|
68
|
+
description="The query parameters of the request. It can also be a reference like ${myTaskExternalId.output.someKey}",
|
|
69
|
+
)
|
|
70
|
+
body: JsonValue | str | None = Field(
|
|
71
|
+
None, description="The body of the request. It can also be a reference like ${myTaskExternalId.output.someKey}"
|
|
72
|
+
)
|
|
73
|
+
request_timeout_in_millis: float | str | None = Field(
|
|
74
|
+
None,
|
|
75
|
+
description="The timeout for the request in milliseconds. It can also be a reference like ${myTaskExternalId.output.someKey}",
|
|
76
|
+
)
|
|
77
|
+
cdf_version_header: Literal["alpha", "beta"] | str | None = Field(
|
|
78
|
+
None, description="The Cognite Data Fusion version header to use for the request."
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class CDFTaskParameters(BaseModelResource):
|
|
83
|
+
cdf_request: CDFRequest
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class DynamicRef(BaseModelResource):
|
|
87
|
+
tasks: str = Field(
|
|
88
|
+
description="A Reference is an expression that allows dynamically injecting input to a task during execution. References can be used to reference the input of the Workflow, the output of a previous task in the Workflow, or the input of a previous task in the Workflow. Note that the injected value must be valid in the context of the property it is injected into. Example Task reference: ${myTaskExternalId.output.someKey} Example Workflow input reference: ${workflow.input.myKey}"
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class DynamicTaskParameters(BaseModelResource):
|
|
93
|
+
dynamic: DynamicRef = Field(description="Reference to another task to use as the definition for this task.")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class SubworkflowRef(BaseModelResource):
|
|
97
|
+
tasks: "WorkflowVersionId | list[Task]" = Field(
|
|
98
|
+
description="Reference to the subworkflow to execute. This can be either a reference to an existing workflow version or an inline definition of tasks."
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class SubworkflowTaskParameters(BaseModelResource):
|
|
103
|
+
subworkflow: SubworkflowRef = Field(description="Reference to the subworkflow to execute.")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class SimulatorInputUnit(BaseModelResource):
|
|
107
|
+
name: str = Field(description="Name of the unit.")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class SimulatorInput(BaseModelResource):
|
|
111
|
+
reference_id: str = Field(description="Reference id of the value to override.")
|
|
112
|
+
value: str | int | float | list[str] | list[int] | list[float] = Field(
|
|
113
|
+
description="Override the value used for a simulation run."
|
|
114
|
+
)
|
|
115
|
+
unit: SimulatorInputUnit | None = Field(None, description="Override the unit of the value")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class SimulationRef(BaseModelResource):
|
|
119
|
+
routine_external_id: str = Field(description="The external id of the routine to be executed.")
|
|
120
|
+
run_time: int | None = Field(
|
|
121
|
+
None,
|
|
122
|
+
description="Run time in milliseconds. Reference timestamp used for data pre-processing and data sampling.",
|
|
123
|
+
)
|
|
124
|
+
inputs: list[SimulatorInput] | None = Field(
|
|
125
|
+
None, description="List of inputs to be provided to the simulation.", max_length=200
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class SimulationTaskParameters(BaseModelResource):
|
|
130
|
+
simulation: SimulationRef = Field(description="Reference to the simulation to execute.")
|
|
131
|
+
|
|
132
|
+
|
|
9
133
|
class TaskId(BaseModelResource):
|
|
10
134
|
external_id: str = Field(
|
|
11
135
|
max_length=255, description="The external ID provided by the client. Must be unique for the resource type."
|
|
12
136
|
)
|
|
13
137
|
|
|
14
138
|
|
|
15
|
-
class TaskDefinition(BaseModelResource):
|
|
139
|
+
class TaskDefinition(BaseModelResource, ABC):
|
|
16
140
|
external_id: str = Field(
|
|
17
141
|
max_length=255,
|
|
18
142
|
description="Identifier for the task. Must be unique within the version. No trailing or leading whitespace and no null characters allowed.",
|
|
19
143
|
)
|
|
20
|
-
type:
|
|
144
|
+
type: str
|
|
21
145
|
name: str | None = Field(
|
|
22
146
|
default=None,
|
|
23
147
|
max_length=255,
|
|
@@ -28,7 +152,6 @@ class TaskDefinition(BaseModelResource):
|
|
|
28
152
|
max_length=500,
|
|
29
153
|
description="Description of the intention of the task",
|
|
30
154
|
)
|
|
31
|
-
parameters: JsonValue = Field()
|
|
32
155
|
retries: int = Field(
|
|
33
156
|
3,
|
|
34
157
|
ge=0,
|
|
@@ -52,13 +175,49 @@ class TaskDefinition(BaseModelResource):
|
|
|
52
175
|
)
|
|
53
176
|
|
|
54
177
|
|
|
178
|
+
class FunctionTask(TaskDefinition):
|
|
179
|
+
type: Literal["function"] = "function"
|
|
180
|
+
parameters: FunctionTaskParameters
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class TransformationTask(TaskDefinition):
|
|
184
|
+
type: Literal["transformation"] = "transformation"
|
|
185
|
+
parameters: TransformationTaskParameters
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class CDFTask(TaskDefinition):
|
|
189
|
+
type: Literal["cdfRequest"] = "cdfRequest"
|
|
190
|
+
parameters: CDFTaskParameters
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class DynamicTask(TaskDefinition):
|
|
194
|
+
type: Literal["dynamic"] = "dynamic"
|
|
195
|
+
parameters: DynamicTaskParameters
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class SubworkflowTask(TaskDefinition):
|
|
199
|
+
type: Literal["subworkflow"] = "subworkflow"
|
|
200
|
+
parameters: SubworkflowTaskParameters
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class SimulationTask(TaskDefinition):
|
|
204
|
+
type: Literal["simulation"] = "simulation"
|
|
205
|
+
parameters: SimulationTaskParameters
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
Task = Annotated[
|
|
209
|
+
FunctionTask | TransformationTask | CDFTask | DynamicTask | SubworkflowTask | SimulationTask,
|
|
210
|
+
Field(discriminator="type"),
|
|
211
|
+
]
|
|
212
|
+
|
|
213
|
+
|
|
55
214
|
class WorkflowDefinition(BaseModelResource):
|
|
56
215
|
description: str | None = Field(
|
|
57
216
|
default=None,
|
|
58
217
|
max_length=500,
|
|
59
218
|
description="The description of the workflow version.",
|
|
60
219
|
)
|
|
61
|
-
tasks: list[
|
|
220
|
+
tasks: list[Task]
|
|
62
221
|
|
|
63
222
|
|
|
64
223
|
class WorkflowVersionYAML(ToolkitResource):
|