cognite-toolkit 0.7.47__py3-none-any.whl → 0.7.49__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +6 -6
- cognite_toolkit/_cdf_tk/client/_toolkit_client.py +6 -4
- cognite_toolkit/_cdf_tk/client/api/instances.py +139 -0
- cognite_toolkit/_cdf_tk/client/api/location_filters.py +177 -0
- cognite_toolkit/_cdf_tk/client/api/raw.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/robotics.py +19 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_capabilities.py +127 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_data_postprocessing.py +138 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_frames.py +122 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_locations.py +127 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_maps.py +122 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_robots.py +122 -0
- cognite_toolkit/_cdf_tk/client/api/search_config.py +101 -0
- cognite_toolkit/_cdf_tk/client/api/streams.py +63 -55
- cognite_toolkit/_cdf_tk/client/api/three_d.py +293 -277
- cognite_toolkit/_cdf_tk/client/cdf_client/api.py +34 -5
- cognite_toolkit/_cdf_tk/client/http_client/_client.py +5 -2
- cognite_toolkit/_cdf_tk/client/http_client/_data_classes2.py +4 -3
- cognite_toolkit/_cdf_tk/client/request_classes/filters.py +45 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/apm_config.py +128 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/cognite_file.py +53 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/__init__.py +4 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_instance.py +22 -11
- cognite_toolkit/_cdf_tk/client/resource_classes/identifiers.py +7 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/location_filter.py +9 -2
- cognite_toolkit/_cdf_tk/client/resource_classes/resource_view_mapping.py +38 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_map.py +6 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_robot.py +10 -5
- cognite_toolkit/_cdf_tk/client/resource_classes/streams.py +1 -20
- cognite_toolkit/_cdf_tk/client/resource_classes/three_d.py +30 -9
- cognite_toolkit/_cdf_tk/client/testing.py +2 -2
- cognite_toolkit/_cdf_tk/commands/_migrate/command.py +103 -108
- cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +6 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +119 -41
- cognite_toolkit/_cdf_tk/commands/_migrate/issues.py +21 -38
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +14 -12
- cognite_toolkit/_cdf_tk/commands/build_v2/_module_parser.py +138 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/_modules_parser.py +163 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py +83 -96
- cognite_toolkit/_cdf_tk/commands/build_v2/{build_input.py → build_parameters.py} +8 -22
- cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_modules.py +27 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_resource.py +22 -0
- cognite_toolkit/_cdf_tk/cruds/__init__.py +11 -5
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py +14 -30
- cognite_toolkit/_cdf_tk/data_classes/__init__.py +3 -0
- cognite_toolkit/_cdf_tk/data_classes/_issues.py +36 -0
- cognite_toolkit/_cdf_tk/data_classes/_module_directories.py +2 -1
- cognite_toolkit/_cdf_tk/storageio/_base.py +2 -0
- cognite_toolkit/_cdf_tk/storageio/logger.py +162 -0
- cognite_toolkit/_cdf_tk/utils/__init__.py +8 -1
- cognite_toolkit/_cdf_tk/utils/interactive_select.py +3 -1
- cognite_toolkit/_cdf_tk/utils/modules.py +7 -0
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.47.dist-info → cognite_toolkit-0.7.49.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.7.47.dist-info → cognite_toolkit-0.7.49.dist-info}/RECORD +61 -43
- cognite_toolkit/_cdf_tk/commands/build_v2/build_issues.py +0 -27
- /cognite_toolkit/_cdf_tk/client/resource_classes/{search_config_resource.py → search_config.py} +0 -0
- {cognite_toolkit-0.7.47.dist-info → cognite_toolkit-0.7.49.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.7.47.dist-info → cognite_toolkit-0.7.49.dist-info}/entry_points.txt +0 -0
|
@@ -30,8 +30,8 @@ from cognite_toolkit._cdf_tk.client.resource_classes.legacy.canvas import (
|
|
|
30
30
|
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.charts import Chart, ChartWrite
|
|
31
31
|
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.migration import ResourceViewMappingApply
|
|
32
32
|
from cognite_toolkit._cdf_tk.client.resource_classes.three_d import (
|
|
33
|
+
AssetMappingClassicResponse,
|
|
33
34
|
AssetMappingDMRequest,
|
|
34
|
-
AssetMappingResponse,
|
|
35
35
|
RevisionStatus,
|
|
36
36
|
ThreeDModelResponse,
|
|
37
37
|
)
|
|
@@ -46,13 +46,13 @@ from cognite_toolkit._cdf_tk.commands._migrate.issues import (
|
|
|
46
46
|
CanvasMigrationIssue,
|
|
47
47
|
ChartMigrationIssue,
|
|
48
48
|
ConversionIssue,
|
|
49
|
-
MigrationIssue,
|
|
50
49
|
ThreeDModelMigrationIssue,
|
|
51
50
|
)
|
|
52
51
|
from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
|
|
53
52
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitMigrationError, ToolkitValueError
|
|
54
53
|
from cognite_toolkit._cdf_tk.protocols import T_ResourceRequest, T_ResourceResponse
|
|
55
54
|
from cognite_toolkit._cdf_tk.storageio._base import T_Selector
|
|
55
|
+
from cognite_toolkit._cdf_tk.storageio.logger import DataLogger, NoOpLogger
|
|
56
56
|
from cognite_toolkit._cdf_tk.storageio.selectors import CanvasSelector, ChartSelector, ThreeDSelector
|
|
57
57
|
from cognite_toolkit._cdf_tk.utils import humanize_collection
|
|
58
58
|
from cognite_toolkit._cdf_tk.utils.useful_types2 import T_AssetCentricResourceExtended
|
|
@@ -62,6 +62,10 @@ from .selectors import AssetCentricMigrationSelector
|
|
|
62
62
|
|
|
63
63
|
|
|
64
64
|
class DataMapper(Generic[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC):
|
|
65
|
+
def __init__(self, client: ToolkitClient) -> None:
|
|
66
|
+
self.client = client
|
|
67
|
+
self.logger: DataLogger = NoOpLogger()
|
|
68
|
+
|
|
65
69
|
def prepare(self, source_selector: T_Selector) -> None:
|
|
66
70
|
"""Prepare the data mapper with the given source selector.
|
|
67
71
|
|
|
@@ -73,14 +77,14 @@ class DataMapper(Generic[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC
|
|
|
73
77
|
pass
|
|
74
78
|
|
|
75
79
|
@abstractmethod
|
|
76
|
-
def map(self, source: Sequence[T_ResourceResponse]) -> Sequence[
|
|
80
|
+
def map(self, source: Sequence[T_ResourceResponse]) -> Sequence[T_ResourceRequest | None]:
|
|
77
81
|
"""Map a chunk of source data to the target format.
|
|
78
82
|
|
|
79
83
|
Args:
|
|
80
84
|
source: The source data chunk to be mapped.
|
|
81
85
|
|
|
82
86
|
Returns:
|
|
83
|
-
A
|
|
87
|
+
A sequence of mapped target data.
|
|
84
88
|
|
|
85
89
|
"""
|
|
86
90
|
raise NotImplementedError("Subclasses must implement this method.")
|
|
@@ -90,7 +94,7 @@ class AssetCentricMapper(
|
|
|
90
94
|
DataMapper[AssetCentricMigrationSelector, AssetCentricMapping[T_AssetCentricResourceExtended], InstanceApply]
|
|
91
95
|
):
|
|
92
96
|
def __init__(self, client: ToolkitClient) -> None:
|
|
93
|
-
|
|
97
|
+
super().__init__(client)
|
|
94
98
|
self._ingestion_view_by_id: dict[ViewId, View] = {}
|
|
95
99
|
self._view_mapping_by_id: dict[str, ResourceViewMappingApply] = {}
|
|
96
100
|
self._direct_relation_cache = DirectRelationCache(client)
|
|
@@ -118,14 +122,37 @@ class AssetCentricMapper(
|
|
|
118
122
|
|
|
119
123
|
def map(
|
|
120
124
|
self, source: Sequence[AssetCentricMapping[T_AssetCentricResourceExtended]]
|
|
121
|
-
) -> Sequence[
|
|
125
|
+
) -> Sequence[InstanceApply | None]:
|
|
122
126
|
"""Map a chunk of asset-centric data to InstanceApplyList format."""
|
|
123
127
|
# We update the direct relation cache in bulk for all resources in the chunk.
|
|
124
128
|
self._direct_relation_cache.update(item.resource for item in source)
|
|
125
|
-
output: list[
|
|
129
|
+
output: list[InstanceApply | None] = []
|
|
130
|
+
issues: list[ConversionIssue] = []
|
|
126
131
|
for item in source:
|
|
127
132
|
instance, conversion_issue = self._map_single_item(item)
|
|
128
|
-
|
|
133
|
+
identifier = str(item.mapping.as_asset_centric_id())
|
|
134
|
+
|
|
135
|
+
if conversion_issue.missing_instance_space:
|
|
136
|
+
self.logger.tracker.add_issue(identifier, "Missing instance space")
|
|
137
|
+
if conversion_issue.failed_conversions:
|
|
138
|
+
self.logger.tracker.add_issue(identifier, "Failed conversions")
|
|
139
|
+
if conversion_issue.invalid_instance_property_types:
|
|
140
|
+
self.logger.tracker.add_issue(identifier, "Invalid instance property types")
|
|
141
|
+
if conversion_issue.missing_asset_centric_properties:
|
|
142
|
+
self.logger.tracker.add_issue(identifier, "Missing asset-centric properties")
|
|
143
|
+
if conversion_issue.missing_instance_properties:
|
|
144
|
+
self.logger.tracker.add_issue(identifier, "Missing instance properties")
|
|
145
|
+
if conversion_issue.ignored_asset_centric_properties:
|
|
146
|
+
self.logger.tracker.add_issue(identifier, "Ignored asset-centric properties")
|
|
147
|
+
|
|
148
|
+
if conversion_issue.has_issues:
|
|
149
|
+
issues.append(conversion_issue)
|
|
150
|
+
|
|
151
|
+
if instance is None:
|
|
152
|
+
self.logger.tracker.finalize_item(identifier, "failure")
|
|
153
|
+
output.append(instance)
|
|
154
|
+
if issues:
|
|
155
|
+
self.logger.log(issues)
|
|
129
156
|
return output
|
|
130
157
|
|
|
131
158
|
def _map_single_item(
|
|
@@ -154,15 +181,29 @@ class AssetCentricMapper(
|
|
|
154
181
|
|
|
155
182
|
|
|
156
183
|
class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
|
|
157
|
-
def
|
|
158
|
-
self.client = client
|
|
159
|
-
|
|
160
|
-
def map(self, source: Sequence[Chart]) -> Sequence[tuple[ChartWrite | None, MigrationIssue]]:
|
|
184
|
+
def map(self, source: Sequence[Chart]) -> Sequence[ChartWrite | None]:
|
|
161
185
|
self._populate_cache(source)
|
|
162
|
-
output: list[
|
|
186
|
+
output: list[ChartWrite | None] = []
|
|
187
|
+
issues: list[ChartMigrationIssue] = []
|
|
163
188
|
for item in source:
|
|
164
189
|
mapped_item, issue = self._map_single_item(item)
|
|
165
|
-
|
|
190
|
+
identifier = item.external_id
|
|
191
|
+
|
|
192
|
+
if issue.missing_timeseries_ids:
|
|
193
|
+
self.logger.tracker.add_issue(identifier, "Missing timeseries IDs")
|
|
194
|
+
if issue.missing_timeseries_external_ids:
|
|
195
|
+
self.logger.tracker.add_issue(identifier, "Missing timeseries external IDs")
|
|
196
|
+
if issue.missing_timeseries_identifier:
|
|
197
|
+
self.logger.tracker.add_issue(identifier, "Missing timeseries identifier")
|
|
198
|
+
|
|
199
|
+
if issue.has_issues:
|
|
200
|
+
issues.append(issue)
|
|
201
|
+
|
|
202
|
+
if mapped_item is None:
|
|
203
|
+
self.logger.tracker.finalize_item(identifier, "failure")
|
|
204
|
+
output.append(mapped_item)
|
|
205
|
+
if issues:
|
|
206
|
+
self.logger.log(issues)
|
|
166
207
|
return output
|
|
167
208
|
|
|
168
209
|
def _populate_cache(self, source: Sequence[Chart]) -> None:
|
|
@@ -184,7 +225,7 @@ class ChartMapper(DataMapper[ChartSelector, Chart, ChartWrite]):
|
|
|
184
225
|
self.client.migration.lookup.time_series(external_id=list(timeseries_external_ids))
|
|
185
226
|
|
|
186
227
|
def _map_single_item(self, item: Chart) -> tuple[ChartWrite | None, ChartMigrationIssue]:
|
|
187
|
-
issue = ChartMigrationIssue(chart_external_id=item.external_id)
|
|
228
|
+
issue = ChartMigrationIssue(chart_external_id=item.external_id, id=item.external_id)
|
|
188
229
|
time_series_collection = item.data.time_series_collection or []
|
|
189
230
|
timeseries_core_collection = self._create_timeseries_core_collection(time_series_collection, issue)
|
|
190
231
|
if issue.has_issues:
|
|
@@ -280,16 +321,30 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
|
|
|
280
321
|
DEFAULT_TIMESERIES_VIEW = ViewId("cdf_cdm", "CogniteTimeSeries", "v1")
|
|
281
322
|
|
|
282
323
|
def __init__(self, client: ToolkitClient, dry_run: bool, skip_on_missing_ref: bool = False) -> None:
|
|
283
|
-
|
|
324
|
+
super().__init__(client)
|
|
284
325
|
self.dry_run = dry_run
|
|
285
326
|
self.skip_on_missing_ref = skip_on_missing_ref
|
|
286
327
|
|
|
287
|
-
def map(self, source: Sequence[IndustrialCanvas]) -> Sequence[
|
|
328
|
+
def map(self, source: Sequence[IndustrialCanvas]) -> Sequence[IndustrialCanvasApply | None]:
|
|
288
329
|
self._populate_cache(source)
|
|
289
|
-
output: list[
|
|
330
|
+
output: list[IndustrialCanvasApply | None] = []
|
|
331
|
+
issues: list[CanvasMigrationIssue] = []
|
|
290
332
|
for item in source:
|
|
291
333
|
mapped_item, issue = self._map_single_item(item)
|
|
292
|
-
|
|
334
|
+
identifier = item.as_id()
|
|
335
|
+
|
|
336
|
+
if issue.missing_reference_ids:
|
|
337
|
+
self.logger.tracker.add_issue(identifier, "Missing reference IDs")
|
|
338
|
+
|
|
339
|
+
if issue.has_issues:
|
|
340
|
+
issues.append(issue)
|
|
341
|
+
|
|
342
|
+
if mapped_item is None:
|
|
343
|
+
self.logger.tracker.finalize_item(identifier, "failure")
|
|
344
|
+
|
|
345
|
+
output.append(mapped_item)
|
|
346
|
+
if issues:
|
|
347
|
+
self.logger.log(issues)
|
|
293
348
|
return output
|
|
294
349
|
|
|
295
350
|
@property
|
|
@@ -340,7 +395,9 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
|
|
|
340
395
|
|
|
341
396
|
def _map_single_item(self, canvas: IndustrialCanvas) -> tuple[IndustrialCanvasApply | None, CanvasMigrationIssue]:
|
|
342
397
|
update = canvas.as_write()
|
|
343
|
-
issue = CanvasMigrationIssue(
|
|
398
|
+
issue = CanvasMigrationIssue(
|
|
399
|
+
canvas_external_id=canvas.canvas.external_id, canvas_name=canvas.canvas.name, id=canvas.canvas.name
|
|
400
|
+
)
|
|
344
401
|
|
|
345
402
|
remaining_container_references: list[ContainerReferenceApply] = []
|
|
346
403
|
new_fdm_references: list[FdmInstanceContainerReferenceApply] = []
|
|
@@ -399,17 +456,26 @@ class CanvasMapper(DataMapper[CanvasSelector, IndustrialCanvas, IndustrialCanvas
|
|
|
399
456
|
|
|
400
457
|
|
|
401
458
|
class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrationRequest]):
|
|
402
|
-
def
|
|
403
|
-
self.client = client
|
|
404
|
-
|
|
405
|
-
def map(
|
|
406
|
-
self, source: Sequence[ThreeDModelResponse]
|
|
407
|
-
) -> Sequence[tuple[ThreeDMigrationRequest | None, MigrationIssue]]:
|
|
459
|
+
def map(self, source: Sequence[ThreeDModelResponse]) -> Sequence[ThreeDMigrationRequest | None]:
|
|
408
460
|
self._populate_cache(source)
|
|
409
|
-
output: list[
|
|
461
|
+
output: list[ThreeDMigrationRequest | None] = []
|
|
462
|
+
issues: list[ThreeDModelMigrationIssue] = []
|
|
410
463
|
for item in source:
|
|
411
464
|
mapped_item, issue = self._map_single_item(item)
|
|
412
|
-
|
|
465
|
+
identifier = item.name
|
|
466
|
+
|
|
467
|
+
if issue.error_message:
|
|
468
|
+
for error in issue.error_message:
|
|
469
|
+
self.logger.tracker.add_issue(identifier, error)
|
|
470
|
+
|
|
471
|
+
if issue.has_issues:
|
|
472
|
+
issues.append(issue)
|
|
473
|
+
|
|
474
|
+
if mapped_item is None:
|
|
475
|
+
self.logger.tracker.finalize_item(identifier, "failure")
|
|
476
|
+
output.append(mapped_item)
|
|
477
|
+
if issues:
|
|
478
|
+
self.logger.log(issues)
|
|
413
479
|
return output
|
|
414
480
|
|
|
415
481
|
def _populate_cache(self, source: Sequence[ThreeDModelResponse]) -> None:
|
|
@@ -422,7 +488,7 @@ class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrati
|
|
|
422
488
|
def _map_single_item(
|
|
423
489
|
self, item: ThreeDModelResponse
|
|
424
490
|
) -> tuple[ThreeDMigrationRequest | None, ThreeDModelMigrationIssue]:
|
|
425
|
-
issue = ThreeDModelMigrationIssue(model_name=item.name, model_id=item.id)
|
|
491
|
+
issue = ThreeDModelMigrationIssue(model_name=item.name, model_id=item.id, id=item.name)
|
|
426
492
|
instance_space: str | None = None
|
|
427
493
|
last_revision_id: int | None = None
|
|
428
494
|
model_type: Literal["CAD", "PointCloud", "Image360"] | None = None
|
|
@@ -477,21 +543,31 @@ class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrati
|
|
|
477
543
|
return None
|
|
478
544
|
|
|
479
545
|
|
|
480
|
-
class ThreeDAssetMapper(DataMapper[ThreeDSelector,
|
|
481
|
-
def
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
def map(
|
|
485
|
-
self, source: Sequence[AssetMappingResponse]
|
|
486
|
-
) -> Sequence[tuple[AssetMappingDMRequest | None, MigrationIssue]]:
|
|
487
|
-
output: list[tuple[AssetMappingDMRequest | None, MigrationIssue]] = []
|
|
546
|
+
class ThreeDAssetMapper(DataMapper[ThreeDSelector, AssetMappingClassicResponse, AssetMappingDMRequest]):
|
|
547
|
+
def map(self, source: Sequence[AssetMappingClassicResponse]) -> Sequence[AssetMappingDMRequest | None]:
|
|
548
|
+
output: list[AssetMappingDMRequest | None] = []
|
|
549
|
+
issues: list[ThreeDModelMigrationIssue] = []
|
|
488
550
|
self._populate_cache(source)
|
|
489
551
|
for item in source:
|
|
490
552
|
mapped_item, issue = self._map_single_item(item)
|
|
491
|
-
|
|
553
|
+
identifier = f"AssetMapping_{item.model_id!s}_{item.revision_id!s}_{item.asset_id!s}"
|
|
554
|
+
|
|
555
|
+
if issue.error_message:
|
|
556
|
+
for error in issue.error_message:
|
|
557
|
+
self.logger.tracker.add_issue(identifier, error)
|
|
558
|
+
|
|
559
|
+
if issue.has_issues:
|
|
560
|
+
issues.append(issue)
|
|
561
|
+
|
|
562
|
+
if mapped_item is None:
|
|
563
|
+
self.logger.tracker.finalize_item(identifier, "failure")
|
|
564
|
+
|
|
565
|
+
output.append(mapped_item)
|
|
566
|
+
if issues:
|
|
567
|
+
self.logger.log(issues)
|
|
492
568
|
return output
|
|
493
569
|
|
|
494
|
-
def _populate_cache(self, source: Sequence[
|
|
570
|
+
def _populate_cache(self, source: Sequence[AssetMappingClassicResponse]) -> None:
|
|
495
571
|
asset_ids: set[int] = set()
|
|
496
572
|
for mapping in source:
|
|
497
573
|
if mapping.asset_id is not None:
|
|
@@ -499,9 +575,11 @@ class ThreeDAssetMapper(DataMapper[ThreeDSelector, AssetMappingResponse, AssetMa
|
|
|
499
575
|
self.client.migration.lookup.assets(list(asset_ids))
|
|
500
576
|
|
|
501
577
|
def _map_single_item(
|
|
502
|
-
self, item:
|
|
578
|
+
self, item: AssetMappingClassicResponse
|
|
503
579
|
) -> tuple[AssetMappingDMRequest | None, ThreeDModelMigrationIssue]:
|
|
504
|
-
issue = ThreeDModelMigrationIssue(
|
|
580
|
+
issue = ThreeDModelMigrationIssue(
|
|
581
|
+
model_name=f"AssetMapping_{item.model_id}", model_id=item.model_id, id=f"AssetMapping_{item.model_id}"
|
|
582
|
+
)
|
|
505
583
|
asset_instance_id = item.asset_instance_id
|
|
506
584
|
if item.asset_id and asset_instance_id is None:
|
|
507
585
|
asset_node_id = self.client.migration.lookup.assets(item.asset_id)
|
|
@@ -1,34 +1,27 @@
|
|
|
1
|
-
import
|
|
2
|
-
from typing import Any, ClassVar
|
|
1
|
+
from typing import Any, Literal
|
|
3
2
|
|
|
4
|
-
from cognite.client.data_classes.data_modeling import NodeId
|
|
5
|
-
from cognite.client.utils._identifier import InstanceId
|
|
6
|
-
from cognite.client.utils._text import to_camel_case
|
|
7
3
|
from pydantic import BaseModel, Field, field_serializer
|
|
4
|
+
from pydantic.alias_generators import to_camel
|
|
8
5
|
|
|
6
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeReference
|
|
9
7
|
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.migration import AssetCentricId
|
|
10
|
-
from cognite_toolkit._cdf_tk.
|
|
8
|
+
from cognite_toolkit._cdf_tk.storageio.logger import LogEntry
|
|
11
9
|
|
|
12
10
|
|
|
13
|
-
class
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class MigrationIssue(MigrationObject):
|
|
11
|
+
class MigrationIssue(LogEntry):
|
|
17
12
|
"""Represents an issue encountered during migration."""
|
|
18
13
|
|
|
19
|
-
type:
|
|
20
|
-
|
|
21
|
-
def dump(self) -> dict[str, JsonVal]:
|
|
22
|
-
# Dump json to ensure it is serializable
|
|
23
|
-
dumped = json.loads(self.model_dump_json(by_alias=True))
|
|
24
|
-
dumped["type"] = self.type
|
|
25
|
-
return dumped
|
|
14
|
+
type: str
|
|
26
15
|
|
|
27
16
|
@property
|
|
28
17
|
def has_issues(self) -> bool:
|
|
29
18
|
"""Check if there are any issues recorded in this MigrationIssue."""
|
|
30
19
|
return True
|
|
31
20
|
|
|
21
|
+
def dump(self) -> dict[str, Any]:
|
|
22
|
+
"""Serialize the MigrationIssue to a dictionary."""
|
|
23
|
+
return self.model_dump(by_alias=True)
|
|
24
|
+
|
|
32
25
|
|
|
33
26
|
class ThreeDModelMigrationIssue(MigrationIssue):
|
|
34
27
|
"""Represents a 3D model migration issue encountered during migration.
|
|
@@ -37,7 +30,7 @@ class ThreeDModelMigrationIssue(MigrationIssue):
|
|
|
37
30
|
model_external_id (str): The external ID of the 3D model that could not be migrated.
|
|
38
31
|
"""
|
|
39
32
|
|
|
40
|
-
type:
|
|
33
|
+
type: Literal["threeDModelMigration"] = "threeDModelMigration"
|
|
41
34
|
model_name: str
|
|
42
35
|
model_id: int
|
|
43
36
|
error_message: list[str] = Field(default_factory=list)
|
|
@@ -55,7 +48,7 @@ class ChartMigrationIssue(MigrationIssue):
|
|
|
55
48
|
chart_external_id (str): The external ID of the chart that could not be migrated.
|
|
56
49
|
"""
|
|
57
50
|
|
|
58
|
-
type:
|
|
51
|
+
type: Literal["chartMigration"] = "chartMigration"
|
|
59
52
|
chart_external_id: str
|
|
60
53
|
missing_timeseries_ids: list[int] = Field(default_factory=list)
|
|
61
54
|
missing_timeseries_external_ids: list[str] = Field(default_factory=list)
|
|
@@ -70,7 +63,7 @@ class ChartMigrationIssue(MigrationIssue):
|
|
|
70
63
|
|
|
71
64
|
|
|
72
65
|
class CanvasMigrationIssue(MigrationIssue):
|
|
73
|
-
type:
|
|
66
|
+
type: Literal["canvasMigration"] = "canvasMigration"
|
|
74
67
|
canvas_external_id: str
|
|
75
68
|
canvas_name: str
|
|
76
69
|
missing_reference_ids: list[AssetCentricId] = Field(default_factory=list)
|
|
@@ -84,7 +77,7 @@ class CanvasMigrationIssue(MigrationIssue):
|
|
|
84
77
|
class ReadIssue(MigrationIssue):
|
|
85
78
|
"""Represents a read issue encountered during migration."""
|
|
86
79
|
|
|
87
|
-
|
|
80
|
+
...
|
|
88
81
|
|
|
89
82
|
|
|
90
83
|
class ReadFileIssue(ReadIssue):
|
|
@@ -95,7 +88,7 @@ class ReadFileIssue(ReadIssue):
|
|
|
95
88
|
error (str | None): An optional error message providing additional details about the read issue.
|
|
96
89
|
"""
|
|
97
90
|
|
|
98
|
-
type:
|
|
91
|
+
type: Literal["fileRead"] = "fileRead"
|
|
99
92
|
|
|
100
93
|
row_no: int
|
|
101
94
|
error: str | None = None
|
|
@@ -109,7 +102,7 @@ class ReadAPIIssue(ReadIssue):
|
|
|
109
102
|
error (str | None): An optional error message providing additional details about the read issue.
|
|
110
103
|
"""
|
|
111
104
|
|
|
112
|
-
type:
|
|
105
|
+
type: Literal["apiRead"] = "apiRead"
|
|
113
106
|
asset_centric_id: AssetCentricId
|
|
114
107
|
error: str | None = None
|
|
115
108
|
|
|
@@ -121,7 +114,7 @@ class ReadAPIIssue(ReadIssue):
|
|
|
121
114
|
}
|
|
122
115
|
|
|
123
116
|
|
|
124
|
-
class FailedConversion(
|
|
117
|
+
class FailedConversion(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
|
|
125
118
|
"""Represents a property that failed to convert during migration.
|
|
126
119
|
|
|
127
120
|
Attributes:
|
|
@@ -135,7 +128,7 @@ class FailedConversion(MigrationObject):
|
|
|
135
128
|
error: str
|
|
136
129
|
|
|
137
130
|
|
|
138
|
-
class InvalidPropertyDataType(
|
|
131
|
+
class InvalidPropertyDataType(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
|
|
139
132
|
"""Represents a property with an invalid type during migration.
|
|
140
133
|
|
|
141
134
|
Attributes:
|
|
@@ -160,9 +153,9 @@ class ConversionIssue(MigrationIssue):
|
|
|
160
153
|
failed_conversions (list[FailedConversion]): List of properties that failed to convert with reasons.
|
|
161
154
|
"""
|
|
162
155
|
|
|
163
|
-
type:
|
|
156
|
+
type: Literal["conversion"] = "conversion"
|
|
164
157
|
asset_centric_id: AssetCentricId
|
|
165
|
-
instance_id:
|
|
158
|
+
instance_id: NodeReference
|
|
166
159
|
missing_asset_centric_properties: list[str] = Field(default_factory=list)
|
|
167
160
|
missing_instance_properties: list[str] = Field(default_factory=list)
|
|
168
161
|
invalid_instance_property_types: list[InvalidPropertyDataType] = Field(default_factory=list)
|
|
@@ -181,10 +174,6 @@ class ConversionIssue(MigrationIssue):
|
|
|
181
174
|
or self.missing_instance_space
|
|
182
175
|
)
|
|
183
176
|
|
|
184
|
-
@field_serializer("instance_id")
|
|
185
|
-
def serialize_instance_id(self, instance_id: NodeId) -> dict[str, str]:
|
|
186
|
-
return instance_id.dump(include_instance_type=True)
|
|
187
|
-
|
|
188
177
|
@field_serializer("asset_centric_id")
|
|
189
178
|
def serialize_asset_centric_id(self, asset_centric_id: AssetCentricId) -> dict[str, Any]:
|
|
190
179
|
return {
|
|
@@ -197,16 +186,10 @@ class WriteIssue(MigrationIssue):
|
|
|
197
186
|
"""Represents a write issue encountered during migration.
|
|
198
187
|
|
|
199
188
|
Attributes:
|
|
200
|
-
instance_id (InstanceId): The InstanceId of the data model instance that could not be written.
|
|
201
189
|
status_code (int): The HTTP status code returned during the write operation.
|
|
202
190
|
message (str | None): An optional message providing additional details about the write issue.
|
|
203
191
|
"""
|
|
204
192
|
|
|
205
|
-
type:
|
|
206
|
-
instance_id: InstanceId
|
|
193
|
+
type: Literal["write"] = "write"
|
|
207
194
|
status_code: int
|
|
208
195
|
message: str | None = None
|
|
209
|
-
|
|
210
|
-
@field_serializer("instance_id")
|
|
211
|
-
def serialize_instance_id(self, instance_id: NodeId) -> dict[str, str]:
|
|
212
|
-
return instance_id.dump(include_instance_type=True)
|
|
@@ -16,8 +16,8 @@ from cognite_toolkit._cdf_tk.client.http_client import (
|
|
|
16
16
|
)
|
|
17
17
|
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.pending_instances_ids import PendingInstanceId
|
|
18
18
|
from cognite_toolkit._cdf_tk.client.resource_classes.three_d import (
|
|
19
|
+
AssetMappingClassicResponse,
|
|
19
20
|
AssetMappingDMRequest,
|
|
20
|
-
AssetMappingResponse,
|
|
21
21
|
ThreeDModelResponse,
|
|
22
22
|
)
|
|
23
23
|
from cognite_toolkit._cdf_tk.commands._migrate.data_classes import ThreeDMigrationRequest
|
|
@@ -78,7 +78,7 @@ class AssetCentricMigrationIO(
|
|
|
78
78
|
self.skip_linking = skip_linking
|
|
79
79
|
|
|
80
80
|
def as_id(self, item: AssetCentricMapping) -> str:
|
|
81
|
-
return
|
|
81
|
+
return str(item.mapping.as_asset_centric_id())
|
|
82
82
|
|
|
83
83
|
def stream_data(self, selector: AssetCentricMigrationSelector, limit: int | None = None) -> Iterator[Page]:
|
|
84
84
|
if isinstance(selector, MigrationCSVFileSelector):
|
|
@@ -181,7 +181,7 @@ class AssetCentricMigrationIO(
|
|
|
181
181
|
|
|
182
182
|
pending_instance_id_endpoint = self.PENDING_INSTANCE_ID_ENDPOINT_BY_KIND[selector.kind]
|
|
183
183
|
results: list[HTTPMessage] = []
|
|
184
|
-
to_upload = self.link_asset_centric(data_chunk, http_client,
|
|
184
|
+
to_upload = self.link_asset_centric(data_chunk, http_client, pending_instance_id_endpoint)
|
|
185
185
|
if to_upload:
|
|
186
186
|
results.extend(list(super().upload_items(to_upload, http_client, None)))
|
|
187
187
|
return results
|
|
@@ -191,7 +191,6 @@ class AssetCentricMigrationIO(
|
|
|
191
191
|
cls,
|
|
192
192
|
data_chunk: Sequence[UploadItem[InstanceApply]],
|
|
193
193
|
http_client: HTTPClient,
|
|
194
|
-
results: list[HTTPMessage],
|
|
195
194
|
pending_instance_id_endpoint: str,
|
|
196
195
|
) -> Sequence[UploadItem[InstanceApply]]:
|
|
197
196
|
"""Links asset-centric resources to their (uncreated) instances using the pending-instance-ids endpoint."""
|
|
@@ -212,7 +211,6 @@ class AssetCentricMigrationIO(
|
|
|
212
211
|
for res in batch_results:
|
|
213
212
|
if isinstance(res, SuccessResponseItems):
|
|
214
213
|
successful_linked.update(res.ids)
|
|
215
|
-
results.extend(batch_results)
|
|
216
214
|
to_upload = [item for item in data_chunk if item.source_id in successful_linked]
|
|
217
215
|
return to_upload
|
|
218
216
|
|
|
@@ -393,7 +391,7 @@ class ThreeDMigrationIO(UploadableStorageIO[ThreeDSelector, ThreeDModelResponse,
|
|
|
393
391
|
self.data_model_type = data_model_type
|
|
394
392
|
|
|
395
393
|
def as_id(self, item: ThreeDModelResponse) -> str:
|
|
396
|
-
return
|
|
394
|
+
return item.name
|
|
397
395
|
|
|
398
396
|
def _is_selected(self, item: ThreeDModelResponse, included_models: set[int] | None) -> bool:
|
|
399
397
|
return self._is_correct_type(item) and (included_models is None or item.id in included_models)
|
|
@@ -415,7 +413,7 @@ class ThreeDMigrationIO(UploadableStorageIO[ThreeDSelector, ThreeDModelResponse,
|
|
|
415
413
|
total = 0
|
|
416
414
|
while True:
|
|
417
415
|
request_limit = min(self.DOWNLOAD_LIMIT, limit - total) if limit is not None else self.DOWNLOAD_LIMIT
|
|
418
|
-
response = self.client.tool.three_d.
|
|
416
|
+
response = self.client.tool.three_d.models_classic.paginate(
|
|
419
417
|
published=published, include_revision_info=True, limit=request_limit, cursor=cursor
|
|
420
418
|
)
|
|
421
419
|
items = [item for item in response.items if self._is_selected(item, included_models)]
|
|
@@ -477,7 +475,9 @@ class ThreeDMigrationIO(UploadableStorageIO[ThreeDSelector, ThreeDModelResponse,
|
|
|
477
475
|
return results
|
|
478
476
|
|
|
479
477
|
|
|
480
|
-
class ThreeDAssetMappingMigrationIO(
|
|
478
|
+
class ThreeDAssetMappingMigrationIO(
|
|
479
|
+
UploadableStorageIO[ThreeDSelector, AssetMappingClassicResponse, AssetMappingDMRequest]
|
|
480
|
+
):
|
|
481
481
|
KIND = "3DMigrationAssetMapping"
|
|
482
482
|
SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
|
|
483
483
|
SUPPORTED_COMPRESSIONS = frozenset({".gz"})
|
|
@@ -493,10 +493,12 @@ class ThreeDAssetMappingMigrationIO(UploadableStorageIO[ThreeDSelector, AssetMap
|
|
|
493
493
|
# We can only migrate asset mappings for 3D models that are already migrated to data modeling.
|
|
494
494
|
self._3D_io = ThreeDMigrationIO(client, data_model_type="data modeling")
|
|
495
495
|
|
|
496
|
-
def as_id(self, item:
|
|
496
|
+
def as_id(self, item: AssetMappingClassicResponse) -> str:
|
|
497
497
|
return f"AssetMapping_{item.model_id!s}_{item.revision_id!s}_{item.asset_id!s}"
|
|
498
498
|
|
|
499
|
-
def stream_data(
|
|
499
|
+
def stream_data(
|
|
500
|
+
self, selector: ThreeDSelector, limit: int | None = None
|
|
501
|
+
) -> Iterable[Page[AssetMappingClassicResponse]]:
|
|
500
502
|
total = 0
|
|
501
503
|
for three_d_page in self._3D_io.stream_data(selector, None):
|
|
502
504
|
for model in three_d_page.items:
|
|
@@ -510,7 +512,7 @@ class ThreeDAssetMappingMigrationIO(UploadableStorageIO[ThreeDSelector, AssetMap
|
|
|
510
512
|
)
|
|
511
513
|
if limit is not None and total >= limit:
|
|
512
514
|
return
|
|
513
|
-
response = self.client.tool.three_d.
|
|
515
|
+
response = self.client.tool.three_d.asset_mappings_classic.paginate(
|
|
514
516
|
model_id=model.id,
|
|
515
517
|
revision_id=model.last_revision_info.revision_id,
|
|
516
518
|
cursor=cursor,
|
|
@@ -562,6 +564,6 @@ class ThreeDAssetMappingMigrationIO(UploadableStorageIO[ThreeDSelector, AssetMap
|
|
|
562
564
|
raise NotImplementedError("Deserializing 3D Asset Mappings from JSON is not supported.")
|
|
563
565
|
|
|
564
566
|
def data_to_json_chunk(
|
|
565
|
-
self, data_chunk: Sequence[
|
|
567
|
+
self, data_chunk: Sequence[AssetMappingClassicResponse], selector: ThreeDSelector | None = None
|
|
566
568
|
) -> list[dict[str, JsonVal]]:
|
|
567
569
|
raise NotImplementedError("Serializing 3D Asset Mappings to JSON is not supported.")
|