cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.30__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. cognite_toolkit/_cdf.py +16 -17
  2. cognite_toolkit/_cdf_tk/apps/__init__.py +2 -0
  3. cognite_toolkit/_cdf_tk/apps/_core_app.py +13 -5
  4. cognite_toolkit/_cdf_tk/apps/_data_app.py +1 -1
  5. cognite_toolkit/_cdf_tk/apps/_dev_app.py +86 -0
  6. cognite_toolkit/_cdf_tk/apps/_download_app.py +692 -24
  7. cognite_toolkit/_cdf_tk/apps/_dump_app.py +43 -101
  8. cognite_toolkit/_cdf_tk/apps/_landing_app.py +18 -4
  9. cognite_toolkit/_cdf_tk/apps/_migrate_app.py +249 -9
  10. cognite_toolkit/_cdf_tk/apps/_modules_app.py +0 -3
  11. cognite_toolkit/_cdf_tk/apps/_purge.py +15 -43
  12. cognite_toolkit/_cdf_tk/apps/_run.py +11 -0
  13. cognite_toolkit/_cdf_tk/apps/_upload_app.py +45 -6
  14. cognite_toolkit/_cdf_tk/builders/__init__.py +2 -2
  15. cognite_toolkit/_cdf_tk/builders/_base.py +28 -42
  16. cognite_toolkit/_cdf_tk/cdf_toml.py +20 -1
  17. cognite_toolkit/_cdf_tk/client/_toolkit_client.py +23 -3
  18. cognite_toolkit/_cdf_tk/client/api/extended_functions.py +6 -9
  19. cognite_toolkit/_cdf_tk/client/api/infield.py +93 -1
  20. cognite_toolkit/_cdf_tk/client/api/migration.py +175 -1
  21. cognite_toolkit/_cdf_tk/client/api/streams.py +84 -0
  22. cognite_toolkit/_cdf_tk/client/api/three_d.py +50 -0
  23. cognite_toolkit/_cdf_tk/client/data_classes/base.py +25 -1
  24. cognite_toolkit/_cdf_tk/client/data_classes/canvas.py +46 -3
  25. cognite_toolkit/_cdf_tk/client/data_classes/charts.py +3 -3
  26. cognite_toolkit/_cdf_tk/client/data_classes/charts_data.py +95 -213
  27. cognite_toolkit/_cdf_tk/client/data_classes/infield.py +32 -18
  28. cognite_toolkit/_cdf_tk/client/data_classes/migration.py +10 -2
  29. cognite_toolkit/_cdf_tk/client/data_classes/streams.py +90 -0
  30. cognite_toolkit/_cdf_tk/client/data_classes/three_d.py +47 -0
  31. cognite_toolkit/_cdf_tk/client/testing.py +18 -2
  32. cognite_toolkit/_cdf_tk/commands/__init__.py +6 -6
  33. cognite_toolkit/_cdf_tk/commands/_changes.py +3 -42
  34. cognite_toolkit/_cdf_tk/commands/_download.py +21 -11
  35. cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py +0 -2
  36. cognite_toolkit/_cdf_tk/commands/_migrate/command.py +22 -20
  37. cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +133 -91
  38. cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py +73 -22
  39. cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +311 -43
  40. cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py +5 -5
  41. cognite_toolkit/_cdf_tk/commands/_migrate/issues.py +33 -0
  42. cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +157 -8
  43. cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py +9 -4
  44. cognite_toolkit/_cdf_tk/commands/_purge.py +27 -28
  45. cognite_toolkit/_cdf_tk/commands/_questionary_style.py +16 -0
  46. cognite_toolkit/_cdf_tk/commands/_upload.py +109 -86
  47. cognite_toolkit/_cdf_tk/commands/about.py +221 -0
  48. cognite_toolkit/_cdf_tk/commands/auth.py +19 -12
  49. cognite_toolkit/_cdf_tk/commands/build_cmd.py +15 -61
  50. cognite_toolkit/_cdf_tk/commands/clean.py +63 -16
  51. cognite_toolkit/_cdf_tk/commands/deploy.py +20 -17
  52. cognite_toolkit/_cdf_tk/commands/dump_resource.py +6 -4
  53. cognite_toolkit/_cdf_tk/commands/init.py +225 -3
  54. cognite_toolkit/_cdf_tk/commands/modules.py +20 -44
  55. cognite_toolkit/_cdf_tk/commands/pull.py +6 -19
  56. cognite_toolkit/_cdf_tk/commands/resources.py +179 -0
  57. cognite_toolkit/_cdf_tk/constants.py +20 -1
  58. cognite_toolkit/_cdf_tk/cruds/__init__.py +19 -5
  59. cognite_toolkit/_cdf_tk/cruds/_base_cruds.py +14 -70
  60. cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +8 -17
  61. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py +4 -1
  62. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py +11 -9
  63. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py +4 -14
  64. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py +44 -43
  65. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py +4 -11
  66. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py +4 -13
  67. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py +205 -66
  68. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py +5 -17
  69. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py +116 -27
  70. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py +6 -27
  71. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py +9 -28
  72. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/hosted_extractors.py +12 -30
  73. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/industrial_tool.py +3 -7
  74. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/location.py +3 -15
  75. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py +4 -12
  76. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/raw.py +4 -10
  77. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py +3 -8
  78. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/robotics.py +15 -44
  79. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py +94 -0
  80. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/three_d_model.py +3 -7
  81. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py +5 -15
  82. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py +39 -31
  83. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/workflow.py +20 -40
  84. cognite_toolkit/_cdf_tk/cruds/_worker.py +24 -36
  85. cognite_toolkit/_cdf_tk/feature_flags.py +16 -36
  86. cognite_toolkit/_cdf_tk/plugins.py +2 -1
  87. cognite_toolkit/_cdf_tk/resource_classes/__init__.py +4 -0
  88. cognite_toolkit/_cdf_tk/resource_classes/capabilities.py +12 -0
  89. cognite_toolkit/_cdf_tk/resource_classes/functions.py +3 -1
  90. cognite_toolkit/_cdf_tk/resource_classes/infield_cdm_location_config.py +109 -0
  91. cognite_toolkit/_cdf_tk/resource_classes/migration.py +8 -17
  92. cognite_toolkit/_cdf_tk/resource_classes/streams.py +29 -0
  93. cognite_toolkit/_cdf_tk/storageio/__init__.py +9 -21
  94. cognite_toolkit/_cdf_tk/storageio/_annotations.py +19 -16
  95. cognite_toolkit/_cdf_tk/storageio/_applications.py +338 -26
  96. cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +67 -104
  97. cognite_toolkit/_cdf_tk/storageio/_base.py +61 -29
  98. cognite_toolkit/_cdf_tk/storageio/_datapoints.py +276 -20
  99. cognite_toolkit/_cdf_tk/storageio/_file_content.py +436 -0
  100. cognite_toolkit/_cdf_tk/storageio/_instances.py +34 -2
  101. cognite_toolkit/_cdf_tk/storageio/_raw.py +26 -0
  102. cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py +62 -4
  103. cognite_toolkit/_cdf_tk/storageio/selectors/_base.py +14 -2
  104. cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py +14 -0
  105. cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py +14 -0
  106. cognite_toolkit/_cdf_tk/storageio/selectors/_datapoints.py +23 -3
  107. cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py +164 -0
  108. cognite_toolkit/_cdf_tk/tk_warnings/other.py +4 -0
  109. cognite_toolkit/_cdf_tk/tracker.py +2 -2
  110. cognite_toolkit/_cdf_tk/utils/dtype_conversion.py +9 -3
  111. cognite_toolkit/_cdf_tk/utils/fileio/__init__.py +2 -0
  112. cognite_toolkit/_cdf_tk/utils/fileio/_base.py +5 -1
  113. cognite_toolkit/_cdf_tk/utils/fileio/_readers.py +112 -20
  114. cognite_toolkit/_cdf_tk/utils/fileio/_writers.py +15 -15
  115. cognite_toolkit/_cdf_tk/utils/http_client/_client.py +284 -18
  116. cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py +50 -4
  117. cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py +187 -0
  118. cognite_toolkit/_cdf_tk/utils/interactive_select.py +9 -14
  119. cognite_toolkit/_cdf_tk/utils/sql_parser.py +2 -3
  120. cognite_toolkit/_cdf_tk/utils/useful_types.py +6 -2
  121. cognite_toolkit/_cdf_tk/validation.py +79 -1
  122. cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
  123. cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
  124. cognite_toolkit/_resources/cdf.toml +5 -4
  125. cognite_toolkit/_version.py +1 -1
  126. cognite_toolkit/config.dev.yaml +13 -0
  127. {cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.30.dist-info}/METADATA +24 -24
  128. {cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.30.dist-info}/RECORD +153 -143
  129. cognite_toolkit-0.7.30.dist-info/WHEEL +4 -0
  130. {cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.30.dist-info}/entry_points.txt +1 -0
  131. cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py +0 -201
  132. cognite_toolkit/_cdf_tk/commands/dump_data.py +0 -489
  133. cognite_toolkit/_cdf_tk/commands/featureflag.py +0 -27
  134. cognite_toolkit/_cdf_tk/utils/table_writers.py +0 -434
  135. cognite_toolkit-0.6.97.dist-info/WHEEL +0 -4
  136. cognite_toolkit-0.6.97.dist-info/licenses/LICENSE +0 -18
@@ -1,6 +1,5 @@
1
- from collections.abc import Mapping, Set
2
- from dataclasses import dataclass
3
- from typing import Any, ClassVar, overload
1
+ from collections.abc import Iterable, Mapping, Set
2
+ from typing import Any, ClassVar, cast
4
3
 
5
4
  from cognite.client.data_classes import Annotation, Asset, Event, FileMetadata, TimeSeries
6
5
  from cognite.client.data_classes.data_modeling import (
@@ -13,8 +12,13 @@ from cognite.client.data_classes.data_modeling import (
13
12
  )
14
13
  from cognite.client.data_classes.data_modeling.instances import EdgeApply, NodeOrEdgeData, PropertyValueWrite
15
14
  from cognite.client.data_classes.data_modeling.views import ViewProperty
15
+ from cognite.client.utils._identifier import InstanceId
16
16
 
17
- from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId, ResourceViewMapping
17
+ from cognite_toolkit._cdf_tk.client import ToolkitClient
18
+ from cognite_toolkit._cdf_tk.client.data_classes.migration import (
19
+ AssetCentricId,
20
+ ResourceViewMappingApply,
21
+ )
18
22
  from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
19
23
  from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
20
24
  asset_centric_convert_to_primary_property,
@@ -22,124 +26,162 @@ from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
22
26
  )
23
27
  from cognite_toolkit._cdf_tk.utils.useful_types import (
24
28
  AssetCentricResourceExtended,
25
- AssetCentricType,
29
+ AssetCentricTypeExtended,
26
30
  )
27
31
 
28
32
  from .data_model import INSTANCE_SOURCE_VIEW_ID
29
33
  from .issues import ConversionIssue, FailedConversion, InvalidPropertyDataType
30
34
 
31
35
 
32
- @dataclass
33
36
  class DirectRelationCache:
34
37
  """Cache for direct relation references to look up target of direct relations.
35
38
 
36
- This is used when creating direct relations from asset-centric resources to CogniteAsset and CogniteSourceSystem.
37
-
38
- Attributes:
39
- asset: Mapping[int, DirectRelationReference]
40
- A mapping from asset IDs to their corresponding DirectRelationReference in the data model.
41
- source: Mapping[str, DirectRelationReference]
42
- A mapping from source strings to their corresponding DirectRelationReference in the data model.
43
-
44
- Methods:
45
- get(resource_type: AssetCentric, property_id: str) -> Mapping[str | int, DirectRelationReference]:
46
- Get the appropriate mapping based on the resource type and property ID.
47
-
48
- Note:
49
- The ASSET_REFERENCE_PROPERTIES and SOURCE_REFERENCE_PROPERTIES class variables define which properties
50
- in asset-centric resources reference assets and sources, respectively.
51
-
39
+ This is used when creating direct relations from asset-centric resources to assets, files, and source systems.
52
40
  """
53
41
 
54
- ASSET_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricType, str]]] = {
42
+ class TableName:
43
+ ASSET_ID = "assetId"
44
+ SOURCE_NAME = "source"
45
+ FILE_ID = "fileId"
46
+ ASSET_EXTERNAL_ID = "assetExternalId"
47
+ FILE_EXTERNAL_ID = "fileExternalId"
48
+
49
+ ASSET_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
55
50
  ("timeseries", "assetId"),
56
51
  ("file", "assetIds"),
57
52
  ("event", "assetIds"),
58
53
  ("sequence", "assetId"),
54
+ ("annotation", "data.assetRef.id"),
59
55
  ("asset", "parentId"),
60
- ("fileAnnotation", "data.assetRef.id"),
61
56
  }
62
- SOURCE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricType, str]]] = {
57
+ SOURCE_NAME_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
63
58
  ("asset", "source"),
64
59
  ("event", "source"),
65
60
  ("file", "source"),
66
61
  }
67
- FILE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricType, str]]] = {
68
- ("fileAnnotation", "data.fileRef.id"),
69
- ("fileAnnotation", "annotatedResourceId"),
62
+ FILE_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
63
+ ("annotation", "data.fileRef.id"),
64
+ ("annotation", "annotatedResourceId"),
70
65
  }
66
+ ASSET_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.assetRef.externalId")}
67
+ FILE_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.fileRef.externalId")}
68
+
69
+ def __init__(self, client: ToolkitClient) -> None:
70
+ self._client = client
71
+ self._cache_map: dict[
72
+ tuple[str, str] | str, dict[str, DirectRelationReference] | dict[int, DirectRelationReference]
73
+ ] = {}
74
+ # Constructing the cache map to be accessed by both table name and property id
75
+ for table_name, properties in [
76
+ (self.TableName.ASSET_ID, self.ASSET_ID_PROPERTIES),
77
+ (self.TableName.SOURCE_NAME, self.SOURCE_NAME_PROPERTIES),
78
+ (self.TableName.FILE_ID, self.FILE_ID_PROPERTIES),
79
+ (self.TableName.ASSET_EXTERNAL_ID, self.ASSET_EXTERNAL_ID_PROPERTIES),
80
+ (self.TableName.FILE_EXTERNAL_ID, self.FILE_EXTERNAL_ID_PROPERTIES),
81
+ ]:
82
+ cache: dict[str, DirectRelationReference] | dict[int, DirectRelationReference] = {}
83
+ self._cache_map[table_name] = cache
84
+ for key in properties:
85
+ self._cache_map[key] = cache
86
+
87
+ def update(self, resources: Iterable[AssetCentricResourceExtended]) -> None:
88
+ """Update the cache with direct relation references for the given asset-centric resources.
89
+
90
+ This is used to bulk update the cache for a chunk of resources before converting them to data model instances.
91
+ """
92
+ asset_ids: set[int] = set()
93
+ source_ids: set[str] = set()
94
+ file_ids: set[int] = set()
95
+ asset_external_ids: set[str] = set()
96
+ file_external_ids: set[str] = set()
97
+ for resource in resources:
98
+ if isinstance(resource, Annotation):
99
+ if resource.annotated_resource_type == "file" and resource.annotated_resource_id:
100
+ file_ids.add(resource.annotated_resource_id)
101
+ if "assetRef" in resource.data:
102
+ asset_ref = resource.data["assetRef"]
103
+ if isinstance(asset_id := asset_ref.get("id"), int):
104
+ asset_ids.add(asset_id)
105
+ if isinstance(asset_external_id := asset_ref.get("externalId"), str):
106
+ asset_external_ids.add(asset_external_id)
107
+ if "fileRef" in resource.data:
108
+ file_ref = resource.data["fileRef"]
109
+ if isinstance(file_id := file_ref.get("id"), int):
110
+ file_ids.add(file_id)
111
+ if isinstance(file_external_id := file_ref.get("externalId"), str):
112
+ file_external_ids.add(file_external_id)
113
+ elif isinstance(resource, Asset):
114
+ if resource.source:
115
+ source_ids.add(resource.source)
116
+ if resource.parent_id is not None:
117
+ asset_ids.add(resource.parent_id)
118
+ elif isinstance(resource, FileMetadata):
119
+ if resource.source:
120
+ source_ids.add(resource.source)
121
+ if resource.asset_ids:
122
+ asset_ids.update(resource.asset_ids)
123
+ elif isinstance(resource, Event):
124
+ if resource.source:
125
+ source_ids.add(resource.source)
126
+ if resource.asset_ids:
127
+ asset_ids.update(resource.asset_ids)
128
+ elif isinstance(resource, TimeSeries):
129
+ if resource.asset_id is not None:
130
+ asset_ids.add(resource.asset_id)
131
+ if asset_ids:
132
+ self._update_cache(self._client.migration.lookup.assets(id=list(asset_ids)), self.TableName.ASSET_ID)
133
+ if source_ids:
134
+ # SourceSystems are not cached in the client, so we have to handle the caching ourselves.
135
+ cache = cast(dict[str, DirectRelationReference], self._cache_map[self.TableName.SOURCE_NAME])
136
+ missing = set(source_ids) - set(cache.keys())
137
+ if missing:
138
+ source_systems = self._client.migration.created_source_system.retrieve(list(missing))
139
+ for source_system in source_systems:
140
+ cache[source_system.source] = source_system.as_direct_relation_reference()
141
+ if file_ids:
142
+ self._update_cache(self._client.migration.lookup.files(id=list(file_ids)), self.TableName.FILE_ID)
143
+ if asset_external_ids:
144
+ self._update_cache(
145
+ self._client.migration.lookup.assets(external_id=list(asset_external_ids)),
146
+ self.TableName.ASSET_EXTERNAL_ID,
147
+ )
148
+ if file_external_ids:
149
+ self._update_cache(
150
+ self._client.migration.lookup.files(external_id=list(file_external_ids)),
151
+ self.TableName.FILE_EXTERNAL_ID,
152
+ )
71
153
 
72
- asset: Mapping[int, DirectRelationReference]
73
- source: Mapping[str, DirectRelationReference]
74
- file: Mapping[int, DirectRelationReference]
75
-
76
- def get(self, resource_type: AssetCentricType, property_id: str) -> Mapping[str | int, DirectRelationReference]:
77
- key = resource_type, property_id
78
- if key in self.ASSET_REFERENCE_PROPERTIES:
79
- return self.asset # type: ignore[return-value]
80
- if key in self.SOURCE_REFERENCE_PROPERTIES:
81
- return self.source # type: ignore[return-value]
82
- if key in self.FILE_REFERENCE_PROPERTIES:
83
- return self.file # type: ignore[return-value]
84
- return {}
85
-
86
-
87
- @overload
88
- def asset_centric_to_dm(
89
- resource: AssetCentricResourceExtended,
90
- instance_id: NodeId,
91
- view_source: ResourceViewMapping,
92
- view_properties: dict[str, ViewProperty],
93
- asset_instance_id_by_id: Mapping[int, DirectRelationReference],
94
- source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
95
- file_instance_id_by_id: Mapping[int, DirectRelationReference],
96
- ) -> tuple[NodeApply | None, ConversionIssue]: ...
97
-
154
+ def _update_cache(self, instance_id_by_id: dict[int, NodeId] | dict[str, NodeId], table_name: str) -> None:
155
+ cache = self._cache_map[table_name]
156
+ for identifier, instance_id in instance_id_by_id.items():
157
+ cache[identifier] = DirectRelationReference(space=instance_id.space, external_id=instance_id.external_id) # type: ignore[index]
98
158
 
99
- @overload
100
- def asset_centric_to_dm(
101
- resource: AssetCentricResourceExtended,
102
- instance_id: EdgeId,
103
- view_source: ResourceViewMapping,
104
- view_properties: dict[str, ViewProperty],
105
- asset_instance_id_by_id: Mapping[int, DirectRelationReference],
106
- source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
107
- file_instance_id_by_id: Mapping[int, DirectRelationReference],
108
- ) -> tuple[EdgeApply | None, ConversionIssue]: ...
159
+ def get_cache(
160
+ self, resource_type: AssetCentricTypeExtended, property_id: str
161
+ ) -> Mapping[str | int, DirectRelationReference] | None:
162
+ """Get the cache for the given resource type and property ID."""
163
+ return self._cache_map.get((resource_type, property_id)) # type: ignore[return-value]
109
164
 
110
165
 
111
166
  def asset_centric_to_dm(
112
167
  resource: AssetCentricResourceExtended,
113
- instance_id: NodeId | EdgeId,
114
- view_source: ResourceViewMapping,
168
+ instance_id: InstanceId,
169
+ view_source: ResourceViewMappingApply,
115
170
  view_properties: dict[str, ViewProperty],
116
- asset_instance_id_by_id: Mapping[int, DirectRelationReference],
117
- source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
118
- file_instance_id_by_id: Mapping[int, DirectRelationReference],
171
+ direct_relation_cache: DirectRelationCache,
119
172
  ) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
120
173
  """Convert an asset-centric resource to a data model instance.
121
174
 
122
175
  Args:
123
176
  resource (CogniteResource): The asset-centric resource to convert.
124
177
  instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
125
- view_source (ResourceViewMapping): The view source defining how to map the resource to the data model.
178
+ view_source (ResourceViewMappingApply): The view source defining how to map the resource to the data model.
126
179
  view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
127
- asset_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from asset IDs to their corresponding
128
- DirectRelationReference in the data model. This is used to create direct relations for resources that
129
- reference assets.
130
- source_instance_id_by_external_id (dict[str, DirectRelationReference]): A mapping from source strings to their
131
- corresponding DirectRelationReference in the data model. This is used to create direct relations for resources
132
- that reference sources.
133
- file_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from file IDs to their corresponding
134
- DirectRelationReference in the data model. This is used to create direct relations for resources that
135
- reference files.
180
+ direct_relation_cache (DirectRelationCache): Cache for direct relation references.
136
181
 
137
182
  Returns:
138
183
  tuple[NodeApply | EdgeApply, ConversionIssue]: A tuple containing the converted NodeApply and any ConversionIssue encountered.
139
184
  """
140
- cache = DirectRelationCache(
141
- asset=asset_instance_id_by_id, source=source_instance_id_by_external_id, file=file_instance_id_by_id
142
- )
143
185
  resource_type = _lookup_resource_type(resource)
144
186
  dumped = resource.dump()
145
187
  try:
@@ -159,13 +201,13 @@ def asset_centric_to_dm(
159
201
  view_source.property_mapping,
160
202
  resource_type,
161
203
  issue=issue,
162
- direct_relation_cache=cache,
204
+ direct_relation_cache=direct_relation_cache,
163
205
  )
164
206
  sources: list[NodeOrEdgeData] = []
165
207
  if properties:
166
208
  sources.append(NodeOrEdgeData(source=view_source.view_id, properties=properties))
167
209
 
168
- if resource_type != "fileAnnotation":
210
+ if resource_type != "annotation":
169
211
  instance_source_properties = {
170
212
  "resourceType": resource_type,
171
213
  "id": id_,
@@ -177,7 +219,7 @@ def asset_centric_to_dm(
177
219
  instance: NodeApply | EdgeApply
178
220
  if isinstance(instance_id, EdgeId):
179
221
  edge_properties = create_edge_properties(
180
- dumped, view_source.property_mapping, resource_type, issue, cache, instance_id.space
222
+ dumped, view_source.property_mapping, resource_type, issue, direct_relation_cache, instance_id.space
181
223
  )
182
224
  if any(key not in edge_properties for key in ("start_node", "end_node", "type")):
183
225
  # Failed conversion of edge properties
@@ -196,7 +238,7 @@ def asset_centric_to_dm(
196
238
  return instance, issue
197
239
 
198
240
 
199
- def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetCentricType:
241
+ def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetCentricTypeExtended:
200
242
  if isinstance(resource_type, Asset):
201
243
  return "asset"
202
244
  elif isinstance(resource_type, FileMetadata):
@@ -210,7 +252,7 @@ def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetC
210
252
  "diagrams.AssetLink",
211
253
  "diagrams.FileLink",
212
254
  ):
213
- return "fileAnnotation"
255
+ return "annotation"
214
256
  raise ValueError(f"Unsupported resource type: {resource_type}")
215
257
 
216
258
 
@@ -218,7 +260,7 @@ def create_properties(
218
260
  dumped: dict[str, Any],
219
261
  view_properties: dict[str, ViewProperty],
220
262
  property_mapping: dict[str, str],
221
- resource_type: AssetCentricType,
263
+ resource_type: AssetCentricTypeExtended,
222
264
  issue: ConversionIssue,
223
265
  direct_relation_cache: DirectRelationCache,
224
266
  ) -> dict[str, PropertyValueWrite]:
@@ -261,7 +303,7 @@ def create_properties(
261
303
  dm_prop.nullable,
262
304
  destination_container_property=(dm_prop.container, dm_prop.container_property_identifier),
263
305
  source_property=(resource_type, prop_json_path),
264
- direct_relation_lookup=direct_relation_cache.get(resource_type, prop_json_path),
306
+ direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
265
307
  )
266
308
  except (ValueError, TypeError, NotImplementedError) as e:
267
309
  issue.failed_conversions.append(
@@ -289,7 +331,7 @@ def create_properties(
289
331
  def create_edge_properties(
290
332
  dumped: dict[str, Any],
291
333
  property_mapping: dict[str, str],
292
- resource_type: AssetCentricType,
334
+ resource_type: AssetCentricTypeExtended,
293
335
  issue: ConversionIssue,
294
336
  direct_relation_cache: DirectRelationCache,
295
337
  default_instance_space: str,
@@ -309,7 +351,7 @@ def create_edge_properties(
309
351
  flatten_dump[prop_json_path],
310
352
  DirectRelation(),
311
353
  False,
312
- direct_relation_lookup=direct_relation_cache.get(resource_type, prop_json_path),
354
+ direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
313
355
  )
314
356
  except (ValueError, TypeError, NotImplementedError) as e:
315
357
  issue.failed_conversions.append(
@@ -1,26 +1,30 @@
1
1
  from dataclasses import dataclass
2
2
  from pathlib import Path
3
- from typing import Any, Generic, Literal
3
+ from typing import Annotated, Any, Generic, Literal
4
4
 
5
5
  from cognite.client.data_classes._base import (
6
6
  WriteableCogniteResource,
7
7
  WriteableCogniteResourceList,
8
8
  )
9
- from cognite.client.data_classes.data_modeling import InstanceApply, NodeId, ViewId
9
+ from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, NodeId, ViewId
10
+ from cognite.client.utils._identifier import InstanceId
10
11
  from cognite.client.utils._text import to_camel_case
11
- from pydantic import BaseModel, field_validator, model_validator
12
+ from pydantic import BaseModel, BeforeValidator, field_validator, model_validator
12
13
 
13
14
  from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
14
15
  from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
15
16
  from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
16
- from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
17
+ from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import (
18
+ ASSET_ANNOTATIONS_ID,
19
+ FILE_ANNOTATIONS_ID,
20
+ create_default_mappings,
21
+ )
17
22
  from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
18
23
  from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
19
24
  from cognite_toolkit._cdf_tk.utils.useful_types import (
20
- AssetCentricKind,
21
- AssetCentricType,
25
+ AssetCentricKindExtended,
22
26
  JsonVal,
23
- T_AssetCentricResource,
27
+ T_AssetCentricResourceExtended,
24
28
  )
25
29
 
26
30
 
@@ -37,8 +41,8 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
37
41
  for example, the Canvas migration to determine which view to use for the resource.
38
42
  """
39
43
 
40
- resource_type: AssetCentricType
41
- instance_id: NodeId
44
+ resource_type: str
45
+ instance_id: InstanceId
42
46
  id: int
43
47
  data_set_id: int | None = None
44
48
  ingestion_view: str | None = None
@@ -56,7 +60,8 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
56
60
  raise ToolkitValueError(f"No default ingestion view specified for resource type '{self.resource_type}'")
57
61
 
58
62
  def as_asset_centric_id(self) -> AssetCentricId:
59
- return AssetCentricId(resource_type=self.resource_type, id_=self.id)
63
+ # MyPy fails to understand that resource_type is AssetCentricKindExtended in subclasses
64
+ return AssetCentricId(resource_type=self.resource_type, id_=self.id) # type: ignore[arg-type]
60
65
 
61
66
  @model_validator(mode="before")
62
67
  def _handle_flat_dict(cls, values: Any) -> Any:
@@ -87,12 +92,6 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
87
92
  return ViewId.load(v)
88
93
  return v
89
94
 
90
- @field_validator("instance_id", mode="before")
91
- def _validate_instance_id(cls, v: Any) -> Any:
92
- if isinstance(v, dict):
93
- return NodeId.load(v)
94
- return v
95
-
96
95
 
97
96
  class MigrationMappingList(ModelList[MigrationMapping]):
98
97
  @classmethod
@@ -113,14 +112,22 @@ class MigrationMappingList(ModelList[MigrationMapping]):
113
112
 
114
113
  def as_node_ids(self) -> list[NodeId]:
115
114
  """Return a list of NodeIds from the migration mappings."""
116
- return [mapping.instance_id for mapping in self]
115
+ return [mapping.instance_id for mapping in self if isinstance(mapping.instance_id, NodeId)]
116
+
117
+ def as_edge_ids(self) -> list[EdgeId]:
118
+ """Return a list of EdgeIds from the migration mappings."""
119
+ return [mapping.instance_id for mapping in self if isinstance(mapping.instance_id, EdgeId)]
117
120
 
118
121
  def spaces(self) -> set[str]:
119
122
  """Return a set of spaces from the migration mappings."""
120
123
  return {mapping.instance_id.space for mapping in self}
121
124
 
122
125
  def as_pending_ids(self) -> list[PendingInstanceId]:
123
- return [PendingInstanceId(pending_instance_id=mapping.instance_id, id=mapping.id) for mapping in self]
126
+ return [
127
+ PendingInstanceId(pending_instance_id=mapping.instance_id, id=mapping.id)
128
+ for mapping in self
129
+ if isinstance(mapping.instance_id, NodeId)
130
+ ]
124
131
 
125
132
  def get_data_set_ids(self) -> set[int]:
126
133
  """Return a list of data set IDs from the migration mappings."""
@@ -131,7 +138,9 @@ class MigrationMappingList(ModelList[MigrationMapping]):
131
138
  return {mapping.id: mapping for mapping in self}
132
139
 
133
140
  @classmethod
134
- def read_csv_file(cls, filepath: Path, resource_type: AssetCentricKind | None = None) -> "MigrationMappingList":
141
+ def read_csv_file(
142
+ cls, filepath: Path, resource_type: AssetCentricKindExtended | None = None
143
+ ) -> "MigrationMappingList":
135
144
  if cls is not MigrationMappingList or resource_type is None:
136
145
  return super().read_csv_file(filepath)
137
146
  cls_by_resource_type: dict[str, type[MigrationMappingList]] = {
@@ -139,6 +148,7 @@ class MigrationMappingList(ModelList[MigrationMapping]):
139
148
  "TimeSeries": TimeSeriesMigrationMappingList,
140
149
  "FileMetadata": FileMigrationMappingList,
141
150
  "Events": EventMigrationMappingList,
151
+ "Annotations": AnnotationMigrationMappingList,
142
152
  }
143
153
  if resource_type not in cls_by_resource_type:
144
154
  raise ToolkitValueError(
@@ -147,20 +157,53 @@ class MigrationMappingList(ModelList[MigrationMapping]):
147
157
  return cls_by_resource_type[resource_type].read_csv_file(filepath, resource_type=None)
148
158
 
149
159
 
160
+ def _validate_node_id(value: Any) -> Any:
161
+ if isinstance(value, dict):
162
+ return NodeId.load(value)
163
+ return value
164
+
165
+
150
166
  class AssetMapping(MigrationMapping):
151
167
  resource_type: Literal["asset"] = "asset"
168
+ instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
152
169
 
153
170
 
154
171
  class EventMapping(MigrationMapping):
155
172
  resource_type: Literal["event"] = "event"
173
+ instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
156
174
 
157
175
 
158
176
  class TimeSeriesMapping(MigrationMapping):
159
177
  resource_type: Literal["timeseries"] = "timeseries"
178
+ instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
160
179
 
161
180
 
162
181
  class FileMapping(MigrationMapping):
163
182
  resource_type: Literal["file"] = "file"
183
+ instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
184
+
185
+
186
+ class AnnotationMapping(MigrationMapping):
187
+ resource_type: Literal["annotation"] = "annotation"
188
+ instance_id: EdgeId
189
+ annotation_type: Literal["diagrams.AssetLink", "diagrams.FileLink"] | None = None
190
+
191
+ def get_ingestion_view(self) -> str:
192
+ """Get the ingestion view for the mapping. If not specified, return the default ingestion view."""
193
+ if self.ingestion_view:
194
+ return self.ingestion_view
195
+ elif self.annotation_type == "diagrams.AssetLink":
196
+ return ASSET_ANNOTATIONS_ID
197
+ elif self.annotation_type == "diagrams.FileLink":
198
+ return FILE_ANNOTATIONS_ID
199
+ else:
200
+ raise ToolkitValueError("Cannot determine default ingestion view for annotation without annotation_type")
201
+
202
+ @field_validator("instance_id", mode="before")
203
+ def _validate_instance_id(cls, v: Any) -> Any:
204
+ if isinstance(v, dict):
205
+ return EdgeId.load(v)
206
+ return v
164
207
 
165
208
 
166
209
  class AssetMigrationMappingList(MigrationMappingList):
@@ -187,10 +230,16 @@ class TimeSeriesMigrationMappingList(MigrationMappingList):
187
230
  return TimeSeriesMapping
188
231
 
189
232
 
233
+ class AnnotationMigrationMappingList(MigrationMappingList):
234
+ @classmethod
235
+ def _get_base_model_cls(cls) -> type[AnnotationMapping]:
236
+ return AnnotationMapping
237
+
238
+
190
239
  @dataclass
191
- class AssetCentricMapping(Generic[T_AssetCentricResource], WriteableCogniteResource[InstanceApply]):
240
+ class AssetCentricMapping(Generic[T_AssetCentricResourceExtended], WriteableCogniteResource[InstanceApply]):
192
241
  mapping: MigrationMapping
193
- resource: T_AssetCentricResource
242
+ resource: T_AssetCentricResourceExtended
194
243
 
195
244
  def as_write(self) -> InstanceApply:
196
245
  raise NotImplementedError()
@@ -205,7 +254,9 @@ class AssetCentricMapping(Generic[T_AssetCentricResource], WriteableCogniteResou
205
254
  }
206
255
 
207
256
 
208
- class AssetCentricMappingList(WriteableCogniteResourceList[InstanceApply, AssetCentricMapping[T_AssetCentricResource]]):
257
+ class AssetCentricMappingList(
258
+ WriteableCogniteResourceList[InstanceApply, AssetCentricMapping[T_AssetCentricResourceExtended]]
259
+ ):
209
260
  _RESOURCE: type = AssetCentricMapping
210
261
 
211
262
  def as_write(self) -> InstanceApplyList: