cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf.py +16 -17
- cognite_toolkit/_cdf_tk/apps/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/apps/_core_app.py +13 -5
- cognite_toolkit/_cdf_tk/apps/_data_app.py +1 -1
- cognite_toolkit/_cdf_tk/apps/_dev_app.py +86 -0
- cognite_toolkit/_cdf_tk/apps/_download_app.py +692 -24
- cognite_toolkit/_cdf_tk/apps/_dump_app.py +43 -101
- cognite_toolkit/_cdf_tk/apps/_landing_app.py +18 -4
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +249 -9
- cognite_toolkit/_cdf_tk/apps/_modules_app.py +0 -3
- cognite_toolkit/_cdf_tk/apps/_purge.py +15 -43
- cognite_toolkit/_cdf_tk/apps/_run.py +11 -0
- cognite_toolkit/_cdf_tk/apps/_upload_app.py +45 -6
- cognite_toolkit/_cdf_tk/builders/__init__.py +2 -2
- cognite_toolkit/_cdf_tk/builders/_base.py +28 -42
- cognite_toolkit/_cdf_tk/cdf_toml.py +20 -1
- cognite_toolkit/_cdf_tk/client/_toolkit_client.py +23 -3
- cognite_toolkit/_cdf_tk/client/api/extended_functions.py +6 -9
- cognite_toolkit/_cdf_tk/client/api/infield.py +93 -1
- cognite_toolkit/_cdf_tk/client/api/migration.py +175 -1
- cognite_toolkit/_cdf_tk/client/api/streams.py +84 -0
- cognite_toolkit/_cdf_tk/client/api/three_d.py +50 -0
- cognite_toolkit/_cdf_tk/client/data_classes/base.py +25 -1
- cognite_toolkit/_cdf_tk/client/data_classes/canvas.py +46 -3
- cognite_toolkit/_cdf_tk/client/data_classes/charts.py +3 -3
- cognite_toolkit/_cdf_tk/client/data_classes/charts_data.py +95 -213
- cognite_toolkit/_cdf_tk/client/data_classes/infield.py +32 -18
- cognite_toolkit/_cdf_tk/client/data_classes/migration.py +10 -2
- cognite_toolkit/_cdf_tk/client/data_classes/streams.py +90 -0
- cognite_toolkit/_cdf_tk/client/data_classes/three_d.py +47 -0
- cognite_toolkit/_cdf_tk/client/testing.py +18 -2
- cognite_toolkit/_cdf_tk/commands/__init__.py +6 -6
- cognite_toolkit/_cdf_tk/commands/_changes.py +3 -42
- cognite_toolkit/_cdf_tk/commands/_download.py +21 -11
- cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py +0 -2
- cognite_toolkit/_cdf_tk/commands/_migrate/command.py +22 -20
- cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +133 -91
- cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py +73 -22
- cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +311 -43
- cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py +5 -5
- cognite_toolkit/_cdf_tk/commands/_migrate/issues.py +33 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +157 -8
- cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py +9 -4
- cognite_toolkit/_cdf_tk/commands/_purge.py +27 -28
- cognite_toolkit/_cdf_tk/commands/_questionary_style.py +16 -0
- cognite_toolkit/_cdf_tk/commands/_upload.py +109 -86
- cognite_toolkit/_cdf_tk/commands/about.py +221 -0
- cognite_toolkit/_cdf_tk/commands/auth.py +19 -12
- cognite_toolkit/_cdf_tk/commands/build_cmd.py +15 -61
- cognite_toolkit/_cdf_tk/commands/clean.py +63 -16
- cognite_toolkit/_cdf_tk/commands/deploy.py +20 -17
- cognite_toolkit/_cdf_tk/commands/dump_resource.py +6 -4
- cognite_toolkit/_cdf_tk/commands/init.py +225 -3
- cognite_toolkit/_cdf_tk/commands/modules.py +20 -44
- cognite_toolkit/_cdf_tk/commands/pull.py +6 -19
- cognite_toolkit/_cdf_tk/commands/resources.py +179 -0
- cognite_toolkit/_cdf_tk/constants.py +20 -1
- cognite_toolkit/_cdf_tk/cruds/__init__.py +19 -5
- cognite_toolkit/_cdf_tk/cruds/_base_cruds.py +14 -70
- cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +8 -17
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py +4 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py +11 -9
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py +4 -14
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py +44 -43
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py +4 -11
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py +4 -13
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py +205 -66
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py +5 -17
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py +116 -27
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py +6 -27
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py +9 -28
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/hosted_extractors.py +12 -30
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/industrial_tool.py +3 -7
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/location.py +3 -15
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py +4 -12
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/raw.py +4 -10
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py +3 -8
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/robotics.py +15 -44
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py +94 -0
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/three_d_model.py +3 -7
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py +5 -15
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py +39 -31
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/workflow.py +20 -40
- cognite_toolkit/_cdf_tk/cruds/_worker.py +24 -36
- cognite_toolkit/_cdf_tk/feature_flags.py +16 -36
- cognite_toolkit/_cdf_tk/plugins.py +2 -1
- cognite_toolkit/_cdf_tk/resource_classes/__init__.py +4 -0
- cognite_toolkit/_cdf_tk/resource_classes/capabilities.py +12 -0
- cognite_toolkit/_cdf_tk/resource_classes/functions.py +3 -1
- cognite_toolkit/_cdf_tk/resource_classes/infield_cdm_location_config.py +109 -0
- cognite_toolkit/_cdf_tk/resource_classes/migration.py +8 -17
- cognite_toolkit/_cdf_tk/resource_classes/streams.py +29 -0
- cognite_toolkit/_cdf_tk/storageio/__init__.py +9 -21
- cognite_toolkit/_cdf_tk/storageio/_annotations.py +19 -16
- cognite_toolkit/_cdf_tk/storageio/_applications.py +338 -26
- cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +67 -104
- cognite_toolkit/_cdf_tk/storageio/_base.py +61 -29
- cognite_toolkit/_cdf_tk/storageio/_datapoints.py +276 -20
- cognite_toolkit/_cdf_tk/storageio/_file_content.py +436 -0
- cognite_toolkit/_cdf_tk/storageio/_instances.py +34 -2
- cognite_toolkit/_cdf_tk/storageio/_raw.py +26 -0
- cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py +62 -4
- cognite_toolkit/_cdf_tk/storageio/selectors/_base.py +14 -2
- cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py +14 -0
- cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py +14 -0
- cognite_toolkit/_cdf_tk/storageio/selectors/_datapoints.py +23 -3
- cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py +164 -0
- cognite_toolkit/_cdf_tk/tk_warnings/other.py +4 -0
- cognite_toolkit/_cdf_tk/tracker.py +2 -2
- cognite_toolkit/_cdf_tk/utils/dtype_conversion.py +9 -3
- cognite_toolkit/_cdf_tk/utils/fileio/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/utils/fileio/_base.py +5 -1
- cognite_toolkit/_cdf_tk/utils/fileio/_readers.py +112 -20
- cognite_toolkit/_cdf_tk/utils/fileio/_writers.py +15 -15
- cognite_toolkit/_cdf_tk/utils/http_client/_client.py +284 -18
- cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py +50 -4
- cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py +187 -0
- cognite_toolkit/_cdf_tk/utils/interactive_select.py +9 -14
- cognite_toolkit/_cdf_tk/utils/sql_parser.py +2 -3
- cognite_toolkit/_cdf_tk/utils/useful_types.py +6 -2
- cognite_toolkit/_cdf_tk/validation.py +79 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +5 -4
- cognite_toolkit/_version.py +1 -1
- cognite_toolkit/config.dev.yaml +13 -0
- {cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.30.dist-info}/METADATA +24 -24
- {cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.30.dist-info}/RECORD +153 -143
- cognite_toolkit-0.7.30.dist-info/WHEEL +4 -0
- {cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.30.dist-info}/entry_points.txt +1 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py +0 -201
- cognite_toolkit/_cdf_tk/commands/dump_data.py +0 -489
- cognite_toolkit/_cdf_tk/commands/featureflag.py +0 -27
- cognite_toolkit/_cdf_tk/utils/table_writers.py +0 -434
- cognite_toolkit-0.6.97.dist-info/WHEEL +0 -4
- cognite_toolkit-0.6.97.dist-info/licenses/LICENSE +0 -18
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
from collections.abc import Mapping, Set
|
|
2
|
-
from
|
|
3
|
-
from typing import Any, ClassVar, overload
|
|
1
|
+
from collections.abc import Iterable, Mapping, Set
|
|
2
|
+
from typing import Any, ClassVar, cast
|
|
4
3
|
|
|
5
4
|
from cognite.client.data_classes import Annotation, Asset, Event, FileMetadata, TimeSeries
|
|
6
5
|
from cognite.client.data_classes.data_modeling import (
|
|
@@ -13,8 +12,13 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
13
12
|
)
|
|
14
13
|
from cognite.client.data_classes.data_modeling.instances import EdgeApply, NodeOrEdgeData, PropertyValueWrite
|
|
15
14
|
from cognite.client.data_classes.data_modeling.views import ViewProperty
|
|
15
|
+
from cognite.client.utils._identifier import InstanceId
|
|
16
16
|
|
|
17
|
-
from cognite_toolkit._cdf_tk.client
|
|
17
|
+
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
18
|
+
from cognite_toolkit._cdf_tk.client.data_classes.migration import (
|
|
19
|
+
AssetCentricId,
|
|
20
|
+
ResourceViewMappingApply,
|
|
21
|
+
)
|
|
18
22
|
from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
|
|
19
23
|
from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
|
|
20
24
|
asset_centric_convert_to_primary_property,
|
|
@@ -22,124 +26,162 @@ from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
|
|
|
22
26
|
)
|
|
23
27
|
from cognite_toolkit._cdf_tk.utils.useful_types import (
|
|
24
28
|
AssetCentricResourceExtended,
|
|
25
|
-
|
|
29
|
+
AssetCentricTypeExtended,
|
|
26
30
|
)
|
|
27
31
|
|
|
28
32
|
from .data_model import INSTANCE_SOURCE_VIEW_ID
|
|
29
33
|
from .issues import ConversionIssue, FailedConversion, InvalidPropertyDataType
|
|
30
34
|
|
|
31
35
|
|
|
32
|
-
@dataclass
|
|
33
36
|
class DirectRelationCache:
|
|
34
37
|
"""Cache for direct relation references to look up target of direct relations.
|
|
35
38
|
|
|
36
|
-
This is used when creating direct relations from asset-centric resources to
|
|
37
|
-
|
|
38
|
-
Attributes:
|
|
39
|
-
asset: Mapping[int, DirectRelationReference]
|
|
40
|
-
A mapping from asset IDs to their corresponding DirectRelationReference in the data model.
|
|
41
|
-
source: Mapping[str, DirectRelationReference]
|
|
42
|
-
A mapping from source strings to their corresponding DirectRelationReference in the data model.
|
|
43
|
-
|
|
44
|
-
Methods:
|
|
45
|
-
get(resource_type: AssetCentric, property_id: str) -> Mapping[str | int, DirectRelationReference]:
|
|
46
|
-
Get the appropriate mapping based on the resource type and property ID.
|
|
47
|
-
|
|
48
|
-
Note:
|
|
49
|
-
The ASSET_REFERENCE_PROPERTIES and SOURCE_REFERENCE_PROPERTIES class variables define which properties
|
|
50
|
-
in asset-centric resources reference assets and sources, respectively.
|
|
51
|
-
|
|
39
|
+
This is used when creating direct relations from asset-centric resources to assets, files, and source systems.
|
|
52
40
|
"""
|
|
53
41
|
|
|
54
|
-
|
|
42
|
+
class TableName:
|
|
43
|
+
ASSET_ID = "assetId"
|
|
44
|
+
SOURCE_NAME = "source"
|
|
45
|
+
FILE_ID = "fileId"
|
|
46
|
+
ASSET_EXTERNAL_ID = "assetExternalId"
|
|
47
|
+
FILE_EXTERNAL_ID = "fileExternalId"
|
|
48
|
+
|
|
49
|
+
ASSET_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
|
|
55
50
|
("timeseries", "assetId"),
|
|
56
51
|
("file", "assetIds"),
|
|
57
52
|
("event", "assetIds"),
|
|
58
53
|
("sequence", "assetId"),
|
|
54
|
+
("annotation", "data.assetRef.id"),
|
|
59
55
|
("asset", "parentId"),
|
|
60
|
-
("fileAnnotation", "data.assetRef.id"),
|
|
61
56
|
}
|
|
62
|
-
|
|
57
|
+
SOURCE_NAME_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
|
|
63
58
|
("asset", "source"),
|
|
64
59
|
("event", "source"),
|
|
65
60
|
("file", "source"),
|
|
66
61
|
}
|
|
67
|
-
|
|
68
|
-
("
|
|
69
|
-
("
|
|
62
|
+
FILE_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {
|
|
63
|
+
("annotation", "data.fileRef.id"),
|
|
64
|
+
("annotation", "annotatedResourceId"),
|
|
70
65
|
}
|
|
66
|
+
ASSET_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.assetRef.externalId")}
|
|
67
|
+
FILE_EXTERNAL_ID_PROPERTIES: ClassVar[Set[tuple[str, str]]] = {("annotation", "data.fileRef.externalId")}
|
|
68
|
+
|
|
69
|
+
def __init__(self, client: ToolkitClient) -> None:
|
|
70
|
+
self._client = client
|
|
71
|
+
self._cache_map: dict[
|
|
72
|
+
tuple[str, str] | str, dict[str, DirectRelationReference] | dict[int, DirectRelationReference]
|
|
73
|
+
] = {}
|
|
74
|
+
# Constructing the cache map to be accessed by both table name and property id
|
|
75
|
+
for table_name, properties in [
|
|
76
|
+
(self.TableName.ASSET_ID, self.ASSET_ID_PROPERTIES),
|
|
77
|
+
(self.TableName.SOURCE_NAME, self.SOURCE_NAME_PROPERTIES),
|
|
78
|
+
(self.TableName.FILE_ID, self.FILE_ID_PROPERTIES),
|
|
79
|
+
(self.TableName.ASSET_EXTERNAL_ID, self.ASSET_EXTERNAL_ID_PROPERTIES),
|
|
80
|
+
(self.TableName.FILE_EXTERNAL_ID, self.FILE_EXTERNAL_ID_PROPERTIES),
|
|
81
|
+
]:
|
|
82
|
+
cache: dict[str, DirectRelationReference] | dict[int, DirectRelationReference] = {}
|
|
83
|
+
self._cache_map[table_name] = cache
|
|
84
|
+
for key in properties:
|
|
85
|
+
self._cache_map[key] = cache
|
|
86
|
+
|
|
87
|
+
def update(self, resources: Iterable[AssetCentricResourceExtended]) -> None:
|
|
88
|
+
"""Update the cache with direct relation references for the given asset-centric resources.
|
|
89
|
+
|
|
90
|
+
This is used to bulk update the cache for a chunk of resources before converting them to data model instances.
|
|
91
|
+
"""
|
|
92
|
+
asset_ids: set[int] = set()
|
|
93
|
+
source_ids: set[str] = set()
|
|
94
|
+
file_ids: set[int] = set()
|
|
95
|
+
asset_external_ids: set[str] = set()
|
|
96
|
+
file_external_ids: set[str] = set()
|
|
97
|
+
for resource in resources:
|
|
98
|
+
if isinstance(resource, Annotation):
|
|
99
|
+
if resource.annotated_resource_type == "file" and resource.annotated_resource_id:
|
|
100
|
+
file_ids.add(resource.annotated_resource_id)
|
|
101
|
+
if "assetRef" in resource.data:
|
|
102
|
+
asset_ref = resource.data["assetRef"]
|
|
103
|
+
if isinstance(asset_id := asset_ref.get("id"), int):
|
|
104
|
+
asset_ids.add(asset_id)
|
|
105
|
+
if isinstance(asset_external_id := asset_ref.get("externalId"), str):
|
|
106
|
+
asset_external_ids.add(asset_external_id)
|
|
107
|
+
if "fileRef" in resource.data:
|
|
108
|
+
file_ref = resource.data["fileRef"]
|
|
109
|
+
if isinstance(file_id := file_ref.get("id"), int):
|
|
110
|
+
file_ids.add(file_id)
|
|
111
|
+
if isinstance(file_external_id := file_ref.get("externalId"), str):
|
|
112
|
+
file_external_ids.add(file_external_id)
|
|
113
|
+
elif isinstance(resource, Asset):
|
|
114
|
+
if resource.source:
|
|
115
|
+
source_ids.add(resource.source)
|
|
116
|
+
if resource.parent_id is not None:
|
|
117
|
+
asset_ids.add(resource.parent_id)
|
|
118
|
+
elif isinstance(resource, FileMetadata):
|
|
119
|
+
if resource.source:
|
|
120
|
+
source_ids.add(resource.source)
|
|
121
|
+
if resource.asset_ids:
|
|
122
|
+
asset_ids.update(resource.asset_ids)
|
|
123
|
+
elif isinstance(resource, Event):
|
|
124
|
+
if resource.source:
|
|
125
|
+
source_ids.add(resource.source)
|
|
126
|
+
if resource.asset_ids:
|
|
127
|
+
asset_ids.update(resource.asset_ids)
|
|
128
|
+
elif isinstance(resource, TimeSeries):
|
|
129
|
+
if resource.asset_id is not None:
|
|
130
|
+
asset_ids.add(resource.asset_id)
|
|
131
|
+
if asset_ids:
|
|
132
|
+
self._update_cache(self._client.migration.lookup.assets(id=list(asset_ids)), self.TableName.ASSET_ID)
|
|
133
|
+
if source_ids:
|
|
134
|
+
# SourceSystems are not cached in the client, so we have to handle the caching ourselves.
|
|
135
|
+
cache = cast(dict[str, DirectRelationReference], self._cache_map[self.TableName.SOURCE_NAME])
|
|
136
|
+
missing = set(source_ids) - set(cache.keys())
|
|
137
|
+
if missing:
|
|
138
|
+
source_systems = self._client.migration.created_source_system.retrieve(list(missing))
|
|
139
|
+
for source_system in source_systems:
|
|
140
|
+
cache[source_system.source] = source_system.as_direct_relation_reference()
|
|
141
|
+
if file_ids:
|
|
142
|
+
self._update_cache(self._client.migration.lookup.files(id=list(file_ids)), self.TableName.FILE_ID)
|
|
143
|
+
if asset_external_ids:
|
|
144
|
+
self._update_cache(
|
|
145
|
+
self._client.migration.lookup.assets(external_id=list(asset_external_ids)),
|
|
146
|
+
self.TableName.ASSET_EXTERNAL_ID,
|
|
147
|
+
)
|
|
148
|
+
if file_external_ids:
|
|
149
|
+
self._update_cache(
|
|
150
|
+
self._client.migration.lookup.files(external_id=list(file_external_ids)),
|
|
151
|
+
self.TableName.FILE_EXTERNAL_ID,
|
|
152
|
+
)
|
|
71
153
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
def get(self, resource_type: AssetCentricType, property_id: str) -> Mapping[str | int, DirectRelationReference]:
|
|
77
|
-
key = resource_type, property_id
|
|
78
|
-
if key in self.ASSET_REFERENCE_PROPERTIES:
|
|
79
|
-
return self.asset # type: ignore[return-value]
|
|
80
|
-
if key in self.SOURCE_REFERENCE_PROPERTIES:
|
|
81
|
-
return self.source # type: ignore[return-value]
|
|
82
|
-
if key in self.FILE_REFERENCE_PROPERTIES:
|
|
83
|
-
return self.file # type: ignore[return-value]
|
|
84
|
-
return {}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
@overload
|
|
88
|
-
def asset_centric_to_dm(
|
|
89
|
-
resource: AssetCentricResourceExtended,
|
|
90
|
-
instance_id: NodeId,
|
|
91
|
-
view_source: ResourceViewMapping,
|
|
92
|
-
view_properties: dict[str, ViewProperty],
|
|
93
|
-
asset_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
94
|
-
source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
|
|
95
|
-
file_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
96
|
-
) -> tuple[NodeApply | None, ConversionIssue]: ...
|
|
97
|
-
|
|
154
|
+
def _update_cache(self, instance_id_by_id: dict[int, NodeId] | dict[str, NodeId], table_name: str) -> None:
|
|
155
|
+
cache = self._cache_map[table_name]
|
|
156
|
+
for identifier, instance_id in instance_id_by_id.items():
|
|
157
|
+
cache[identifier] = DirectRelationReference(space=instance_id.space, external_id=instance_id.external_id) # type: ignore[index]
|
|
98
158
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
view_properties: dict[str, ViewProperty],
|
|
105
|
-
asset_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
106
|
-
source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
|
|
107
|
-
file_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
108
|
-
) -> tuple[EdgeApply | None, ConversionIssue]: ...
|
|
159
|
+
def get_cache(
|
|
160
|
+
self, resource_type: AssetCentricTypeExtended, property_id: str
|
|
161
|
+
) -> Mapping[str | int, DirectRelationReference] | None:
|
|
162
|
+
"""Get the cache for the given resource type and property ID."""
|
|
163
|
+
return self._cache_map.get((resource_type, property_id)) # type: ignore[return-value]
|
|
109
164
|
|
|
110
165
|
|
|
111
166
|
def asset_centric_to_dm(
|
|
112
167
|
resource: AssetCentricResourceExtended,
|
|
113
|
-
instance_id:
|
|
114
|
-
view_source:
|
|
168
|
+
instance_id: InstanceId,
|
|
169
|
+
view_source: ResourceViewMappingApply,
|
|
115
170
|
view_properties: dict[str, ViewProperty],
|
|
116
|
-
|
|
117
|
-
source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
|
|
118
|
-
file_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
171
|
+
direct_relation_cache: DirectRelationCache,
|
|
119
172
|
) -> tuple[NodeApply | EdgeApply | None, ConversionIssue]:
|
|
120
173
|
"""Convert an asset-centric resource to a data model instance.
|
|
121
174
|
|
|
122
175
|
Args:
|
|
123
176
|
resource (CogniteResource): The asset-centric resource to convert.
|
|
124
177
|
instance_id (NodeId | EdgeApply): The ID of the instance to create or update.
|
|
125
|
-
view_source (
|
|
178
|
+
view_source (ResourceViewMappingApply): The view source defining how to map the resource to the data model.
|
|
126
179
|
view_properties (dict[str, ViewProperty]): The defined properties referenced in the view source mapping.
|
|
127
|
-
|
|
128
|
-
DirectRelationReference in the data model. This is used to create direct relations for resources that
|
|
129
|
-
reference assets.
|
|
130
|
-
source_instance_id_by_external_id (dict[str, DirectRelationReference]): A mapping from source strings to their
|
|
131
|
-
corresponding DirectRelationReference in the data model. This is used to create direct relations for resources
|
|
132
|
-
that reference sources.
|
|
133
|
-
file_instance_id_by_id (dict[int, DirectRelationReference]): A mapping from file IDs to their corresponding
|
|
134
|
-
DirectRelationReference in the data model. This is used to create direct relations for resources that
|
|
135
|
-
reference files.
|
|
180
|
+
direct_relation_cache (DirectRelationCache): Cache for direct relation references.
|
|
136
181
|
|
|
137
182
|
Returns:
|
|
138
183
|
tuple[NodeApply | EdgeApply, ConversionIssue]: A tuple containing the converted NodeApply and any ConversionIssue encountered.
|
|
139
184
|
"""
|
|
140
|
-
cache = DirectRelationCache(
|
|
141
|
-
asset=asset_instance_id_by_id, source=source_instance_id_by_external_id, file=file_instance_id_by_id
|
|
142
|
-
)
|
|
143
185
|
resource_type = _lookup_resource_type(resource)
|
|
144
186
|
dumped = resource.dump()
|
|
145
187
|
try:
|
|
@@ -159,13 +201,13 @@ def asset_centric_to_dm(
|
|
|
159
201
|
view_source.property_mapping,
|
|
160
202
|
resource_type,
|
|
161
203
|
issue=issue,
|
|
162
|
-
direct_relation_cache=
|
|
204
|
+
direct_relation_cache=direct_relation_cache,
|
|
163
205
|
)
|
|
164
206
|
sources: list[NodeOrEdgeData] = []
|
|
165
207
|
if properties:
|
|
166
208
|
sources.append(NodeOrEdgeData(source=view_source.view_id, properties=properties))
|
|
167
209
|
|
|
168
|
-
if resource_type != "
|
|
210
|
+
if resource_type != "annotation":
|
|
169
211
|
instance_source_properties = {
|
|
170
212
|
"resourceType": resource_type,
|
|
171
213
|
"id": id_,
|
|
@@ -177,7 +219,7 @@ def asset_centric_to_dm(
|
|
|
177
219
|
instance: NodeApply | EdgeApply
|
|
178
220
|
if isinstance(instance_id, EdgeId):
|
|
179
221
|
edge_properties = create_edge_properties(
|
|
180
|
-
dumped, view_source.property_mapping, resource_type, issue,
|
|
222
|
+
dumped, view_source.property_mapping, resource_type, issue, direct_relation_cache, instance_id.space
|
|
181
223
|
)
|
|
182
224
|
if any(key not in edge_properties for key in ("start_node", "end_node", "type")):
|
|
183
225
|
# Failed conversion of edge properties
|
|
@@ -196,7 +238,7 @@ def asset_centric_to_dm(
|
|
|
196
238
|
return instance, issue
|
|
197
239
|
|
|
198
240
|
|
|
199
|
-
def _lookup_resource_type(resource_type: AssetCentricResourceExtended) ->
|
|
241
|
+
def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetCentricTypeExtended:
|
|
200
242
|
if isinstance(resource_type, Asset):
|
|
201
243
|
return "asset"
|
|
202
244
|
elif isinstance(resource_type, FileMetadata):
|
|
@@ -210,7 +252,7 @@ def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetC
|
|
|
210
252
|
"diagrams.AssetLink",
|
|
211
253
|
"diagrams.FileLink",
|
|
212
254
|
):
|
|
213
|
-
return "
|
|
255
|
+
return "annotation"
|
|
214
256
|
raise ValueError(f"Unsupported resource type: {resource_type}")
|
|
215
257
|
|
|
216
258
|
|
|
@@ -218,7 +260,7 @@ def create_properties(
|
|
|
218
260
|
dumped: dict[str, Any],
|
|
219
261
|
view_properties: dict[str, ViewProperty],
|
|
220
262
|
property_mapping: dict[str, str],
|
|
221
|
-
resource_type:
|
|
263
|
+
resource_type: AssetCentricTypeExtended,
|
|
222
264
|
issue: ConversionIssue,
|
|
223
265
|
direct_relation_cache: DirectRelationCache,
|
|
224
266
|
) -> dict[str, PropertyValueWrite]:
|
|
@@ -261,7 +303,7 @@ def create_properties(
|
|
|
261
303
|
dm_prop.nullable,
|
|
262
304
|
destination_container_property=(dm_prop.container, dm_prop.container_property_identifier),
|
|
263
305
|
source_property=(resource_type, prop_json_path),
|
|
264
|
-
direct_relation_lookup=direct_relation_cache.
|
|
306
|
+
direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
|
|
265
307
|
)
|
|
266
308
|
except (ValueError, TypeError, NotImplementedError) as e:
|
|
267
309
|
issue.failed_conversions.append(
|
|
@@ -289,7 +331,7 @@ def create_properties(
|
|
|
289
331
|
def create_edge_properties(
|
|
290
332
|
dumped: dict[str, Any],
|
|
291
333
|
property_mapping: dict[str, str],
|
|
292
|
-
resource_type:
|
|
334
|
+
resource_type: AssetCentricTypeExtended,
|
|
293
335
|
issue: ConversionIssue,
|
|
294
336
|
direct_relation_cache: DirectRelationCache,
|
|
295
337
|
default_instance_space: str,
|
|
@@ -309,7 +351,7 @@ def create_edge_properties(
|
|
|
309
351
|
flatten_dump[prop_json_path],
|
|
310
352
|
DirectRelation(),
|
|
311
353
|
False,
|
|
312
|
-
direct_relation_lookup=direct_relation_cache.
|
|
354
|
+
direct_relation_lookup=direct_relation_cache.get_cache(resource_type, prop_json_path),
|
|
313
355
|
)
|
|
314
356
|
except (ValueError, TypeError, NotImplementedError) as e:
|
|
315
357
|
issue.failed_conversions.append(
|
|
@@ -1,26 +1,30 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from typing import Any, Generic, Literal
|
|
3
|
+
from typing import Annotated, Any, Generic, Literal
|
|
4
4
|
|
|
5
5
|
from cognite.client.data_classes._base import (
|
|
6
6
|
WriteableCogniteResource,
|
|
7
7
|
WriteableCogniteResourceList,
|
|
8
8
|
)
|
|
9
|
-
from cognite.client.data_classes.data_modeling import InstanceApply, NodeId, ViewId
|
|
9
|
+
from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, NodeId, ViewId
|
|
10
|
+
from cognite.client.utils._identifier import InstanceId
|
|
10
11
|
from cognite.client.utils._text import to_camel_case
|
|
11
|
-
from pydantic import BaseModel, field_validator, model_validator
|
|
12
|
+
from pydantic import BaseModel, BeforeValidator, field_validator, model_validator
|
|
12
13
|
|
|
13
14
|
from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
|
|
14
15
|
from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
|
|
15
16
|
from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
|
|
16
|
-
from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import
|
|
17
|
+
from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import (
|
|
18
|
+
ASSET_ANNOTATIONS_ID,
|
|
19
|
+
FILE_ANNOTATIONS_ID,
|
|
20
|
+
create_default_mappings,
|
|
21
|
+
)
|
|
17
22
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
|
18
23
|
from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
|
|
19
24
|
from cognite_toolkit._cdf_tk.utils.useful_types import (
|
|
20
|
-
|
|
21
|
-
AssetCentricType,
|
|
25
|
+
AssetCentricKindExtended,
|
|
22
26
|
JsonVal,
|
|
23
|
-
|
|
27
|
+
T_AssetCentricResourceExtended,
|
|
24
28
|
)
|
|
25
29
|
|
|
26
30
|
|
|
@@ -37,8 +41,8 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
|
|
|
37
41
|
for example, the Canvas migration to determine which view to use for the resource.
|
|
38
42
|
"""
|
|
39
43
|
|
|
40
|
-
resource_type:
|
|
41
|
-
instance_id:
|
|
44
|
+
resource_type: str
|
|
45
|
+
instance_id: InstanceId
|
|
42
46
|
id: int
|
|
43
47
|
data_set_id: int | None = None
|
|
44
48
|
ingestion_view: str | None = None
|
|
@@ -56,7 +60,8 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
|
|
|
56
60
|
raise ToolkitValueError(f"No default ingestion view specified for resource type '{self.resource_type}'")
|
|
57
61
|
|
|
58
62
|
def as_asset_centric_id(self) -> AssetCentricId:
|
|
59
|
-
|
|
63
|
+
# MyPy fails to understand that resource_type is AssetCentricKindExtended in subclasses
|
|
64
|
+
return AssetCentricId(resource_type=self.resource_type, id_=self.id) # type: ignore[arg-type]
|
|
60
65
|
|
|
61
66
|
@model_validator(mode="before")
|
|
62
67
|
def _handle_flat_dict(cls, values: Any) -> Any:
|
|
@@ -87,12 +92,6 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
|
|
|
87
92
|
return ViewId.load(v)
|
|
88
93
|
return v
|
|
89
94
|
|
|
90
|
-
@field_validator("instance_id", mode="before")
|
|
91
|
-
def _validate_instance_id(cls, v: Any) -> Any:
|
|
92
|
-
if isinstance(v, dict):
|
|
93
|
-
return NodeId.load(v)
|
|
94
|
-
return v
|
|
95
|
-
|
|
96
95
|
|
|
97
96
|
class MigrationMappingList(ModelList[MigrationMapping]):
|
|
98
97
|
@classmethod
|
|
@@ -113,14 +112,22 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
113
112
|
|
|
114
113
|
def as_node_ids(self) -> list[NodeId]:
|
|
115
114
|
"""Return a list of NodeIds from the migration mappings."""
|
|
116
|
-
return [mapping.instance_id for mapping in self]
|
|
115
|
+
return [mapping.instance_id for mapping in self if isinstance(mapping.instance_id, NodeId)]
|
|
116
|
+
|
|
117
|
+
def as_edge_ids(self) -> list[EdgeId]:
|
|
118
|
+
"""Return a list of EdgeIds from the migration mappings."""
|
|
119
|
+
return [mapping.instance_id for mapping in self if isinstance(mapping.instance_id, EdgeId)]
|
|
117
120
|
|
|
118
121
|
def spaces(self) -> set[str]:
|
|
119
122
|
"""Return a set of spaces from the migration mappings."""
|
|
120
123
|
return {mapping.instance_id.space for mapping in self}
|
|
121
124
|
|
|
122
125
|
def as_pending_ids(self) -> list[PendingInstanceId]:
|
|
123
|
-
return [
|
|
126
|
+
return [
|
|
127
|
+
PendingInstanceId(pending_instance_id=mapping.instance_id, id=mapping.id)
|
|
128
|
+
for mapping in self
|
|
129
|
+
if isinstance(mapping.instance_id, NodeId)
|
|
130
|
+
]
|
|
124
131
|
|
|
125
132
|
def get_data_set_ids(self) -> set[int]:
|
|
126
133
|
"""Return a list of data set IDs from the migration mappings."""
|
|
@@ -131,7 +138,9 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
131
138
|
return {mapping.id: mapping for mapping in self}
|
|
132
139
|
|
|
133
140
|
@classmethod
|
|
134
|
-
def read_csv_file(
|
|
141
|
+
def read_csv_file(
|
|
142
|
+
cls, filepath: Path, resource_type: AssetCentricKindExtended | None = None
|
|
143
|
+
) -> "MigrationMappingList":
|
|
135
144
|
if cls is not MigrationMappingList or resource_type is None:
|
|
136
145
|
return super().read_csv_file(filepath)
|
|
137
146
|
cls_by_resource_type: dict[str, type[MigrationMappingList]] = {
|
|
@@ -139,6 +148,7 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
139
148
|
"TimeSeries": TimeSeriesMigrationMappingList,
|
|
140
149
|
"FileMetadata": FileMigrationMappingList,
|
|
141
150
|
"Events": EventMigrationMappingList,
|
|
151
|
+
"Annotations": AnnotationMigrationMappingList,
|
|
142
152
|
}
|
|
143
153
|
if resource_type not in cls_by_resource_type:
|
|
144
154
|
raise ToolkitValueError(
|
|
@@ -147,20 +157,53 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
147
157
|
return cls_by_resource_type[resource_type].read_csv_file(filepath, resource_type=None)
|
|
148
158
|
|
|
149
159
|
|
|
160
|
+
def _validate_node_id(value: Any) -> Any:
|
|
161
|
+
if isinstance(value, dict):
|
|
162
|
+
return NodeId.load(value)
|
|
163
|
+
return value
|
|
164
|
+
|
|
165
|
+
|
|
150
166
|
class AssetMapping(MigrationMapping):
|
|
151
167
|
resource_type: Literal["asset"] = "asset"
|
|
168
|
+
instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
|
|
152
169
|
|
|
153
170
|
|
|
154
171
|
class EventMapping(MigrationMapping):
|
|
155
172
|
resource_type: Literal["event"] = "event"
|
|
173
|
+
instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
|
|
156
174
|
|
|
157
175
|
|
|
158
176
|
class TimeSeriesMapping(MigrationMapping):
|
|
159
177
|
resource_type: Literal["timeseries"] = "timeseries"
|
|
178
|
+
instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
|
|
160
179
|
|
|
161
180
|
|
|
162
181
|
class FileMapping(MigrationMapping):
|
|
163
182
|
resource_type: Literal["file"] = "file"
|
|
183
|
+
instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class AnnotationMapping(MigrationMapping):
|
|
187
|
+
resource_type: Literal["annotation"] = "annotation"
|
|
188
|
+
instance_id: EdgeId
|
|
189
|
+
annotation_type: Literal["diagrams.AssetLink", "diagrams.FileLink"] | None = None
|
|
190
|
+
|
|
191
|
+
def get_ingestion_view(self) -> str:
|
|
192
|
+
"""Get the ingestion view for the mapping. If not specified, return the default ingestion view."""
|
|
193
|
+
if self.ingestion_view:
|
|
194
|
+
return self.ingestion_view
|
|
195
|
+
elif self.annotation_type == "diagrams.AssetLink":
|
|
196
|
+
return ASSET_ANNOTATIONS_ID
|
|
197
|
+
elif self.annotation_type == "diagrams.FileLink":
|
|
198
|
+
return FILE_ANNOTATIONS_ID
|
|
199
|
+
else:
|
|
200
|
+
raise ToolkitValueError("Cannot determine default ingestion view for annotation without annotation_type")
|
|
201
|
+
|
|
202
|
+
@field_validator("instance_id", mode="before")
|
|
203
|
+
def _validate_instance_id(cls, v: Any) -> Any:
|
|
204
|
+
if isinstance(v, dict):
|
|
205
|
+
return EdgeId.load(v)
|
|
206
|
+
return v
|
|
164
207
|
|
|
165
208
|
|
|
166
209
|
class AssetMigrationMappingList(MigrationMappingList):
|
|
@@ -187,10 +230,16 @@ class TimeSeriesMigrationMappingList(MigrationMappingList):
|
|
|
187
230
|
return TimeSeriesMapping
|
|
188
231
|
|
|
189
232
|
|
|
233
|
+
class AnnotationMigrationMappingList(MigrationMappingList):
|
|
234
|
+
@classmethod
|
|
235
|
+
def _get_base_model_cls(cls) -> type[AnnotationMapping]:
|
|
236
|
+
return AnnotationMapping
|
|
237
|
+
|
|
238
|
+
|
|
190
239
|
@dataclass
|
|
191
|
-
class AssetCentricMapping(Generic[
|
|
240
|
+
class AssetCentricMapping(Generic[T_AssetCentricResourceExtended], WriteableCogniteResource[InstanceApply]):
|
|
192
241
|
mapping: MigrationMapping
|
|
193
|
-
resource:
|
|
242
|
+
resource: T_AssetCentricResourceExtended
|
|
194
243
|
|
|
195
244
|
def as_write(self) -> InstanceApply:
|
|
196
245
|
raise NotImplementedError()
|
|
@@ -205,7 +254,9 @@ class AssetCentricMapping(Generic[T_AssetCentricResource], WriteableCogniteResou
|
|
|
205
254
|
}
|
|
206
255
|
|
|
207
256
|
|
|
208
|
-
class AssetCentricMappingList(
|
|
257
|
+
class AssetCentricMappingList(
|
|
258
|
+
WriteableCogniteResourceList[InstanceApply, AssetCentricMapping[T_AssetCentricResourceExtended]]
|
|
259
|
+
):
|
|
209
260
|
_RESOURCE: type = AssetCentricMapping
|
|
210
261
|
|
|
211
262
|
def as_write(self) -> InstanceApplyList:
|