cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +2 -2
- cognite/neat/core/_data_model/exporters/__init__.py +6 -6
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +14 -14
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +8 -8
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +50 -25
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/__init__.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/{_imf2rules.py → _imf2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/{_inference2rules.py → _inference2rdata_model.py} +59 -59
- cognite/neat/core/_data_model/importers/_rdf/{_owl2rules.py → _owl2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +76 -19
- cognite/neat/core/_data_model/models/__init__.py +11 -9
- cognite/neat/core/_data_model/models/_base_unverified.py +12 -12
- cognite/neat/core/_data_model/models/_base_verified.py +9 -14
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +88 -78
- cognite/neat/core/_data_model/models/conceptual/_verified.py +54 -52
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +75 -55
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +17 -15
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +29 -25
- cognite/neat/core/_data_model/transformers/_base.py +27 -20
- cognite/neat/core/_data_model/transformers/_converters.py +707 -622
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +64 -55
- cognite/neat/core/_instances/extractors/_base.py +2 -2
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +9 -9
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +107 -103
- cognite/neat/core/_instances/loaders/_base.py +3 -3
- cognite/neat/core/_instances/loaders/_rdf2dms.py +22 -22
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_rdfpath.py +1 -1
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/_factory.py +1 -1
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_resources.py +1 -1
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +1 -1
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +42 -36
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_experimental.py +1 -1
- cognite/neat/session/_inspect.py +13 -13
- cognite/neat/session/_mapping.py +15 -9
- cognite/neat/session/_read.py +39 -37
- cognite/neat/session/_set.py +6 -6
- cognite/neat/session/_show.py +24 -21
- cognite/neat/session/_state/README.md +1 -1
- cognite/neat/session/_state.py +27 -27
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +23 -21
- cognite/neat/session/_to.py +42 -42
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/METADATA +14 -7
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/RECORD +102 -100
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -6,14 +6,18 @@ from typing import Any, ClassVar, Literal
|
|
|
6
6
|
from cognite.client import data_modeling as dm
|
|
7
7
|
|
|
8
8
|
from cognite.neat.core._client import NeatClient
|
|
9
|
-
from cognite.neat.core._data_model.models import
|
|
9
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel, SheetList
|
|
10
10
|
from cognite.neat.core._data_model.models.data_types import Enum
|
|
11
|
-
from cognite.neat.core._data_model.models.dms import DMSContainer, DMSEnum, DMSProperty
|
|
12
11
|
from cognite.neat.core._data_model.models.entities import (
|
|
13
|
-
|
|
12
|
+
ConceptEntity,
|
|
14
13
|
ContainerEntity,
|
|
15
14
|
ViewEntity,
|
|
16
15
|
)
|
|
16
|
+
from cognite.neat.core._data_model.models.physical import (
|
|
17
|
+
PhysicalContainer,
|
|
18
|
+
PhysicalEnum,
|
|
19
|
+
PhysicalProperty,
|
|
20
|
+
)
|
|
17
21
|
from cognite.neat.core._issues.errors import (
|
|
18
22
|
CDFMissingClientError,
|
|
19
23
|
NeatValueError,
|
|
@@ -21,10 +25,10 @@ from cognite.neat.core._issues.errors import (
|
|
|
21
25
|
)
|
|
22
26
|
from cognite.neat.core._issues.warnings import PropertyOverwritingWarning
|
|
23
27
|
|
|
24
|
-
from ._base import
|
|
28
|
+
from ._base import VerifiedDataModelTransformer
|
|
25
29
|
|
|
26
30
|
|
|
27
|
-
class MapOntoTransformers(
|
|
31
|
+
class MapOntoTransformers(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel], ABC):
|
|
28
32
|
"""Base class for transformers that map one rule onto another."""
|
|
29
33
|
|
|
30
34
|
...
|
|
@@ -33,7 +37,7 @@ class MapOntoTransformers(VerifiedRulesTransformer[DMSRules, DMSRules], ABC):
|
|
|
33
37
|
class MapOneToOne(MapOntoTransformers):
|
|
34
38
|
"""Takes transform data models and makes it into an extension of the reference data model.
|
|
35
39
|
|
|
36
|
-
Note this transformer mutates the input
|
|
40
|
+
Note this transformer mutates the input data model.
|
|
37
41
|
|
|
38
42
|
The argument view_extension_mapping is a dictionary where the keys are views of this data model,
|
|
39
43
|
and each value is the view of the reference data model that the view should extend. For example:
|
|
@@ -55,14 +59,17 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
55
59
|
"""
|
|
56
60
|
|
|
57
61
|
def __init__(
|
|
58
|
-
self,
|
|
62
|
+
self,
|
|
63
|
+
reference: PhysicalDataModel,
|
|
64
|
+
view_extension_mapping: dict[str, str],
|
|
65
|
+
default_extension: str | None = None,
|
|
59
66
|
) -> None:
|
|
60
67
|
self.reference = reference
|
|
61
68
|
self.view_extension_mapping = view_extension_mapping
|
|
62
69
|
self.default_extension = default_extension
|
|
63
70
|
|
|
64
|
-
def transform(self,
|
|
65
|
-
solution:
|
|
71
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
72
|
+
solution: PhysicalDataModel = data_model
|
|
66
73
|
view_by_external_id = {view.view.external_id: view for view in solution.views}
|
|
67
74
|
ref_view_by_external_id = {view.view.external_id: view for view in self.reference.views}
|
|
68
75
|
|
|
@@ -73,11 +80,11 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
73
80
|
if self.default_extension and self.default_extension not in ref_view_by_external_id:
|
|
74
81
|
raise ValueError(f"Default extension view not in the reference data model {self.default_extension}")
|
|
75
82
|
|
|
76
|
-
properties_by_view_external_id: dict[str, dict[str,
|
|
83
|
+
properties_by_view_external_id: dict[str, dict[str, PhysicalProperty]] = defaultdict(dict)
|
|
77
84
|
for prop in solution.properties:
|
|
78
85
|
properties_by_view_external_id[prop.view.external_id][prop.view_property] = prop
|
|
79
86
|
|
|
80
|
-
ref_properties_by_view_external_id: dict[str, dict[str,
|
|
87
|
+
ref_properties_by_view_external_id: dict[str, dict[str, PhysicalProperty]] = defaultdict(dict)
|
|
81
88
|
for prop in self.reference.properties:
|
|
82
89
|
ref_properties_by_view_external_id[prop.view.external_id][prop.view_property] = prop
|
|
83
90
|
|
|
@@ -108,14 +115,15 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
108
115
|
return solution
|
|
109
116
|
|
|
110
117
|
|
|
111
|
-
class
|
|
118
|
+
class PhysicalDataModelMapper(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
112
119
|
"""Maps properties and classes using the given mapping.
|
|
113
120
|
|
|
114
121
|
Args:
|
|
115
|
-
mapping: The mapping to use represented as a
|
|
122
|
+
mapping: The mapping to use represented as a physical data model object.
|
|
116
123
|
data_type_conflict: How to handle data type conflicts. The default is "overwrite".
|
|
117
124
|
A data type conflicts occurs when the data type of a property in the mapping is different from the
|
|
118
|
-
data type of the property in the input
|
|
125
|
+
data type of the property in the input data model. If "overwrite" the data type
|
|
126
|
+
in the input data model is overwritten
|
|
119
127
|
with the data type in the mapping.
|
|
120
128
|
"""
|
|
121
129
|
|
|
@@ -123,37 +131,41 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
123
131
|
["connection", "value_type", "min_count", "immutable", "max_count", "default", "index", "constraint"]
|
|
124
132
|
)
|
|
125
133
|
|
|
126
|
-
def __init__(
|
|
134
|
+
def __init__(
|
|
135
|
+
self,
|
|
136
|
+
mapping: PhysicalDataModel,
|
|
137
|
+
data_type_conflict: Literal["overwrite"] = "overwrite",
|
|
138
|
+
) -> None:
|
|
127
139
|
self.mapping = mapping
|
|
128
140
|
self.data_type_conflict = data_type_conflict
|
|
129
141
|
|
|
130
|
-
def transform(self,
|
|
142
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
131
143
|
if self.data_type_conflict != "overwrite":
|
|
132
144
|
raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
|
|
133
|
-
|
|
134
|
-
|
|
145
|
+
input_data_model = data_model
|
|
146
|
+
new_data_model = input_data_model.model_copy(deep=True)
|
|
135
147
|
|
|
136
|
-
views_by_external_id = {view.view.external_id: view for view in
|
|
148
|
+
views_by_external_id = {view.view.external_id: view for view in new_data_model.views}
|
|
137
149
|
new_views: set[ViewEntity] = set()
|
|
138
150
|
for mapping_view in self.mapping.views:
|
|
139
151
|
if existing_view := views_by_external_id.get(mapping_view.view.external_id):
|
|
140
152
|
existing_view.implements = mapping_view.implements
|
|
141
153
|
else:
|
|
142
|
-
# We need to add all the views in the mapping that are not in the input
|
|
143
|
-
# This is to ensure that all ValueTypes are present in the resulting
|
|
154
|
+
# We need to add all the views in the mapping that are not in the input data model.
|
|
155
|
+
# This is to ensure that all ValueTypes are present in the resulting data model.
|
|
144
156
|
# For example, if a property is a direct relation to an Equipment view, we need to add
|
|
145
|
-
# the Equipment view to the
|
|
146
|
-
|
|
157
|
+
# the Equipment view to the data model.
|
|
158
|
+
new_data_model.views.append(mapping_view)
|
|
147
159
|
new_views.add(mapping_view.view)
|
|
148
160
|
|
|
149
161
|
properties_by_view_property = {
|
|
150
|
-
(prop.view.external_id, prop.view_property): prop for prop in
|
|
162
|
+
(prop.view.external_id, prop.view_property): prop for prop in new_data_model.properties
|
|
151
163
|
}
|
|
152
|
-
existing_enum_collections = {item.collection for item in
|
|
153
|
-
mapping_enums_by_collection: dict[
|
|
164
|
+
existing_enum_collections = {item.collection for item in new_data_model.enum or []}
|
|
165
|
+
mapping_enums_by_collection: dict[ConceptEntity, list[PhysicalEnum]] = defaultdict(list)
|
|
154
166
|
for item in self.mapping.enum or []:
|
|
155
167
|
mapping_enums_by_collection[item.collection].append(item)
|
|
156
|
-
existing_containers = {container.container for container in
|
|
168
|
+
existing_containers = {container.container for container in new_data_model.containers or []}
|
|
157
169
|
mapping_containers_by_id = {container.container: container for container in self.mapping.containers or []}
|
|
158
170
|
for mapping_prop in self.mapping.properties:
|
|
159
171
|
if existing_prop := properties_by_view_property.get(
|
|
@@ -163,7 +175,10 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
163
175
|
if conflicts and self.data_type_conflict == "overwrite":
|
|
164
176
|
warnings.warn(
|
|
165
177
|
PropertyOverwritingWarning(
|
|
166
|
-
existing_prop.view.as_id(),
|
|
178
|
+
existing_prop.view.as_id(),
|
|
179
|
+
"view",
|
|
180
|
+
existing_prop.view_property,
|
|
181
|
+
tuple(conflicts),
|
|
167
182
|
),
|
|
168
183
|
stacklevel=2,
|
|
169
184
|
)
|
|
@@ -177,24 +192,24 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
177
192
|
existing_prop.container = mapping_prop.container
|
|
178
193
|
existing_prop.container_property = mapping_prop.container_property
|
|
179
194
|
elif isinstance(mapping_prop.value_type, ViewEntity):
|
|
180
|
-
# All connections must be included in the
|
|
195
|
+
# All connections must be included in the data model. This is to update the
|
|
181
196
|
# ValueTypes of the implemented views.
|
|
182
|
-
|
|
197
|
+
new_data_model.properties.append(mapping_prop)
|
|
183
198
|
elif "guid" in mapping_prop.view_property.casefold():
|
|
184
199
|
# All guid properties are included. Theses are necessary to get an appropriate
|
|
185
200
|
# filter on the resulting view.
|
|
186
|
-
|
|
201
|
+
new_data_model.properties.append(mapping_prop)
|
|
187
202
|
else:
|
|
188
|
-
# Skipping mapped properties that are not in the input
|
|
203
|
+
# Skipping mapped properties that are not in the input data model.
|
|
189
204
|
continue
|
|
190
205
|
|
|
191
206
|
if (
|
|
192
207
|
isinstance(mapping_prop.value_type, Enum)
|
|
193
208
|
and mapping_prop.value_type.collection not in existing_enum_collections
|
|
194
209
|
):
|
|
195
|
-
if not
|
|
196
|
-
|
|
197
|
-
|
|
210
|
+
if not new_data_model.enum:
|
|
211
|
+
new_data_model.enum = SheetList[PhysicalEnum]([])
|
|
212
|
+
new_data_model.enum.extend(mapping_enums_by_collection[mapping_prop.value_type.collection])
|
|
198
213
|
|
|
199
214
|
if (
|
|
200
215
|
mapping_prop.container
|
|
@@ -202,13 +217,15 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
202
217
|
and (new_container := mapping_containers_by_id.get(mapping_prop.container))
|
|
203
218
|
):
|
|
204
219
|
# Mapping can include new containers for GUID properties
|
|
205
|
-
if not
|
|
206
|
-
|
|
207
|
-
|
|
220
|
+
if not new_data_model.containers:
|
|
221
|
+
new_data_model.containers = SheetList[PhysicalContainer]([])
|
|
222
|
+
new_data_model.containers.append(new_container)
|
|
208
223
|
|
|
209
|
-
return
|
|
224
|
+
return new_data_model
|
|
210
225
|
|
|
211
|
-
def _find_overwrites(
|
|
226
|
+
def _find_overwrites(
|
|
227
|
+
self, prop: PhysicalProperty, mapping_prop: PhysicalProperty
|
|
228
|
+
) -> tuple[dict[str, Any], list[str]]:
|
|
212
229
|
"""Finds the properties that need to be overwritten and returns them.
|
|
213
230
|
|
|
214
231
|
In addition, conflicting properties are returned. Note that overwriting properties that are
|
|
@@ -240,7 +257,7 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
240
257
|
return f"Mapping to {self.mapping.metadata.as_data_model_id()!r}."
|
|
241
258
|
|
|
242
259
|
|
|
243
|
-
class AsParentPropertyId(
|
|
260
|
+
class AsParentPropertyId(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
244
261
|
"""Looks up all view properties that map to the same container property,
|
|
245
262
|
and changes the child view property id to match the parent property id.
|
|
246
263
|
"""
|
|
@@ -248,34 +265,36 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
248
265
|
def __init__(self, client: NeatClient | None = None) -> None:
|
|
249
266
|
self._client = client
|
|
250
267
|
|
|
251
|
-
def transform(self,
|
|
252
|
-
|
|
253
|
-
|
|
268
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
269
|
+
input_data_model = data_model
|
|
270
|
+
new_data_model = input_data_model.model_copy(deep=True)
|
|
254
271
|
|
|
255
|
-
path_by_view = self._inheritance_path_by_view(
|
|
256
|
-
view_by_container_property = self._view_by_container_properties(
|
|
272
|
+
path_by_view = self._inheritance_path_by_view(new_data_model)
|
|
273
|
+
view_by_container_property = self._view_by_container_properties(new_data_model)
|
|
257
274
|
|
|
258
275
|
parent_view_property_by_container_property = self._get_parent_view_property_by_container_property(
|
|
259
276
|
path_by_view, view_by_container_property
|
|
260
277
|
)
|
|
261
278
|
|
|
262
|
-
for prop in
|
|
279
|
+
for prop in new_data_model.properties:
|
|
263
280
|
if prop.container and prop.container_property:
|
|
264
281
|
if parent_name := parent_view_property_by_container_property.get(
|
|
265
282
|
(prop.container, prop.container_property)
|
|
266
283
|
):
|
|
267
284
|
prop.view_property = parent_name
|
|
268
285
|
|
|
269
|
-
return
|
|
286
|
+
return new_data_model
|
|
270
287
|
|
|
271
288
|
# Todo: Move into Probe class. Note this means that the Probe class must take a NeatClient as an argument.
|
|
272
|
-
def _inheritance_path_by_view(self,
|
|
273
|
-
parents_by_view: dict[ViewEntity, list[ViewEntity]] = {
|
|
289
|
+
def _inheritance_path_by_view(self, data_model: PhysicalDataModel) -> dict[ViewEntity, list[ViewEntity]]:
|
|
290
|
+
parents_by_view: dict[ViewEntity, list[ViewEntity]] = {
|
|
291
|
+
view.view: view.implements or [] for view in data_model.views
|
|
292
|
+
}
|
|
274
293
|
|
|
275
294
|
path_by_view: dict[ViewEntity, list[ViewEntity]] = {}
|
|
276
|
-
for view in
|
|
295
|
+
for view in data_model.views:
|
|
277
296
|
path_by_view[view.view] = self._get_inheritance_path(
|
|
278
|
-
view.view, parents_by_view,
|
|
297
|
+
view.view, parents_by_view, data_model.metadata.as_data_model_id()
|
|
279
298
|
)
|
|
280
299
|
return path_by_view
|
|
281
300
|
|
|
@@ -312,13 +331,13 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
312
331
|
return inheritance_path
|
|
313
332
|
|
|
314
333
|
def _view_by_container_properties(
|
|
315
|
-
self,
|
|
334
|
+
self, data_model: PhysicalDataModel
|
|
316
335
|
) -> dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]:
|
|
317
336
|
view_properties_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]] = (
|
|
318
337
|
defaultdict(list)
|
|
319
338
|
)
|
|
320
339
|
view_with_properties: set[ViewEntity] = set()
|
|
321
|
-
for prop in
|
|
340
|
+
for prop in data_model.properties:
|
|
322
341
|
if not prop.container or not prop.container_property:
|
|
323
342
|
continue
|
|
324
343
|
view_properties_by_container_properties[(prop.container, prop.container_property)].append(
|
|
@@ -327,7 +346,7 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
327
346
|
view_with_properties.add(prop.view)
|
|
328
347
|
|
|
329
348
|
# We need to look up all parent properties.
|
|
330
|
-
to_lookup = {view.view.as_id() for view in
|
|
349
|
+
to_lookup = {view.view.as_id() for view in data_model.views if view.view not in view_with_properties}
|
|
331
350
|
if to_lookup and self._client is None:
|
|
332
351
|
raise CDFMissingClientError(
|
|
333
352
|
f"Views {to_lookup} are not in the data model. Please provide a client to lookup the views."
|
|
@@ -1,53 +1,58 @@
|
|
|
1
1
|
from abc import ABC
|
|
2
|
+
from typing import cast
|
|
2
3
|
|
|
3
4
|
from cognite.neat.core._client import NeatClient
|
|
4
5
|
from cognite.neat.core._data_model._shared import (
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
ImportedDataModel,
|
|
7
|
+
T_ImportedUnverifiedDataModel,
|
|
8
|
+
T_VerifiedDataModel,
|
|
9
|
+
VerifiedDataModel,
|
|
9
10
|
)
|
|
10
11
|
from cognite.neat.core._data_model.models import (
|
|
11
12
|
ConceptualDataModel,
|
|
12
|
-
|
|
13
|
-
DMSRules,
|
|
13
|
+
PhysicalDataModel,
|
|
14
14
|
UnverifiedConceptualDataModel,
|
|
15
|
+
UnverifiedPhysicalDataModel,
|
|
15
16
|
)
|
|
16
|
-
from cognite.neat.core._data_model.models.conceptual import
|
|
17
|
-
from cognite.neat.core._data_model.models.
|
|
17
|
+
from cognite.neat.core._data_model.models.conceptual import ConceptualValidation
|
|
18
|
+
from cognite.neat.core._data_model.models.physical import PhysicalValidation
|
|
18
19
|
from cognite.neat.core._issues import MultiValueError, catch_issues
|
|
19
20
|
from cognite.neat.core._issues.errors import NeatTypeError, NeatValueError
|
|
20
21
|
|
|
21
|
-
from ._base import
|
|
22
|
+
from ._base import DataModelTransformer
|
|
22
23
|
|
|
23
24
|
|
|
24
|
-
class VerificationTransformer(
|
|
25
|
+
class VerificationTransformer(DataModelTransformer[T_ImportedUnverifiedDataModel, T_VerifiedDataModel], ABC):
|
|
25
26
|
"""Base class for all verification transformers."""
|
|
26
27
|
|
|
27
|
-
|
|
28
|
+
_data_model_cls: type[T_VerifiedDataModel]
|
|
28
29
|
_validation_cls: type
|
|
29
30
|
|
|
30
31
|
def __init__(self, validate: bool = True, client: NeatClient | None = None) -> None:
|
|
31
32
|
self.validate = validate
|
|
32
33
|
self._client = client
|
|
33
34
|
|
|
34
|
-
def transform(self,
|
|
35
|
-
in_ =
|
|
35
|
+
def transform(self, data_model: T_ImportedUnverifiedDataModel) -> T_VerifiedDataModel:
|
|
36
|
+
in_ = data_model.unverified_data_model
|
|
36
37
|
if in_ is None:
|
|
37
|
-
raise NeatValueError("Cannot verify
|
|
38
|
-
|
|
38
|
+
raise NeatValueError("Cannot verify data model. The reading of the data model failed.")
|
|
39
|
+
verified_data_model: T_VerifiedDataModel | None = None
|
|
39
40
|
# We need to catch issues as we use the error args to provide extra context for the errors/warnings
|
|
40
41
|
# For example, which row in the spreadsheet the error occurred.
|
|
41
|
-
with catch_issues(
|
|
42
|
-
|
|
42
|
+
with catch_issues(data_model.context) as issues:
|
|
43
|
+
data_model_cls = self._get_data_model_cls(data_model)
|
|
43
44
|
dumped = in_.dump()
|
|
44
|
-
|
|
45
|
+
verified_data_model = data_model_cls.model_validate(dumped) # type: ignore[assignment]
|
|
45
46
|
if self.validate:
|
|
46
|
-
validation_cls = self._get_validation_cls(
|
|
47
|
-
if issubclass(validation_cls,
|
|
48
|
-
validation_issues =
|
|
49
|
-
|
|
50
|
-
|
|
47
|
+
validation_cls = self._get_validation_cls(verified_data_model) # type: ignore[arg-type]
|
|
48
|
+
if issubclass(validation_cls, PhysicalValidation):
|
|
49
|
+
validation_issues = PhysicalValidation(
|
|
50
|
+
cast(PhysicalDataModel, verified_data_model),
|
|
51
|
+
self._client,
|
|
52
|
+
data_model.context,
|
|
53
|
+
).validate() # type: ignore[arg-type]
|
|
54
|
+
elif issubclass(validation_cls, ConceptualValidation):
|
|
55
|
+
validation_issues = ConceptualValidation(verified_data_model, data_model.context).validate() # type: ignore[arg-type]
|
|
51
56
|
else:
|
|
52
57
|
raise NeatValueError("Unsupported rule type")
|
|
53
58
|
issues.extend(validation_issues)
|
|
@@ -56,56 +61,60 @@ class VerificationTransformer(RulesTransformer[T_ReadInputRules, T_VerifiedRules
|
|
|
56
61
|
issues.trigger_warnings()
|
|
57
62
|
if issues.has_errors:
|
|
58
63
|
raise MultiValueError(issues.errors)
|
|
59
|
-
if
|
|
60
|
-
raise NeatValueError("
|
|
61
|
-
return
|
|
64
|
+
if verified_data_model is None:
|
|
65
|
+
raise NeatValueError("Data model was not verified")
|
|
66
|
+
return verified_data_model
|
|
62
67
|
|
|
63
|
-
def
|
|
64
|
-
return self.
|
|
68
|
+
def _get_data_model_cls(self, in_: T_ImportedUnverifiedDataModel) -> type[T_VerifiedDataModel]:
|
|
69
|
+
return self._data_model_cls
|
|
65
70
|
|
|
66
|
-
def _get_validation_cls(self,
|
|
71
|
+
def _get_validation_cls(self, data_model: T_VerifiedDataModel) -> type:
|
|
67
72
|
return self._validation_cls
|
|
68
73
|
|
|
69
74
|
@property
|
|
70
75
|
def description(self) -> str:
|
|
71
|
-
return "Verify
|
|
76
|
+
return "Verify data model"
|
|
72
77
|
|
|
73
78
|
|
|
74
|
-
class
|
|
75
|
-
|
|
79
|
+
class VerifyPhysicalDataModel(
|
|
80
|
+
VerificationTransformer[ImportedDataModel[UnverifiedPhysicalDataModel], PhysicalDataModel]
|
|
81
|
+
):
|
|
82
|
+
"""Class to verify physical data model."""
|
|
76
83
|
|
|
77
|
-
|
|
78
|
-
_validation_cls =
|
|
84
|
+
_data_model_cls = PhysicalDataModel
|
|
85
|
+
_validation_cls = PhysicalValidation
|
|
79
86
|
|
|
80
|
-
def transform(self,
|
|
81
|
-
return super().transform(
|
|
87
|
+
def transform(self, data_model: ImportedDataModel[UnverifiedPhysicalDataModel]) -> PhysicalDataModel:
|
|
88
|
+
return super().transform(data_model)
|
|
82
89
|
|
|
83
90
|
|
|
84
|
-
class
|
|
85
|
-
|
|
91
|
+
class VerifyConceptualDataModel(
|
|
92
|
+
VerificationTransformer[ImportedDataModel[UnverifiedConceptualDataModel], ConceptualDataModel]
|
|
93
|
+
):
|
|
94
|
+
"""Class to verify conceptual data model."""
|
|
86
95
|
|
|
87
|
-
|
|
88
|
-
_validation_cls =
|
|
96
|
+
_data_model_cls = ConceptualDataModel
|
|
97
|
+
_validation_cls = ConceptualValidation
|
|
89
98
|
|
|
90
|
-
def transform(self,
|
|
91
|
-
return super().transform(
|
|
99
|
+
def transform(self, data_model: ImportedDataModel[UnverifiedConceptualDataModel]) -> ConceptualDataModel:
|
|
100
|
+
return super().transform(data_model)
|
|
92
101
|
|
|
93
102
|
|
|
94
|
-
class
|
|
95
|
-
"""Class to verify arbitrary
|
|
103
|
+
class VerifyAnyDataModel(VerificationTransformer[T_ImportedUnverifiedDataModel, VerifiedDataModel]):
|
|
104
|
+
"""Class to verify arbitrary data model"""
|
|
96
105
|
|
|
97
|
-
def
|
|
98
|
-
if isinstance(in_.
|
|
106
|
+
def _get_data_model_cls(self, in_: T_ImportedUnverifiedDataModel) -> type[VerifiedDataModel]:
|
|
107
|
+
if isinstance(in_.unverified_data_model, UnverifiedConceptualDataModel):
|
|
99
108
|
return ConceptualDataModel
|
|
100
|
-
elif isinstance(in_.
|
|
101
|
-
return
|
|
109
|
+
elif isinstance(in_.unverified_data_model, UnverifiedPhysicalDataModel):
|
|
110
|
+
return PhysicalDataModel
|
|
102
111
|
else:
|
|
103
|
-
raise NeatTypeError(f"Unsupported
|
|
112
|
+
raise NeatTypeError(f"Unsupported data model type: {type(in_)}")
|
|
104
113
|
|
|
105
|
-
def _get_validation_cls(self,
|
|
106
|
-
if isinstance(
|
|
107
|
-
return
|
|
108
|
-
elif isinstance(
|
|
109
|
-
return
|
|
114
|
+
def _get_validation_cls(self, data_model: VerifiedDataModel) -> type:
|
|
115
|
+
if isinstance(data_model, ConceptualDataModel):
|
|
116
|
+
return ConceptualValidation
|
|
117
|
+
elif isinstance(data_model, PhysicalDataModel):
|
|
118
|
+
return PhysicalValidation
|
|
110
119
|
else:
|
|
111
|
-
raise NeatTypeError(f"Unsupported
|
|
120
|
+
raise NeatTypeError(f"Unsupported data model type: {type(data_model)}")
|
|
@@ -38,8 +38,8 @@ class KnowledgeGraphExtractor(BaseExtractor):
|
|
|
38
38
|
"""A knowledge graph extractor extracts triples with a schema"""
|
|
39
39
|
|
|
40
40
|
@abstractmethod
|
|
41
|
-
def
|
|
42
|
-
"""Returns the
|
|
41
|
+
def get_conceptual_data_model(self) -> ConceptualDataModel:
|
|
42
|
+
"""Returns the conceptual data model that the extractor uses."""
|
|
43
43
|
raise NotImplementedError()
|
|
44
44
|
|
|
45
45
|
@property
|
|
@@ -14,7 +14,7 @@ from cognite.neat.core._constants import (
|
|
|
14
14
|
DEFAULT_NAMESPACE,
|
|
15
15
|
get_default_prefixes_and_namespaces,
|
|
16
16
|
)
|
|
17
|
-
from cognite.neat.core._data_model._shared import
|
|
17
|
+
from cognite.neat.core._data_model._shared import ImportedDataModel
|
|
18
18
|
from cognite.neat.core._data_model.catalog import classic_model
|
|
19
19
|
from cognite.neat.core._data_model.models import (
|
|
20
20
|
ConceptualDataModel,
|
|
@@ -208,18 +208,18 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
|
|
|
208
208
|
|
|
209
209
|
yield from self._extract_asset_parent_data_sets()
|
|
210
210
|
|
|
211
|
-
def
|
|
211
|
+
def get_conceptual_data_model(self) -> ConceptualDataModel:
|
|
212
212
|
# To avoid circular imports
|
|
213
213
|
from cognite.neat.core._data_model.importers import ExcelImporter
|
|
214
214
|
|
|
215
215
|
unverified = cast(
|
|
216
|
-
|
|
217
|
-
ExcelImporter(classic_model).
|
|
216
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
217
|
+
ExcelImporter(classic_model).to_data_model(),
|
|
218
218
|
)
|
|
219
|
-
if unverified.
|
|
220
|
-
raise NeatValueError(f"Could not read the classic model
|
|
219
|
+
if unverified.unverified_data_model is None:
|
|
220
|
+
raise NeatValueError(f"Could not read the classic data model from {classic_model}.")
|
|
221
221
|
|
|
222
|
-
verified = unverified.
|
|
222
|
+
verified = unverified.unverified_data_model.as_verified_data_model()
|
|
223
223
|
prefixes = get_default_prefixes_and_namespaces()
|
|
224
224
|
instance_prefix: str | None = next((k for k, v in prefixes.items() if v == self._namespace), None)
|
|
225
225
|
if instance_prefix is None:
|
|
@@ -234,8 +234,8 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
|
|
|
234
234
|
if is_snake_case:
|
|
235
235
|
prop_id = to_snake_case(prop_id)
|
|
236
236
|
prop.instance_source = [self._namespace[prop_id]]
|
|
237
|
-
for cls_ in verified.
|
|
238
|
-
cls_id = cls_.
|
|
237
|
+
for cls_ in verified.concepts:
|
|
238
|
+
cls_id = cls_.concept.suffix
|
|
239
239
|
if is_snake_case:
|
|
240
240
|
cls_id = to_snake_case(cls_id)
|
|
241
241
|
cls_.instance_source = self._namespace[cls_id]
|