cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +2 -2
- cognite/neat/core/_data_model/exporters/__init__.py +6 -6
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +14 -14
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +8 -8
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +50 -25
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/__init__.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/{_imf2rules.py → _imf2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/{_inference2rules.py → _inference2rdata_model.py} +59 -59
- cognite/neat/core/_data_model/importers/_rdf/{_owl2rules.py → _owl2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +76 -19
- cognite/neat/core/_data_model/models/__init__.py +11 -9
- cognite/neat/core/_data_model/models/_base_unverified.py +12 -12
- cognite/neat/core/_data_model/models/_base_verified.py +9 -14
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +88 -78
- cognite/neat/core/_data_model/models/conceptual/_verified.py +54 -52
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +75 -55
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +17 -15
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +29 -25
- cognite/neat/core/_data_model/transformers/_base.py +27 -20
- cognite/neat/core/_data_model/transformers/_converters.py +707 -622
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +64 -55
- cognite/neat/core/_instances/extractors/_base.py +2 -2
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +9 -9
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +107 -103
- cognite/neat/core/_instances/loaders/_base.py +3 -3
- cognite/neat/core/_instances/loaders/_rdf2dms.py +22 -22
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_rdfpath.py +1 -1
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/_factory.py +1 -1
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_resources.py +1 -1
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +1 -1
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +42 -36
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_experimental.py +1 -1
- cognite/neat/session/_inspect.py +13 -13
- cognite/neat/session/_mapping.py +15 -9
- cognite/neat/session/_read.py +39 -37
- cognite/neat/session/_set.py +6 -6
- cognite/neat/session/_show.py +24 -21
- cognite/neat/session/_state/README.md +1 -1
- cognite/neat/session/_state.py +27 -27
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +23 -21
- cognite/neat/session/_to.py +42 -42
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/METADATA +14 -7
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/RECORD +102 -100
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -31,25 +31,25 @@ from cognite.neat.core._constants import (
|
|
|
31
31
|
)
|
|
32
32
|
from cognite.neat.core._data_model._constants import PATTERNS, get_reserved_words
|
|
33
33
|
from cognite.neat.core._data_model._shared import (
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
ImportedDataModel,
|
|
35
|
+
ImportedUnverifiedDataModel,
|
|
36
|
+
VerifiedDataModel,
|
|
37
37
|
)
|
|
38
|
-
from cognite.neat.core._data_model.analysis import
|
|
38
|
+
from cognite.neat.core._data_model.analysis import DataModelAnalysis
|
|
39
39
|
from cognite.neat.core._data_model.importers import DMSImporter
|
|
40
40
|
from cognite.neat.core._data_model.models import (
|
|
41
41
|
ConceptualDataModel,
|
|
42
|
-
|
|
43
|
-
DMSRules,
|
|
42
|
+
PhysicalDataModel,
|
|
44
43
|
SheetList,
|
|
45
44
|
UnverifiedConceptualDataModel,
|
|
45
|
+
UnverifiedPhysicalDataModel,
|
|
46
46
|
data_types,
|
|
47
47
|
)
|
|
48
48
|
from cognite.neat.core._data_model.models.conceptual import (
|
|
49
|
-
|
|
49
|
+
Concept,
|
|
50
50
|
ConceptualMetadata,
|
|
51
51
|
ConceptualProperty,
|
|
52
|
-
|
|
52
|
+
UnverifiedConcept,
|
|
53
53
|
UnverifiedConceptualProperty,
|
|
54
54
|
)
|
|
55
55
|
from cognite.neat.core._data_model.models.data_types import (
|
|
@@ -60,24 +60,28 @@ from cognite.neat.core._data_model.models.data_types import (
|
|
|
60
60
|
String,
|
|
61
61
|
Timeseries,
|
|
62
62
|
)
|
|
63
|
-
from cognite.neat.core._data_model.models.dms import (
|
|
64
|
-
DMSMetadata,
|
|
65
|
-
DMSProperty,
|
|
66
|
-
DMSValidation,
|
|
67
|
-
DMSView,
|
|
68
|
-
)
|
|
69
|
-
from cognite.neat.core._data_model.models.dms._rules import DMSContainer, DMSEnum, DMSNode
|
|
70
63
|
from cognite.neat.core._data_model.models.entities import (
|
|
71
|
-
|
|
64
|
+
ConceptEntity,
|
|
72
65
|
ContainerEntity,
|
|
73
|
-
DMSUnknownEntity,
|
|
74
66
|
EdgeEntity,
|
|
75
67
|
HasDataFilter,
|
|
76
68
|
MultiValueTypeInfo,
|
|
69
|
+
PhysicalUnknownEntity,
|
|
77
70
|
ReverseConnectionEntity,
|
|
78
71
|
UnknownEntity,
|
|
79
72
|
ViewEntity,
|
|
80
73
|
)
|
|
74
|
+
from cognite.neat.core._data_model.models.physical import (
|
|
75
|
+
PhysicalMetadata,
|
|
76
|
+
PhysicalProperty,
|
|
77
|
+
PhysicalValidation,
|
|
78
|
+
PhysicalView,
|
|
79
|
+
)
|
|
80
|
+
from cognite.neat.core._data_model.models.physical._verified import (
|
|
81
|
+
PhysicalContainer,
|
|
82
|
+
PhysicalEnum,
|
|
83
|
+
PhysicalNodeType,
|
|
84
|
+
)
|
|
81
85
|
from cognite.neat.core._issues import IssueList
|
|
82
86
|
from cognite.neat.core._issues._factory import from_pydantic_errors
|
|
83
87
|
from cognite.neat.core._issues.errors import CDFMissingClientError, NeatValueError
|
|
@@ -98,29 +102,32 @@ from cognite.neat.core._utils.text import (
|
|
|
98
102
|
to_words,
|
|
99
103
|
)
|
|
100
104
|
|
|
101
|
-
from ._base import
|
|
102
|
-
|
|
105
|
+
from ._base import (
|
|
106
|
+
DataModelTransformer,
|
|
107
|
+
T_VerifiedIn,
|
|
108
|
+
T_VerifiedOut,
|
|
109
|
+
VerifiedDataModelTransformer,
|
|
110
|
+
)
|
|
111
|
+
from ._verification import VerifyPhysicalDataModel
|
|
103
112
|
|
|
104
|
-
|
|
105
|
-
|
|
113
|
+
T_UnverifiedInDataModel = TypeVar("T_UnverifiedInDataModel", bound=ImportedUnverifiedDataModel)
|
|
114
|
+
T_UnverifiedOutDataModel = TypeVar("T_UnverifiedOutDataModel", bound=ImportedUnverifiedDataModel)
|
|
106
115
|
|
|
107
116
|
|
|
108
|
-
class ConversionTransformer(
|
|
117
|
+
class ConversionTransformer(VerifiedDataModelTransformer[T_VerifiedIn, T_VerifiedOut], ABC):
|
|
109
118
|
"""Base class for all conversion transformers."""
|
|
110
119
|
|
|
111
120
|
...
|
|
112
121
|
|
|
113
122
|
|
|
114
123
|
class ToDMSCompliantEntities(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
124
|
+
DataModelTransformer[
|
|
125
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
126
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
118
127
|
]
|
|
119
128
|
):
|
|
120
|
-
"""
|
|
129
|
+
"""Makes concept and property ids compliant with DMS regex restrictions.
|
|
121
130
|
|
|
122
|
-
This is typically used with importers from arbitrary sources to ensure that classes and properties have valid
|
|
123
|
-
names.
|
|
124
131
|
|
|
125
132
|
Args:
|
|
126
133
|
rename_warning: How to handle renaming of entities that are not compliant with the Information Model.
|
|
@@ -135,70 +142,74 @@ class ToDMSCompliantEntities(
|
|
|
135
142
|
def description(self) -> str:
|
|
136
143
|
return "Ensures that all entities are compliant with the Information Model."
|
|
137
144
|
|
|
138
|
-
def transform(
|
|
139
|
-
|
|
140
|
-
|
|
145
|
+
def transform(
|
|
146
|
+
self, data_model: ImportedDataModel[UnverifiedConceptualDataModel]
|
|
147
|
+
) -> ImportedDataModel[UnverifiedConceptualDataModel]:
|
|
148
|
+
if data_model.unverified_data_model is None:
|
|
149
|
+
return data_model
|
|
141
150
|
# Doing dump to obtain a copy, and ensure that all entities are created. Input allows
|
|
142
151
|
# string for entities, the dump call will convert these to entities.
|
|
143
|
-
dumped =
|
|
152
|
+
dumped = data_model.unverified_data_model.dump()
|
|
144
153
|
copy = UnverifiedConceptualDataModel.load(dumped)
|
|
145
154
|
|
|
146
|
-
|
|
147
|
-
for
|
|
148
|
-
|
|
149
|
-
if not PATTERNS.view_id_compliance.match(
|
|
150
|
-
new_suffix = self.
|
|
155
|
+
new_by_old_concept_suffix: dict[str, str] = {}
|
|
156
|
+
for concept in copy.concepts:
|
|
157
|
+
concept_entity = cast(ConceptEntity, concept.concept) # Safe due to the dump above
|
|
158
|
+
if not PATTERNS.view_id_compliance.match(concept_entity.suffix):
|
|
159
|
+
new_suffix = self._fix_concept_suffix(concept_entity.suffix)
|
|
151
160
|
if self._renaming == "raise":
|
|
152
161
|
warnings.warn(
|
|
153
|
-
NeatValueWarning(f"Invalid class name {
|
|
162
|
+
NeatValueWarning(f"Invalid class name {concept_entity.suffix!r}.Renaming to {new_suffix}"),
|
|
154
163
|
stacklevel=2,
|
|
155
164
|
)
|
|
156
|
-
|
|
165
|
+
concept.concept.suffix = new_suffix # type: ignore[union-attr]
|
|
157
166
|
|
|
158
|
-
for
|
|
159
|
-
if
|
|
160
|
-
for i, parent in enumerate(
|
|
161
|
-
if isinstance(parent,
|
|
162
|
-
|
|
167
|
+
for concept in copy.concepts:
|
|
168
|
+
if concept.implements:
|
|
169
|
+
for i, parent in enumerate(concept.implements):
|
|
170
|
+
if isinstance(parent, ConceptEntity) and parent.suffix in new_by_old_concept_suffix:
|
|
171
|
+
concept.implements[i].suffix = new_by_old_concept_suffix[parent.suffix] # type: ignore[union-attr]
|
|
163
172
|
|
|
164
173
|
for prop in copy.properties:
|
|
165
|
-
if not PATTERNS.
|
|
174
|
+
if not PATTERNS.physical_property_id_compliance.match(prop.property_):
|
|
166
175
|
new_property = self._fix_property(prop.property_)
|
|
167
176
|
if self._renaming == "warning":
|
|
168
177
|
warnings.warn(
|
|
169
178
|
NeatValueWarning(
|
|
170
|
-
f"Invalid property name {prop.
|
|
179
|
+
f"Invalid property name {prop.concept.suffix}.{prop.property_!r}."
|
|
180
|
+
f" Renaming to {new_property}"
|
|
181
|
+
# type: ignore[union-attr]
|
|
171
182
|
),
|
|
172
183
|
stacklevel=2,
|
|
173
184
|
)
|
|
174
185
|
prop.property_ = new_property
|
|
175
186
|
|
|
176
|
-
if isinstance(prop.
|
|
177
|
-
prop.
|
|
187
|
+
if isinstance(prop.concept, ConceptEntity) and prop.concept.suffix in new_by_old_concept_suffix:
|
|
188
|
+
prop.concept.suffix = new_by_old_concept_suffix[prop.concept.suffix]
|
|
178
189
|
|
|
179
|
-
if isinstance(prop.value_type,
|
|
180
|
-
prop.value_type.suffix =
|
|
190
|
+
if isinstance(prop.value_type, ConceptEntity) and prop.value_type.suffix in new_by_old_concept_suffix:
|
|
191
|
+
prop.value_type.suffix = new_by_old_concept_suffix[prop.value_type.suffix]
|
|
181
192
|
|
|
182
193
|
if isinstance(prop.value_type, MultiValueTypeInfo):
|
|
183
194
|
for i, value_type in enumerate(prop.value_type.types):
|
|
184
|
-
if isinstance(value_type,
|
|
185
|
-
prop.value_type.types[i].suffix =
|
|
195
|
+
if isinstance(value_type, ConceptEntity) and value_type.suffix in new_by_old_concept_suffix:
|
|
196
|
+
prop.value_type.types[i].suffix = new_by_old_concept_suffix[value_type.suffix] # type: ignore[union-attr]
|
|
186
197
|
|
|
187
|
-
return
|
|
198
|
+
return ImportedDataModel(unverified_data_model=copy, context=data_model.context)
|
|
188
199
|
|
|
189
200
|
@cached_property
|
|
190
|
-
def
|
|
191
|
-
return set(get_reserved_words("
|
|
201
|
+
def _reserved_concept_words(self) -> set[str]:
|
|
202
|
+
return set(get_reserved_words("concept"))
|
|
192
203
|
|
|
193
204
|
@cached_property
|
|
194
205
|
def _reserved_property_words(self) -> set[str]:
|
|
195
206
|
return set(get_reserved_words("property"))
|
|
196
207
|
|
|
197
|
-
def
|
|
198
|
-
if suffix in self.
|
|
208
|
+
def _fix_concept_suffix(self, suffix: str) -> str:
|
|
209
|
+
if suffix in self._reserved_concept_words:
|
|
199
210
|
return f"My{suffix}"
|
|
200
211
|
suffix = urllib.parse.unquote(suffix)
|
|
201
|
-
suffix = NamingStandardization.
|
|
212
|
+
suffix = NamingStandardization.standardize_concept_str(suffix)
|
|
202
213
|
if len(suffix) > 252:
|
|
203
214
|
suffix = suffix[:252]
|
|
204
215
|
return suffix
|
|
@@ -213,8 +224,8 @@ class ToDMSCompliantEntities(
|
|
|
213
224
|
return property_
|
|
214
225
|
|
|
215
226
|
|
|
216
|
-
class StandardizeSpaceAndVersion(
|
|
217
|
-
"""This transformer standardizes the space and version of the
|
|
227
|
+
class StandardizeSpaceAndVersion(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]): # type: ignore[misc]
|
|
228
|
+
"""This transformer standardizes the space and version of the physical data model.
|
|
218
229
|
|
|
219
230
|
typically used to ensure all the views are moved to the same version as the data model.
|
|
220
231
|
|
|
@@ -224,8 +235,8 @@ class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
224
235
|
def description(self) -> str:
|
|
225
236
|
return "Ensures uniform version and space of the views belonging to the data model."
|
|
226
237
|
|
|
227
|
-
def transform(self,
|
|
228
|
-
copy =
|
|
238
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
239
|
+
copy = data_model.model_copy(deep=True)
|
|
229
240
|
|
|
230
241
|
space = copy.metadata.space
|
|
231
242
|
version = copy.metadata.version
|
|
@@ -234,7 +245,7 @@ class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
234
245
|
copy.properties = self._standardize_properties(copy.properties, space, version)
|
|
235
246
|
return copy
|
|
236
247
|
|
|
237
|
-
def _standardize_views(self, views: SheetList[
|
|
248
|
+
def _standardize_views(self, views: SheetList[PhysicalView], space: str, version: str) -> SheetList[PhysicalView]:
|
|
238
249
|
for view in views:
|
|
239
250
|
if view.view.space not in COGNITE_SPACES:
|
|
240
251
|
view.view.version = version
|
|
@@ -248,8 +259,8 @@ class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
248
259
|
return views
|
|
249
260
|
|
|
250
261
|
def _standardize_properties(
|
|
251
|
-
self, properties: SheetList[
|
|
252
|
-
) -> SheetList[
|
|
262
|
+
self, properties: SheetList[PhysicalProperty], space: str, version: str
|
|
263
|
+
) -> SheetList[PhysicalProperty]:
|
|
253
264
|
for property_ in properties:
|
|
254
265
|
if property_.view.space not in COGNITE_SPACES:
|
|
255
266
|
property_.view.version = version
|
|
@@ -272,17 +283,17 @@ class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
272
283
|
return properties
|
|
273
284
|
|
|
274
285
|
|
|
275
|
-
class ToCompliantEntities(
|
|
276
|
-
"""Converts input
|
|
286
|
+
class ToCompliantEntities(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]): # type: ignore[misc]
|
|
287
|
+
"""Converts input data_model to data_model with compliant entity IDs that match regex patters used
|
|
277
288
|
by DMS schema components."""
|
|
278
289
|
|
|
279
290
|
@property
|
|
280
291
|
def description(self) -> str:
|
|
281
292
|
return "Ensures externalIDs are compliant with CDF"
|
|
282
293
|
|
|
283
|
-
def transform(self,
|
|
284
|
-
copy =
|
|
285
|
-
copy.
|
|
294
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
295
|
+
copy = data_model.model_copy(deep=True)
|
|
296
|
+
copy.concepts = self._fix_concepts(copy.concepts)
|
|
286
297
|
copy.properties = self._fix_properties(copy.properties)
|
|
287
298
|
return copy
|
|
288
299
|
|
|
@@ -301,30 +312,30 @@ class ToCompliantEntities(VerifiedRulesTransformer[ConceptualDataModel, Conceptu
|
|
|
301
312
|
return re.sub(r"[^a-zA-Z0-9]+", "_", entity)
|
|
302
313
|
|
|
303
314
|
@classmethod
|
|
304
|
-
def
|
|
305
|
-
if isinstance(
|
|
306
|
-
|
|
307
|
-
prefix=cls._fix_entity(
|
|
308
|
-
suffix=cls._fix_entity(
|
|
315
|
+
def _fix_concept(cls, concept: ConceptEntity) -> ConceptEntity:
|
|
316
|
+
if isinstance(concept, ConceptEntity) and type(concept.prefix) is str:
|
|
317
|
+
concept = ConceptEntity(
|
|
318
|
+
prefix=cls._fix_entity(concept.prefix),
|
|
319
|
+
suffix=cls._fix_entity(concept.suffix),
|
|
309
320
|
)
|
|
310
321
|
|
|
311
|
-
return
|
|
322
|
+
return concept
|
|
312
323
|
|
|
313
324
|
@classmethod
|
|
314
325
|
def _fix_value_type(
|
|
315
|
-
cls, value_type: DataType |
|
|
316
|
-
) -> DataType |
|
|
317
|
-
fixed_value_type: DataType |
|
|
326
|
+
cls, value_type: DataType | ConceptEntity | MultiValueTypeInfo
|
|
327
|
+
) -> DataType | ConceptEntity | MultiValueTypeInfo:
|
|
328
|
+
fixed_value_type: DataType | ConceptEntity | MultiValueTypeInfo
|
|
318
329
|
|
|
319
330
|
# value type specified as MultiValueTypeInfo
|
|
320
331
|
if isinstance(value_type, MultiValueTypeInfo):
|
|
321
332
|
fixed_value_type = MultiValueTypeInfo(
|
|
322
|
-
types=[cast(DataType |
|
|
333
|
+
types=[cast(DataType | ConceptEntity, cls._fix_value_type(type_)) for type_ in value_type.types],
|
|
323
334
|
)
|
|
324
335
|
|
|
325
336
|
# value type specified as ClassEntity instance
|
|
326
|
-
elif isinstance(value_type,
|
|
327
|
-
fixed_value_type = cls.
|
|
337
|
+
elif isinstance(value_type, ConceptEntity):
|
|
338
|
+
fixed_value_type = cls._fix_concept(value_type)
|
|
328
339
|
|
|
329
340
|
# this is a DataType instance but also we should default to original value
|
|
330
341
|
else:
|
|
@@ -333,10 +344,10 @@ class ToCompliantEntities(VerifiedRulesTransformer[ConceptualDataModel, Conceptu
|
|
|
333
344
|
return fixed_value_type
|
|
334
345
|
|
|
335
346
|
@classmethod
|
|
336
|
-
def
|
|
337
|
-
fixed_definitions = SheetList[
|
|
347
|
+
def _fix_concepts(cls, definitions: SheetList[Concept]) -> SheetList[Concept]:
|
|
348
|
+
fixed_definitions = SheetList[Concept]()
|
|
338
349
|
for definition in definitions:
|
|
339
|
-
definition.
|
|
350
|
+
definition.concept = cls._fix_concept(definition.concept)
|
|
340
351
|
fixed_definitions.append(definition)
|
|
341
352
|
return fixed_definitions
|
|
342
353
|
|
|
@@ -344,7 +355,7 @@ class ToCompliantEntities(VerifiedRulesTransformer[ConceptualDataModel, Conceptu
|
|
|
344
355
|
def _fix_properties(cls, definitions: SheetList[ConceptualProperty]) -> SheetList[ConceptualProperty]:
|
|
345
356
|
fixed_definitions = SheetList[ConceptualProperty]()
|
|
346
357
|
for definition in definitions:
|
|
347
|
-
definition.
|
|
358
|
+
definition.concept = cls._fix_concept(definition.concept)
|
|
348
359
|
definition.property_ = cls._fix_entity(definition.property_)
|
|
349
360
|
definition.value_type = cls._fix_value_type(definition.value_type)
|
|
350
361
|
fixed_definitions.append(definition)
|
|
@@ -362,20 +373,20 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
362
373
|
return f"Prefixes all entities with {self._prefix!r} prefix if they are in the same space as data model."
|
|
363
374
|
|
|
364
375
|
@overload
|
|
365
|
-
def transform(self,
|
|
376
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel: ...
|
|
366
377
|
|
|
367
378
|
@overload
|
|
368
|
-
def transform(self,
|
|
379
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel: ...
|
|
369
380
|
|
|
370
|
-
def transform(self,
|
|
371
|
-
copy: ConceptualDataModel |
|
|
381
|
+
def transform(self, data_model: ConceptualDataModel | PhysicalDataModel) -> ConceptualDataModel | PhysicalDataModel:
|
|
382
|
+
copy: ConceptualDataModel | PhysicalDataModel = data_model.model_copy(deep=True)
|
|
372
383
|
|
|
373
|
-
# Case: Prefix
|
|
384
|
+
# Case: Prefix Conceptual Data Model
|
|
374
385
|
if isinstance(copy, ConceptualDataModel):
|
|
375
386
|
# prefix classes
|
|
376
|
-
for cls in copy.
|
|
377
|
-
if cls.
|
|
378
|
-
cls.
|
|
387
|
+
for cls in copy.concepts:
|
|
388
|
+
if cls.concept.prefix == copy.metadata.prefix:
|
|
389
|
+
cls.concept = self._with_prefix(cls.concept)
|
|
379
390
|
|
|
380
391
|
if cls.implements:
|
|
381
392
|
# prefix parents
|
|
@@ -384,21 +395,21 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
384
395
|
cls.implements[i] = self._with_prefix(parent_class)
|
|
385
396
|
|
|
386
397
|
for prop in copy.properties:
|
|
387
|
-
if prop.
|
|
388
|
-
prop.
|
|
398
|
+
if prop.concept.prefix == copy.metadata.prefix:
|
|
399
|
+
prop.concept = self._with_prefix(prop.concept)
|
|
389
400
|
|
|
390
401
|
# value type property is not multi and it is ClassEntity
|
|
391
402
|
|
|
392
|
-
if isinstance(prop.value_type,
|
|
393
|
-
prop.value_type = self._with_prefix(cast(
|
|
403
|
+
if isinstance(prop.value_type, ConceptEntity) and prop.value_type.prefix == copy.metadata.prefix:
|
|
404
|
+
prop.value_type = self._with_prefix(cast(ConceptEntity, prop.value_type))
|
|
394
405
|
elif isinstance(prop.value_type, MultiValueTypeInfo):
|
|
395
406
|
for i, value_type in enumerate(prop.value_type.types):
|
|
396
|
-
if isinstance(value_type,
|
|
397
|
-
prop.value_type.types[i] = self._with_prefix(cast(
|
|
407
|
+
if isinstance(value_type, ConceptEntity) and value_type.prefix == copy.metadata.prefix:
|
|
408
|
+
prop.value_type.types[i] = self._with_prefix(cast(ConceptEntity, value_type))
|
|
398
409
|
return copy
|
|
399
410
|
|
|
400
|
-
# Case: Prefix
|
|
401
|
-
elif isinstance(copy,
|
|
411
|
+
# Case: Prefix Physical Data Model
|
|
412
|
+
elif isinstance(copy, PhysicalDataModel):
|
|
402
413
|
for view in copy.views:
|
|
403
414
|
if view.view.space == copy.metadata.space:
|
|
404
415
|
view.view = self._with_prefix(view.view)
|
|
@@ -408,15 +419,21 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
408
419
|
if parent_view.space == copy.metadata.space:
|
|
409
420
|
view.implements[i] = self._with_prefix(parent_view)
|
|
410
421
|
|
|
411
|
-
for
|
|
412
|
-
if
|
|
413
|
-
|
|
422
|
+
for physical_prop in copy.properties:
|
|
423
|
+
if physical_prop.view.space == copy.metadata.space:
|
|
424
|
+
physical_prop.view = self._with_prefix(physical_prop.view)
|
|
414
425
|
|
|
415
|
-
if
|
|
416
|
-
|
|
426
|
+
if (
|
|
427
|
+
isinstance(physical_prop.value_type, ViewEntity)
|
|
428
|
+
and physical_prop.value_type.space == copy.metadata.space
|
|
429
|
+
):
|
|
430
|
+
physical_prop.value_type = self._with_prefix(physical_prop.value_type)
|
|
417
431
|
|
|
418
|
-
if
|
|
419
|
-
|
|
432
|
+
if (
|
|
433
|
+
isinstance(physical_prop.container, ContainerEntity)
|
|
434
|
+
and physical_prop.container.space == copy.metadata.space
|
|
435
|
+
):
|
|
436
|
+
physical_prop.container = self._with_prefix(physical_prop.container)
|
|
420
437
|
|
|
421
438
|
if copy.containers:
|
|
422
439
|
for container in copy.containers:
|
|
@@ -424,10 +441,10 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
424
441
|
container.container = self._with_prefix(container.container)
|
|
425
442
|
return copy
|
|
426
443
|
|
|
427
|
-
raise NeatValueError(f"Unsupported
|
|
444
|
+
raise NeatValueError(f"Unsupported data_model type: {type(copy)}")
|
|
428
445
|
|
|
429
446
|
@overload
|
|
430
|
-
def _with_prefix(self, entity:
|
|
447
|
+
def _with_prefix(self, entity: ConceptEntity) -> ConceptEntity: ...
|
|
431
448
|
|
|
432
449
|
@overload
|
|
433
450
|
def _with_prefix(self, entity: ViewEntity) -> ViewEntity: ...
|
|
@@ -436,9 +453,9 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
436
453
|
def _with_prefix(self, entity: ContainerEntity) -> ContainerEntity: ...
|
|
437
454
|
|
|
438
455
|
def _with_prefix(
|
|
439
|
-
self, entity: ViewEntity | ContainerEntity |
|
|
440
|
-
) -> ViewEntity | ContainerEntity |
|
|
441
|
-
if isinstance(entity, ViewEntity | ContainerEntity |
|
|
456
|
+
self, entity: ViewEntity | ContainerEntity | ConceptEntity
|
|
457
|
+
) -> ViewEntity | ContainerEntity | ConceptEntity:
|
|
458
|
+
if isinstance(entity, ViewEntity | ContainerEntity | ConceptEntity):
|
|
442
459
|
entity.suffix = f"{self._prefix}{entity.suffix}"
|
|
443
460
|
|
|
444
461
|
else:
|
|
@@ -455,61 +472,61 @@ class StandardizeNaming(ConversionTransformer):
|
|
|
455
472
|
return "Sets views/classes/containers names to PascalCase and properties to camelCase."
|
|
456
473
|
|
|
457
474
|
@overload
|
|
458
|
-
def transform(self,
|
|
475
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel: ...
|
|
459
476
|
|
|
460
477
|
@overload
|
|
461
|
-
def transform(self,
|
|
478
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel: ...
|
|
462
479
|
|
|
463
|
-
def transform(self,
|
|
464
|
-
output =
|
|
480
|
+
def transform(self, data_model: ConceptualDataModel | PhysicalDataModel) -> ConceptualDataModel | PhysicalDataModel:
|
|
481
|
+
output = data_model.model_copy(deep=True)
|
|
465
482
|
if isinstance(output, ConceptualDataModel):
|
|
466
|
-
return self.
|
|
467
|
-
elif isinstance(output,
|
|
468
|
-
return self.
|
|
469
|
-
raise NeatValueError(f"Unsupported
|
|
470
|
-
|
|
471
|
-
def
|
|
472
|
-
|
|
473
|
-
for cls in
|
|
474
|
-
new_suffix = NamingStandardization.
|
|
475
|
-
|
|
476
|
-
cls.
|
|
477
|
-
|
|
478
|
-
for cls in
|
|
483
|
+
return self._standardize_conceptual_data_model(output)
|
|
484
|
+
elif isinstance(output, PhysicalDataModel):
|
|
485
|
+
return self._standardize_physical_data_model(output)
|
|
486
|
+
raise NeatValueError(f"Unsupported data_model type: {type(output)}")
|
|
487
|
+
|
|
488
|
+
def _standardize_conceptual_data_model(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
489
|
+
new_by_old_concept_suffix: dict[str, str] = {}
|
|
490
|
+
for cls in data_model.concepts:
|
|
491
|
+
new_suffix = NamingStandardization.standardize_concept_str(cls.concept.suffix)
|
|
492
|
+
new_by_old_concept_suffix[cls.concept.suffix] = new_suffix
|
|
493
|
+
cls.concept.suffix = new_suffix
|
|
494
|
+
|
|
495
|
+
for cls in data_model.concepts:
|
|
479
496
|
if cls.implements:
|
|
480
497
|
for i, parent in enumerate(cls.implements):
|
|
481
|
-
if parent.suffix in
|
|
482
|
-
cls.implements[i].suffix =
|
|
498
|
+
if parent.suffix in new_by_old_concept_suffix:
|
|
499
|
+
cls.implements[i].suffix = new_by_old_concept_suffix[parent.suffix]
|
|
483
500
|
|
|
484
|
-
for prop in
|
|
501
|
+
for prop in data_model.properties:
|
|
485
502
|
prop.property_ = NamingStandardization.standardize_property_str(prop.property_)
|
|
486
|
-
if prop.
|
|
487
|
-
prop.
|
|
503
|
+
if prop.concept.suffix in new_by_old_concept_suffix:
|
|
504
|
+
prop.concept.suffix = new_by_old_concept_suffix[prop.concept.suffix]
|
|
488
505
|
|
|
489
|
-
if isinstance(prop.value_type,
|
|
490
|
-
prop.value_type.suffix =
|
|
506
|
+
if isinstance(prop.value_type, ConceptEntity) and prop.value_type.suffix in new_by_old_concept_suffix:
|
|
507
|
+
prop.value_type.suffix = new_by_old_concept_suffix[prop.value_type.suffix]
|
|
491
508
|
|
|
492
509
|
if isinstance(prop.value_type, MultiValueTypeInfo):
|
|
493
510
|
for i, value_type in enumerate(prop.value_type.types):
|
|
494
|
-
if isinstance(value_type,
|
|
495
|
-
prop.value_type.types[i].suffix =
|
|
511
|
+
if isinstance(value_type, ConceptEntity) and value_type.suffix in new_by_old_concept_suffix:
|
|
512
|
+
prop.value_type.types[i].suffix = new_by_old_concept_suffix[value_type.suffix] # type: ignore[union-attr]
|
|
496
513
|
|
|
497
|
-
return
|
|
514
|
+
return data_model
|
|
498
515
|
|
|
499
|
-
def
|
|
516
|
+
def _standardize_physical_data_model(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
500
517
|
new_by_old_view: dict[str, str] = {}
|
|
501
|
-
for view in
|
|
502
|
-
new_suffix = NamingStandardization.
|
|
518
|
+
for view in data_model.views:
|
|
519
|
+
new_suffix = NamingStandardization.standardize_concept_str(view.view.suffix)
|
|
503
520
|
new_by_old_view[view.view.suffix] = new_suffix
|
|
504
521
|
view.view.suffix = new_suffix
|
|
505
522
|
new_by_old_container: dict[str, str] = {}
|
|
506
|
-
if
|
|
507
|
-
for container in
|
|
508
|
-
new_suffix = NamingStandardization.
|
|
523
|
+
if data_model.containers:
|
|
524
|
+
for container in data_model.containers:
|
|
525
|
+
new_suffix = NamingStandardization.standardize_concept_str(container.container.suffix)
|
|
509
526
|
new_by_old_container[container.container.suffix] = new_suffix
|
|
510
527
|
container.container.suffix = new_suffix
|
|
511
528
|
|
|
512
|
-
for view in
|
|
529
|
+
for view in data_model.views:
|
|
513
530
|
if view.implements:
|
|
514
531
|
for i, parent in enumerate(view.implements):
|
|
515
532
|
if parent.suffix in new_by_old_view:
|
|
@@ -520,14 +537,14 @@ class StandardizeNaming(ConversionTransformer):
|
|
|
520
537
|
view.filter_.inner[i].suffix = new_by_old_container[item.suffix]
|
|
521
538
|
if isinstance(item, ViewEntity) and item.suffix in new_by_old_view:
|
|
522
539
|
view.filter_.inner[i].suffix = new_by_old_view[item.suffix]
|
|
523
|
-
if
|
|
524
|
-
for container in
|
|
540
|
+
if data_model.containers:
|
|
541
|
+
for container in data_model.containers:
|
|
525
542
|
if container.constraint:
|
|
526
543
|
for i, constraint in enumerate(container.constraint):
|
|
527
544
|
if constraint.suffix in new_by_old_container:
|
|
528
545
|
container.constraint[i].suffix = new_by_old_container[constraint.suffix]
|
|
529
546
|
new_property_by_view_by_old_property: dict[ViewEntity, dict[str, str]] = defaultdict(dict)
|
|
530
|
-
for prop in
|
|
547
|
+
for prop in data_model.properties:
|
|
531
548
|
if prop.view.suffix in new_by_old_view:
|
|
532
549
|
prop.view.suffix = new_by_old_view[prop.view.suffix]
|
|
533
550
|
new_view_property = NamingStandardization.standardize_property_str(prop.view_property)
|
|
@@ -545,7 +562,7 @@ class StandardizeNaming(ConversionTransformer):
|
|
|
545
562
|
prop.container.suffix = new_by_old_container[prop.container.suffix]
|
|
546
563
|
if prop.container_property:
|
|
547
564
|
prop.container_property = NamingStandardization.standardize_property_str(prop.container_property)
|
|
548
|
-
for prop in
|
|
565
|
+
for prop in data_model.properties:
|
|
549
566
|
if (
|
|
550
567
|
isinstance(prop.connection, ReverseConnectionEntity)
|
|
551
568
|
and isinstance(prop.value_type, ViewEntity)
|
|
@@ -554,11 +571,11 @@ class StandardizeNaming(ConversionTransformer):
|
|
|
554
571
|
new_by_old_property = new_property_by_view_by_old_property[prop.value_type]
|
|
555
572
|
if prop.connection.property_ in new_by_old_property:
|
|
556
573
|
prop.connection.property_ = new_by_old_property[prop.connection.property_]
|
|
557
|
-
return
|
|
574
|
+
return data_model
|
|
558
575
|
|
|
559
576
|
|
|
560
|
-
class
|
|
561
|
-
"""Converts
|
|
577
|
+
class ConceptualToPhysical(ConversionTransformer[ConceptualDataModel, PhysicalDataModel]):
|
|
578
|
+
"""Converts conceptual to physical data model."""
|
|
562
579
|
|
|
563
580
|
def __init__(
|
|
564
581
|
self,
|
|
@@ -570,42 +587,42 @@ class InformationToDMS(ConversionTransformer[ConceptualDataModel, DMSRules]):
|
|
|
570
587
|
self.reserved_properties = reserved_properties
|
|
571
588
|
self.client = client
|
|
572
589
|
|
|
573
|
-
def transform(self,
|
|
574
|
-
return
|
|
590
|
+
def transform(self, data_model: ConceptualDataModel) -> PhysicalDataModel:
|
|
591
|
+
return _ConceptualDataModelConverter(data_model, self.client).as_physical_data_model(
|
|
575
592
|
self.ignore_undefined_value_types, self.reserved_properties
|
|
576
593
|
)
|
|
577
594
|
|
|
578
595
|
|
|
579
|
-
class
|
|
580
|
-
"""Converts
|
|
596
|
+
class PhysicalToConceptual(ConversionTransformer[PhysicalDataModel, ConceptualDataModel]):
|
|
597
|
+
"""Converts Physical to Conceptual data model."""
|
|
581
598
|
|
|
582
599
|
def __init__(self, instance_namespace: Namespace | None = None):
|
|
583
600
|
self.instance_namespace = instance_namespace
|
|
584
601
|
|
|
585
|
-
def transform(self,
|
|
586
|
-
return
|
|
602
|
+
def transform(self, data_model: PhysicalDataModel) -> ConceptualDataModel:
|
|
603
|
+
return _PhysicalDataModelConverter(data_model, self.instance_namespace).as_conceptual_data_model()
|
|
587
604
|
|
|
588
605
|
|
|
589
|
-
class
|
|
590
|
-
"""Converts any
|
|
606
|
+
class ConvertToDataModel(ConversionTransformer[VerifiedDataModel, VerifiedDataModel]):
|
|
607
|
+
"""Converts any data_model to any data_model."""
|
|
591
608
|
|
|
592
|
-
def __init__(self, out_cls: type[
|
|
609
|
+
def __init__(self, out_cls: type[VerifiedDataModel]):
|
|
593
610
|
self._out_cls = out_cls
|
|
594
611
|
|
|
595
|
-
def transform(self,
|
|
596
|
-
if isinstance(
|
|
597
|
-
return
|
|
598
|
-
if isinstance(
|
|
599
|
-
return
|
|
600
|
-
if isinstance(
|
|
601
|
-
return
|
|
602
|
-
raise ValueError(f"Unsupported conversion from {type(
|
|
612
|
+
def transform(self, data_model: VerifiedDataModel) -> VerifiedDataModel:
|
|
613
|
+
if isinstance(data_model, self._out_cls):
|
|
614
|
+
return data_model
|
|
615
|
+
if isinstance(data_model, ConceptualDataModel) and self._out_cls is PhysicalDataModel:
|
|
616
|
+
return ConceptualToPhysical().transform(data_model)
|
|
617
|
+
if isinstance(data_model, PhysicalDataModel) and self._out_cls is ConceptualDataModel:
|
|
618
|
+
return PhysicalToConceptual().transform(data_model)
|
|
619
|
+
raise ValueError(f"Unsupported conversion from {type(data_model)} to {self._out_cls}")
|
|
603
620
|
|
|
604
621
|
|
|
605
|
-
_T_Entity = TypeVar("_T_Entity", bound=
|
|
622
|
+
_T_Entity = TypeVar("_T_Entity", bound=ConceptEntity | ViewEntity)
|
|
606
623
|
|
|
607
624
|
|
|
608
|
-
class SetIDDMSModel(
|
|
625
|
+
class SetIDDMSModel(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
609
626
|
def __init__(self, new_id: DataModelId | tuple[str, str, str], name: str | None = None):
|
|
610
627
|
self.new_id = DataModelId.load(new_id)
|
|
611
628
|
self.name = name
|
|
@@ -614,23 +631,23 @@ class SetIDDMSModel(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
614
631
|
def description(self) -> str:
|
|
615
632
|
return f"Sets the Data Model ID to {self.new_id.as_tuple()}"
|
|
616
633
|
|
|
617
|
-
def transform(self,
|
|
634
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
618
635
|
if self.new_id.version is None:
|
|
619
636
|
raise NeatValueError("Version is required when setting a new Data Model ID")
|
|
620
|
-
dump =
|
|
637
|
+
dump = data_model.dump()
|
|
621
638
|
dump["metadata"]["space"] = self.new_id.space
|
|
622
639
|
dump["metadata"]["external_id"] = self.new_id.external_id
|
|
623
640
|
dump["metadata"]["version"] = self.new_id.version
|
|
624
641
|
dump["metadata"]["name"] = self.name or self._generate_name()
|
|
625
642
|
# Serialize and deserialize to set the new space and external_id
|
|
626
643
|
# as the default values for the new model.
|
|
627
|
-
return
|
|
644
|
+
return PhysicalDataModel.model_validate(UnverifiedPhysicalDataModel.load(dump).dump())
|
|
628
645
|
|
|
629
646
|
def _generate_name(self) -> str:
|
|
630
647
|
return title(to_words(self.new_id.external_id))
|
|
631
648
|
|
|
632
649
|
|
|
633
|
-
class ToExtensionModel(
|
|
650
|
+
class ToExtensionModel(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel], ABC):
|
|
634
651
|
type_: ClassVar[str]
|
|
635
652
|
|
|
636
653
|
def __init__(self, new_model_id: DataModelIdentifier) -> None:
|
|
@@ -658,10 +675,10 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
658
675
|
self.org_name = org_name
|
|
659
676
|
self.move_connections = move_connections
|
|
660
677
|
|
|
661
|
-
def transform(self,
|
|
662
|
-
return self._to_enterprise(
|
|
678
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
679
|
+
return self._to_enterprise(data_model)
|
|
663
680
|
|
|
664
|
-
def _to_enterprise(self, reference_model:
|
|
681
|
+
def _to_enterprise(self, reference_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
665
682
|
enterprise_model = reference_model.model_copy(deep=True)
|
|
666
683
|
|
|
667
684
|
enterprise_model.metadata.name = f"{self.org_name} {self.type_} data model"
|
|
@@ -702,7 +719,9 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
702
719
|
return enterprise_model
|
|
703
720
|
|
|
704
721
|
@staticmethod
|
|
705
|
-
def _create_connection_properties(
|
|
722
|
+
def _create_connection_properties(
|
|
723
|
+
data_model: PhysicalDataModel, new_views: SheetList[PhysicalView]
|
|
724
|
+
) -> SheetList[PhysicalProperty]:
|
|
706
725
|
"""Creates a new connection property for each connection property in the reference model.
|
|
707
726
|
|
|
708
727
|
This is for example when you create an enterprise model from CogniteCore, you ensure that your
|
|
@@ -710,8 +729,8 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
710
729
|
"""
|
|
711
730
|
# Note all new news have an implements attribute that points to the original view
|
|
712
731
|
previous_by_new_view = {view.implements[0]: view.view for view in new_views if view.implements}
|
|
713
|
-
connection_properties = SheetList[
|
|
714
|
-
for prop in
|
|
732
|
+
connection_properties = SheetList[PhysicalProperty]()
|
|
733
|
+
for prop in data_model.properties:
|
|
715
734
|
if (
|
|
716
735
|
isinstance(prop.value_type, ViewEntity)
|
|
717
736
|
and prop.view in previous_by_new_view
|
|
@@ -725,23 +744,27 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
725
744
|
return connection_properties
|
|
726
745
|
|
|
727
746
|
def _create_new_views(
|
|
728
|
-
self,
|
|
729
|
-
) -> tuple[
|
|
747
|
+
self, data_model: PhysicalDataModel
|
|
748
|
+
) -> tuple[
|
|
749
|
+
SheetList[PhysicalView],
|
|
750
|
+
SheetList[PhysicalContainer],
|
|
751
|
+
SheetList[PhysicalProperty],
|
|
752
|
+
]:
|
|
730
753
|
"""Creates new views for the new model.
|
|
731
754
|
|
|
732
755
|
If the dummy property is provided, it will also create a new container for each view
|
|
733
756
|
with a single property that is the dummy property.
|
|
734
757
|
"""
|
|
735
|
-
new_views = SheetList[
|
|
736
|
-
new_containers = SheetList[
|
|
737
|
-
new_properties = SheetList[
|
|
758
|
+
new_views = SheetList[PhysicalView]()
|
|
759
|
+
new_containers = SheetList[PhysicalContainer]()
|
|
760
|
+
new_properties = SheetList[PhysicalProperty]()
|
|
738
761
|
|
|
739
|
-
for definition in
|
|
762
|
+
for definition in data_model.views:
|
|
740
763
|
view_entity = self._remove_cognite_affix(definition.view)
|
|
741
764
|
view_entity.version = cast(str, self.new_model_id.version)
|
|
742
765
|
view_entity.prefix = self.new_model_id.space
|
|
743
766
|
new_views.append(
|
|
744
|
-
|
|
767
|
+
PhysicalView(
|
|
745
768
|
view=view_entity,
|
|
746
769
|
implements=[definition.view],
|
|
747
770
|
in_model=True,
|
|
@@ -754,10 +777,10 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
754
777
|
|
|
755
778
|
container_entity = ContainerEntity(space=view_entity.prefix, externalId=view_entity.external_id)
|
|
756
779
|
|
|
757
|
-
container =
|
|
780
|
+
container = PhysicalContainer(container=container_entity)
|
|
758
781
|
|
|
759
782
|
property_id = f"{to_camel_case(view_entity.suffix)}{self.dummy_property}"
|
|
760
|
-
property_ =
|
|
783
|
+
property_ = PhysicalProperty(
|
|
761
784
|
view=view_entity,
|
|
762
785
|
view_property=property_id,
|
|
763
786
|
value_type=String(),
|
|
@@ -815,8 +838,8 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
815
838
|
self.exclude_views_in_other_spaces = exclude_views_in_other_spaces
|
|
816
839
|
self.skip_cognite_views = skip_cognite_views
|
|
817
840
|
|
|
818
|
-
def transform(self,
|
|
819
|
-
reference_model =
|
|
841
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
842
|
+
reference_model = data_model
|
|
820
843
|
reference_model_id = reference_model.metadata.as_data_model_id()
|
|
821
844
|
if reference_model_id in COGNITE_MODELS:
|
|
822
845
|
warnings.warn(
|
|
@@ -825,13 +848,13 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
825
848
|
)
|
|
826
849
|
return self._to_solution(reference_model)
|
|
827
850
|
|
|
828
|
-
def _to_solution(self,
|
|
829
|
-
"""For creation of solution data model /
|
|
830
|
-
|
|
851
|
+
def _to_solution(self, reference_data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
852
|
+
"""For creation of solution data model / data_model specifically for mapping over existing containers."""
|
|
853
|
+
reference_data_model = self._expand_properties(reference_data_model.model_copy(deep=True))
|
|
831
854
|
|
|
832
|
-
new_views, new_properties, read_view_by_new_view = self._create_views(
|
|
855
|
+
new_views, new_properties, read_view_by_new_view = self._create_views(reference_data_model)
|
|
833
856
|
new_containers, new_container_properties = self._create_containers_update_view_filter(
|
|
834
|
-
new_views,
|
|
857
|
+
new_views, reference_data_model, read_view_by_new_view
|
|
835
858
|
)
|
|
836
859
|
new_properties.extend(new_container_properties)
|
|
837
860
|
|
|
@@ -846,7 +869,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
846
869
|
else:
|
|
847
870
|
new_properties.sort(key=lambda prop: (prop.view.external_id, prop.view_property))
|
|
848
871
|
|
|
849
|
-
metadata =
|
|
872
|
+
metadata = reference_data_model.metadata.model_copy(
|
|
850
873
|
deep=True,
|
|
851
874
|
update={
|
|
852
875
|
"space": self.new_model_id.space,
|
|
@@ -855,18 +878,18 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
855
878
|
"name": f"{self.type_} data model",
|
|
856
879
|
},
|
|
857
880
|
)
|
|
858
|
-
return
|
|
881
|
+
return PhysicalDataModel(
|
|
859
882
|
metadata=metadata,
|
|
860
883
|
properties=new_properties,
|
|
861
884
|
views=new_views,
|
|
862
885
|
containers=new_containers or None,
|
|
863
|
-
enum=
|
|
864
|
-
nodes=
|
|
886
|
+
enum=reference_data_model.enum,
|
|
887
|
+
nodes=reference_data_model.nodes,
|
|
865
888
|
)
|
|
866
889
|
|
|
867
890
|
@staticmethod
|
|
868
|
-
def _expand_properties(
|
|
869
|
-
probe =
|
|
891
|
+
def _expand_properties(data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
892
|
+
probe = DataModelAnalysis(physical=data_model)
|
|
870
893
|
ancestor_properties_by_view = probe.properties_by_view(
|
|
871
894
|
include_ancestors=True,
|
|
872
895
|
include_different_space=True,
|
|
@@ -885,15 +908,19 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
885
908
|
# original property will point to the parent view, and not the child.
|
|
886
909
|
continue
|
|
887
910
|
if prop.view_property not in property_ids:
|
|
888
|
-
|
|
911
|
+
data_model.properties.append(prop)
|
|
889
912
|
property_ids.add(prop.view_property)
|
|
890
|
-
return
|
|
913
|
+
return data_model
|
|
891
914
|
|
|
892
915
|
def _create_views(
|
|
893
|
-
self, reference:
|
|
894
|
-
) -> tuple[
|
|
916
|
+
self, reference: PhysicalDataModel
|
|
917
|
+
) -> tuple[
|
|
918
|
+
SheetList[PhysicalView],
|
|
919
|
+
SheetList[PhysicalProperty],
|
|
920
|
+
dict[ViewEntity, ViewEntity],
|
|
921
|
+
]:
|
|
895
922
|
renaming: dict[ViewEntity, ViewEntity] = {}
|
|
896
|
-
new_views = SheetList[
|
|
923
|
+
new_views = SheetList[PhysicalView]()
|
|
897
924
|
read_view_by_new_view: dict[ViewEntity, ViewEntity] = {}
|
|
898
925
|
for ref_view in reference.views:
|
|
899
926
|
if (self.skip_cognite_views and ref_view.view.space in COGNITE_SPACES) or (
|
|
@@ -911,7 +938,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
911
938
|
# This will be used to point to the one view in the Enterprise model,
|
|
912
939
|
# while the new view will to be written to.
|
|
913
940
|
new_entity.suffix = f"{self.view_prefix}{ref_view.view.suffix}"
|
|
914
|
-
new_view =
|
|
941
|
+
new_view = PhysicalView(
|
|
915
942
|
view=ViewEntity(
|
|
916
943
|
# MyPy we validate that version is string in the constructor
|
|
917
944
|
space=self.new_model_id.space,
|
|
@@ -930,7 +957,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
930
957
|
renaming[ref_view.view] = new_entity
|
|
931
958
|
new_views.append(ref_view.model_copy(deep=True, update={"implements": None, "view": new_entity}))
|
|
932
959
|
|
|
933
|
-
new_properties = SheetList[
|
|
960
|
+
new_properties = SheetList[PhysicalProperty]()
|
|
934
961
|
new_view_entities = {view.view for view in new_views}
|
|
935
962
|
for prop in reference.properties:
|
|
936
963
|
new_property = prop.model_copy(deep=True)
|
|
@@ -945,10 +972,13 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
945
972
|
return new_views, new_properties, read_view_by_new_view
|
|
946
973
|
|
|
947
974
|
def _create_containers_update_view_filter(
|
|
948
|
-
self,
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
975
|
+
self,
|
|
976
|
+
new_views: SheetList[PhysicalView],
|
|
977
|
+
reference: PhysicalDataModel,
|
|
978
|
+
read_view_by_new_view: dict[ViewEntity, ViewEntity],
|
|
979
|
+
) -> tuple[SheetList[PhysicalContainer], SheetList[PhysicalProperty]]:
|
|
980
|
+
new_containers = SheetList[PhysicalContainer]()
|
|
981
|
+
container_properties: SheetList[PhysicalProperty] = SheetList[PhysicalProperty]()
|
|
952
982
|
ref_containers_by_ref_view: dict[ViewEntity, set[ContainerEntity]] = defaultdict(set)
|
|
953
983
|
ref_views_by_external_id = {
|
|
954
984
|
view.view.external_id: view for view in reference.views if view.view.space == reference.metadata.space
|
|
@@ -964,7 +994,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
964
994
|
container_entity = ContainerEntity(space=self.new_model_id.space, externalId=view.view.external_id)
|
|
965
995
|
prefix = to_camel_case(view.view.suffix)
|
|
966
996
|
if self.properties == "repeat" and self.dummy_property:
|
|
967
|
-
property_ =
|
|
997
|
+
property_ = PhysicalProperty(
|
|
968
998
|
view=view.view,
|
|
969
999
|
view_property=f"{prefix}{self.dummy_property}",
|
|
970
1000
|
value_type=String(),
|
|
@@ -974,7 +1004,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
974
1004
|
container=container_entity,
|
|
975
1005
|
container_property=f"{prefix}{self.dummy_property}",
|
|
976
1006
|
)
|
|
977
|
-
new_containers.append(
|
|
1007
|
+
new_containers.append(PhysicalContainer(container=container_entity))
|
|
978
1008
|
container_properties.append(property_)
|
|
979
1009
|
elif self.properties == "repeat" and self.dummy_property is None:
|
|
980
1010
|
# For this case we set the filter. This is used by the DataProductModel.
|
|
@@ -983,7 +1013,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
983
1013
|
if ref_view := ref_views_by_external_id.get(view.view.external_id):
|
|
984
1014
|
self._set_view_filter(view, ref_containers_by_ref_view, ref_view)
|
|
985
1015
|
elif self.properties == "connection" and self.direct_property:
|
|
986
|
-
property_ =
|
|
1016
|
+
property_ = PhysicalProperty(
|
|
987
1017
|
view=view.view,
|
|
988
1018
|
view_property=self.direct_property,
|
|
989
1019
|
value_type=read_view,
|
|
@@ -994,7 +1024,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
994
1024
|
container_property=self.direct_property,
|
|
995
1025
|
connection="direct",
|
|
996
1026
|
)
|
|
997
|
-
new_containers.append(
|
|
1027
|
+
new_containers.append(PhysicalContainer(container=container_entity))
|
|
998
1028
|
container_properties.append(property_)
|
|
999
1029
|
else:
|
|
1000
1030
|
raise NeatValueError(f"Unsupported properties mode: {self.properties}")
|
|
@@ -1007,7 +1037,10 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
1007
1037
|
return new_containers, container_properties
|
|
1008
1038
|
|
|
1009
1039
|
def _set_view_filter(
|
|
1010
|
-
self,
|
|
1040
|
+
self,
|
|
1041
|
+
view: PhysicalView,
|
|
1042
|
+
ref_containers_by_ref_view: dict[ViewEntity, set[ContainerEntity]],
|
|
1043
|
+
ref_view: PhysicalView,
|
|
1011
1044
|
) -> None:
|
|
1012
1045
|
if self.filter_type == "view":
|
|
1013
1046
|
view.filter_ = HasDataFilter(inner=[ref_view.view])
|
|
@@ -1047,12 +1080,12 @@ class ToDataProductModel(ToSolutionModel):
|
|
|
1047
1080
|
)
|
|
1048
1081
|
self.include = include
|
|
1049
1082
|
|
|
1050
|
-
def transform(self,
|
|
1083
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1051
1084
|
# Overwrite transform to avoid the warning.
|
|
1052
|
-
return self._to_solution(
|
|
1085
|
+
return self._to_solution(data_model)
|
|
1053
1086
|
|
|
1054
1087
|
|
|
1055
|
-
class DropModelViews(
|
|
1088
|
+
class DropModelViews(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
1056
1089
|
_ASSET_VIEW = ViewId("cdf_cdm", "CogniteAsset", "v1")
|
|
1057
1090
|
_VIEW_BY_COLLECTION: Mapping[Literal["3D", "Annotation", "BaseViews"], frozenset[ViewId]] = {
|
|
1058
1091
|
"3D": frozenset(
|
|
@@ -1110,22 +1143,22 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1110
1143
|
)
|
|
1111
1144
|
)
|
|
1112
1145
|
|
|
1113
|
-
def transform(self,
|
|
1146
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1114
1147
|
exclude_views: set[ViewEntity] = {
|
|
1115
|
-
view.view for view in
|
|
1148
|
+
view.view for view in data_model.views if view.view.suffix in self.drop_external_ids
|
|
1116
1149
|
}
|
|
1117
|
-
if
|
|
1150
|
+
if data_model.metadata.as_data_model_id() in COGNITE_MODELS:
|
|
1118
1151
|
exclude_views |= {
|
|
1119
1152
|
ViewEntity.from_id(view_id, "v1")
|
|
1120
1153
|
for collection in self.drop_collection
|
|
1121
1154
|
for view_id in self._VIEW_BY_COLLECTION[collection]
|
|
1122
1155
|
}
|
|
1123
|
-
new_model =
|
|
1156
|
+
new_model = data_model.model_copy(deep=True)
|
|
1124
1157
|
|
|
1125
|
-
properties_by_view =
|
|
1158
|
+
properties_by_view = DataModelAnalysis(physical=new_model).properties_by_view(include_ancestors=True)
|
|
1126
1159
|
|
|
1127
|
-
new_model.views = SheetList[
|
|
1128
|
-
new_properties = SheetList[
|
|
1160
|
+
new_model.views = SheetList[PhysicalView]([view for view in new_model.views if view.view not in exclude_views])
|
|
1161
|
+
new_properties = SheetList[PhysicalProperty]()
|
|
1129
1162
|
mapped_containers: set[ContainerEntity] = set()
|
|
1130
1163
|
for view in new_model.views:
|
|
1131
1164
|
for prop in properties_by_view[view.view]:
|
|
@@ -1140,7 +1173,7 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1140
1173
|
|
|
1141
1174
|
new_model.properties = new_properties
|
|
1142
1175
|
new_model.containers = (
|
|
1143
|
-
SheetList[
|
|
1176
|
+
SheetList[PhysicalContainer](
|
|
1144
1177
|
[container for container in new_model.containers or [] if container.container in mapped_containers]
|
|
1145
1178
|
)
|
|
1146
1179
|
or None
|
|
@@ -1149,7 +1182,7 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1149
1182
|
return new_model
|
|
1150
1183
|
|
|
1151
1184
|
@classmethod
|
|
1152
|
-
def _is_asset_3D_property(cls, prop:
|
|
1185
|
+
def _is_asset_3D_property(cls, prop: PhysicalProperty) -> bool:
|
|
1153
1186
|
return prop.view.as_id() == cls._ASSET_VIEW and prop.view_property == "object3D"
|
|
1154
1187
|
|
|
1155
1188
|
@property
|
|
@@ -1157,37 +1190,37 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1157
1190
|
return f"Removed {len(self.drop_external_ids) + len(self.drop_collection)} views from data model"
|
|
1158
1191
|
|
|
1159
1192
|
|
|
1160
|
-
class IncludeReferenced(
|
|
1193
|
+
class IncludeReferenced(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
1161
1194
|
def __init__(self, client: NeatClient, include_properties: bool = False) -> None:
|
|
1162
1195
|
self._client = client
|
|
1163
1196
|
self.include_properties = include_properties
|
|
1164
1197
|
|
|
1165
|
-
def transform(self,
|
|
1166
|
-
|
|
1167
|
-
view_ids, container_ids =
|
|
1198
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1199
|
+
physical_data_model = data_model
|
|
1200
|
+
view_ids, container_ids = PhysicalValidation(physical_data_model).imported_views_and_containers_ids()
|
|
1168
1201
|
if not (view_ids or container_ids):
|
|
1169
1202
|
warnings.warn(
|
|
1170
1203
|
NeatValueWarning(
|
|
1171
|
-
f"Data model {
|
|
1204
|
+
f"Data model {physical_data_model.metadata.as_data_model_id()} does not have any "
|
|
1172
1205
|
"referenced views or containers."
|
|
1173
1206
|
"that is not already included in the data model."
|
|
1174
1207
|
),
|
|
1175
1208
|
stacklevel=2,
|
|
1176
1209
|
)
|
|
1177
|
-
return
|
|
1210
|
+
return physical_data_model
|
|
1178
1211
|
|
|
1179
1212
|
schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
|
|
1180
|
-
copy_ =
|
|
1213
|
+
copy_ = physical_data_model.model_copy(deep=True)
|
|
1181
1214
|
# Sorting to ensure deterministic order
|
|
1182
1215
|
schema.containers = ContainerApplyDict(sorted(schema.containers.items(), key=lambda x: x[0].as_tuple()))
|
|
1183
1216
|
schema.views = ViewApplyDict(sorted(schema.views.items(), key=lambda x: x[0].as_tuple()))
|
|
1184
1217
|
importer = DMSImporter(schema)
|
|
1185
1218
|
|
|
1186
|
-
imported = importer.
|
|
1187
|
-
if imported.
|
|
1219
|
+
imported = importer.to_data_model()
|
|
1220
|
+
if imported.unverified_data_model is None:
|
|
1188
1221
|
raise NeatValueError("Could not import the referenced views and containers.")
|
|
1189
1222
|
|
|
1190
|
-
verified =
|
|
1223
|
+
verified = VerifyPhysicalDataModel(validate=False).transform(imported)
|
|
1191
1224
|
if copy_.containers is None:
|
|
1192
1225
|
copy_.containers = verified.containers
|
|
1193
1226
|
else:
|
|
@@ -1208,17 +1241,16 @@ class IncludeReferenced(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1208
1241
|
return "Included referenced views and containers in the data model."
|
|
1209
1242
|
|
|
1210
1243
|
|
|
1211
|
-
class
|
|
1244
|
+
class AddConceptImplements(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
1212
1245
|
def __init__(self, implements: str, suffix: str):
|
|
1213
1246
|
self.implements = implements
|
|
1214
1247
|
self.suffix = suffix
|
|
1215
1248
|
|
|
1216
|
-
def transform(self,
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
class_.implements = [ClassEntity(prefix=class_.class_.prefix, suffix=self.implements)]
|
|
1249
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
1250
|
+
output = data_model.model_copy(deep=True)
|
|
1251
|
+
for concept in output.concepts:
|
|
1252
|
+
if concept.concept.suffix.endswith(self.suffix):
|
|
1253
|
+
concept.implements = [ConceptEntity(prefix=concept.concept.prefix, suffix=self.implements)]
|
|
1222
1254
|
return output
|
|
1223
1255
|
|
|
1224
1256
|
@property
|
|
@@ -1226,7 +1258,7 @@ class AddClassImplements(VerifiedRulesTransformer[ConceptualDataModel, Conceptua
|
|
|
1226
1258
|
return f"Added implements property to classes with suffix {self.suffix}"
|
|
1227
1259
|
|
|
1228
1260
|
|
|
1229
|
-
class ClassicPrepareCore(
|
|
1261
|
+
class ClassicPrepareCore(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
1230
1262
|
"""Update the classic data model with the following:
|
|
1231
1263
|
|
|
1232
1264
|
This is a special purpose transformer that is only intended to be used with when reading
|
|
@@ -1255,32 +1287,32 @@ class ClassicPrepareCore(VerifiedRulesTransformer[ConceptualDataModel, Conceptua
|
|
|
1255
1287
|
def description(self) -> str:
|
|
1256
1288
|
return "Update the classic data model to the data types in Cognite Core."
|
|
1257
1289
|
|
|
1258
|
-
def transform(self,
|
|
1259
|
-
output =
|
|
1290
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
1291
|
+
output = data_model.model_copy(deep=True)
|
|
1260
1292
|
for prop in output.properties:
|
|
1261
|
-
if prop.
|
|
1293
|
+
if prop.concept.suffix == "Timeseries" and prop.property_ == "isString":
|
|
1262
1294
|
prop.value_type = String()
|
|
1263
1295
|
prefix = output.metadata.prefix
|
|
1264
1296
|
namespace = output.metadata.namespace
|
|
1265
|
-
source_system_class =
|
|
1266
|
-
|
|
1297
|
+
source_system_class = Concept(
|
|
1298
|
+
concept=ConceptEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
1267
1299
|
description="A source system that provides data to the data model.",
|
|
1268
1300
|
neatId=namespace["ClassicSourceSystem"],
|
|
1269
1301
|
instance_source=self.instance_namespace["ClassicSourceSystem"],
|
|
1270
1302
|
)
|
|
1271
|
-
output.
|
|
1303
|
+
output.concepts.append(source_system_class)
|
|
1272
1304
|
for prop in output.properties:
|
|
1273
|
-
if prop.property_ == "source" and prop.
|
|
1274
|
-
prop.value_type =
|
|
1305
|
+
if prop.property_ == "source" and prop.concept.suffix != "ClassicSourceSystem":
|
|
1306
|
+
prop.value_type = ConceptEntity(prefix=prefix, suffix="ClassicSourceSystem")
|
|
1275
1307
|
elif prop.property_ == "externalId":
|
|
1276
1308
|
prop.property_ = "classicExternalId"
|
|
1277
|
-
if self.reference_timeseries and prop.
|
|
1309
|
+
if self.reference_timeseries and prop.concept.suffix == "ClassicTimeSeries":
|
|
1278
1310
|
prop.value_type = Timeseries()
|
|
1279
|
-
elif self.reference_files and prop.
|
|
1311
|
+
elif self.reference_files and prop.concept.suffix == "ClassicFile":
|
|
1280
1312
|
prop.value_type = File()
|
|
1281
|
-
elif prop.property_ == "sourceExternalId" and prop.
|
|
1313
|
+
elif prop.property_ == "sourceExternalId" and prop.concept.suffix == "ClassicRelationship":
|
|
1282
1314
|
prop.property_ = "startNode"
|
|
1283
|
-
elif prop.property_ == "targetExternalId" and prop.
|
|
1315
|
+
elif prop.property_ == "targetExternalId" and prop.concept.suffix == "ClassicRelationship":
|
|
1284
1316
|
prop.property_ = "endNode"
|
|
1285
1317
|
instance_prefix = next(
|
|
1286
1318
|
(prefix for prefix, namespace in output.prefixes.items() if namespace == self.instance_namespace), None
|
|
@@ -1293,7 +1325,7 @@ class ClassicPrepareCore(VerifiedRulesTransformer[ConceptualDataModel, Conceptua
|
|
|
1293
1325
|
neatId=namespace["ClassicSourceSystem/name"],
|
|
1294
1326
|
property_="name",
|
|
1295
1327
|
value_type=String(),
|
|
1296
|
-
|
|
1328
|
+
concept=ConceptEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
1297
1329
|
max_count=1,
|
|
1298
1330
|
instance_source=[self.instance_namespace["name"]],
|
|
1299
1331
|
)
|
|
@@ -1301,13 +1333,13 @@ class ClassicPrepareCore(VerifiedRulesTransformer[ConceptualDataModel, Conceptua
|
|
|
1301
1333
|
return output
|
|
1302
1334
|
|
|
1303
1335
|
|
|
1304
|
-
class ChangeViewPrefix(
|
|
1336
|
+
class ChangeViewPrefix(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
1305
1337
|
def __init__(self, old: str, new: str) -> None:
|
|
1306
1338
|
self.old = old
|
|
1307
1339
|
self.new = new
|
|
1308
1340
|
|
|
1309
|
-
def transform(self,
|
|
1310
|
-
output =
|
|
1341
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1342
|
+
output = data_model.model_copy(deep=True)
|
|
1311
1343
|
new_by_old: dict[ViewEntity, ViewEntity] = {}
|
|
1312
1344
|
for view in output.views:
|
|
1313
1345
|
if view.view.external_id.startswith(self.old):
|
|
@@ -1326,12 +1358,12 @@ class ChangeViewPrefix(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1326
1358
|
return output
|
|
1327
1359
|
|
|
1328
1360
|
|
|
1329
|
-
class
|
|
1330
|
-
def __init__(self, extra:
|
|
1361
|
+
class MergePhysicalDataModels(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
1362
|
+
def __init__(self, extra: PhysicalDataModel) -> None:
|
|
1331
1363
|
self.extra = extra
|
|
1332
1364
|
|
|
1333
|
-
def transform(self,
|
|
1334
|
-
output =
|
|
1365
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1366
|
+
output = data_model.model_copy(deep=True)
|
|
1335
1367
|
existing_views = {view.view for view in output.views}
|
|
1336
1368
|
for view in self.extra.views:
|
|
1337
1369
|
if view.view not in existing_views:
|
|
@@ -1347,18 +1379,18 @@ class MergeDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1347
1379
|
output.properties.append(prop)
|
|
1348
1380
|
if prop.container and prop.container not in existing_containers:
|
|
1349
1381
|
if output.containers is None:
|
|
1350
|
-
output.containers = SheetList[
|
|
1382
|
+
output.containers = SheetList[PhysicalContainer]()
|
|
1351
1383
|
output.containers.append(new_containers_by_entity[prop.container])
|
|
1352
1384
|
if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections:
|
|
1353
1385
|
if output.enum is None:
|
|
1354
|
-
output.enum = SheetList[
|
|
1386
|
+
output.enum = SheetList[PhysicalEnum]()
|
|
1355
1387
|
output.enum.append(new_enum_collections_by_entity[prop.value_type.collection])
|
|
1356
1388
|
|
|
1357
1389
|
existing_nodes = {node.node for node in output.nodes or []}
|
|
1358
1390
|
for node in self.extra.nodes or []:
|
|
1359
1391
|
if node.node not in existing_nodes:
|
|
1360
1392
|
if output.nodes is None:
|
|
1361
|
-
output.nodes = SheetList[
|
|
1393
|
+
output.nodes = SheetList[PhysicalNodeType]()
|
|
1362
1394
|
output.nodes.append(node)
|
|
1363
1395
|
|
|
1364
1396
|
return output
|
|
@@ -1368,19 +1400,19 @@ class MergeDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1368
1400
|
return f"Merged with {self.extra.metadata.as_data_model_id()}"
|
|
1369
1401
|
|
|
1370
1402
|
|
|
1371
|
-
class
|
|
1403
|
+
class MergeConceptualDataModels(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
1372
1404
|
def __init__(self, extra: ConceptualDataModel) -> None:
|
|
1373
1405
|
self.extra = extra
|
|
1374
1406
|
|
|
1375
|
-
def transform(self,
|
|
1376
|
-
output =
|
|
1377
|
-
existing_classes = {cls.
|
|
1378
|
-
for cls in self.extra.
|
|
1379
|
-
if cls.
|
|
1380
|
-
output.
|
|
1381
|
-
existing_properties = {(prop.
|
|
1407
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
1408
|
+
output = data_model.model_copy(deep=True)
|
|
1409
|
+
existing_classes = {cls.concept for cls in output.concepts}
|
|
1410
|
+
for cls in self.extra.concepts:
|
|
1411
|
+
if cls.concept not in existing_classes:
|
|
1412
|
+
output.concepts.append(cls)
|
|
1413
|
+
existing_properties = {(prop.concept, prop.property_) for prop in output.properties}
|
|
1382
1414
|
for prop in self.extra.properties:
|
|
1383
|
-
if (prop.
|
|
1415
|
+
if (prop.concept, prop.property_) not in existing_properties:
|
|
1384
1416
|
output.properties.append(prop)
|
|
1385
1417
|
for prefix, namespace in self.extra.prefixes.items():
|
|
1386
1418
|
if prefix not in output.prefixes:
|
|
@@ -1388,57 +1420,63 @@ class MergeInformationRules(VerifiedRulesTransformer[ConceptualDataModel, Concep
|
|
|
1388
1420
|
return output
|
|
1389
1421
|
|
|
1390
1422
|
|
|
1391
|
-
class
|
|
1423
|
+
class _ConceptualDataModelConverter:
|
|
1392
1424
|
_start_or_end_node: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
|
|
1393
1425
|
|
|
1394
|
-
def __init__(
|
|
1395
|
-
self
|
|
1426
|
+
def __init__(
|
|
1427
|
+
self,
|
|
1428
|
+
conceptual_data_model: ConceptualDataModel,
|
|
1429
|
+
client: NeatClient | None = None,
|
|
1430
|
+
):
|
|
1431
|
+
self.conceptual_data_model = conceptual_data_model
|
|
1396
1432
|
self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
|
|
1397
1433
|
self.client = client
|
|
1398
1434
|
|
|
1399
|
-
def
|
|
1400
|
-
self,
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1435
|
+
def as_physical_data_model(
|
|
1436
|
+
self,
|
|
1437
|
+
ignore_undefined_value_types: bool = False,
|
|
1438
|
+
reserved_properties: Literal["error", "warning"] = "error",
|
|
1439
|
+
) -> "PhysicalDataModel":
|
|
1440
|
+
from cognite.neat.core._data_model.models.physical._verified import (
|
|
1441
|
+
PhysicalContainer,
|
|
1442
|
+
PhysicalDataModel,
|
|
1443
|
+
PhysicalProperty,
|
|
1444
|
+
PhysicalView,
|
|
1407
1445
|
)
|
|
1408
1446
|
|
|
1409
|
-
|
|
1410
|
-
default_version =
|
|
1411
|
-
default_space = self._to_space(
|
|
1412
|
-
|
|
1447
|
+
conceptual_metadata = self.conceptual_data_model.metadata
|
|
1448
|
+
default_version = conceptual_metadata.version
|
|
1449
|
+
default_space = self._to_space(conceptual_metadata.prefix)
|
|
1450
|
+
physical_metadata = self._convert_conceptual_to_physical_metadata(conceptual_metadata)
|
|
1413
1451
|
|
|
1414
|
-
|
|
1415
|
-
for prop in self.
|
|
1416
|
-
|
|
1452
|
+
properties_by_concept: dict[ConceptEntity, set[str]] = defaultdict(set)
|
|
1453
|
+
for prop in self.conceptual_data_model.properties:
|
|
1454
|
+
properties_by_concept[prop.concept].add(prop.property_)
|
|
1417
1455
|
|
|
1418
1456
|
# Edge Classes is defined by having both startNode and endNode properties
|
|
1419
1457
|
edge_classes = {
|
|
1420
|
-
|
|
1421
|
-
for
|
|
1422
|
-
if ({"startNode", "start_node"} &
|
|
1458
|
+
concept
|
|
1459
|
+
for concept, concept_properties in properties_by_concept.items()
|
|
1460
|
+
if ({"startNode", "start_node"} & concept_properties) and ({"endNode", "end_node"} & concept_properties)
|
|
1423
1461
|
}
|
|
1424
|
-
|
|
1425
|
-
(prop.
|
|
1426
|
-
for prop in self.
|
|
1427
|
-
if prop.value_type in edge_classes and isinstance(prop.value_type,
|
|
1462
|
+
edge_value_types_by_concept_property_pair = {
|
|
1463
|
+
(prop.concept, prop.property_): prop.value_type
|
|
1464
|
+
for prop in self.conceptual_data_model.properties
|
|
1465
|
+
if prop.value_type in edge_classes and isinstance(prop.value_type, ConceptEntity)
|
|
1428
1466
|
}
|
|
1429
1467
|
end_node_by_edge = {
|
|
1430
|
-
prop.
|
|
1431
|
-
for prop in self.
|
|
1432
|
-
if prop.
|
|
1468
|
+
prop.concept: prop.value_type
|
|
1469
|
+
for prop in self.conceptual_data_model.properties
|
|
1470
|
+
if prop.concept in edge_classes
|
|
1433
1471
|
and (prop.property_ == "endNode" or prop.property_ == "end_node")
|
|
1434
|
-
and isinstance(prop.value_type,
|
|
1472
|
+
and isinstance(prop.value_type, ConceptEntity)
|
|
1435
1473
|
}
|
|
1436
1474
|
ancestors_by_view: dict[ViewEntity, set[ViewEntity]] = {}
|
|
1437
|
-
|
|
1475
|
+
parents_by_concept = DataModelAnalysis(self.conceptual_data_model).parents_by_concept(
|
|
1438
1476
|
include_ancestors=True, include_different_space=True
|
|
1439
1477
|
)
|
|
1440
|
-
for
|
|
1441
|
-
view_type =
|
|
1478
|
+
for concept, parents in parents_by_concept.items():
|
|
1479
|
+
view_type = concept.as_view_entity(default_space, default_version)
|
|
1442
1480
|
parent_views = {parent.as_view_entity(default_space, default_version) for parent in parents}
|
|
1443
1481
|
if view_type in ancestors_by_view:
|
|
1444
1482
|
ancestors_by_view[view_type].update(parent_views)
|
|
@@ -1452,14 +1490,14 @@ class _InformationRulesConverter:
|
|
|
1452
1490
|
else:
|
|
1453
1491
|
ancestors_by_view[view] = cognite_views[view]
|
|
1454
1492
|
|
|
1455
|
-
|
|
1456
|
-
used_containers: dict[ContainerEntity, Counter[
|
|
1457
|
-
used_cognite_containers: dict[ContainerEntity,
|
|
1493
|
+
properties_by_concept: dict[ConceptEntity, list[PhysicalProperty]] = defaultdict(list)
|
|
1494
|
+
used_containers: dict[ContainerEntity, Counter[ConceptEntity]] = defaultdict(Counter)
|
|
1495
|
+
used_cognite_containers: dict[ContainerEntity, PhysicalContainer] = {}
|
|
1458
1496
|
|
|
1459
|
-
for prop in self.
|
|
1497
|
+
for prop in self.conceptual_data_model.properties:
|
|
1460
1498
|
if ignore_undefined_value_types and isinstance(prop.value_type, UnknownEntity):
|
|
1461
1499
|
continue
|
|
1462
|
-
if prop.
|
|
1500
|
+
if prop.concept in edge_classes and prop.property_ in self._start_or_end_node:
|
|
1463
1501
|
continue
|
|
1464
1502
|
if prop.property_ in DMS_RESERVED_PROPERTIES:
|
|
1465
1503
|
msg = f"Property {prop.property_} is a reserved property in DMS."
|
|
@@ -1469,72 +1507,77 @@ class _InformationRulesConverter:
|
|
|
1469
1507
|
continue
|
|
1470
1508
|
|
|
1471
1509
|
if cognite_property := self._find_cognite_property(
|
|
1472
|
-
prop.property_,
|
|
1510
|
+
prop.property_, parents_by_concept[prop.concept], cognite_properties
|
|
1473
1511
|
):
|
|
1474
|
-
|
|
1512
|
+
physical_property = self._customize_physical_property(
|
|
1475
1513
|
prop,
|
|
1476
1514
|
cognite_property,
|
|
1477
|
-
prop.
|
|
1515
|
+
prop.concept,
|
|
1478
1516
|
default_space,
|
|
1479
1517
|
default_version,
|
|
1480
1518
|
ancestors_by_view,
|
|
1481
1519
|
)
|
|
1482
|
-
if
|
|
1483
|
-
if
|
|
1484
|
-
used_cognite_containers[
|
|
1520
|
+
if physical_property.container:
|
|
1521
|
+
if physical_property.container not in used_cognite_containers:
|
|
1522
|
+
used_cognite_containers[physical_property.container] = cognite_containers[
|
|
1523
|
+
physical_property.container
|
|
1524
|
+
]
|
|
1485
1525
|
else:
|
|
1486
1526
|
# Not matching any parent.
|
|
1487
|
-
|
|
1527
|
+
physical_property = self._as_physical_property(
|
|
1488
1528
|
prop,
|
|
1489
1529
|
default_space,
|
|
1490
1530
|
default_version,
|
|
1491
1531
|
edge_classes,
|
|
1492
|
-
|
|
1532
|
+
edge_value_types_by_concept_property_pair,
|
|
1493
1533
|
end_node_by_edge,
|
|
1494
1534
|
)
|
|
1495
|
-
if
|
|
1496
|
-
used_containers[
|
|
1535
|
+
if physical_property.container:
|
|
1536
|
+
used_containers[physical_property.container][prop.concept] += 1
|
|
1497
1537
|
|
|
1498
|
-
|
|
1538
|
+
properties_by_concept[prop.concept].append(physical_property)
|
|
1499
1539
|
|
|
1500
|
-
views: list[
|
|
1540
|
+
views: list[PhysicalView] = []
|
|
1501
1541
|
|
|
1502
|
-
for
|
|
1503
|
-
|
|
1504
|
-
name=
|
|
1505
|
-
view=
|
|
1506
|
-
description=
|
|
1507
|
-
implements=self._get_view_implements(
|
|
1542
|
+
for concept in self.conceptual_data_model.concepts:
|
|
1543
|
+
physical_view = PhysicalView(
|
|
1544
|
+
name=concept.name,
|
|
1545
|
+
view=concept.concept.as_view_entity(default_space, default_version),
|
|
1546
|
+
description=concept.description,
|
|
1547
|
+
implements=self._get_view_implements(concept, conceptual_metadata),
|
|
1508
1548
|
)
|
|
1509
1549
|
|
|
1510
|
-
|
|
1511
|
-
views.append(
|
|
1550
|
+
physical_view.conceptual = concept.neatId
|
|
1551
|
+
views.append(physical_view)
|
|
1512
1552
|
|
|
1513
|
-
|
|
1553
|
+
concept_by_concept_entity = {cls_.concept: cls_ for cls_ in self.conceptual_data_model.concepts}
|
|
1514
1554
|
|
|
1515
1555
|
existing_containers: set[ContainerEntity] = set()
|
|
1516
1556
|
|
|
1517
|
-
containers: list[
|
|
1518
|
-
for container_entity,
|
|
1557
|
+
containers: list[PhysicalContainer] = []
|
|
1558
|
+
for container_entity, concept_entities in used_containers.items():
|
|
1519
1559
|
if container_entity in existing_containers:
|
|
1520
1560
|
continue
|
|
1521
1561
|
constrains = self._create_container_constraint(
|
|
1522
|
-
|
|
1562
|
+
concept_entities,
|
|
1563
|
+
default_space,
|
|
1564
|
+
concept_by_concept_entity,
|
|
1565
|
+
used_containers,
|
|
1523
1566
|
)
|
|
1524
|
-
|
|
1525
|
-
|
|
1567
|
+
most_used_concept_entity = concept_entities.most_common(1)[0][0]
|
|
1568
|
+
concept = concept_by_concept_entity[most_used_concept_entity]
|
|
1526
1569
|
|
|
1527
|
-
if len(set(
|
|
1570
|
+
if len(set(concept_entities) - set(edge_classes)) == 0:
|
|
1528
1571
|
used_for: Literal["node", "edge", "all"] = "edge"
|
|
1529
|
-
elif len(set(
|
|
1572
|
+
elif len(set(concept_entities) - set(edge_classes)) == len(concept_entities):
|
|
1530
1573
|
used_for = "node"
|
|
1531
1574
|
else:
|
|
1532
1575
|
used_for = "all"
|
|
1533
1576
|
|
|
1534
|
-
container =
|
|
1577
|
+
container = PhysicalContainer(
|
|
1535
1578
|
container=container_entity,
|
|
1536
|
-
name=
|
|
1537
|
-
description=
|
|
1579
|
+
name=concept.name,
|
|
1580
|
+
description=concept.description,
|
|
1538
1581
|
constraint=constrains or None,
|
|
1539
1582
|
used_for=used_for,
|
|
1540
1583
|
)
|
|
@@ -1543,69 +1586,75 @@ class _InformationRulesConverter:
|
|
|
1543
1586
|
if used_cognite_containers:
|
|
1544
1587
|
containers.extend(used_cognite_containers.values())
|
|
1545
1588
|
|
|
1546
|
-
|
|
1547
|
-
metadata=
|
|
1548
|
-
properties=SheetList[
|
|
1549
|
-
|
|
1550
|
-
|
|
1589
|
+
physical_data_model = PhysicalDataModel(
|
|
1590
|
+
metadata=physical_metadata,
|
|
1591
|
+
properties=SheetList[PhysicalProperty](
|
|
1592
|
+
[prop for prop_set in properties_by_concept.values() for prop in prop_set]
|
|
1593
|
+
),
|
|
1594
|
+
views=SheetList[PhysicalView](views),
|
|
1595
|
+
containers=SheetList[PhysicalContainer](containers),
|
|
1551
1596
|
)
|
|
1552
1597
|
|
|
1553
|
-
self.
|
|
1598
|
+
self.conceptual_data_model.sync_with_physical_data_model(physical_data_model)
|
|
1554
1599
|
|
|
1555
|
-
return
|
|
1600
|
+
return physical_data_model
|
|
1556
1601
|
|
|
1557
1602
|
def _get_cognite_components(
|
|
1558
1603
|
self,
|
|
1559
1604
|
) -> tuple[
|
|
1560
|
-
dict[tuple[
|
|
1561
|
-
dict[ContainerEntity,
|
|
1605
|
+
dict[tuple[ConceptEntity, str], PhysicalProperty],
|
|
1606
|
+
dict[ContainerEntity, PhysicalContainer],
|
|
1562
1607
|
dict[ViewEntity, set[ViewEntity]],
|
|
1563
1608
|
]:
|
|
1564
1609
|
cognite_concepts = self._get_cognite_concepts()
|
|
1565
|
-
cognite_properties: dict[tuple[
|
|
1566
|
-
cognite_containers: dict[ContainerEntity,
|
|
1610
|
+
cognite_properties: dict[tuple[ConceptEntity, str], PhysicalProperty] = {}
|
|
1611
|
+
cognite_containers: dict[ContainerEntity, PhysicalContainer] = {}
|
|
1567
1612
|
cognite_views: dict[ViewEntity, set[ViewEntity]] = {}
|
|
1568
1613
|
if cognite_concepts:
|
|
1569
1614
|
if self.client is None:
|
|
1570
1615
|
raise CDFMissingClientError(
|
|
1571
|
-
f"Cannot convert {self.
|
|
1616
|
+
f"Cannot convert {self.conceptual_data_model.metadata.as_data_model_id()}. Missing Cognite Client."
|
|
1572
1617
|
f"This is required as the data model is referencing cognite concepts in the implements"
|
|
1573
1618
|
f"{humanize_collection(cognite_concepts)}"
|
|
1574
1619
|
)
|
|
1575
|
-
|
|
1620
|
+
cognite_data_model = self._get_cognite_physical_data_model(cognite_concepts, self.client)
|
|
1576
1621
|
|
|
1577
1622
|
cognite_properties = {
|
|
1578
|
-
(
|
|
1623
|
+
(
|
|
1624
|
+
physical_property.view.as_concept_entity(),
|
|
1625
|
+
physical_property.view_property,
|
|
1626
|
+
): physical_property
|
|
1627
|
+
for physical_property in cognite_data_model.properties
|
|
1579
1628
|
}
|
|
1580
|
-
cognite_containers = {container.container: container for container in
|
|
1581
|
-
cognite_views =
|
|
1629
|
+
cognite_containers = {container.container: container for container in cognite_data_model.containers or []}
|
|
1630
|
+
cognite_views = DataModelAnalysis(physical=cognite_data_model).implements_by_view(
|
|
1582
1631
|
include_ancestors=True, include_different_space=True
|
|
1583
1632
|
)
|
|
1584
1633
|
return cognite_properties, cognite_containers, cognite_views
|
|
1585
1634
|
|
|
1586
1635
|
@staticmethod
|
|
1587
1636
|
def _create_container_constraint(
|
|
1588
|
-
|
|
1637
|
+
concept_entities: Counter[ConceptEntity],
|
|
1589
1638
|
default_space: str,
|
|
1590
|
-
|
|
1639
|
+
concept_by_concept_entity: dict[ConceptEntity, Concept],
|
|
1591
1640
|
referenced_containers: Collection[ContainerEntity],
|
|
1592
1641
|
) -> list[ContainerEntity]:
|
|
1593
1642
|
constrains: list[ContainerEntity] = []
|
|
1594
|
-
for entity in
|
|
1595
|
-
|
|
1596
|
-
for parent in
|
|
1643
|
+
for entity in concept_entities:
|
|
1644
|
+
concept = concept_by_concept_entity[entity]
|
|
1645
|
+
for parent in concept.implements or []:
|
|
1597
1646
|
parent_entity = parent.as_container_entity(default_space)
|
|
1598
1647
|
if parent_entity in referenced_containers:
|
|
1599
1648
|
constrains.append(parent_entity)
|
|
1600
1649
|
return constrains
|
|
1601
1650
|
|
|
1602
1651
|
@classmethod
|
|
1603
|
-
def
|
|
1604
|
-
from cognite.neat.core._data_model.models.
|
|
1605
|
-
|
|
1652
|
+
def _convert_conceptual_to_physical_metadata(cls, metadata: ConceptualMetadata) -> "PhysicalMetadata":
|
|
1653
|
+
from cognite.neat.core._data_model.models.physical._verified import (
|
|
1654
|
+
PhysicalMetadata,
|
|
1606
1655
|
)
|
|
1607
1656
|
|
|
1608
|
-
|
|
1657
|
+
physical_metadata = PhysicalMetadata(
|
|
1609
1658
|
space=metadata.space,
|
|
1610
1659
|
version=metadata.version,
|
|
1611
1660
|
external_id=metadata.external_id,
|
|
@@ -1615,23 +1664,23 @@ class _InformationRulesConverter:
|
|
|
1615
1664
|
updated=metadata.updated,
|
|
1616
1665
|
)
|
|
1617
1666
|
|
|
1618
|
-
|
|
1619
|
-
return
|
|
1667
|
+
physical_metadata.conceptual = metadata.identifier
|
|
1668
|
+
return physical_metadata
|
|
1620
1669
|
|
|
1621
|
-
def
|
|
1670
|
+
def _as_physical_property(
|
|
1622
1671
|
self,
|
|
1623
|
-
|
|
1672
|
+
conceptual_property: ConceptualProperty,
|
|
1624
1673
|
default_space: str,
|
|
1625
1674
|
default_version: str,
|
|
1626
|
-
edge_classes: set[
|
|
1627
|
-
|
|
1628
|
-
end_node_by_edge: dict[
|
|
1629
|
-
) -> "
|
|
1630
|
-
from cognite.neat.core._data_model.models.
|
|
1675
|
+
edge_classes: set[ConceptEntity],
|
|
1676
|
+
edge_value_types_by_concept_property_pair: dict[tuple[ConceptEntity, str], ConceptEntity],
|
|
1677
|
+
end_node_by_edge: dict[ConceptEntity, ConceptEntity],
|
|
1678
|
+
) -> "PhysicalProperty":
|
|
1679
|
+
from cognite.neat.core._data_model.models.physical._verified import PhysicalProperty
|
|
1631
1680
|
|
|
1632
1681
|
# returns property type, which can be ObjectProperty or DatatypeProperty
|
|
1633
1682
|
value_type = self._get_value_type(
|
|
1634
|
-
|
|
1683
|
+
conceptual_property,
|
|
1635
1684
|
default_space,
|
|
1636
1685
|
default_version,
|
|
1637
1686
|
edge_classes,
|
|
@@ -1639,14 +1688,18 @@ class _InformationRulesConverter:
|
|
|
1639
1688
|
)
|
|
1640
1689
|
|
|
1641
1690
|
connection = self._get_connection(
|
|
1642
|
-
|
|
1691
|
+
conceptual_property,
|
|
1692
|
+
value_type,
|
|
1693
|
+
edge_value_types_by_concept_property_pair,
|
|
1694
|
+
default_space,
|
|
1695
|
+
default_version,
|
|
1643
1696
|
)
|
|
1644
1697
|
|
|
1645
1698
|
container: ContainerEntity | None = None
|
|
1646
1699
|
container_property: str | None = None
|
|
1647
1700
|
# DMS should have min count of either 0 or 1
|
|
1648
|
-
min_count = min(1, max(0,
|
|
1649
|
-
max_count =
|
|
1701
|
+
min_count = min(1, max(0, conceptual_property.min_count or 0))
|
|
1702
|
+
max_count = conceptual_property.max_count
|
|
1650
1703
|
if isinstance(connection, EdgeEntity):
|
|
1651
1704
|
min_count = 0
|
|
1652
1705
|
max_count = 1 if max_count == 1 else float("inf")
|
|
@@ -1655,74 +1708,82 @@ class _InformationRulesConverter:
|
|
|
1655
1708
|
max_count = 1 if max_count == 1 else float("inf")
|
|
1656
1709
|
elif connection == "direct":
|
|
1657
1710
|
min_count = 0
|
|
1658
|
-
container, container_property = self._get_container(
|
|
1711
|
+
container, container_property = self._get_container(conceptual_property, default_space)
|
|
1659
1712
|
else:
|
|
1660
|
-
container, container_property = self._get_container(
|
|
1713
|
+
container, container_property = self._get_container(conceptual_property, default_space)
|
|
1661
1714
|
|
|
1662
|
-
|
|
1663
|
-
name=
|
|
1715
|
+
physical_property = PhysicalProperty(
|
|
1716
|
+
name=conceptual_property.name,
|
|
1664
1717
|
value_type=value_type,
|
|
1665
1718
|
min_count=min_count,
|
|
1666
1719
|
max_count=max_count,
|
|
1667
1720
|
connection=connection,
|
|
1668
|
-
default=
|
|
1721
|
+
default=conceptual_property.default,
|
|
1669
1722
|
container=container,
|
|
1670
1723
|
container_property=container_property,
|
|
1671
|
-
view=
|
|
1672
|
-
view_property=
|
|
1724
|
+
view=conceptual_property.concept.as_view_entity(default_space, default_version),
|
|
1725
|
+
view_property=conceptual_property.property_,
|
|
1673
1726
|
)
|
|
1674
1727
|
|
|
1675
1728
|
# linking
|
|
1676
|
-
|
|
1729
|
+
physical_property.conceptual = conceptual_property.neatId
|
|
1677
1730
|
|
|
1678
|
-
return
|
|
1731
|
+
return physical_property
|
|
1679
1732
|
|
|
1680
1733
|
@staticmethod
|
|
1681
|
-
def
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1734
|
+
def _customize_physical_property(
|
|
1735
|
+
conceptual_property: ConceptualProperty,
|
|
1736
|
+
physical_property: PhysicalProperty,
|
|
1737
|
+
concept: ConceptEntity,
|
|
1685
1738
|
default_space: str,
|
|
1686
1739
|
default_version: str,
|
|
1687
1740
|
ancestors_by_view: dict[ViewEntity, set[ViewEntity]],
|
|
1688
|
-
) ->
|
|
1689
|
-
"""Customize the
|
|
1690
|
-
This means updating the name and description of the
|
|
1691
|
-
In addition, the value type can be updated given that the value type is matches the
|
|
1741
|
+
) -> PhysicalProperty:
|
|
1742
|
+
"""Customize the physical property to match the conceptual property.
|
|
1743
|
+
This means updating the name and description of the physical property with the conceptual property.
|
|
1744
|
+
In addition, the value type can be updated given that the value type is matches the physical property value
|
|
1692
1745
|
type or in the case of a View Value type a derivative of the cognite property value type.
|
|
1693
1746
|
|
|
1694
1747
|
Args:
|
|
1695
1748
|
prop: Information property
|
|
1696
1749
|
cognite_prop: Cognite property
|
|
1697
|
-
|
|
1750
|
+
concept: Concept entity
|
|
1698
1751
|
default_space: The default space
|
|
1699
1752
|
default_version: The default version
|
|
1700
1753
|
ancestors_by_view: Ancestors by view
|
|
1701
1754
|
|
|
1702
1755
|
Returns:
|
|
1703
|
-
|
|
1756
|
+
PhysicalProperty: The customized physical property
|
|
1704
1757
|
|
|
1705
1758
|
"""
|
|
1706
|
-
value_type: DataType | ViewEntity |
|
|
1707
|
-
if isinstance(
|
|
1759
|
+
value_type: DataType | ViewEntity | PhysicalUnknownEntity = physical_property.value_type
|
|
1760
|
+
if isinstance(conceptual_property.value_type, DataType) and conceptual_property.value_type != value_type:
|
|
1708
1761
|
warnings.warn(
|
|
1709
|
-
PropertyOverwritingWarning(
|
|
1762
|
+
PropertyOverwritingWarning(
|
|
1763
|
+
conceptual_property.property_,
|
|
1764
|
+
"property",
|
|
1765
|
+
"value type",
|
|
1766
|
+
(str(conceptual_property.value_type),),
|
|
1767
|
+
),
|
|
1710
1768
|
stacklevel=2,
|
|
1711
1769
|
)
|
|
1712
|
-
elif isinstance(
|
|
1770
|
+
elif isinstance(conceptual_property.value_type, DataType):
|
|
1713
1771
|
# User set the same value type as core concept.
|
|
1714
1772
|
pass
|
|
1715
|
-
elif isinstance(
|
|
1716
|
-
|
|
1773
|
+
elif isinstance(conceptual_property.value_type, ConceptEntity) and isinstance(
|
|
1774
|
+
physical_property.value_type, ViewEntity
|
|
1775
|
+
):
|
|
1776
|
+
view_type = conceptual_property.value_type.as_view_entity(default_space, default_version)
|
|
1717
1777
|
ancestors = ancestors_by_view.get(view_type, set())
|
|
1718
|
-
if view_type ==
|
|
1778
|
+
if view_type == physical_property.value_type or physical_property.value_type in ancestors:
|
|
1719
1779
|
value_type = view_type
|
|
1720
1780
|
else:
|
|
1721
1781
|
warnings.warn(
|
|
1722
1782
|
NeatValueWarning(
|
|
1723
1783
|
f"Invalid Value Type. The view {view_type} must implement "
|
|
1724
1784
|
f"{humanize_collection(ancestors, bind_word='or')} "
|
|
1725
|
-
|
|
1785
|
+
"to be used as the Value Type in the "
|
|
1786
|
+
f"{conceptual_property.concept!s}.{conceptual_property.property_}. "
|
|
1726
1787
|
f"Skipping..."
|
|
1727
1788
|
),
|
|
1728
1789
|
stacklevel=2,
|
|
@@ -1730,101 +1791,112 @@ class _InformationRulesConverter:
|
|
|
1730
1791
|
else:
|
|
1731
1792
|
warnings.warn(
|
|
1732
1793
|
NeatValueWarning(
|
|
1733
|
-
f"Invalid Value Type. The {
|
|
1734
|
-
f"
|
|
1794
|
+
f"Invalid Value Type. The {conceptual_property.value_type} is "
|
|
1795
|
+
f"not supported as {conceptual_property.concept} implements"
|
|
1796
|
+
"a cognite concepts. Will skip this, and use "
|
|
1797
|
+
f"the {physical_property.value_type} instead."
|
|
1735
1798
|
),
|
|
1736
1799
|
stacklevel=2,
|
|
1737
1800
|
)
|
|
1738
1801
|
|
|
1739
|
-
return
|
|
1802
|
+
return physical_property.model_copy(
|
|
1740
1803
|
update={
|
|
1741
|
-
"view":
|
|
1742
|
-
"name":
|
|
1743
|
-
"description":
|
|
1804
|
+
"view": concept.as_view_entity(default_space, default_version),
|
|
1805
|
+
"name": conceptual_property.name or physical_property.name,
|
|
1806
|
+
"description": conceptual_property.description or physical_property.description,
|
|
1744
1807
|
"value_type": value_type,
|
|
1745
1808
|
}
|
|
1746
1809
|
)
|
|
1747
1810
|
|
|
1748
1811
|
@staticmethod
|
|
1749
1812
|
def _get_connection(
|
|
1750
|
-
|
|
1751
|
-
value_type: DataType | ViewEntity |
|
|
1752
|
-
|
|
1813
|
+
conceptual_property: ConceptualProperty,
|
|
1814
|
+
value_type: DataType | ViewEntity | PhysicalUnknownEntity,
|
|
1815
|
+
edge_value_types_by_concept_property_pair: dict[tuple[ConceptEntity, str], ConceptEntity],
|
|
1753
1816
|
default_space: str,
|
|
1754
1817
|
default_version: str,
|
|
1755
1818
|
) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
|
|
1756
1819
|
if (
|
|
1757
1820
|
isinstance(value_type, ViewEntity)
|
|
1758
|
-
and (
|
|
1821
|
+
and (conceptual_property.concept, conceptual_property.property_)
|
|
1822
|
+
in edge_value_types_by_concept_property_pair
|
|
1759
1823
|
):
|
|
1760
|
-
edge_value_type =
|
|
1824
|
+
edge_value_type = edge_value_types_by_concept_property_pair[
|
|
1825
|
+
(conceptual_property.concept, conceptual_property.property_)
|
|
1826
|
+
]
|
|
1761
1827
|
return EdgeEntity(properties=edge_value_type.as_view_entity(default_space, default_version))
|
|
1762
1828
|
if isinstance(value_type, ViewEntity) and (
|
|
1763
|
-
|
|
1829
|
+
conceptual_property.max_count in {float("inf"), None}
|
|
1830
|
+
or (isinstance(conceptual_property.max_count, int | float) and conceptual_property.max_count > 1)
|
|
1764
1831
|
):
|
|
1765
1832
|
return EdgeEntity()
|
|
1766
1833
|
elif isinstance(value_type, ViewEntity):
|
|
1767
1834
|
return "direct"
|
|
1768
1835
|
# defaulting to direct connection
|
|
1769
|
-
elif isinstance(value_type,
|
|
1836
|
+
elif isinstance(value_type, PhysicalUnknownEntity):
|
|
1770
1837
|
return "direct"
|
|
1771
1838
|
return None
|
|
1772
1839
|
|
|
1773
1840
|
def _get_value_type(
|
|
1774
1841
|
self,
|
|
1775
|
-
|
|
1842
|
+
conceptual_property: ConceptualProperty,
|
|
1776
1843
|
default_space: str,
|
|
1777
1844
|
default_version: str,
|
|
1778
|
-
edge_classes: set[
|
|
1779
|
-
end_node_by_edge: dict[
|
|
1780
|
-
) -> DataType | ViewEntity |
|
|
1781
|
-
if isinstance(
|
|
1782
|
-
return
|
|
1845
|
+
edge_classes: set[ConceptEntity],
|
|
1846
|
+
end_node_by_edge: dict[ConceptEntity, ConceptEntity],
|
|
1847
|
+
) -> DataType | ViewEntity | PhysicalUnknownEntity:
|
|
1848
|
+
if isinstance(conceptual_property.value_type, DataType):
|
|
1849
|
+
return conceptual_property.value_type
|
|
1783
1850
|
|
|
1784
1851
|
# UnknownEntity should resolve to DMSUnknownEntity
|
|
1785
1852
|
# meaning end node type is unknown
|
|
1786
|
-
elif isinstance(
|
|
1787
|
-
return
|
|
1853
|
+
elif isinstance(conceptual_property.value_type, UnknownEntity):
|
|
1854
|
+
return PhysicalUnknownEntity()
|
|
1788
1855
|
|
|
1789
|
-
elif isinstance(
|
|
1790
|
-
|
|
1791
|
-
|
|
1856
|
+
elif isinstance(conceptual_property.value_type, ConceptEntity) and (
|
|
1857
|
+
conceptual_property.value_type in edge_classes
|
|
1858
|
+
):
|
|
1859
|
+
if conceptual_property.value_type in end_node_by_edge:
|
|
1860
|
+
return end_node_by_edge[conceptual_property.value_type].as_view_entity(default_space, default_version)
|
|
1792
1861
|
# This occurs if the end node is not pointing to a class
|
|
1793
1862
|
warnings.warn(
|
|
1794
1863
|
NeatValueWarning(
|
|
1795
|
-
f"Edge class {
|
|
1864
|
+
f"Edge class {conceptual_property.value_type} does not "
|
|
1865
|
+
"have 'endNode' property, defaulting to DMSUnknownEntity"
|
|
1796
1866
|
),
|
|
1797
1867
|
stacklevel=2,
|
|
1798
1868
|
)
|
|
1799
|
-
return
|
|
1800
|
-
elif isinstance(
|
|
1801
|
-
return
|
|
1869
|
+
return PhysicalUnknownEntity()
|
|
1870
|
+
elif isinstance(conceptual_property.value_type, ConceptEntity):
|
|
1871
|
+
return conceptual_property.value_type.as_view_entity(default_space, default_version)
|
|
1802
1872
|
|
|
1803
|
-
elif isinstance(
|
|
1873
|
+
elif isinstance(conceptual_property.value_type, MultiValueTypeInfo):
|
|
1804
1874
|
# Multi Object type should resolve to DMSUnknownEntity
|
|
1805
1875
|
# meaning end node type is unknown
|
|
1806
|
-
if
|
|
1807
|
-
non_unknown = [
|
|
1876
|
+
if conceptual_property.value_type.is_multi_object_type():
|
|
1877
|
+
non_unknown = [
|
|
1878
|
+
type_ for type_ in conceptual_property.value_type.types if isinstance(type_, UnknownEntity)
|
|
1879
|
+
]
|
|
1808
1880
|
if list(non_unknown) == 1:
|
|
1809
1881
|
#
|
|
1810
1882
|
return non_unknown[0].as_view_entity(default_space, default_version)
|
|
1811
|
-
return
|
|
1883
|
+
return PhysicalUnknownEntity()
|
|
1812
1884
|
|
|
1813
1885
|
# Multi Data type should resolve to a single data type, or it should
|
|
1814
|
-
elif
|
|
1815
|
-
return self.convert_multi_data_type(
|
|
1886
|
+
elif conceptual_property.value_type.is_multi_data_type():
|
|
1887
|
+
return self.convert_multi_data_type(conceptual_property.value_type)
|
|
1816
1888
|
|
|
1817
1889
|
# Mixed types default to string
|
|
1818
1890
|
else:
|
|
1819
|
-
non_any_uri = [type_ for type_ in
|
|
1891
|
+
non_any_uri = [type_ for type_ in conceptual_property.value_type.types if type_ != AnyURI()]
|
|
1820
1892
|
if list(non_any_uri) == 1:
|
|
1821
|
-
if isinstance(non_any_uri[0],
|
|
1893
|
+
if isinstance(non_any_uri[0], ConceptEntity):
|
|
1822
1894
|
return non_any_uri[0].as_view_entity(default_space, default_version)
|
|
1823
1895
|
else:
|
|
1824
1896
|
return non_any_uri[0]
|
|
1825
1897
|
return String()
|
|
1826
1898
|
|
|
1827
|
-
raise ValueError(f"Unsupported value type: {
|
|
1899
|
+
raise ValueError(f"Unsupported value type: {conceptual_property.value_type.type_}")
|
|
1828
1900
|
|
|
1829
1901
|
@classmethod
|
|
1830
1902
|
def _to_space(cls, prefix: str) -> str:
|
|
@@ -1838,7 +1910,7 @@ class _InformationRulesConverter:
|
|
|
1838
1910
|
return prefix
|
|
1839
1911
|
|
|
1840
1912
|
def _get_container(self, prop: ConceptualProperty, default_space: str) -> tuple[ContainerEntity, str]:
|
|
1841
|
-
container_entity = prop.
|
|
1913
|
+
container_entity = prop.concept.as_container_entity(default_space)
|
|
1842
1914
|
|
|
1843
1915
|
while self.property_count_by_container[container_entity] >= DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
|
|
1844
1916
|
container_entity.suffix = self._bump_suffix(container_entity.suffix)
|
|
@@ -1846,7 +1918,7 @@ class _InformationRulesConverter:
|
|
|
1846
1918
|
self.property_count_by_container[container_entity] += 1
|
|
1847
1919
|
return container_entity, prop.property_
|
|
1848
1920
|
|
|
1849
|
-
def _get_view_implements(self, cls_:
|
|
1921
|
+
def _get_view_implements(self, cls_: Concept, metadata: ConceptualMetadata) -> list[ViewEntity]:
|
|
1850
1922
|
implements = []
|
|
1851
1923
|
for parent in cls_.implements or []:
|
|
1852
1924
|
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
@@ -1885,16 +1957,18 @@ class _InformationRulesConverter:
|
|
|
1885
1957
|
|
|
1886
1958
|
return data_types.String()
|
|
1887
1959
|
|
|
1888
|
-
def _get_cognite_concepts(self) -> set[
|
|
1889
|
-
return {
|
|
1960
|
+
def _get_cognite_concepts(self) -> set[ConceptEntity]:
|
|
1961
|
+
return {
|
|
1962
|
+
cls_.concept for cls_ in self.conceptual_data_model.concepts if str(cls_.concept.prefix) in COGNITE_SPACES
|
|
1963
|
+
} | {
|
|
1890
1964
|
parent
|
|
1891
|
-
for cls_ in self.
|
|
1965
|
+
for cls_ in self.conceptual_data_model.concepts
|
|
1892
1966
|
for parent in cls_.implements or []
|
|
1893
1967
|
if str(parent.prefix) in COGNITE_SPACES
|
|
1894
1968
|
}
|
|
1895
1969
|
|
|
1896
1970
|
@staticmethod
|
|
1897
|
-
def
|
|
1971
|
+
def _get_cognite_physical_data_model(concepts: set[ConceptEntity], client: NeatClient) -> PhysicalDataModel:
|
|
1898
1972
|
view_ids = [dm.ViewId(str(cls_.prefix), cls_.suffix, cls_.version) for cls_ in concepts]
|
|
1899
1973
|
views = client.loaders.views.retrieve(view_ids, format="read", include_connected=True, include_ancestor=True)
|
|
1900
1974
|
spaces = Counter(view.space for view in views)
|
|
@@ -1912,15 +1986,17 @@ class _InformationRulesConverter:
|
|
|
1912
1986
|
"purpose.",
|
|
1913
1987
|
views=list(views),
|
|
1914
1988
|
)
|
|
1915
|
-
unverified = DMSImporter.from_data_model(client, model).
|
|
1916
|
-
if unverified.
|
|
1989
|
+
unverified = DMSImporter.from_data_model(client, model).to_data_model()
|
|
1990
|
+
if unverified.unverified_data_model is None:
|
|
1917
1991
|
raise NeatValueError("Failed to create CogniteConcepts")
|
|
1918
|
-
return unverified.
|
|
1992
|
+
return unverified.unverified_data_model.as_verified_data_model()
|
|
1919
1993
|
|
|
1920
1994
|
@staticmethod
|
|
1921
1995
|
def _find_cognite_property(
|
|
1922
|
-
property_: str,
|
|
1923
|
-
|
|
1996
|
+
property_: str,
|
|
1997
|
+
parents: set[ConceptEntity],
|
|
1998
|
+
cognite_properties: dict[tuple[ConceptEntity, str], PhysicalProperty],
|
|
1999
|
+
) -> PhysicalProperty | None:
|
|
1924
2000
|
"""Find the parent class that has the property in the cognite properties"""
|
|
1925
2001
|
for parent in parents:
|
|
1926
2002
|
if (parent, property_) in cognite_properties:
|
|
@@ -1928,41 +2004,39 @@ class _InformationRulesConverter:
|
|
|
1928
2004
|
return None
|
|
1929
2005
|
|
|
1930
2006
|
|
|
1931
|
-
class
|
|
1932
|
-
def __init__(self,
|
|
1933
|
-
self.
|
|
2007
|
+
class _PhysicalDataModelConverter:
|
|
2008
|
+
def __init__(self, data_model: PhysicalDataModel, instance_namespace: Namespace | None = None) -> None:
|
|
2009
|
+
self.physical_data_model = data_model
|
|
1934
2010
|
self.instance_namespace = instance_namespace
|
|
1935
2011
|
|
|
1936
|
-
def
|
|
2012
|
+
def as_conceptual_data_model(
|
|
1937
2013
|
self,
|
|
1938
2014
|
) -> "ConceptualDataModel":
|
|
1939
2015
|
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
1940
|
-
|
|
2016
|
+
Concept,
|
|
1941
2017
|
ConceptualDataModel,
|
|
1942
2018
|
ConceptualProperty,
|
|
1943
2019
|
)
|
|
1944
2020
|
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
metadata = self._convert_metadata_to_info(dms)
|
|
2021
|
+
metadata = self._convert_physical_to_conceptual_metadata(self.physical_data_model.metadata)
|
|
1948
2022
|
|
|
1949
|
-
|
|
1950
|
-
for view in self.
|
|
1951
|
-
|
|
2023
|
+
concepts: list[Concept] = []
|
|
2024
|
+
for view in self.physical_data_model.views:
|
|
2025
|
+
concept = Concept(
|
|
1952
2026
|
# we do not want a version in class as we use URI for the class
|
|
1953
|
-
|
|
2027
|
+
concept=ConceptEntity(prefix=view.view.prefix, suffix=view.view.suffix),
|
|
1954
2028
|
description=view.description,
|
|
1955
2029
|
name=view.name,
|
|
1956
2030
|
implements=[
|
|
1957
2031
|
# we do not want a version in class as we use URI for the class
|
|
1958
|
-
implemented_view.
|
|
2032
|
+
implemented_view.as_concept_entity(skip_version=True)
|
|
1959
2033
|
for implemented_view in view.implements or []
|
|
1960
2034
|
],
|
|
1961
2035
|
)
|
|
1962
2036
|
|
|
1963
2037
|
# Linking
|
|
1964
|
-
|
|
1965
|
-
|
|
2038
|
+
concept.physical = view.neatId
|
|
2039
|
+
concepts.append(concept)
|
|
1966
2040
|
|
|
1967
2041
|
prefixes = get_default_prefixes_and_namespaces()
|
|
1968
2042
|
if self.instance_namespace:
|
|
@@ -1973,23 +2047,23 @@ class _DMSRulesConverter:
|
|
|
1973
2047
|
prefixes[instance_prefix] = self.instance_namespace
|
|
1974
2048
|
|
|
1975
2049
|
properties: list[ConceptualProperty] = []
|
|
1976
|
-
value_type: DataType |
|
|
1977
|
-
for property_ in self.
|
|
2050
|
+
value_type: DataType | ConceptEntity | str
|
|
2051
|
+
for property_ in self.physical_data_model.properties:
|
|
1978
2052
|
if isinstance(property_.value_type, DataType):
|
|
1979
2053
|
value_type = property_.value_type
|
|
1980
2054
|
elif isinstance(property_.value_type, ViewEntity):
|
|
1981
|
-
value_type =
|
|
2055
|
+
value_type = ConceptEntity(
|
|
1982
2056
|
prefix=property_.value_type.prefix,
|
|
1983
2057
|
suffix=property_.value_type.suffix,
|
|
1984
2058
|
)
|
|
1985
|
-
elif isinstance(property_.value_type,
|
|
2059
|
+
elif isinstance(property_.value_type, PhysicalUnknownEntity):
|
|
1986
2060
|
value_type = UnknownEntity()
|
|
1987
2061
|
else:
|
|
1988
2062
|
raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
|
|
1989
2063
|
|
|
1990
|
-
|
|
2064
|
+
conceptual_property = ConceptualProperty(
|
|
1991
2065
|
# Removing version
|
|
1992
|
-
|
|
2066
|
+
concept=ConceptEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
|
|
1993
2067
|
property_=property_.view_property,
|
|
1994
2068
|
name=property_.name,
|
|
1995
2069
|
value_type=value_type,
|
|
@@ -1999,23 +2073,23 @@ class _DMSRulesConverter:
|
|
|
1999
2073
|
)
|
|
2000
2074
|
|
|
2001
2075
|
# Linking
|
|
2002
|
-
|
|
2076
|
+
conceptual_property.physical = property_.neatId
|
|
2003
2077
|
|
|
2004
|
-
properties.append(
|
|
2078
|
+
properties.append(conceptual_property)
|
|
2005
2079
|
|
|
2006
|
-
|
|
2080
|
+
conceptual_data_model = ConceptualDataModel(
|
|
2007
2081
|
metadata=metadata,
|
|
2008
2082
|
properties=SheetList[ConceptualProperty](properties),
|
|
2009
|
-
|
|
2083
|
+
concepts=SheetList[Concept](concepts),
|
|
2010
2084
|
prefixes=prefixes,
|
|
2011
2085
|
)
|
|
2012
2086
|
|
|
2013
|
-
self.
|
|
2087
|
+
self.physical_data_model.sync_with_conceptual_data_model(conceptual_data_model)
|
|
2014
2088
|
|
|
2015
|
-
return
|
|
2089
|
+
return conceptual_data_model
|
|
2016
2090
|
|
|
2017
2091
|
@classmethod
|
|
2018
|
-
def
|
|
2092
|
+
def _convert_physical_to_conceptual_metadata(cls, metadata: PhysicalMetadata) -> "ConceptualMetadata":
|
|
2019
2093
|
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
2020
2094
|
ConceptualMetadata,
|
|
2021
2095
|
)
|
|
@@ -2032,8 +2106,8 @@ class _DMSRulesConverter:
|
|
|
2032
2106
|
)
|
|
2033
2107
|
|
|
2034
2108
|
|
|
2035
|
-
class
|
|
2036
|
-
"""Subsets editable CDM
|
|
2109
|
+
class _SubsetEditableCDMPhysicalDataModel(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
2110
|
+
"""Subsets editable CDM data model to only include desired set of CDM concepts.
|
|
2037
2111
|
|
|
2038
2112
|
!!! note "Platypus UI limitations"
|
|
2039
2113
|
This is temporal solution to enable cleaner extension of core data model,
|
|
@@ -2049,65 +2123,65 @@ class _SubsetEditableCDMRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
2049
2123
|
|
|
2050
2124
|
self._views = views
|
|
2051
2125
|
|
|
2052
|
-
def transform(self,
|
|
2126
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
2053
2127
|
# should check to make sure data model is based on the editable CDM
|
|
2054
2128
|
# if not raise an error
|
|
2055
2129
|
|
|
2056
|
-
|
|
2057
|
-
"metadata":
|
|
2058
|
-
"views": SheetList[
|
|
2059
|
-
"properties": SheetList[
|
|
2060
|
-
"containers": SheetList[
|
|
2061
|
-
"enum":
|
|
2062
|
-
"nodes":
|
|
2130
|
+
subsetted_data_model: dict[str, Any] = {
|
|
2131
|
+
"metadata": data_model.metadata.model_copy(),
|
|
2132
|
+
"views": SheetList[PhysicalView](),
|
|
2133
|
+
"properties": SheetList[PhysicalProperty](),
|
|
2134
|
+
"containers": SheetList[PhysicalContainer](),
|
|
2135
|
+
"enum": data_model.enum,
|
|
2136
|
+
"nodes": data_model.nodes,
|
|
2063
2137
|
}
|
|
2064
2138
|
|
|
2065
2139
|
containers_to_keep = set()
|
|
2066
2140
|
|
|
2067
|
-
if editable_views_to_keep := self._editable_views_to_keep(
|
|
2068
|
-
for view in
|
|
2141
|
+
if editable_views_to_keep := self._editable_views_to_keep(data_model):
|
|
2142
|
+
for view in data_model.views:
|
|
2069
2143
|
if view.view in editable_views_to_keep or view.view.space in COGNITE_SPACES:
|
|
2070
|
-
|
|
2144
|
+
subsetted_data_model["views"].append(view)
|
|
2071
2145
|
|
|
2072
|
-
for property_ in
|
|
2146
|
+
for property_ in data_model.properties:
|
|
2073
2147
|
if property_.view in editable_views_to_keep and (
|
|
2074
2148
|
isinstance(property_.value_type, DataType)
|
|
2075
|
-
or isinstance(property_.value_type,
|
|
2149
|
+
or isinstance(property_.value_type, PhysicalUnknownEntity)
|
|
2076
2150
|
or (isinstance(property_.value_type, ViewEntity) and property_.value_type in editable_views_to_keep)
|
|
2077
2151
|
):
|
|
2078
|
-
|
|
2152
|
+
subsetted_data_model["properties"].append(property_)
|
|
2079
2153
|
if property_.container:
|
|
2080
2154
|
containers_to_keep.add(property_.container)
|
|
2081
2155
|
|
|
2082
|
-
if
|
|
2083
|
-
for container in
|
|
2156
|
+
if data_model.containers:
|
|
2157
|
+
for container in data_model.containers:
|
|
2084
2158
|
if container.container in containers_to_keep:
|
|
2085
|
-
|
|
2159
|
+
subsetted_data_model["containers"].append(container)
|
|
2086
2160
|
try:
|
|
2087
|
-
return
|
|
2161
|
+
return PhysicalDataModel.model_validate(subsetted_data_model)
|
|
2088
2162
|
except ValidationError as e:
|
|
2089
|
-
raise NeatValueError(f"Cannot subset
|
|
2163
|
+
raise NeatValueError(f"Cannot subset data_model: {e}") from e
|
|
2090
2164
|
else:
|
|
2091
|
-
raise NeatValueError("Cannot subset
|
|
2165
|
+
raise NeatValueError("Cannot subset data_model: provided data model is not based on Core Data Model")
|
|
2092
2166
|
|
|
2093
|
-
def _editable_views_to_keep(self,
|
|
2167
|
+
def _editable_views_to_keep(self, data_model: PhysicalDataModel) -> set[ViewEntity]:
|
|
2094
2168
|
return {
|
|
2095
2169
|
view.view
|
|
2096
|
-
for view in
|
|
2170
|
+
for view in data_model.views
|
|
2097
2171
|
if view.view.space not in COGNITE_SPACES
|
|
2098
2172
|
and view.implements
|
|
2099
2173
|
and any(implemented in self._views for implemented in view.implements)
|
|
2100
2174
|
}
|
|
2101
2175
|
|
|
2102
2176
|
|
|
2103
|
-
class
|
|
2104
|
-
"""Subsets
|
|
2177
|
+
class SubsetPhysicalDataModel(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
2178
|
+
"""Subsets physical data model to only include the specified views."""
|
|
2105
2179
|
|
|
2106
2180
|
def __init__(self, views: set[ViewEntity]):
|
|
2107
2181
|
self._views = views
|
|
2108
2182
|
|
|
2109
|
-
def transform(self,
|
|
2110
|
-
analysis =
|
|
2183
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
2184
|
+
analysis = DataModelAnalysis(physical=data_model)
|
|
2111
2185
|
|
|
2112
2186
|
views_by_view = analysis.view_by_view_entity
|
|
2113
2187
|
implements_by_view = analysis.implements_by_view()
|
|
@@ -2121,26 +2195,26 @@ class SubsetDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
2121
2195
|
subset = subset.union(ancestors)
|
|
2122
2196
|
|
|
2123
2197
|
if not subset:
|
|
2124
|
-
raise NeatValueError("None of the requested views are defined in the
|
|
2198
|
+
raise NeatValueError("None of the requested views are defined in the data_model!")
|
|
2125
2199
|
|
|
2126
2200
|
if nonexisting := self._views - subset:
|
|
2127
2201
|
raise NeatValueError(
|
|
2128
2202
|
"Following requested views do not exist"
|
|
2129
|
-
f" in the
|
|
2203
|
+
f" in the data_model: [{','.join([view.external_id for view in nonexisting])}]. Aborting."
|
|
2130
2204
|
)
|
|
2131
2205
|
|
|
2132
|
-
|
|
2133
|
-
"metadata":
|
|
2134
|
-
"views": SheetList[
|
|
2135
|
-
"properties": SheetList[
|
|
2136
|
-
"containers": SheetList[
|
|
2137
|
-
"enum":
|
|
2138
|
-
"nodes":
|
|
2206
|
+
subsetted_data_model: dict[str, Any] = {
|
|
2207
|
+
"metadata": data_model.metadata.model_copy(),
|
|
2208
|
+
"views": SheetList[PhysicalView](),
|
|
2209
|
+
"properties": SheetList[PhysicalProperty](),
|
|
2210
|
+
"containers": SheetList[PhysicalContainer](),
|
|
2211
|
+
"enum": data_model.enum,
|
|
2212
|
+
"nodes": data_model.nodes,
|
|
2139
2213
|
}
|
|
2140
2214
|
|
|
2141
2215
|
# add views
|
|
2142
2216
|
for view in subset:
|
|
2143
|
-
|
|
2217
|
+
subsetted_data_model["views"].append(views_by_view[view])
|
|
2144
2218
|
|
|
2145
2219
|
used_containers = set()
|
|
2146
2220
|
|
|
@@ -2152,85 +2226,85 @@ class SubsetDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
2152
2226
|
for property_ in properties:
|
|
2153
2227
|
if (
|
|
2154
2228
|
isinstance(property_.value_type, DataType)
|
|
2155
|
-
or isinstance(property_.value_type,
|
|
2229
|
+
or isinstance(property_.value_type, PhysicalUnknownEntity)
|
|
2156
2230
|
or (isinstance(property_.value_type, ViewEntity) and property_.value_type in subset)
|
|
2157
2231
|
):
|
|
2158
|
-
|
|
2232
|
+
subsetted_data_model["properties"].append(property_)
|
|
2159
2233
|
|
|
2160
2234
|
if property_.container:
|
|
2161
2235
|
used_containers.add(property_.container)
|
|
2162
2236
|
|
|
2163
2237
|
# add containers
|
|
2164
|
-
if
|
|
2165
|
-
for container in
|
|
2238
|
+
if data_model.containers:
|
|
2239
|
+
for container in data_model.containers:
|
|
2166
2240
|
if container.container in used_containers:
|
|
2167
|
-
|
|
2241
|
+
subsetted_data_model["containers"].append(container)
|
|
2168
2242
|
|
|
2169
2243
|
try:
|
|
2170
|
-
return
|
|
2244
|
+
return PhysicalDataModel.model_validate(subsetted_data_model)
|
|
2171
2245
|
except ValidationError as e:
|
|
2172
|
-
raise NeatValueError(f"Cannot subset
|
|
2246
|
+
raise NeatValueError(f"Cannot subset data_model: {e}") from e
|
|
2173
2247
|
|
|
2174
2248
|
|
|
2175
|
-
class
|
|
2176
|
-
"""Subsets
|
|
2249
|
+
class SubsetConceptualDataModel(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
2250
|
+
"""Subsets Conceptual Data Model to only include the specified concepts."""
|
|
2177
2251
|
|
|
2178
|
-
def __init__(self,
|
|
2179
|
-
self.
|
|
2252
|
+
def __init__(self, concepts: set[ConceptEntity]):
|
|
2253
|
+
self._concepts = concepts
|
|
2180
2254
|
|
|
2181
|
-
def transform(self,
|
|
2182
|
-
analysis =
|
|
2255
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
2256
|
+
analysis = DataModelAnalysis(conceptual=data_model)
|
|
2183
2257
|
|
|
2184
|
-
|
|
2185
|
-
|
|
2258
|
+
concept_by_concept_entity = analysis.concept_by_concept_entity
|
|
2259
|
+
parent_entity_by_concept_entity = analysis.parents_by_concept()
|
|
2186
2260
|
|
|
2187
|
-
available = analysis.
|
|
2188
|
-
subset = available.intersection(self.
|
|
2261
|
+
available = analysis.defined_concepts(include_ancestors=True)
|
|
2262
|
+
subset = available.intersection(self._concepts)
|
|
2189
2263
|
|
|
2190
2264
|
# need to add all the parent classes of the desired classes to the possible classes
|
|
2191
|
-
ancestors: set[
|
|
2192
|
-
for
|
|
2265
|
+
ancestors: set[ConceptEntity] = set()
|
|
2266
|
+
for concept in subset:
|
|
2193
2267
|
ancestors = ancestors.union(
|
|
2194
|
-
{ancestor for ancestor in get_inheritance_path(
|
|
2268
|
+
{ancestor for ancestor in get_inheritance_path(concept, parent_entity_by_concept_entity)}
|
|
2195
2269
|
)
|
|
2196
2270
|
subset = subset.union(ancestors)
|
|
2197
2271
|
|
|
2198
2272
|
if not subset:
|
|
2199
|
-
raise NeatValueError("None of the requested
|
|
2273
|
+
raise NeatValueError("None of the requested concepts are defined in the data_model!")
|
|
2200
2274
|
|
|
2201
|
-
if nonexisting := self.
|
|
2275
|
+
if nonexisting := self._concepts - subset:
|
|
2202
2276
|
raise NeatValueError(
|
|
2203
|
-
"Following requested
|
|
2204
|
-
f" in the
|
|
2277
|
+
"Following requested concepts do not exist"
|
|
2278
|
+
f" in the data_model: [{','.join([concept.suffix for concept in nonexisting])}]"
|
|
2205
2279
|
". Aborting."
|
|
2206
2280
|
)
|
|
2207
2281
|
|
|
2208
|
-
|
|
2209
|
-
"metadata":
|
|
2210
|
-
"prefixes": (
|
|
2211
|
-
"
|
|
2282
|
+
subsetted_data_model: dict[str, Any] = {
|
|
2283
|
+
"metadata": data_model.metadata.model_copy(),
|
|
2284
|
+
"prefixes": (data_model.prefixes or {}).copy(),
|
|
2285
|
+
"concepts": SheetList[Concept](),
|
|
2212
2286
|
"properties": SheetList[ConceptualProperty](),
|
|
2213
2287
|
}
|
|
2214
2288
|
|
|
2215
|
-
for
|
|
2216
|
-
|
|
2289
|
+
for concept in subset:
|
|
2290
|
+
subsetted_data_model["concepts"].append(concept_by_concept_entity[concept])
|
|
2217
2291
|
|
|
2218
|
-
for
|
|
2219
|
-
if
|
|
2292
|
+
for concept, properties in analysis.properties_by_concepts(include_ancestors=False).items():
|
|
2293
|
+
if concept not in subset:
|
|
2220
2294
|
continue
|
|
2221
2295
|
for property_ in properties:
|
|
2222
2296
|
# datatype property can be added directly
|
|
2223
2297
|
if (
|
|
2224
2298
|
isinstance(property_.value_type, DataType)
|
|
2225
|
-
or (isinstance(property_.value_type,
|
|
2299
|
+
or (isinstance(property_.value_type, ConceptEntity) and property_.value_type in subset)
|
|
2226
2300
|
or isinstance(property_.value_type, UnknownEntity)
|
|
2227
2301
|
):
|
|
2228
|
-
|
|
2302
|
+
subsetted_data_model["properties"].append(property_)
|
|
2229
2303
|
# object property can be added if the value type is in the subset
|
|
2230
2304
|
elif isinstance(property_.value_type, MultiValueTypeInfo):
|
|
2231
2305
|
allowed = [t for t in property_.value_type.types if t in subset or isinstance(t, DataType)]
|
|
2232
2306
|
if allowed:
|
|
2233
|
-
|
|
2307
|
+
subsetted_data_model["properties"].append(
|
|
2234
2308
|
property_.model_copy(
|
|
2235
2309
|
deep=True,
|
|
2236
2310
|
update={"value_type": MultiValueTypeInfo(types=allowed)},
|
|
@@ -2238,15 +2312,15 @@ class SubsetInformationRules(VerifiedRulesTransformer[ConceptualDataModel, Conce
|
|
|
2238
2312
|
)
|
|
2239
2313
|
|
|
2240
2314
|
try:
|
|
2241
|
-
return ConceptualDataModel.model_validate(
|
|
2315
|
+
return ConceptualDataModel.model_validate(subsetted_data_model)
|
|
2242
2316
|
except ValidationError as e:
|
|
2243
|
-
raise NeatValueError(f"Cannot subset
|
|
2317
|
+
raise NeatValueError(f"Cannot subset data_model: {e}") from e
|
|
2244
2318
|
|
|
2245
2319
|
|
|
2246
2320
|
class AddCogniteProperties(
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2321
|
+
DataModelTransformer[
|
|
2322
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
2323
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
2250
2324
|
]
|
|
2251
2325
|
):
|
|
2252
2326
|
"""This transformer looks at the implements of the classes and adds all properties
|
|
@@ -2267,114 +2341,125 @@ class AddCogniteProperties(
|
|
|
2267
2341
|
"""Get the description of the transformer."""
|
|
2268
2342
|
return "Add Cognite properties for all concepts that implements a Cognite concept."
|
|
2269
2343
|
|
|
2270
|
-
def transform(
|
|
2271
|
-
|
|
2344
|
+
def transform(
|
|
2345
|
+
self, data_model: ImportedDataModel[UnverifiedConceptualDataModel]
|
|
2346
|
+
) -> ImportedDataModel[UnverifiedConceptualDataModel]:
|
|
2347
|
+
input_ = data_model.unverified_data_model
|
|
2272
2348
|
if input_ is None:
|
|
2273
|
-
raise NeatValueError("Rule read failed. Cannot add cognite properties to None
|
|
2349
|
+
raise NeatValueError("Rule read failed. Cannot add cognite properties to None data_model.")
|
|
2274
2350
|
|
|
2275
2351
|
default_space = input_.metadata.space
|
|
2276
2352
|
default_version = input_.metadata.version
|
|
2277
2353
|
|
|
2278
|
-
|
|
2279
|
-
properties_by_class = self._get_properties_by_class(input_.properties, rules.read_context, default_space)
|
|
2280
|
-
cognite_implements_concepts = self._get_cognite_concepts(dependencies_by_class)
|
|
2281
|
-
views_by_class_entity = self._get_views_by_class(cognite_implements_concepts, default_space, default_version)
|
|
2354
|
+
dependencies_by_concept = self._get_dependencies_by_concepts(input_.concepts, data_model.context, default_space)
|
|
2282
2355
|
|
|
2283
|
-
|
|
2356
|
+
properties_by_concepts = self._get_properties_by_concepts(input_.properties, data_model.context, default_space)
|
|
2357
|
+
|
|
2358
|
+
cognite_implements_concepts = self._get_cognite_concepts(dependencies_by_concept)
|
|
2359
|
+
views_by_concept_entity = self._get_views_by_concept(
|
|
2360
|
+
cognite_implements_concepts, default_space, default_version
|
|
2361
|
+
)
|
|
2362
|
+
|
|
2363
|
+
for concept_entity, view in views_by_concept_entity.items():
|
|
2284
2364
|
for prop_id, view_prop in view.properties.items():
|
|
2285
|
-
if prop_id in
|
|
2365
|
+
if prop_id in properties_by_concepts[concept_entity]:
|
|
2286
2366
|
continue
|
|
2287
|
-
|
|
2288
|
-
|
|
2367
|
+
properties_by_concepts[concept_entity][prop_id] = DMSImporter.as_unverified_conceptual_property(
|
|
2368
|
+
concept_entity, prop_id, view_prop
|
|
2289
2369
|
)
|
|
2290
2370
|
|
|
2291
2371
|
try:
|
|
2292
|
-
topological_order = TopologicalSorter(
|
|
2372
|
+
topological_order = TopologicalSorter(dependencies_by_concept).static_order()
|
|
2293
2373
|
except CycleError as e:
|
|
2294
2374
|
raise NeatValueError(f"Cycle detected in the class hierarchy: {e}") from e
|
|
2295
2375
|
|
|
2296
2376
|
new_properties: list[UnverifiedConceptualProperty] = input_.properties.copy()
|
|
2297
|
-
for
|
|
2298
|
-
if
|
|
2377
|
+
for concept_entity in topological_order:
|
|
2378
|
+
if concept_entity not in dependencies_by_concept:
|
|
2299
2379
|
continue
|
|
2300
|
-
for parent in
|
|
2301
|
-
for prop in
|
|
2302
|
-
if prop.property_ not in
|
|
2303
|
-
new_prop = prop.copy(
|
|
2380
|
+
for parent in dependencies_by_concept[concept_entity]:
|
|
2381
|
+
for prop in properties_by_concepts[parent].values():
|
|
2382
|
+
if prop.property_ not in properties_by_concepts[concept_entity]:
|
|
2383
|
+
new_prop = prop.copy(
|
|
2384
|
+
update={"Concept": concept_entity},
|
|
2385
|
+
default_prefix=default_space,
|
|
2386
|
+
)
|
|
2304
2387
|
new_properties.append(new_prop)
|
|
2305
|
-
|
|
2388
|
+
properties_by_concepts[concept_entity][prop.property_] = new_prop
|
|
2306
2389
|
|
|
2307
2390
|
if self._dummy_property:
|
|
2308
2391
|
new_properties.append(
|
|
2309
2392
|
UnverifiedConceptualProperty(
|
|
2310
|
-
|
|
2311
|
-
property_=f"{to_camel_case(
|
|
2393
|
+
concept=concept_entity,
|
|
2394
|
+
property_=f"{to_camel_case(concept_entity.suffix)}{self._dummy_property}",
|
|
2312
2395
|
value_type=String(),
|
|
2313
2396
|
min_count=0,
|
|
2314
2397
|
max_count=1,
|
|
2315
2398
|
)
|
|
2316
2399
|
)
|
|
2317
2400
|
|
|
2318
|
-
new_classes: list[
|
|
2319
|
-
existing_classes = {cls.
|
|
2320
|
-
for
|
|
2321
|
-
if
|
|
2322
|
-
new_classes.append(DMSImporter.
|
|
2323
|
-
existing_classes.add(
|
|
2401
|
+
new_classes: list[UnverifiedConcept] = input_.concepts.copy()
|
|
2402
|
+
existing_classes = {cls.concept for cls in input_.concepts}
|
|
2403
|
+
for concept_entity, view in views_by_concept_entity.items():
|
|
2404
|
+
if concept_entity not in existing_classes:
|
|
2405
|
+
new_classes.append(DMSImporter.as_unverified_concept(view))
|
|
2406
|
+
existing_classes.add(concept_entity)
|
|
2324
2407
|
|
|
2325
|
-
return
|
|
2326
|
-
|
|
2408
|
+
return ImportedDataModel(
|
|
2409
|
+
unverified_data_model=UnverifiedConceptualDataModel(
|
|
2327
2410
|
metadata=input_.metadata,
|
|
2328
2411
|
properties=new_properties,
|
|
2329
|
-
|
|
2412
|
+
concepts=new_classes,
|
|
2330
2413
|
prefixes=input_.prefixes,
|
|
2331
2414
|
),
|
|
2332
|
-
|
|
2415
|
+
context={},
|
|
2333
2416
|
)
|
|
2334
2417
|
|
|
2335
2418
|
@staticmethod
|
|
2336
|
-
def
|
|
2419
|
+
def _get_properties_by_concepts(
|
|
2337
2420
|
properties: list[UnverifiedConceptualProperty],
|
|
2338
2421
|
read_context: dict[str, SpreadsheetRead],
|
|
2339
2422
|
default_space: str,
|
|
2340
|
-
) -> dict[
|
|
2423
|
+
) -> dict[ConceptEntity, dict[str, UnverifiedConceptualProperty]]:
|
|
2341
2424
|
issues = IssueList()
|
|
2342
|
-
properties_by_class: dict[
|
|
2425
|
+
properties_by_class: dict[ConceptEntity, dict[str, UnverifiedConceptualProperty]] = defaultdict(dict)
|
|
2343
2426
|
for prop in properties:
|
|
2344
2427
|
try:
|
|
2345
2428
|
dumped = prop.dump(default_prefix=default_space)
|
|
2346
2429
|
except ValidationError as e:
|
|
2347
2430
|
issues.extend(from_pydantic_errors(e.errors(), read_context))
|
|
2348
2431
|
continue
|
|
2349
|
-
|
|
2350
|
-
properties_by_class[
|
|
2432
|
+
concept_entity = cast(ConceptEntity, dumped["Concept"])
|
|
2433
|
+
properties_by_class[concept_entity][prop.property_] = prop
|
|
2351
2434
|
if issues.has_errors:
|
|
2352
2435
|
raise issues.as_errors(operation="Reading properties")
|
|
2353
2436
|
return properties_by_class
|
|
2354
2437
|
|
|
2355
2438
|
@staticmethod
|
|
2356
|
-
def
|
|
2357
|
-
|
|
2439
|
+
def _get_dependencies_by_concepts(
|
|
2440
|
+
concepts: list[UnverifiedConcept],
|
|
2358
2441
|
read_context: dict[str, SpreadsheetRead],
|
|
2359
2442
|
default_space: str,
|
|
2360
|
-
) -> dict[
|
|
2361
|
-
|
|
2443
|
+
) -> dict[ConceptEntity, set[ConceptEntity]]:
|
|
2444
|
+
dependencies_by_concepts: dict[ConceptEntity, set[ConceptEntity]] = {}
|
|
2362
2445
|
issues = IssueList()
|
|
2363
|
-
for raw in
|
|
2446
|
+
for raw in concepts:
|
|
2364
2447
|
try:
|
|
2365
2448
|
dumped = raw.dump(default_prefix=default_space)
|
|
2366
2449
|
except ValidationError as e:
|
|
2367
2450
|
issues.extend(from_pydantic_errors(e.errors(), read_context))
|
|
2368
2451
|
continue
|
|
2369
|
-
|
|
2370
|
-
implements = cast(list[
|
|
2371
|
-
|
|
2452
|
+
concept_entity = cast(ConceptEntity, dumped["Concept"])
|
|
2453
|
+
implements = cast(list[ConceptEntity] | None, dumped["Implements"])
|
|
2454
|
+
dependencies_by_concepts[concept_entity] = set(implements or [])
|
|
2372
2455
|
if issues.has_errors:
|
|
2373
2456
|
raise issues.as_errors(operation="Reading classes")
|
|
2374
|
-
return
|
|
2457
|
+
return dependencies_by_concepts
|
|
2375
2458
|
|
|
2376
2459
|
@staticmethod
|
|
2377
|
-
def _get_cognite_concepts(
|
|
2460
|
+
def _get_cognite_concepts(
|
|
2461
|
+
dependencies_by_class: dict[ConceptEntity, set[ConceptEntity]],
|
|
2462
|
+
) -> set[ConceptEntity]:
|
|
2378
2463
|
cognite_implements_concepts = {
|
|
2379
2464
|
dependency
|
|
2380
2465
|
for dependencies in dependencies_by_class.values()
|
|
@@ -2385,9 +2470,9 @@ class AddCogniteProperties(
|
|
|
2385
2470
|
raise NeatValueError("None of the classes implement Cognite Core concepts.")
|
|
2386
2471
|
return cognite_implements_concepts
|
|
2387
2472
|
|
|
2388
|
-
def
|
|
2389
|
-
self,
|
|
2390
|
-
) -> dict[
|
|
2391
|
-
view_ids = [
|
|
2473
|
+
def _get_views_by_concept(
|
|
2474
|
+
self, concepts: set[ConceptEntity], default_space: str, default_version: str
|
|
2475
|
+
) -> dict[ConceptEntity, View]:
|
|
2476
|
+
view_ids = [concept.as_view_entity(default_space, default_version).as_id() for concept in concepts]
|
|
2392
2477
|
views = self._client.loaders.views.retrieve(view_ids, include_ancestor=True, include_connected=True)
|
|
2393
|
-
return {
|
|
2478
|
+
return {ConceptEntity(prefix=view.space, suffix=view.external_id, version=view.version): view for view in views}
|