cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +10 -27
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +21 -45
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +3 -17
- cognite/neat/_rules/models/_base_rules.py +118 -62
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +20 -178
- cognite/neat/_rules/models/dms/_rules.py +65 -128
- cognite/neat/_rules/models/dms/_rules_input.py +72 -56
- cognite/neat/_rules/models/dms/_validation.py +16 -109
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +19 -122
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +3 -6
- cognite/neat/_rules/transformers/_converters.py +128 -206
- cognite/neat/_rules/transformers/_mapping.py +105 -34
- cognite/neat/_rules/transformers/_verification.py +5 -16
- cognite/neat/_session/_base.py +83 -21
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +22 -10
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +222 -27
- cognite/neat/_session/_read.py +109 -19
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +27 -14
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
|
@@ -4,11 +4,10 @@ from abc import ABC, abstractmethod
|
|
|
4
4
|
from collections import Counter, defaultdict
|
|
5
5
|
from collections.abc import Collection, Mapping
|
|
6
6
|
from datetime import date, datetime
|
|
7
|
-
from typing import Literal, TypeVar, cast
|
|
7
|
+
from typing import Literal, TypeVar, cast, overload
|
|
8
8
|
|
|
9
9
|
from cognite.client.data_classes import data_modeling as dms
|
|
10
10
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
|
|
11
|
-
from rdflib import Namespace
|
|
12
11
|
|
|
13
12
|
from cognite.neat._constants import (
|
|
14
13
|
COGNITE_MODELS,
|
|
@@ -20,8 +19,6 @@ from cognite.neat._issues.warnings._models import (
|
|
|
20
19
|
EnterpriseModelNotBuildOnTopOfCDMWarning,
|
|
21
20
|
SolutionModelBuildOnTopOfCDMWarning,
|
|
22
21
|
)
|
|
23
|
-
from cognite.neat._issues.warnings.user_modeling import ParentInDifferentSpaceWarning
|
|
24
|
-
from cognite.neat._rules._constants import EntityTypes
|
|
25
22
|
from cognite.neat._rules._shared import (
|
|
26
23
|
InputRules,
|
|
27
24
|
JustRules,
|
|
@@ -31,13 +28,9 @@ from cognite.neat._rules._shared import (
|
|
|
31
28
|
)
|
|
32
29
|
from cognite.neat._rules.analysis import DMSAnalysis
|
|
33
30
|
from cognite.neat._rules.models import (
|
|
34
|
-
AssetRules,
|
|
35
31
|
DMSInputRules,
|
|
36
32
|
DMSRules,
|
|
37
|
-
DomainRules,
|
|
38
|
-
ExtensionCategory,
|
|
39
33
|
InformationRules,
|
|
40
|
-
SchemaCompleteness,
|
|
41
34
|
SheetList,
|
|
42
35
|
data_types,
|
|
43
36
|
)
|
|
@@ -45,16 +38,14 @@ from cognite.neat._rules.models.data_types import DataType, String
|
|
|
45
38
|
from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSView
|
|
46
39
|
from cognite.neat._rules.models.dms._rules import DMSContainer
|
|
47
40
|
from cognite.neat._rules.models.entities import (
|
|
48
|
-
AssetEntity,
|
|
49
|
-
AssetFields,
|
|
50
41
|
ClassEntity,
|
|
51
42
|
ContainerEntity,
|
|
52
43
|
DMSUnknownEntity,
|
|
53
44
|
EdgeEntity,
|
|
45
|
+
Entity,
|
|
54
46
|
MultiValueTypeInfo,
|
|
55
|
-
ReferenceEntity,
|
|
56
|
-
RelationshipEntity,
|
|
57
47
|
ReverseConnectionEntity,
|
|
48
|
+
T_Entity,
|
|
58
49
|
UnknownEntity,
|
|
59
50
|
ViewEntity,
|
|
60
51
|
)
|
|
@@ -97,10 +88,8 @@ class ToCompliantEntities(RulesTransformer[InformationInputRules, InformationInp
|
|
|
97
88
|
return ReadRules(self._transform(self._to_rules(rules)), IssueList(), {})
|
|
98
89
|
|
|
99
90
|
def _transform(self, rules: InformationInputRules) -> InformationInputRules:
|
|
100
|
-
rules.metadata.prefix = self._fix_entity(rules.metadata.prefix)
|
|
101
91
|
rules.classes = self._fix_classes(rules.classes)
|
|
102
92
|
rules.properties = self._fix_properties(rules.properties)
|
|
103
|
-
|
|
104
93
|
rules.metadata.version += "_dms_compliant"
|
|
105
94
|
|
|
106
95
|
return rules
|
|
@@ -189,6 +178,74 @@ class ToCompliantEntities(RulesTransformer[InformationInputRules, InformationInp
|
|
|
189
178
|
return fixed_definitions
|
|
190
179
|
|
|
191
180
|
|
|
181
|
+
class PrefixEntities(RulesTransformer[InputRules, InputRules]): # type: ignore[misc]
|
|
182
|
+
"""Prefixes all entities with a given prefix."""
|
|
183
|
+
|
|
184
|
+
def __init__(self, prefix: str) -> None:
|
|
185
|
+
self._prefix = prefix
|
|
186
|
+
|
|
187
|
+
def transform(self, rules: InputRules | OutRules[InputRules]) -> ReadRules[InputRules]:
|
|
188
|
+
return ReadRules(self._transform(self._to_rules(rules)), IssueList(), {})
|
|
189
|
+
|
|
190
|
+
def _transform(self, rules: InputRules) -> InputRules:
|
|
191
|
+
rules.metadata.version += f"_prefixed_{self._prefix}"
|
|
192
|
+
|
|
193
|
+
if isinstance(rules, InformationInputRules):
|
|
194
|
+
# Todo Make Not mutate input class
|
|
195
|
+
prefixed_by_class: dict[str, str] = {}
|
|
196
|
+
for cls in rules.classes:
|
|
197
|
+
prefixed = str(self._with_prefix(cls.class_))
|
|
198
|
+
prefixed_by_class[str(cls.class_)] = prefixed
|
|
199
|
+
cls.class_ = prefixed
|
|
200
|
+
for prop in rules.properties:
|
|
201
|
+
prop.class_ = self._with_prefix(prop.class_)
|
|
202
|
+
if str(prop.value_type) in prefixed_by_class:
|
|
203
|
+
prop.value_type = prefixed_by_class[str(prop.value_type)]
|
|
204
|
+
return rules
|
|
205
|
+
elif isinstance(rules, DMSInputRules):
|
|
206
|
+
# Todo not mutate input class new_dms = copy.deepcopy(rules)
|
|
207
|
+
prefixed_by_view: dict[str, str] = {}
|
|
208
|
+
for view in rules.views:
|
|
209
|
+
prefixed = str(self._with_prefix(view.view))
|
|
210
|
+
prefixed_by_view[str(view.view)] = prefixed
|
|
211
|
+
view.view = prefixed
|
|
212
|
+
for dms_prop in rules.properties:
|
|
213
|
+
dms_prop.view = self._with_prefix(dms_prop.view)
|
|
214
|
+
if str(dms_prop.value_type) in prefixed_by_view:
|
|
215
|
+
dms_prop.value_type = prefixed_by_view[str(dms_prop.value_type)]
|
|
216
|
+
if rules.containers:
|
|
217
|
+
for container in rules.containers:
|
|
218
|
+
container.container = self._with_prefix(container.container)
|
|
219
|
+
return rules
|
|
220
|
+
raise NeatValueError(f"Unsupported rules type: {type(rules)}")
|
|
221
|
+
|
|
222
|
+
@overload
|
|
223
|
+
def _with_prefix(self, raw: str) -> str: ...
|
|
224
|
+
|
|
225
|
+
@overload
|
|
226
|
+
def _with_prefix(self, raw: T_Entity) -> T_Entity: ...
|
|
227
|
+
|
|
228
|
+
def _with_prefix(self, raw: str | T_Entity) -> str | T_Entity:
|
|
229
|
+
is_entity_format = not isinstance(raw, str)
|
|
230
|
+
entity = Entity.load(raw)
|
|
231
|
+
output: ClassEntity | ViewEntity | ContainerEntity
|
|
232
|
+
if isinstance(entity, ClassEntity):
|
|
233
|
+
output = ClassEntity(prefix=entity.prefix, suffix=f"{self._prefix}{entity.suffix}", version=entity.version)
|
|
234
|
+
elif isinstance(entity, ViewEntity):
|
|
235
|
+
output = ViewEntity(
|
|
236
|
+
space=entity.space, externalId=f"{self._prefix}{entity.external_id}", version=entity.version
|
|
237
|
+
)
|
|
238
|
+
elif isinstance(entity, ContainerEntity):
|
|
239
|
+
output = ContainerEntity(space=entity.space, externalId=f"{self._prefix}{entity.external_id}")
|
|
240
|
+
elif isinstance(entity, UnknownEntity | Entity):
|
|
241
|
+
return f"{self._prefix}{raw}"
|
|
242
|
+
else:
|
|
243
|
+
raise NeatValueError(f"Unsupported entity type: {type(entity)}")
|
|
244
|
+
if is_entity_format:
|
|
245
|
+
return cast(T_Entity, output)
|
|
246
|
+
return str(output)
|
|
247
|
+
|
|
248
|
+
|
|
192
249
|
class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
193
250
|
"""Converts InformationRules to DMSRules."""
|
|
194
251
|
|
|
@@ -199,20 +256,6 @@ class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
|
199
256
|
return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types)
|
|
200
257
|
|
|
201
258
|
|
|
202
|
-
class InformationToAsset(ConversionTransformer[InformationRules, AssetRules]):
|
|
203
|
-
"""Converts InformationRules to AssetRules."""
|
|
204
|
-
|
|
205
|
-
def _transform(self, rules: InformationRules) -> AssetRules:
|
|
206
|
-
return _InformationRulesConverter(rules).as_asset_architect_rules()
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
class AssetToInformation(ConversionTransformer[AssetRules, InformationRules]):
|
|
210
|
-
"""Converts AssetRules to InformationRules."""
|
|
211
|
-
|
|
212
|
-
def _transform(self, rules: AssetRules) -> InformationRules:
|
|
213
|
-
return InformationRules.model_validate(rules.model_dump())
|
|
214
|
-
|
|
215
|
-
|
|
216
259
|
class DMSToInformation(ConversionTransformer[DMSRules, InformationRules]):
|
|
217
260
|
"""Converts DMSRules to InformationRules."""
|
|
218
261
|
|
|
@@ -231,16 +274,8 @@ class ConvertToRules(ConversionTransformer[VerifiedRules, VerifiedRules]):
|
|
|
231
274
|
return rules
|
|
232
275
|
if isinstance(rules, InformationRules) and self._out_cls is DMSRules:
|
|
233
276
|
return InformationToDMS().transform(rules).rules
|
|
234
|
-
if isinstance(rules, InformationRules) and self._out_cls is AssetRules:
|
|
235
|
-
return InformationToAsset().transform(rules).rules
|
|
236
|
-
if isinstance(rules, AssetRules) and self._out_cls is InformationRules:
|
|
237
|
-
return AssetToInformation().transform(rules).rules
|
|
238
|
-
if isinstance(rules, AssetRules) and self._out_cls is DMSRules:
|
|
239
|
-
return InformationToDMS().transform(AssetToInformation().transform(rules)).rules
|
|
240
277
|
if isinstance(rules, DMSRules) and self._out_cls is InformationRules:
|
|
241
278
|
return DMSToInformation().transform(rules).rules
|
|
242
|
-
if isinstance(rules, DMSRules) and self._out_cls is AssetRules:
|
|
243
|
-
return InformationToAsset().transform(DMSToInformation().transform(rules)).rules
|
|
244
279
|
raise ValueError(f"Unsupported conversion from {type(rules)} to {self._out_cls}")
|
|
245
280
|
|
|
246
281
|
|
|
@@ -268,10 +303,11 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
268
303
|
self,
|
|
269
304
|
new_model_id: DataModelIdentifier,
|
|
270
305
|
org_name: str = "My",
|
|
271
|
-
type_: Literal["enterprise", "solution"] = "enterprise",
|
|
306
|
+
type_: Literal["enterprise", "solution", "data_product"] = "enterprise",
|
|
272
307
|
mode: Literal["read", "write"] = "read",
|
|
273
308
|
dummy_property: str = "GUID",
|
|
274
309
|
move_connections: bool = False,
|
|
310
|
+
include: Literal["same-space", "all"] = "same-space",
|
|
275
311
|
):
|
|
276
312
|
self.new_model_id = DataModelId.load(new_model_id)
|
|
277
313
|
if not self.new_model_id.version:
|
|
@@ -282,6 +318,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
282
318
|
self.type_ = type_
|
|
283
319
|
self.dummy_property = dummy_property
|
|
284
320
|
self.move_connections = move_connections
|
|
321
|
+
self.include = include
|
|
285
322
|
|
|
286
323
|
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
287
324
|
# Copy to ensure immutability
|
|
@@ -309,6 +346,16 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
309
346
|
)
|
|
310
347
|
|
|
311
348
|
return self._to_enterprise(reference_model)
|
|
349
|
+
elif self.type_ == "data_product":
|
|
350
|
+
expanded = self._expand_properties(reference_model.model_copy(deep=True))
|
|
351
|
+
if self.include == "same-space":
|
|
352
|
+
expanded.properties = SheetList[DMSProperty](
|
|
353
|
+
[prop for prop in expanded.properties if prop.view.space == expanded.metadata.space]
|
|
354
|
+
)
|
|
355
|
+
expanded.views = SheetList[DMSView](
|
|
356
|
+
[view for view in expanded.views if view.view.space == expanded.metadata.space]
|
|
357
|
+
)
|
|
358
|
+
return self._to_solution(expanded, remove_views_in_other_space=False)
|
|
312
359
|
|
|
313
360
|
else:
|
|
314
361
|
raise NeatValueError(f"Unsupported data model type: {self.type_}")
|
|
@@ -316,29 +363,22 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
316
363
|
def _has_views_in_multiple_space(self, rules: DMSRules) -> bool:
|
|
317
364
|
return any(view.view.space != rules.metadata.space for view in rules.views)
|
|
318
365
|
|
|
319
|
-
def _to_solution(self, reference_rules: DMSRules) -> JustRules[DMSRules]:
|
|
366
|
+
def _to_solution(self, reference_rules: DMSRules, remove_views_in_other_space: bool = True) -> JustRules[DMSRules]:
|
|
320
367
|
"""For creation of solution data model / rules specifically for mapping over existing containers."""
|
|
321
368
|
|
|
322
369
|
dump = reference_rules.dump()
|
|
323
370
|
|
|
324
371
|
# Prepare new model metadata prior validation
|
|
325
|
-
dump["metadata"]["schema_"] = SchemaCompleteness.partial.value
|
|
326
|
-
dump["metadata"]["data_model_type"] = self.type_
|
|
327
372
|
dump["metadata"]["name"] = f"{self.org_name} {self.type_} data model"
|
|
328
373
|
dump["metadata"]["space"] = self.new_model_id.space
|
|
329
374
|
dump["metadata"]["external_id"] = self.new_model_id.external_id
|
|
330
375
|
dump["metadata"]["version"] = self.new_model_id.version
|
|
331
376
|
|
|
332
|
-
# dropping reference and last from the dump as they can cause validation
|
|
333
|
-
# issues especially if reference is enterprise model build on top of CDM
|
|
334
|
-
dump.pop("reference", None)
|
|
335
|
-
dump.pop("last", None)
|
|
336
|
-
|
|
337
377
|
# Set implement to NONE for all views
|
|
338
378
|
for view in dump["views"]:
|
|
339
379
|
view["implements"] = None
|
|
340
380
|
|
|
341
|
-
if self._has_views_in_multiple_space(reference_rules):
|
|
381
|
+
if remove_views_in_other_space and self._has_views_in_multiple_space(reference_rules):
|
|
342
382
|
views_to_remove = []
|
|
343
383
|
for view in dump["views"]:
|
|
344
384
|
if ":" in view["view"]:
|
|
@@ -349,7 +389,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
349
389
|
solution_model = DMSRules.model_validate(DMSInputRules.load(dump).dump())
|
|
350
390
|
|
|
351
391
|
# Dropping containers coming from reference model
|
|
352
|
-
solution_model.containers =
|
|
392
|
+
solution_model.containers = None
|
|
353
393
|
|
|
354
394
|
# We want to map properties to existing containers allowing extension
|
|
355
395
|
for prop in solution_model.properties:
|
|
@@ -365,19 +405,17 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
365
405
|
# Remove Cognite affix in view external_id / suffix.
|
|
366
406
|
for prop in solution_model.properties:
|
|
367
407
|
prop.view = self._remove_cognite_affix(prop.view)
|
|
368
|
-
prop.class_ = self._remove_cognite_affix(prop.class_)
|
|
369
408
|
if isinstance(prop.value_type, ViewEntity):
|
|
370
409
|
prop.value_type = self._remove_cognite_affix(prop.value_type)
|
|
371
410
|
for view in solution_model.views:
|
|
372
411
|
view.view = self._remove_cognite_affix(view.view)
|
|
373
|
-
view.class_ = self._remove_cognite_affix(view.class_)
|
|
374
412
|
|
|
375
413
|
if self.mode == "write":
|
|
376
414
|
_, new_containers, new_properties = self._get_new_components(solution_model)
|
|
377
415
|
|
|
378
416
|
# Here we add ONLY dummy properties of the solution model and
|
|
379
417
|
# corresponding solution model space containers to hold them
|
|
380
|
-
solution_model.containers
|
|
418
|
+
solution_model.containers = new_containers
|
|
381
419
|
solution_model.properties.extend(new_properties)
|
|
382
420
|
|
|
383
421
|
return JustRules(solution_model)
|
|
@@ -385,9 +423,6 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
385
423
|
def _to_enterprise(self, reference_model: DMSRules) -> JustRules[DMSRules]:
|
|
386
424
|
dump = reference_model.dump()
|
|
387
425
|
|
|
388
|
-
# This is must prior model validation to avoid validation issues
|
|
389
|
-
dump["metadata"]["schema_"] = SchemaCompleteness.partial.value
|
|
390
|
-
|
|
391
426
|
# This will create reference model components in the enterprise model space
|
|
392
427
|
enterprise_model = DMSRules.model_validate(DMSInputRules.load(dump).dump())
|
|
393
428
|
|
|
@@ -398,9 +433,6 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
398
433
|
enterprise_model.metadata.external_id = self.new_model_id.external_id
|
|
399
434
|
enterprise_model.metadata.version = cast(str, self.new_model_id.version)
|
|
400
435
|
|
|
401
|
-
if reference_model.metadata.as_data_model_id() in COGNITE_MODELS:
|
|
402
|
-
enterprise_model.reference = reference_model
|
|
403
|
-
|
|
404
436
|
# Here we are creating enterprise specific components
|
|
405
437
|
enterprise_views, enterprise_containers, enterprise_properties = self._get_new_components(enterprise_model)
|
|
406
438
|
|
|
@@ -422,6 +454,28 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
422
454
|
|
|
423
455
|
return JustRules(enterprise_model)
|
|
424
456
|
|
|
457
|
+
@staticmethod
|
|
458
|
+
def _expand_properties(rules: DMSRules) -> DMSRules:
|
|
459
|
+
probe = DMSAnalysis(rules)
|
|
460
|
+
ancestor_properties_by_view = probe.classes_with_properties(
|
|
461
|
+
consider_inheritance=True, allow_different_namespace=True
|
|
462
|
+
)
|
|
463
|
+
property_ids_by_view = {
|
|
464
|
+
view: {prop.view_property for prop in properties}
|
|
465
|
+
for view, properties in probe.classes_with_properties(consider_inheritance=False).items()
|
|
466
|
+
}
|
|
467
|
+
for view, property_ids in property_ids_by_view.items():
|
|
468
|
+
ancestor_properties = ancestor_properties_by_view.get(view, [])
|
|
469
|
+
for prop in ancestor_properties:
|
|
470
|
+
if isinstance(prop.connection, ReverseConnectionEntity):
|
|
471
|
+
# If you try to add a reverse direct relation of a parent, it will fail as the ValueType of the
|
|
472
|
+
# original property will point to the parent view, and not the child.
|
|
473
|
+
continue
|
|
474
|
+
if prop.view_property not in property_ids:
|
|
475
|
+
rules.properties.append(prop)
|
|
476
|
+
property_ids.add(prop.view_property)
|
|
477
|
+
return rules
|
|
478
|
+
|
|
425
479
|
def _remove_cognite_affix(self, entity: _T_Entity) -> _T_Entity:
|
|
426
480
|
"""This method removes `Cognite` affix from the entity."""
|
|
427
481
|
new_suffix = entity.suffix.replace("Cognite", self.org_name or "")
|
|
@@ -444,19 +498,16 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
444
498
|
view_entity.version = cast(str, self.new_model_id.version)
|
|
445
499
|
view_entity.prefix = self.new_model_id.space
|
|
446
500
|
container_entity = ContainerEntity(space=view_entity.prefix, externalId=view_entity.external_id)
|
|
447
|
-
class_entity = ClassEntity(prefix=view_entity.prefix, suffix=view_entity.suffix)
|
|
448
501
|
|
|
449
502
|
view = DMSView(
|
|
450
503
|
view=view_entity,
|
|
451
504
|
implements=[definition.view],
|
|
452
505
|
in_model=True,
|
|
453
|
-
class_=class_entity,
|
|
454
506
|
name=definition.name,
|
|
455
507
|
)
|
|
456
508
|
|
|
457
509
|
container = DMSContainer(
|
|
458
510
|
container=container_entity,
|
|
459
|
-
class_=class_entity,
|
|
460
511
|
)
|
|
461
512
|
|
|
462
513
|
property_ = DMSProperty(
|
|
@@ -468,8 +519,6 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
468
519
|
is_list=False,
|
|
469
520
|
container=container_entity,
|
|
470
521
|
container_property=f"{to_camel(view_entity.suffix)}{self.dummy_property}",
|
|
471
|
-
class_=class_entity,
|
|
472
|
-
property_=f"{to_camel(view_entity.suffix)}{self.dummy_property}",
|
|
473
522
|
)
|
|
474
523
|
|
|
475
524
|
new_properties.append(property_)
|
|
@@ -569,12 +618,14 @@ class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
|
|
|
569
618
|
[view for view in new_model.views if view.view.as_id() not in exclude_views]
|
|
570
619
|
)
|
|
571
620
|
new_properties = SheetList[DMSProperty]()
|
|
621
|
+
|
|
572
622
|
for view in new_model.views:
|
|
573
623
|
for prop in properties_by_view[view.view]:
|
|
574
624
|
if self._is_asset_3D_property(prop):
|
|
575
625
|
# We filter out the 3D property of asset
|
|
576
626
|
continue
|
|
577
627
|
new_properties.append(prop)
|
|
628
|
+
|
|
578
629
|
new_model.properties = new_properties
|
|
579
630
|
|
|
580
631
|
return JustRules(new_model)
|
|
@@ -582,51 +633,14 @@ class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
|
|
|
582
633
|
def _is_asset_3D_property(self, prop: DMSProperty) -> bool:
|
|
583
634
|
if "3D" not in self.drop_collection:
|
|
584
635
|
return False
|
|
585
|
-
return prop.view.as_id() == self._ASSET_VIEW and prop.
|
|
636
|
+
return prop.view.as_id() == self._ASSET_VIEW and prop.view_property == "object3D"
|
|
586
637
|
|
|
587
638
|
|
|
588
639
|
class _InformationRulesConverter:
|
|
589
640
|
def __init__(self, information: InformationRules):
|
|
590
641
|
self.rules = information
|
|
591
|
-
self.is_addition = (
|
|
592
|
-
self.rules.metadata.schema_ is SchemaCompleteness.extended
|
|
593
|
-
and self.rules.metadata.extension is ExtensionCategory.addition
|
|
594
|
-
)
|
|
595
|
-
self.is_reshape = (
|
|
596
|
-
self.rules.metadata.schema_ is SchemaCompleteness.extended
|
|
597
|
-
and self.rules.metadata.extension is ExtensionCategory.reshape
|
|
598
|
-
)
|
|
599
|
-
if self.rules.last:
|
|
600
|
-
self.last_classes = {class_.class_: class_ for class_ in self.rules.last.classes}
|
|
601
|
-
else:
|
|
602
|
-
self.last_classes = {}
|
|
603
642
|
self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
|
|
604
643
|
|
|
605
|
-
def as_domain_rules(self) -> DomainRules:
|
|
606
|
-
raise NotImplementedError("DomainRules not implemented yet")
|
|
607
|
-
|
|
608
|
-
def as_asset_architect_rules(self) -> "AssetRules":
|
|
609
|
-
from cognite.neat._rules.models.asset._rules import AssetClass, AssetMetadata, AssetProperty, AssetRules
|
|
610
|
-
|
|
611
|
-
classes: SheetList[AssetClass] = SheetList[AssetClass](
|
|
612
|
-
[AssetClass(**class_.model_dump()) for class_ in self.rules.classes]
|
|
613
|
-
)
|
|
614
|
-
properties: SheetList[AssetProperty] = SheetList[AssetProperty]()
|
|
615
|
-
for prop_ in self.rules.properties:
|
|
616
|
-
if prop_.type_ == EntityTypes.data_property:
|
|
617
|
-
properties.append(
|
|
618
|
-
AssetProperty(**prop_.model_dump(), implementation=[AssetEntity(property=AssetFields.metadata)])
|
|
619
|
-
)
|
|
620
|
-
elif prop_.type_ == EntityTypes.object_property:
|
|
621
|
-
properties.append(AssetProperty(**prop_.model_dump(), implementation=[RelationshipEntity()]))
|
|
622
|
-
|
|
623
|
-
return AssetRules(
|
|
624
|
-
metadata=AssetMetadata(**self.rules.metadata.model_dump()),
|
|
625
|
-
properties=properties,
|
|
626
|
-
classes=classes,
|
|
627
|
-
prefixes=self.rules.prefixes,
|
|
628
|
-
)
|
|
629
|
-
|
|
630
644
|
def as_dms_rules(self, ignore_undefined_value_types: bool = False) -> "DMSRules":
|
|
631
645
|
from cognite.neat._rules.models.dms._rules import (
|
|
632
646
|
DMSContainer,
|
|
@@ -652,31 +666,17 @@ class _InformationRulesConverter:
|
|
|
652
666
|
|
|
653
667
|
views: list[DMSView] = [
|
|
654
668
|
DMSView(
|
|
655
|
-
class_=cls_.class_,
|
|
656
669
|
name=cls_.name,
|
|
657
670
|
view=cls_.class_.as_view_entity(default_space, default_version),
|
|
658
671
|
description=cls_.description,
|
|
659
|
-
reference=cls_.reference,
|
|
660
672
|
implements=self._get_view_implements(cls_, info_metadata),
|
|
661
673
|
)
|
|
662
674
|
for cls_ in self.rules.classes
|
|
663
675
|
]
|
|
664
676
|
|
|
665
|
-
last_dms_rules = _InformationRulesConverter(self.rules.last).as_dms_rules() if self.rules.last else None
|
|
666
|
-
ref_dms_rules = (
|
|
667
|
-
_InformationRulesConverter(self.rules.reference).as_dms_rules() if self.rules.reference else None
|
|
668
|
-
)
|
|
669
|
-
|
|
670
677
|
class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
|
|
671
|
-
if self.rules.last:
|
|
672
|
-
for cls_ in self.rules.last.classes:
|
|
673
|
-
if cls_.class_ not in class_by_entity:
|
|
674
|
-
class_by_entity[cls_.class_] = cls_
|
|
675
678
|
|
|
676
679
|
existing_containers: set[ContainerEntity] = set()
|
|
677
|
-
for rule_set in [last_dms_rules, ref_dms_rules]:
|
|
678
|
-
if rule_set:
|
|
679
|
-
existing_containers.update({c.container for c in rule_set.containers or []})
|
|
680
680
|
|
|
681
681
|
containers: list[DMSContainer] = []
|
|
682
682
|
for container_entity, class_entities in referenced_containers.items():
|
|
@@ -688,7 +688,6 @@ class _InformationRulesConverter:
|
|
|
688
688
|
most_used_class_entity = class_entities.most_common(1)[0][0]
|
|
689
689
|
class_ = class_by_entity[most_used_class_entity]
|
|
690
690
|
container = DMSContainer(
|
|
691
|
-
class_=class_.class_,
|
|
692
691
|
container=container_entity,
|
|
693
692
|
name=class_.name,
|
|
694
693
|
description=class_.description,
|
|
@@ -701,8 +700,6 @@ class _InformationRulesConverter:
|
|
|
701
700
|
properties=SheetList[DMSProperty]([prop for prop_set in properties_by_class.values() for prop in prop_set]),
|
|
702
701
|
views=SheetList[DMSView](views),
|
|
703
702
|
containers=SheetList[DMSContainer](containers),
|
|
704
|
-
last=last_dms_rules,
|
|
705
|
-
reference=ref_dms_rules,
|
|
706
703
|
)
|
|
707
704
|
|
|
708
705
|
@staticmethod
|
|
@@ -715,7 +712,7 @@ class _InformationRulesConverter:
|
|
|
715
712
|
constrains: list[ContainerEntity] = []
|
|
716
713
|
for entity in class_entities:
|
|
717
714
|
class_ = class_by_entity[entity]
|
|
718
|
-
for parent in class_.
|
|
715
|
+
for parent in class_.implements or []:
|
|
719
716
|
parent_entity = parent.as_container_entity(default_space)
|
|
720
717
|
if parent_entity in referenced_containers:
|
|
721
718
|
constrains.append(parent_entity)
|
|
@@ -727,14 +724,10 @@ class _InformationRulesConverter:
|
|
|
727
724
|
DMSMetadata,
|
|
728
725
|
)
|
|
729
726
|
|
|
730
|
-
space = cls._to_space(metadata.prefix)
|
|
731
|
-
|
|
732
727
|
return DMSMetadata(
|
|
733
|
-
|
|
734
|
-
space=space,
|
|
735
|
-
data_model_type=metadata.data_model_type,
|
|
728
|
+
space=metadata.space,
|
|
736
729
|
version=metadata.version,
|
|
737
|
-
external_id=metadata.
|
|
730
|
+
external_id=metadata.external_id,
|
|
738
731
|
creator=metadata.creator,
|
|
739
732
|
name=metadata.name,
|
|
740
733
|
created=metadata.created,
|
|
@@ -798,15 +791,12 @@ class _InformationRulesConverter:
|
|
|
798
791
|
container, container_property = self._get_container(prop, default_space)
|
|
799
792
|
|
|
800
793
|
return DMSProperty(
|
|
801
|
-
class_=prop.class_,
|
|
802
794
|
name=prop.name,
|
|
803
|
-
property_=prop.property_,
|
|
804
795
|
value_type=value_type,
|
|
805
796
|
nullable=nullable,
|
|
806
797
|
is_list=is_list,
|
|
807
798
|
connection=connection,
|
|
808
799
|
default=prop.default,
|
|
809
|
-
reference=prop.reference,
|
|
810
800
|
container=container,
|
|
811
801
|
container_property=container_property,
|
|
812
802
|
view=prop.class_.as_view_entity(default_space, default_version),
|
|
@@ -825,18 +815,7 @@ class _InformationRulesConverter:
|
|
|
825
815
|
return prefix
|
|
826
816
|
|
|
827
817
|
def _get_container(self, prop: InformationProperty, default_space: str) -> tuple[ContainerEntity, str]:
|
|
828
|
-
|
|
829
|
-
return (
|
|
830
|
-
prop.reference.as_container_entity(default_space),
|
|
831
|
-
prop.reference.property_ or prop.property_,
|
|
832
|
-
)
|
|
833
|
-
elif (self.is_addition or self.is_reshape) and prop.class_ in self.last_classes:
|
|
834
|
-
# We need to create a new container for the property, as we cannot change
|
|
835
|
-
# the existing container in the last schema
|
|
836
|
-
container_entity = prop.class_.as_container_entity(default_space)
|
|
837
|
-
container_entity.suffix = self._bump_suffix(container_entity.suffix)
|
|
838
|
-
else:
|
|
839
|
-
container_entity = prop.class_.as_container_entity(default_space)
|
|
818
|
+
container_entity = prop.class_.as_container_entity(default_space)
|
|
840
819
|
|
|
841
820
|
while self.property_count_by_container[container_entity] >= DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
|
|
842
821
|
container_entity.suffix = self._bump_suffix(container_entity.suffix)
|
|
@@ -845,27 +824,9 @@ class _InformationRulesConverter:
|
|
|
845
824
|
return container_entity, prop.property_
|
|
846
825
|
|
|
847
826
|
def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
implements = [
|
|
852
|
-
cls_.reference.as_view_entity(
|
|
853
|
-
self.rules.reference.metadata.prefix, self.rules.reference.metadata.version
|
|
854
|
-
)
|
|
855
|
-
]
|
|
856
|
-
else:
|
|
857
|
-
implements = [
|
|
858
|
-
cls_.reference.as_view_entity(metadata.prefix, metadata.version),
|
|
859
|
-
]
|
|
860
|
-
else:
|
|
861
|
-
implements = []
|
|
862
|
-
for parent in cls_.parent or []:
|
|
863
|
-
if self.rules.reference and parent.prefix == self.rules.reference.metadata.prefix:
|
|
864
|
-
view_entity = parent.as_view_entity(
|
|
865
|
-
self.rules.reference.metadata.prefix, self.rules.reference.metadata.version
|
|
866
|
-
)
|
|
867
|
-
else:
|
|
868
|
-
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
827
|
+
implements = []
|
|
828
|
+
for parent in cls_.implements or []:
|
|
829
|
+
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
869
830
|
implements.append(view_entity)
|
|
870
831
|
return implements
|
|
871
832
|
|
|
@@ -906,9 +867,6 @@ class _DMSRulesConverter:
|
|
|
906
867
|
def __init__(self, dms: DMSRules):
|
|
907
868
|
self.dms = dms
|
|
908
869
|
|
|
909
|
-
def as_domain_rules(self) -> "DomainRules":
|
|
910
|
-
raise NotImplementedError("DomainRules not implemented yet")
|
|
911
|
-
|
|
912
870
|
def as_information_rules(
|
|
913
871
|
self,
|
|
914
872
|
) -> "InformationRules":
|
|
@@ -925,16 +883,13 @@ class _DMSRulesConverter:
|
|
|
925
883
|
classes = [
|
|
926
884
|
InformationClass(
|
|
927
885
|
# we do not want a version in class as we use URI for the class
|
|
928
|
-
class_=ClassEntity(prefix=view.
|
|
886
|
+
class_=ClassEntity(prefix=view.view.prefix, suffix=view.view.suffix),
|
|
929
887
|
description=view.description,
|
|
930
|
-
|
|
888
|
+
implements=[
|
|
931
889
|
# we do not want a version in class as we use URI for the class
|
|
932
890
|
implemented_view.as_class(skip_version=True)
|
|
933
|
-
# We only want parents in the same namespace, parent in a different namespace is a reference
|
|
934
891
|
for implemented_view in view.implements or []
|
|
935
|
-
if implemented_view.prefix == view.class_.prefix
|
|
936
892
|
],
|
|
937
|
-
reference=self._get_class_reference(view),
|
|
938
893
|
)
|
|
939
894
|
for view in self.dms.views
|
|
940
895
|
]
|
|
@@ -957,13 +912,12 @@ class _DMSRulesConverter:
|
|
|
957
912
|
properties.append(
|
|
958
913
|
InformationProperty(
|
|
959
914
|
# Removing version
|
|
960
|
-
class_=ClassEntity(suffix=property_.
|
|
915
|
+
class_=ClassEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
|
|
961
916
|
property_=property_.view_property,
|
|
962
917
|
value_type=value_type,
|
|
963
918
|
description=property_.description,
|
|
964
|
-
min_count=0 if property_.nullable or property_.nullable is None else 1,
|
|
965
|
-
max_count=float("inf") if property_.is_list or property_.nullable is None else 1,
|
|
966
|
-
reference=self._get_property_reference(property_),
|
|
919
|
+
min_count=(0 if property_.nullable or property_.nullable is None else 1),
|
|
920
|
+
max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
|
|
967
921
|
)
|
|
968
922
|
)
|
|
969
923
|
|
|
@@ -971,51 +925,19 @@ class _DMSRulesConverter:
|
|
|
971
925
|
metadata=metadata,
|
|
972
926
|
properties=SheetList[InformationProperty](properties),
|
|
973
927
|
classes=SheetList[InformationClass](classes),
|
|
974
|
-
last=_DMSRulesConverter(self.dms.last).as_information_rules() if self.dms.last else None,
|
|
975
|
-
reference=_DMSRulesConverter(self.dms.reference).as_information_rules() if self.dms.reference else None,
|
|
976
928
|
)
|
|
977
929
|
|
|
978
930
|
@classmethod
|
|
979
931
|
def _convert_metadata_to_info(cls, metadata: DMSMetadata) -> "InformationMetadata":
|
|
980
932
|
from cognite.neat._rules.models.information._rules import InformationMetadata
|
|
981
933
|
|
|
982
|
-
prefix = metadata.space
|
|
983
934
|
return InformationMetadata(
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
extension=metadata.extension,
|
|
987
|
-
prefix=prefix,
|
|
988
|
-
namespace=Namespace(f"https://purl.orgl/neat/{prefix}/"),
|
|
935
|
+
space=metadata.space,
|
|
936
|
+
external_id=metadata.external_id,
|
|
989
937
|
version=metadata.version,
|
|
990
938
|
description=metadata.description,
|
|
991
|
-
name=metadata.name
|
|
939
|
+
name=metadata.name,
|
|
992
940
|
creator=metadata.creator,
|
|
993
941
|
created=metadata.created,
|
|
994
942
|
updated=metadata.updated,
|
|
995
943
|
)
|
|
996
|
-
|
|
997
|
-
@classmethod
|
|
998
|
-
def _get_class_reference(cls, view: DMSView) -> ReferenceEntity | None:
|
|
999
|
-
parents_other_namespace = [parent for parent in view.implements or [] if parent.prefix != view.class_.prefix]
|
|
1000
|
-
if len(parents_other_namespace) == 0:
|
|
1001
|
-
return None
|
|
1002
|
-
if len(parents_other_namespace) > 1:
|
|
1003
|
-
warnings.warn(
|
|
1004
|
-
ParentInDifferentSpaceWarning(view.view.as_id()),
|
|
1005
|
-
stacklevel=2,
|
|
1006
|
-
)
|
|
1007
|
-
other_parent = parents_other_namespace[0]
|
|
1008
|
-
|
|
1009
|
-
return ReferenceEntity(prefix=other_parent.prefix, suffix=other_parent.suffix)
|
|
1010
|
-
|
|
1011
|
-
@classmethod
|
|
1012
|
-
def _get_property_reference(cls, property_: DMSProperty) -> ReferenceEntity | None:
|
|
1013
|
-
has_container_other_namespace = property_.container and property_.container.prefix != property_.class_.prefix
|
|
1014
|
-
if not has_container_other_namespace:
|
|
1015
|
-
return None
|
|
1016
|
-
container = cast(ContainerEntity, property_.container)
|
|
1017
|
-
return ReferenceEntity(
|
|
1018
|
-
prefix=container.prefix,
|
|
1019
|
-
suffix=container.suffix,
|
|
1020
|
-
property=property_.container_property,
|
|
1021
|
-
)
|