cognite-neat 0.121.1__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +1 -1
- cognite/neat/core/_data_model/exporters/__init__.py +5 -5
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +4 -4
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +6 -6
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +40 -20
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +9 -9
- cognite/neat/core/_data_model/importers/_rdf/_imf2rules.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/_inference2rules.py +36 -36
- cognite/neat/core/_data_model/importers/_rdf/_owl2rules.py +12 -12
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +72 -12
- cognite/neat/core/_data_model/models/__init__.py +8 -8
- cognite/neat/core/_data_model/models/_base_unverified.py +1 -1
- cognite/neat/core/_data_model/models/_base_verified.py +3 -3
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +87 -77
- cognite/neat/core/_data_model/models/conceptual/_verified.py +53 -51
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +71 -52
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +13 -11
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +27 -23
- cognite/neat/core/_data_model/transformers/_base.py +26 -19
- cognite/neat/core/_data_model/transformers/_converters.py +703 -618
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +63 -54
- cognite/neat/core/_instances/extractors/_base.py +1 -1
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +8 -8
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +98 -95
- cognite/neat/core/_instances/loaders/_base.py +2 -2
- cognite/neat/core/_instances/loaders/_rdf2dms.py +6 -6
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +22 -20
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +8 -6
- cognite/neat/session/_read.py +2 -2
- cognite/neat/session/_set.py +3 -3
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +13 -13
- cognite/neat/session/_subset.py +12 -9
- cognite/neat/session/_template.py +13 -13
- cognite/neat/session/_to.py +17 -17
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/RECORD +95 -93
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -33,34 +33,34 @@ from cognite.neat.core._constants import (
|
|
|
33
33
|
DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
|
|
34
34
|
DMS_PRIMITIVE_LIST_DEFAULT_LIMIT,
|
|
35
35
|
)
|
|
36
|
-
from cognite.neat.core._data_model._shared import
|
|
36
|
+
from cognite.neat.core._data_model._shared import ImportedDataModel
|
|
37
37
|
from cognite.neat.core._data_model.importers._base import BaseImporter
|
|
38
38
|
from cognite.neat.core._data_model.models import (
|
|
39
|
-
DMSInputRules,
|
|
40
39
|
DMSSchema,
|
|
40
|
+
UnverifiedPhysicalDataModel,
|
|
41
41
|
)
|
|
42
42
|
from cognite.neat.core._data_model.models.conceptual import (
|
|
43
|
-
|
|
43
|
+
UnverifiedConcept,
|
|
44
44
|
UnverifiedConceptualProperty,
|
|
45
45
|
)
|
|
46
46
|
from cognite.neat.core._data_model.models.data_types import DataType, Enum, String
|
|
47
|
-
from cognite.neat.core._data_model.models.dms import (
|
|
48
|
-
DMSInputContainer,
|
|
49
|
-
DMSInputEnum,
|
|
50
|
-
DMSInputMetadata,
|
|
51
|
-
DMSInputNode,
|
|
52
|
-
DMSInputProperty,
|
|
53
|
-
DMSInputView,
|
|
54
|
-
)
|
|
55
47
|
from cognite.neat.core._data_model.models.entities import (
|
|
56
|
-
|
|
48
|
+
ConceptEntity,
|
|
57
49
|
ContainerEntity,
|
|
58
50
|
DMSNodeEntity,
|
|
59
|
-
DMSUnknownEntity,
|
|
60
51
|
EdgeEntity,
|
|
52
|
+
PhysicalUnknownEntity,
|
|
61
53
|
ReverseConnectionEntity,
|
|
62
54
|
ViewEntity,
|
|
63
55
|
)
|
|
56
|
+
from cognite.neat.core._data_model.models.physical import (
|
|
57
|
+
UnverifiedPhysicalContainer,
|
|
58
|
+
UnverifiedPhysicalEnum,
|
|
59
|
+
UnverifiedPhysicalMetadata,
|
|
60
|
+
UnverifiedPhysicalNodeType,
|
|
61
|
+
UnverifiedPhysicalProperty,
|
|
62
|
+
UnverifiedPhysicalView,
|
|
63
|
+
)
|
|
64
64
|
from cognite.neat.core._issues import (
|
|
65
65
|
IssueList,
|
|
66
66
|
MultiValueError,
|
|
@@ -85,7 +85,7 @@ from cognite.neat.core._issues.warnings import (
|
|
|
85
85
|
)
|
|
86
86
|
|
|
87
87
|
|
|
88
|
-
class DMSImporter(BaseImporter[
|
|
88
|
+
class DMSImporter(BaseImporter[UnverifiedPhysicalDataModel]):
|
|
89
89
|
"""Imports a Data Model from Cognite Data Fusion.
|
|
90
90
|
|
|
91
91
|
Args:
|
|
@@ -99,7 +99,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
99
99
|
self,
|
|
100
100
|
schema: DMSSchema,
|
|
101
101
|
read_issues: Sequence[NeatIssue] | None = None,
|
|
102
|
-
metadata:
|
|
102
|
+
metadata: UnverifiedPhysicalMetadata | None = None,
|
|
103
103
|
referenced_containers: Iterable[dm.ContainerApply] | None = None,
|
|
104
104
|
):
|
|
105
105
|
self.schema = schema
|
|
@@ -127,7 +127,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
127
127
|
client: NeatClient,
|
|
128
128
|
data_model_id: DataModelIdentifier,
|
|
129
129
|
) -> "DMSImporter":
|
|
130
|
-
"""Create a DMSImporter ready to convert the given data model to
|
|
130
|
+
"""Create a DMSImporter ready to convert the given DMS data model to neat representation.
|
|
131
131
|
|
|
132
132
|
Args:
|
|
133
133
|
client: Instantiated CogniteClient to retrieve data model.
|
|
@@ -188,8 +188,8 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
188
188
|
def _create_metadata_from_model(
|
|
189
189
|
cls,
|
|
190
190
|
model: dm.DataModel[dm.View] | dm.DataModelApply,
|
|
191
|
-
) ->
|
|
192
|
-
description, creator =
|
|
191
|
+
) -> UnverifiedPhysicalMetadata:
|
|
192
|
+
description, creator = UnverifiedPhysicalMetadata._get_description_and_creator(model.description)
|
|
193
193
|
|
|
194
194
|
if isinstance(model, dm.DataModel):
|
|
195
195
|
created = ms_to_datetime(model.created_time)
|
|
@@ -198,7 +198,8 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
198
198
|
now = datetime.now().replace(microsecond=0)
|
|
199
199
|
created = now
|
|
200
200
|
updated = now
|
|
201
|
-
|
|
201
|
+
|
|
202
|
+
return UnverifiedPhysicalMetadata(
|
|
202
203
|
space=model.space,
|
|
203
204
|
external_id=model.external_id,
|
|
204
205
|
name=model.name or model.external_id,
|
|
@@ -213,7 +214,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
213
214
|
def from_directory(cls, directory: str | Path, client: NeatClient | None = None) -> "DMSImporter":
|
|
214
215
|
with catch_issues() as issue_list:
|
|
215
216
|
schema = DMSSchema.from_directory(directory)
|
|
216
|
-
# If there were errors during the import, the
|
|
217
|
+
# If there were errors during the import, the to_data_model will raise them.
|
|
217
218
|
return cls(
|
|
218
219
|
schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
|
|
219
220
|
)
|
|
@@ -255,9 +256,9 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
255
256
|
else:
|
|
256
257
|
raise NeatValueError(f"Unsupported YAML format: {format}")
|
|
257
258
|
|
|
258
|
-
def
|
|
259
|
+
def to_data_model(self) -> ImportedDataModel[UnverifiedPhysicalDataModel]:
|
|
259
260
|
if self.issue_list.has_errors:
|
|
260
|
-
# In case there were errors during the import, the
|
|
261
|
+
# In case there were errors during the import, the to_data_model method will return None
|
|
261
262
|
self.issue_list.trigger_warnings()
|
|
262
263
|
raise MultiValueError(self.issue_list.errors)
|
|
263
264
|
|
|
@@ -268,25 +269,25 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
268
269
|
|
|
269
270
|
model = self.schema.data_model
|
|
270
271
|
|
|
271
|
-
|
|
272
|
+
user_data_model = self._create_rule_components(model, self.schema, self.metadata)
|
|
272
273
|
|
|
273
274
|
self.issue_list.trigger_warnings()
|
|
274
275
|
if self.issue_list.has_errors:
|
|
275
276
|
raise MultiValueError(self.issue_list.errors)
|
|
276
|
-
return
|
|
277
|
+
return ImportedDataModel(user_data_model, {})
|
|
277
278
|
|
|
278
279
|
def _create_rule_components(
|
|
279
280
|
self,
|
|
280
281
|
data_model: dm.DataModelApply,
|
|
281
282
|
schema: DMSSchema,
|
|
282
|
-
metadata:
|
|
283
|
-
) ->
|
|
283
|
+
metadata: UnverifiedPhysicalMetadata | None = None,
|
|
284
|
+
) -> UnverifiedPhysicalDataModel:
|
|
284
285
|
enum_by_container_property = self._create_enum_collections(self._all_containers_by_id.values())
|
|
285
286
|
enum_collection_by_container_property = {
|
|
286
287
|
key: enum_list[0].collection for key, enum_list in enum_by_container_property.items() if enum_list
|
|
287
288
|
}
|
|
288
289
|
|
|
289
|
-
properties: list[
|
|
290
|
+
properties: list[UnverifiedPhysicalProperty] = []
|
|
290
291
|
for view_id, view in schema.views.items():
|
|
291
292
|
view_entity = ViewEntity.from_id(view_id)
|
|
292
293
|
for prop_id, prop in (view.properties or {}).items():
|
|
@@ -300,17 +301,19 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
300
301
|
view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
|
|
301
302
|
}
|
|
302
303
|
|
|
303
|
-
metadata = metadata or
|
|
304
|
+
metadata = metadata or UnverifiedPhysicalMetadata.from_data_model(data_model)
|
|
304
305
|
|
|
305
|
-
return
|
|
306
|
+
return UnverifiedPhysicalDataModel(
|
|
306
307
|
metadata=metadata,
|
|
307
308
|
properties=properties,
|
|
308
|
-
containers=[
|
|
309
|
+
containers=[
|
|
310
|
+
UnverifiedPhysicalContainer.from_container(container) for container in schema.containers.values()
|
|
311
|
+
],
|
|
309
312
|
views=[
|
|
310
|
-
|
|
313
|
+
UnverifiedPhysicalView.from_view(view, in_model=view_id in data_model_view_ids)
|
|
311
314
|
for view_id, view in schema.views.items()
|
|
312
315
|
],
|
|
313
|
-
nodes=[
|
|
316
|
+
nodes=[UnverifiedPhysicalNodeType.from_node_type(node_type) for node_type in schema.node_types.values()],
|
|
314
317
|
enum=[enum for enum_list in enum_by_container_property.values() for enum in enum_list] or None,
|
|
315
318
|
)
|
|
316
319
|
|
|
@@ -320,7 +323,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
320
323
|
prop: ViewPropertyApply,
|
|
321
324
|
view_entity: ViewEntity,
|
|
322
325
|
enum_collection_by_container_property: dict[tuple[dm.ContainerId, str], str],
|
|
323
|
-
) ->
|
|
326
|
+
) -> UnverifiedPhysicalProperty | None:
|
|
324
327
|
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
|
|
325
328
|
self.issue_list.append(
|
|
326
329
|
ResourceNotFoundWarning[dm.ContainerId, dm.PropertyId](
|
|
@@ -364,7 +367,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
364
367
|
if isinstance(value_type, ViewEntity) and value_type.as_id() not in self._all_views_by_id:
|
|
365
368
|
self.issue_list.append(ResourceUnknownWarning(prop.source, "view", view_entity.as_id(), "view"))
|
|
366
369
|
|
|
367
|
-
return
|
|
370
|
+
return UnverifiedPhysicalProperty(
|
|
368
371
|
description=prop.description,
|
|
369
372
|
name=prop.name,
|
|
370
373
|
connection=self._get_connection_type(prop),
|
|
@@ -411,8 +414,8 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
411
414
|
cls,
|
|
412
415
|
prop: ViewPropertyApply | ViewProperty,
|
|
413
416
|
container_property: dm.ContainerProperty | None = None,
|
|
414
|
-
enum_collection_by_container_property: dict[tuple[dm.ContainerId, str], str] | None = None,
|
|
415
|
-
) -> DataType | ViewEntity |
|
|
417
|
+
enum_collection_by_container_property: (dict[tuple[dm.ContainerId, str], str] | None) = None,
|
|
418
|
+
) -> DataType | ViewEntity | PhysicalUnknownEntity | None:
|
|
416
419
|
if isinstance(
|
|
417
420
|
prop,
|
|
418
421
|
SingleEdgeConnectionApply
|
|
@@ -434,7 +437,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
434
437
|
prop_type = prop.type
|
|
435
438
|
if isinstance(prop_type, dm.DirectRelation):
|
|
436
439
|
if prop.source is None:
|
|
437
|
-
return
|
|
440
|
+
return PhysicalUnknownEntity()
|
|
438
441
|
else:
|
|
439
442
|
return ViewEntity.from_id(prop.source)
|
|
440
443
|
elif isinstance(prop_type, PropertyTypeWithUnit) and prop_type.unit:
|
|
@@ -451,7 +454,10 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
451
454
|
f"BUG in Neat: Enum for {prop.container}.{prop.container_property_identifier} not found."
|
|
452
455
|
)
|
|
453
456
|
|
|
454
|
-
return Enum(
|
|
457
|
+
return Enum(
|
|
458
|
+
collection=ConceptEntity(suffix=collection),
|
|
459
|
+
unknownValue=prop_type.unknown_value,
|
|
460
|
+
)
|
|
455
461
|
else:
|
|
456
462
|
return DataType.load(prop_type._type)
|
|
457
463
|
else:
|
|
@@ -614,8 +620,8 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
614
620
|
@staticmethod
|
|
615
621
|
def _create_enum_collections(
|
|
616
622
|
containers: Collection[dm.ContainerApply],
|
|
617
|
-
) -> dict[tuple[dm.ContainerId, str], list[
|
|
618
|
-
enum_by_container_property: dict[tuple[dm.ContainerId, str], list[
|
|
623
|
+
) -> dict[tuple[dm.ContainerId, str], list[UnverifiedPhysicalEnum]]:
|
|
624
|
+
enum_by_container_property: dict[tuple[dm.ContainerId, str], list[UnverifiedPhysicalEnum]] = defaultdict(list)
|
|
619
625
|
|
|
620
626
|
is_external_id_unique = len({container.external_id for container in containers}) == len(containers)
|
|
621
627
|
|
|
@@ -630,15 +636,18 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
630
636
|
collection = f"{container.space}:{container.external_id}.{prop_id}"
|
|
631
637
|
for identifier, value in prop.type.values.items():
|
|
632
638
|
enum_by_container_property[(container_id, prop_id)].append(
|
|
633
|
-
|
|
634
|
-
collection=collection,
|
|
639
|
+
UnverifiedPhysicalEnum(
|
|
640
|
+
collection=collection,
|
|
641
|
+
value=identifier,
|
|
642
|
+
name=value.name,
|
|
643
|
+
description=value.description,
|
|
635
644
|
)
|
|
636
645
|
)
|
|
637
646
|
return enum_by_container_property
|
|
638
647
|
|
|
639
648
|
@classmethod
|
|
640
|
-
def
|
|
641
|
-
cls, entity:
|
|
649
|
+
def as_unverified_conceptual_property(
|
|
650
|
+
cls, entity: ConceptEntity, prop_id: str, view_property: ViewProperty
|
|
642
651
|
) -> UnverifiedConceptualProperty:
|
|
643
652
|
if not isinstance(view_property, dm.MappedProperty | dm.EdgeConnection | ReverseDirectRelation):
|
|
644
653
|
raise PropertyTypeNotSupportedError(
|
|
@@ -653,7 +662,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
653
662
|
raise NeatValueError(f"Failed to get value type for {entity} property {prop_id}")
|
|
654
663
|
|
|
655
664
|
return UnverifiedConceptualProperty(
|
|
656
|
-
|
|
665
|
+
concept=entity,
|
|
657
666
|
property_=prop_id,
|
|
658
667
|
value_type=str(value_type),
|
|
659
668
|
name=view_property.name,
|
|
@@ -664,13 +673,13 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
664
673
|
)
|
|
665
674
|
|
|
666
675
|
@classmethod
|
|
667
|
-
def
|
|
668
|
-
return
|
|
669
|
-
|
|
676
|
+
def as_unverified_concept(cls, view: View) -> UnverifiedConcept:
|
|
677
|
+
return UnverifiedConcept(
|
|
678
|
+
concept=ConceptEntity(prefix=view.space, suffix=view.external_id, version=view.version),
|
|
670
679
|
name=view.name,
|
|
671
680
|
description=view.description,
|
|
672
681
|
implements=[
|
|
673
|
-
|
|
682
|
+
ConceptEntity(
|
|
674
683
|
prefix=parent.space,
|
|
675
684
|
suffix=parent.external_id,
|
|
676
685
|
version=parent.version,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
3
|
|
|
4
|
-
from cognite.neat.core._data_model.importers.
|
|
4
|
+
from cognite.neat.core._data_model.importers._dtdl2data_model.spec import (
|
|
5
5
|
DTMI,
|
|
6
6
|
Command,
|
|
7
7
|
CommandV2,
|
|
@@ -19,7 +19,7 @@ from cognite.neat.core._data_model.importers._dtdl2rules.spec import (
|
|
|
19
19
|
TelemetryV2,
|
|
20
20
|
)
|
|
21
21
|
from cognite.neat.core._data_model.models.conceptual import (
|
|
22
|
-
|
|
22
|
+
UnverifiedConcept,
|
|
23
23
|
UnverifiedConceptualProperty,
|
|
24
24
|
)
|
|
25
25
|
from cognite.neat.core._data_model.models.data_types import (
|
|
@@ -28,7 +28,7 @@ from cognite.neat.core._data_model.models.data_types import (
|
|
|
28
28
|
Json,
|
|
29
29
|
String,
|
|
30
30
|
)
|
|
31
|
-
from cognite.neat.core._data_model.models.entities import
|
|
31
|
+
from cognite.neat.core._data_model.models.entities import ConceptEntity
|
|
32
32
|
from cognite.neat.core._issues import IssueList
|
|
33
33
|
from cognite.neat.core._issues.errors import (
|
|
34
34
|
PropertyTypeNotSupportedError,
|
|
@@ -45,7 +45,7 @@ class _DTDLConverter:
|
|
|
45
45
|
def __init__(self, issues: IssueList | None = None) -> None:
|
|
46
46
|
self.issues = IssueList(issues or [])
|
|
47
47
|
self.properties: list[UnverifiedConceptualProperty] = []
|
|
48
|
-
self.classes: list[
|
|
48
|
+
self.classes: list[UnverifiedConcept] = []
|
|
49
49
|
self._item_by_id: dict[DTMI, DTDLBase] = {}
|
|
50
50
|
|
|
51
51
|
self._method_by_type = {
|
|
@@ -66,8 +66,8 @@ class _DTDLConverter:
|
|
|
66
66
|
raise ValueError("No classes found")
|
|
67
67
|
counted = Counter(
|
|
68
68
|
class_.prefix
|
|
69
|
-
for class_ in (cls_.
|
|
70
|
-
if isinstance(class_,
|
|
69
|
+
for class_ in (cls_.concept for cls_ in self.classes)
|
|
70
|
+
if isinstance(class_, ConceptEntity) and isinstance(class_.prefix, str)
|
|
71
71
|
)
|
|
72
72
|
if not counted:
|
|
73
73
|
raise ValueError("No prefixes found")
|
|
@@ -103,11 +103,11 @@ class _DTDLConverter:
|
|
|
103
103
|
)
|
|
104
104
|
|
|
105
105
|
def convert_interface(self, item: Interface, _: str | None) -> None:
|
|
106
|
-
class_ =
|
|
107
|
-
|
|
106
|
+
class_ = UnverifiedConcept(
|
|
107
|
+
concept=item.id_.as_concept_entity(),
|
|
108
108
|
name=item.display_name,
|
|
109
109
|
description=item.description,
|
|
110
|
-
implements=[parent.
|
|
110
|
+
implements=[parent.as_concept_entity() for parent in item.extends or []] or None,
|
|
111
111
|
)
|
|
112
112
|
self.classes.append(class_)
|
|
113
113
|
for sub_item_or_id in item.contents or []:
|
|
@@ -122,9 +122,9 @@ class _DTDLConverter:
|
|
|
122
122
|
)
|
|
123
123
|
elif isinstance(sub_item_or_id, DTMI):
|
|
124
124
|
sub_item = self._item_by_id[sub_item_or_id]
|
|
125
|
-
self.convert_item(sub_item, class_.
|
|
125
|
+
self.convert_item(sub_item, class_.concept_str)
|
|
126
126
|
else:
|
|
127
|
-
self.convert_item(sub_item_or_id, class_.
|
|
127
|
+
self.convert_item(sub_item_or_id, class_.concept_str)
|
|
128
128
|
# interface.schema objects are handled in the convert method
|
|
129
129
|
|
|
130
130
|
def convert_property(
|
|
@@ -138,7 +138,7 @@ class _DTDLConverter:
|
|
|
138
138
|
return None
|
|
139
139
|
|
|
140
140
|
prop = UnverifiedConceptualProperty(
|
|
141
|
-
|
|
141
|
+
concept=ConceptEntity.load(parent),
|
|
142
142
|
property_=item.name,
|
|
143
143
|
name=item.display_name,
|
|
144
144
|
description=item.description,
|
|
@@ -183,7 +183,7 @@ class _DTDLConverter:
|
|
|
183
183
|
if value_type is None:
|
|
184
184
|
return
|
|
185
185
|
prop = UnverifiedConceptualProperty(
|
|
186
|
-
|
|
186
|
+
concept=ConceptEntity.load(parent),
|
|
187
187
|
property_=item.name,
|
|
188
188
|
name=item.display_name,
|
|
189
189
|
description=item.description,
|
|
@@ -202,7 +202,7 @@ class _DTDLConverter:
|
|
|
202
202
|
if value_type is None:
|
|
203
203
|
return
|
|
204
204
|
prop = UnverifiedConceptualProperty(
|
|
205
|
-
|
|
205
|
+
concept=ConceptEntity.load(parent),
|
|
206
206
|
property_=item.name,
|
|
207
207
|
name=item.display_name,
|
|
208
208
|
description=item.description,
|
|
@@ -217,9 +217,9 @@ class _DTDLConverter:
|
|
|
217
217
|
self._missing_parent_warning(item)
|
|
218
218
|
return None
|
|
219
219
|
if item.target is not None:
|
|
220
|
-
value_type: DataType |
|
|
220
|
+
value_type: DataType | ConceptEntity
|
|
221
221
|
if item.target in self._item_by_id:
|
|
222
|
-
value_type = item.target.
|
|
222
|
+
value_type = item.target.as_concept_entity()
|
|
223
223
|
else:
|
|
224
224
|
# Falling back to json
|
|
225
225
|
self.issues.append(
|
|
@@ -231,7 +231,7 @@ class _DTDLConverter:
|
|
|
231
231
|
value_type = Json()
|
|
232
232
|
|
|
233
233
|
prop = UnverifiedConceptualProperty(
|
|
234
|
-
|
|
234
|
+
concept=ConceptEntity.load(parent),
|
|
235
235
|
property_=item.name,
|
|
236
236
|
name=item.display_name,
|
|
237
237
|
description=item.description,
|
|
@@ -254,8 +254,8 @@ class _DTDLConverter:
|
|
|
254
254
|
)
|
|
255
255
|
return None
|
|
256
256
|
|
|
257
|
-
class_ =
|
|
258
|
-
|
|
257
|
+
class_ = UnverifiedConcept(
|
|
258
|
+
concept=item.id_.as_concept_entity(),
|
|
259
259
|
name=item.display_name,
|
|
260
260
|
description=item.description,
|
|
261
261
|
)
|
|
@@ -266,7 +266,7 @@ class _DTDLConverter:
|
|
|
266
266
|
if value_type is None:
|
|
267
267
|
continue
|
|
268
268
|
prop = UnverifiedConceptualProperty(
|
|
269
|
-
|
|
269
|
+
concept=class_.concept,
|
|
270
270
|
name=field_.name,
|
|
271
271
|
description=field_.description,
|
|
272
272
|
property_=field_.name,
|
|
@@ -278,7 +278,7 @@ class _DTDLConverter:
|
|
|
278
278
|
|
|
279
279
|
def schema_to_value_type(
|
|
280
280
|
self, schema: Schema | Interface | DTMI | None, item: DTDLBase
|
|
281
|
-
) -> DataType |
|
|
281
|
+
) -> DataType | ConceptEntity | None:
|
|
282
282
|
input_type = self._item_by_id.get(schema) if isinstance(schema, DTMI) else schema
|
|
283
283
|
|
|
284
284
|
if isinstance(input_type, Enum):
|
|
@@ -307,7 +307,7 @@ class _DTDLConverter:
|
|
|
307
307
|
else:
|
|
308
308
|
if isinstance(input_type, Object):
|
|
309
309
|
self.convert_object(input_type, None)
|
|
310
|
-
return input_type.id_.
|
|
310
|
+
return input_type.id_.as_concept_entity()
|
|
311
311
|
else:
|
|
312
312
|
self.issues.append(
|
|
313
313
|
PropertyTypeNotSupportedWarning(
|
|
@@ -5,10 +5,10 @@ from pathlib import Path
|
|
|
5
5
|
|
|
6
6
|
from pydantic import ValidationError
|
|
7
7
|
|
|
8
|
-
from cognite.neat.core._data_model._shared import
|
|
8
|
+
from cognite.neat.core._data_model._shared import ImportedDataModel
|
|
9
9
|
from cognite.neat.core._data_model.importers._base import BaseImporter
|
|
10
|
-
from cognite.neat.core._data_model.importers.
|
|
11
|
-
from cognite.neat.core._data_model.importers.
|
|
10
|
+
from cognite.neat.core._data_model.importers._dtdl2data_model.dtdl_converter import _DTDLConverter
|
|
11
|
+
from cognite.neat.core._data_model.importers._dtdl2data_model.spec import (
|
|
12
12
|
DTDL_CLS_BY_TYPE_BY_SPEC,
|
|
13
13
|
DTDLBase,
|
|
14
14
|
Interface,
|
|
@@ -126,7 +126,7 @@ class DTDLImporter(BaseImporter[UnverifiedConceptualDataModel]):
|
|
|
126
126
|
items.append(item)
|
|
127
127
|
return cls(items, zip_file.stem, read_issues=issues)
|
|
128
128
|
|
|
129
|
-
def
|
|
129
|
+
def to_data_model(self) -> ImportedDataModel[UnverifiedConceptualDataModel]:
|
|
130
130
|
converter = _DTDLConverter(self._read_issues)
|
|
131
131
|
|
|
132
132
|
converter.convert(self._items)
|
|
@@ -143,13 +143,13 @@ class DTDLImporter(BaseImporter[UnverifiedConceptualDataModel]):
|
|
|
143
143
|
else:
|
|
144
144
|
metadata["space"] = most_common_prefix
|
|
145
145
|
|
|
146
|
-
|
|
146
|
+
data_model = UnverifiedConceptualDataModel(
|
|
147
147
|
metadata=UnverifiedConceptualMetadata.load(metadata),
|
|
148
148
|
properties=converter.properties,
|
|
149
|
-
|
|
149
|
+
concepts=converter.classes,
|
|
150
150
|
)
|
|
151
151
|
converter.issues.trigger_warnings()
|
|
152
152
|
if converter.issues.has_errors:
|
|
153
153
|
raise MultiValueError(converter.issues.errors)
|
|
154
154
|
|
|
155
|
-
return
|
|
155
|
+
return ImportedDataModel(data_model, {})
|
|
@@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeAlias
|
|
|
15
15
|
from pydantic import BaseModel, Field, field_validator, model_serializer, model_validator
|
|
16
16
|
from pydantic.fields import FieldInfo
|
|
17
17
|
|
|
18
|
-
from cognite.neat.core._data_model.models.entities import
|
|
18
|
+
from cognite.neat.core._data_model.models.entities import ConceptEntity
|
|
19
19
|
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
21
|
from pydantic.type_adapter import IncEx
|
|
@@ -35,8 +35,8 @@ class DTMI(BaseModel):
|
|
|
35
35
|
path: list[str]
|
|
36
36
|
version: str
|
|
37
37
|
|
|
38
|
-
def
|
|
39
|
-
return
|
|
38
|
+
def as_concept_entity(self) -> ConceptEntity:
|
|
39
|
+
return ConceptEntity(prefix="_".join(self.path[:-1]), suffix=self.path[-1], version=self.version)
|
|
40
40
|
|
|
41
41
|
def __hash__(self) -> int:
|
|
42
42
|
return hash(self.to_string())
|
|
@@ -7,7 +7,7 @@ from rdflib import Graph, Namespace, URIRef
|
|
|
7
7
|
from typing_extensions import Self
|
|
8
8
|
|
|
9
9
|
from cognite.neat.core._constants import get_default_prefixes_and_namespaces
|
|
10
|
-
from cognite.neat.core._data_model._shared import
|
|
10
|
+
from cognite.neat.core._data_model._shared import ImportedDataModel
|
|
11
11
|
from cognite.neat.core._data_model.importers._base import BaseImporter
|
|
12
12
|
from cognite.neat.core._data_model.models._base_verified import RoleTypes
|
|
13
13
|
from cognite.neat.core._data_model.models.conceptual import (
|
|
@@ -18,7 +18,7 @@ from cognite.neat.core._data_model.models.entities import UnknownEntity
|
|
|
18
18
|
from cognite.neat.core._issues import IssueList, MultiValueError
|
|
19
19
|
from cognite.neat.core._issues.errors import FileReadError
|
|
20
20
|
from cognite.neat.core._issues.errors._general import NeatValueError
|
|
21
|
-
from cognite.neat.core._store import
|
|
21
|
+
from cognite.neat.core._store import NeatInstanceStore
|
|
22
22
|
from cognite.neat.core._utils.rdf_ import get_namespace
|
|
23
23
|
|
|
24
24
|
DEFAULT_NON_EXISTING_NODE_TYPE = AnyURI()
|
|
@@ -68,7 +68,7 @@ class BaseRDFImporter(BaseImporter[UnverifiedConceptualDataModel]):
|
|
|
68
68
|
@classmethod
|
|
69
69
|
def from_graph_store(
|
|
70
70
|
cls,
|
|
71
|
-
store:
|
|
71
|
+
store: NeatInstanceStore,
|
|
72
72
|
data_model_id: (dm.DataModelId | tuple[str, str, str]) = DEFAULT_RDF_DATA_MODEL_ID,
|
|
73
73
|
max_number_of_instance: int = -1,
|
|
74
74
|
non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
|
|
@@ -115,24 +115,24 @@ class BaseRDFImporter(BaseImporter[UnverifiedConceptualDataModel]):
|
|
|
115
115
|
source_name=source_name,
|
|
116
116
|
)
|
|
117
117
|
|
|
118
|
-
def
|
|
118
|
+
def to_data_model(
|
|
119
119
|
self,
|
|
120
|
-
) ->
|
|
120
|
+
) -> ImportedDataModel[UnverifiedConceptualDataModel]:
|
|
121
121
|
"""
|
|
122
122
|
Creates `Rules` object from the data for target role.
|
|
123
123
|
"""
|
|
124
124
|
if self.issue_list.has_errors:
|
|
125
|
-
# In case there were errors during the import, the
|
|
125
|
+
# In case there were errors during the import, the to_data_model method will return None
|
|
126
126
|
self.issue_list.trigger_warnings()
|
|
127
127
|
raise MultiValueError(self.issue_list.errors)
|
|
128
128
|
|
|
129
|
-
rules_dict = self.
|
|
129
|
+
rules_dict = self._to_data_model_components()
|
|
130
130
|
|
|
131
131
|
rules = UnverifiedConceptualDataModel.load(rules_dict)
|
|
132
132
|
self.issue_list.trigger_warnings()
|
|
133
|
-
return
|
|
133
|
+
return ImportedDataModel(rules, {})
|
|
134
134
|
|
|
135
|
-
def
|
|
135
|
+
def _to_data_model_components(self) -> dict:
|
|
136
136
|
raise NotImplementedError()
|
|
137
137
|
|
|
138
138
|
@classmethod
|
|
@@ -3,26 +3,26 @@ there are loaders to TransformationRules pydantic class."""
|
|
|
3
3
|
|
|
4
4
|
from cognite.neat.core._data_model.importers._rdf._base import BaseRDFImporter
|
|
5
5
|
from cognite.neat.core._data_model.importers._rdf._shared import (
|
|
6
|
-
|
|
6
|
+
parse_concepts,
|
|
7
7
|
parse_properties,
|
|
8
8
|
)
|
|
9
9
|
|
|
10
10
|
CLASSES_QUERY = """
|
|
11
|
-
SELECT ?
|
|
11
|
+
SELECT ?concept ?name ?description ?implements
|
|
12
12
|
WHERE {{
|
|
13
13
|
VALUES ?type {{ imf:BlockType imf:TerminalType imf:AttributeType }}
|
|
14
|
-
?
|
|
14
|
+
?concept a ?type .
|
|
15
15
|
|
|
16
|
-
OPTIONAL {{?
|
|
17
|
-
OPTIONAL {{?
|
|
18
|
-
OPTIONAL {{?
|
|
16
|
+
OPTIONAL {{?concept rdfs:subClassOf ?parent }}.
|
|
17
|
+
OPTIONAL {{?concept rdfs:label|skos:prefLabel ?name }}.
|
|
18
|
+
OPTIONAL {{?concept rdfs:comment|skos:definition ?description}}.
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
# Add imf:Attribute as parent class when no parent is found
|
|
22
22
|
BIND(IF(!bound(?parent) && ?type = imf:AttributeType, imf:Attribute, ?parent) AS ?implements)
|
|
23
23
|
|
|
24
24
|
# FILTERS
|
|
25
|
-
FILTER (!isBlank(?
|
|
25
|
+
FILTER (!isBlank(?concept))
|
|
26
26
|
FILTER (!bound(?implements) || !isBlank(?implements))
|
|
27
27
|
|
|
28
28
|
FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
|
|
@@ -31,13 +31,13 @@ CLASSES_QUERY = """
|
|
|
31
31
|
"""
|
|
32
32
|
|
|
33
33
|
PROPERTIES_QUERY = """
|
|
34
|
-
SELECT ?
|
|
34
|
+
SELECT ?concept ?property_ ?name ?description ?value_type ?min_count ?max_count ?default
|
|
35
35
|
WHERE
|
|
36
36
|
{{
|
|
37
37
|
# CASE 1: Handling Blocks and Terminals
|
|
38
38
|
{{
|
|
39
39
|
VALUES ?type {{ imf:BlockType imf:TerminalType }}
|
|
40
|
-
?
|
|
40
|
+
?concept a ?type ;
|
|
41
41
|
sh:property ?propertyShape .
|
|
42
42
|
?propertyShape sh:path ?property_ .
|
|
43
43
|
|
|
@@ -55,10 +55,10 @@ PROPERTIES_QUERY = """
|
|
|
55
55
|
|
|
56
56
|
# CASE 2: Handling Attributes
|
|
57
57
|
{{
|
|
58
|
-
?
|
|
58
|
+
?concept a imf:AttributeType .
|
|
59
59
|
BIND(xsd:anyURI AS ?valueShape)
|
|
60
60
|
BIND(imf:predicate AS ?property_)
|
|
61
|
-
?
|
|
61
|
+
?concept ?property_ ?defaultURI .
|
|
62
62
|
BIND(STR(?defaultURI) AS ?default)
|
|
63
63
|
|
|
64
64
|
}}
|
|
@@ -67,7 +67,7 @@ PROPERTIES_QUERY = """
|
|
|
67
67
|
BIND(IF(BOUND(?valueShape), ?valueShape, IF(BOUND(?range) , ?range , ?valueShape)) AS ?value_type)
|
|
68
68
|
|
|
69
69
|
FILTER (!isBlank(?property_))
|
|
70
|
-
FILTER (!bound(?
|
|
70
|
+
FILTER (!bound(?concept) || !isBlank(?concept))
|
|
71
71
|
FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "{language}"))
|
|
72
72
|
FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "{language}"))
|
|
73
73
|
}}
|
|
@@ -81,17 +81,17 @@ class IMFImporter(BaseRDFImporter):
|
|
|
81
81
|
def description(self) -> str:
|
|
82
82
|
return f"IMF Types {self.source_name} read as unverified data model"
|
|
83
83
|
|
|
84
|
-
def
|
|
84
|
+
def _to_data_model_components(
|
|
85
85
|
self,
|
|
86
86
|
) -> dict:
|
|
87
|
-
classes, issue_list =
|
|
87
|
+
classes, issue_list = parse_concepts(self.graph, CLASSES_QUERY, self.language, self.issue_list)
|
|
88
88
|
self.issue_list = issue_list
|
|
89
89
|
properties, issue_list = parse_properties(self.graph, PROPERTIES_QUERY, self.language, self.issue_list)
|
|
90
90
|
self.issue_list = issue_list
|
|
91
91
|
|
|
92
92
|
components = {
|
|
93
93
|
"Metadata": self._metadata,
|
|
94
|
-
"
|
|
94
|
+
"Concepts": list(classes.values()) if classes else [],
|
|
95
95
|
"Properties": list(properties.values()) if properties else [],
|
|
96
96
|
}
|
|
97
97
|
|