cognite-neat 0.121.0__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/{_rules → _data_model}/_constants.py +25 -18
- cognite/neat/core/_data_model/_shared.py +59 -0
- cognite/neat/core/_data_model/analysis/__init__.py +3 -0
- cognite/neat/core/{_rules → _data_model}/analysis/_base.py +202 -195
- cognite/neat/core/{_rules → _data_model}/catalog/__init__.py +1 -1
- cognite/neat/core/{_rules → _data_model}/exporters/__init__.py +5 -5
- cognite/neat/core/{_rules → _data_model}/exporters/_base.py +10 -8
- cognite/neat/core/{_rules/exporters/_rules2dms.py → _data_model/exporters/_data_model2dms.py} +22 -18
- cognite/neat/core/{_rules/exporters/_rules2excel.py → _data_model/exporters/_data_model2excel.py} +61 -56
- cognite/neat/core/{_rules/exporters/_rules2instance_template.py → _data_model/exporters/_data_model2instance_template.py} +11 -9
- cognite/neat/core/{_rules/exporters/_rules2ontology.py → _data_model/exporters/_data_model2ontology.py} +64 -61
- cognite/neat/core/{_rules/exporters/_rules2yaml.py → _data_model/exporters/_data_model2yaml.py} +21 -18
- cognite/neat/core/{_rules → _data_model}/importers/__init__.py +6 -8
- cognite/neat/core/{_rules → _data_model}/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/{_rules/importers/_yaml2rules.py → _data_model/importers/_dict2data_model.py} +41 -21
- cognite/neat/core/{_rules/importers/_dms2rules.py → _data_model/importers/_dms2data_model.py} +79 -66
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_converter.py +41 -41
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_importer.py +16 -16
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_base.py +18 -16
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_imf2rules.py +17 -17
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_inference2rules.py +50 -50
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_owl2rules.py +14 -14
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/{_rules/importers/_spreadsheet2rules.py → _data_model/importers/_spreadsheet2data_model.py} +69 -38
- cognite/neat/core/_data_model/models/__init__.py +36 -0
- cognite/neat/core/{_rules/models/_base_input.py → _data_model/models/_base_unverified.py} +12 -12
- cognite/neat/core/{_rules/models/_base_rules.py → _data_model/models/_base_verified.py} +13 -13
- cognite/neat/core/{_rules → _data_model}/models/_types.py +13 -13
- cognite/neat/core/_data_model/models/conceptual/__init__.py +25 -0
- cognite/neat/core/{_rules/models/information/_rules_input.py → _data_model/models/conceptual/_unverified.py} +46 -43
- cognite/neat/core/{_rules/models/information → _data_model/models/conceptual}/_validation.py +93 -79
- cognite/neat/core/{_rules/models/information/_rules.py → _data_model/models/conceptual/_verified.py} +83 -83
- cognite/neat/core/{_rules → _data_model}/models/data_types.py +4 -4
- cognite/neat/core/{_rules → _data_model}/models/entities/__init__.py +8 -8
- cognite/neat/core/{_rules → _data_model}/models/entities/_loaders.py +12 -11
- cognite/neat/core/{_rules → _data_model}/models/entities/_multi_value.py +7 -7
- cognite/neat/core/{_rules → _data_model}/models/entities/_single_value.py +45 -39
- cognite/neat/core/{_rules → _data_model}/models/entities/_types.py +9 -3
- cognite/neat/core/{_rules → _data_model}/models/entities/_wrapped.py +3 -3
- cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_exporter.py +83 -64
- cognite/neat/core/{_rules/models/dms/_rules_input.py → _data_model/models/physical/_unverified.py} +56 -44
- cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_validation.py +20 -17
- cognite/neat/core/{_rules/models/dms/_rules.py → _data_model/models/physical/_verified.py} +79 -71
- cognite/neat/core/{_rules → _data_model}/transformers/__init__.py +27 -23
- cognite/neat/core/{_rules → _data_model}/transformers/_base.py +29 -19
- cognite/neat/core/{_rules → _data_model}/transformers/_converters.py +758 -659
- cognite/neat/core/{_rules → _data_model}/transformers/_mapping.py +79 -60
- cognite/neat/core/_data_model/transformers/_verification.py +120 -0
- cognite/neat/core/{_graph → _instances}/extractors/_base.py +2 -2
- cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_base.py +1 -1
- cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_classic.py +17 -11
- cognite/neat/core/{_graph → _instances}/extractors/_dms_graph.py +47 -39
- cognite/neat/core/{_graph → _instances}/extractors/_mock_graph_generator.py +102 -99
- cognite/neat/core/{_graph → _instances}/extractors/_rdf_file.py +2 -2
- cognite/neat/core/{_graph → _instances}/loaders/_base.py +2 -2
- cognite/neat/core/{_graph → _instances}/loaders/_rdf2dms.py +16 -14
- cognite/neat/core/{_graph → _instances}/transformers/_base.py +7 -4
- cognite/neat/core/{_graph → _instances}/transformers/_classic_cdf.py +1 -1
- cognite/neat/core/{_graph → _instances}/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/__init__.py +2 -0
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +7 -0
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +128 -121
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +7 -8
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +4 -4
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/spreadsheet.py +1 -1
- cognite/neat/core/_utils/text.py +2 -2
- cognite/neat/session/_base.py +29 -25
- cognite/neat/session/_drop.py +3 -3
- cognite/neat/session/_fix.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +11 -9
- cognite/neat/session/_prepare.py +4 -4
- cognite/neat/session/_read.py +15 -15
- cognite/neat/session/_set.py +5 -5
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +17 -17
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +19 -19
- cognite/neat/session/_to.py +21 -21
- cognite/neat/session/_wizard.py +1 -1
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- cognite_neat-0.121.2.dist-info/RECORD +189 -0
- cognite/neat/core/_rules/_shared.py +0 -43
- cognite/neat/core/_rules/analysis/__init__.py +0 -3
- cognite/neat/core/_rules/exporters/_validation.py +0 -14
- cognite/neat/core/_rules/models/__init__.py +0 -34
- cognite/neat/core/_rules/models/dms/__init__.py +0 -32
- cognite/neat/core/_rules/models/information/__init__.py +0 -20
- cognite/neat/core/_rules/transformers/_verification.py +0 -111
- cognite_neat-0.121.0.dist-info/RECORD +0 -187
- /cognite/neat/core/{_graph → _data_model}/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/catalog/classic_model.xlsx +0 -0
- /cognite/neat/core/{_rules/catalog/info-rules-imf.xlsx → _data_model/catalog/conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/{_rules → _data_model}/catalog/hello_world_pump.xlsx +0 -0
- /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/_unit_lookup.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/importers/_rdf/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/entities/_constants.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/mapping/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.yaml +0 -0
- /cognite/neat/core/{_graph/extractors/_classic_cdf → _instances}/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_shared.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/base.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/log.py +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44.xml +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/__init__.py +0 -0
- /cognite/neat/core/{_rules → _instances/extractors/_classic_cdf}/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_assets.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_data_sets.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_events.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_files.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_labels.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_relationships.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_sequences.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_timeseries.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_dict.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_dms.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_raw.py +0 -0
- /cognite/neat/core/{_graph → _instances}/loaders/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_base.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_queries.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_select.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_update.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/_prune_graph.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/_rdfpath.py +0 -0
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -6,25 +6,29 @@ from typing import Any, ClassVar, Literal
|
|
|
6
6
|
from cognite.client import data_modeling as dm
|
|
7
7
|
|
|
8
8
|
from cognite.neat.core._client import NeatClient
|
|
9
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel, SheetList
|
|
10
|
+
from cognite.neat.core._data_model.models.data_types import Enum
|
|
11
|
+
from cognite.neat.core._data_model.models.entities import (
|
|
12
|
+
ConceptEntity,
|
|
13
|
+
ContainerEntity,
|
|
14
|
+
ViewEntity,
|
|
15
|
+
)
|
|
16
|
+
from cognite.neat.core._data_model.models.physical import (
|
|
17
|
+
PhysicalContainer,
|
|
18
|
+
PhysicalEnum,
|
|
19
|
+
PhysicalProperty,
|
|
20
|
+
)
|
|
9
21
|
from cognite.neat.core._issues.errors import (
|
|
10
22
|
CDFMissingClientError,
|
|
11
23
|
NeatValueError,
|
|
12
24
|
ResourceNotFoundError,
|
|
13
25
|
)
|
|
14
26
|
from cognite.neat.core._issues.warnings import PropertyOverwritingWarning
|
|
15
|
-
from cognite.neat.core._rules.models import DMSRules, SheetList
|
|
16
|
-
from cognite.neat.core._rules.models.data_types import Enum
|
|
17
|
-
from cognite.neat.core._rules.models.dms import DMSContainer, DMSEnum, DMSProperty
|
|
18
|
-
from cognite.neat.core._rules.models.entities import (
|
|
19
|
-
ClassEntity,
|
|
20
|
-
ContainerEntity,
|
|
21
|
-
ViewEntity,
|
|
22
|
-
)
|
|
23
27
|
|
|
24
|
-
from ._base import
|
|
28
|
+
from ._base import VerifiedDataModelTransformer
|
|
25
29
|
|
|
26
30
|
|
|
27
|
-
class MapOntoTransformers(
|
|
31
|
+
class MapOntoTransformers(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel], ABC):
|
|
28
32
|
"""Base class for transformers that map one rule onto another."""
|
|
29
33
|
|
|
30
34
|
...
|
|
@@ -33,7 +37,7 @@ class MapOntoTransformers(VerifiedRulesTransformer[DMSRules, DMSRules], ABC):
|
|
|
33
37
|
class MapOneToOne(MapOntoTransformers):
|
|
34
38
|
"""Takes transform data models and makes it into an extension of the reference data model.
|
|
35
39
|
|
|
36
|
-
Note this transformer mutates the input
|
|
40
|
+
Note this transformer mutates the input data model.
|
|
37
41
|
|
|
38
42
|
The argument view_extension_mapping is a dictionary where the keys are views of this data model,
|
|
39
43
|
and each value is the view of the reference data model that the view should extend. For example:
|
|
@@ -55,14 +59,17 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
55
59
|
"""
|
|
56
60
|
|
|
57
61
|
def __init__(
|
|
58
|
-
self,
|
|
62
|
+
self,
|
|
63
|
+
reference: PhysicalDataModel,
|
|
64
|
+
view_extension_mapping: dict[str, str],
|
|
65
|
+
default_extension: str | None = None,
|
|
59
66
|
) -> None:
|
|
60
67
|
self.reference = reference
|
|
61
68
|
self.view_extension_mapping = view_extension_mapping
|
|
62
69
|
self.default_extension = default_extension
|
|
63
70
|
|
|
64
|
-
def transform(self,
|
|
65
|
-
solution:
|
|
71
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
72
|
+
solution: PhysicalDataModel = data_model
|
|
66
73
|
view_by_external_id = {view.view.external_id: view for view in solution.views}
|
|
67
74
|
ref_view_by_external_id = {view.view.external_id: view for view in self.reference.views}
|
|
68
75
|
|
|
@@ -73,11 +80,11 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
73
80
|
if self.default_extension and self.default_extension not in ref_view_by_external_id:
|
|
74
81
|
raise ValueError(f"Default extension view not in the reference data model {self.default_extension}")
|
|
75
82
|
|
|
76
|
-
properties_by_view_external_id: dict[str, dict[str,
|
|
83
|
+
properties_by_view_external_id: dict[str, dict[str, PhysicalProperty]] = defaultdict(dict)
|
|
77
84
|
for prop in solution.properties:
|
|
78
85
|
properties_by_view_external_id[prop.view.external_id][prop.view_property] = prop
|
|
79
86
|
|
|
80
|
-
ref_properties_by_view_external_id: dict[str, dict[str,
|
|
87
|
+
ref_properties_by_view_external_id: dict[str, dict[str, PhysicalProperty]] = defaultdict(dict)
|
|
81
88
|
for prop in self.reference.properties:
|
|
82
89
|
ref_properties_by_view_external_id[prop.view.external_id][prop.view_property] = prop
|
|
83
90
|
|
|
@@ -108,14 +115,15 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
108
115
|
return solution
|
|
109
116
|
|
|
110
117
|
|
|
111
|
-
class
|
|
118
|
+
class PhysicalDataModelMapper(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
112
119
|
"""Maps properties and classes using the given mapping.
|
|
113
120
|
|
|
114
121
|
Args:
|
|
115
|
-
mapping: The mapping to use represented as a
|
|
122
|
+
mapping: The mapping to use represented as a physical data model object.
|
|
116
123
|
data_type_conflict: How to handle data type conflicts. The default is "overwrite".
|
|
117
124
|
A data type conflicts occurs when the data type of a property in the mapping is different from the
|
|
118
|
-
data type of the property in the input
|
|
125
|
+
data type of the property in the input data model. If "overwrite" the data type
|
|
126
|
+
in the input data model is overwritten
|
|
119
127
|
with the data type in the mapping.
|
|
120
128
|
"""
|
|
121
129
|
|
|
@@ -123,37 +131,41 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
123
131
|
["connection", "value_type", "min_count", "immutable", "max_count", "default", "index", "constraint"]
|
|
124
132
|
)
|
|
125
133
|
|
|
126
|
-
def __init__(
|
|
134
|
+
def __init__(
|
|
135
|
+
self,
|
|
136
|
+
mapping: PhysicalDataModel,
|
|
137
|
+
data_type_conflict: Literal["overwrite"] = "overwrite",
|
|
138
|
+
) -> None:
|
|
127
139
|
self.mapping = mapping
|
|
128
140
|
self.data_type_conflict = data_type_conflict
|
|
129
141
|
|
|
130
|
-
def transform(self,
|
|
142
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
131
143
|
if self.data_type_conflict != "overwrite":
|
|
132
144
|
raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
|
|
133
|
-
|
|
134
|
-
|
|
145
|
+
input_data_model = data_model
|
|
146
|
+
new_data_model = input_data_model.model_copy(deep=True)
|
|
135
147
|
|
|
136
|
-
views_by_external_id = {view.view.external_id: view for view in
|
|
148
|
+
views_by_external_id = {view.view.external_id: view for view in new_data_model.views}
|
|
137
149
|
new_views: set[ViewEntity] = set()
|
|
138
150
|
for mapping_view in self.mapping.views:
|
|
139
151
|
if existing_view := views_by_external_id.get(mapping_view.view.external_id):
|
|
140
152
|
existing_view.implements = mapping_view.implements
|
|
141
153
|
else:
|
|
142
|
-
# We need to add all the views in the mapping that are not in the input
|
|
143
|
-
# This is to ensure that all ValueTypes are present in the resulting
|
|
154
|
+
# We need to add all the views in the mapping that are not in the input data model.
|
|
155
|
+
# This is to ensure that all ValueTypes are present in the resulting data model.
|
|
144
156
|
# For example, if a property is a direct relation to an Equipment view, we need to add
|
|
145
|
-
# the Equipment view to the
|
|
146
|
-
|
|
157
|
+
# the Equipment view to the data model.
|
|
158
|
+
new_data_model.views.append(mapping_view)
|
|
147
159
|
new_views.add(mapping_view.view)
|
|
148
160
|
|
|
149
161
|
properties_by_view_property = {
|
|
150
|
-
(prop.view.external_id, prop.view_property): prop for prop in
|
|
162
|
+
(prop.view.external_id, prop.view_property): prop for prop in new_data_model.properties
|
|
151
163
|
}
|
|
152
|
-
existing_enum_collections = {item.collection for item in
|
|
153
|
-
mapping_enums_by_collection: dict[
|
|
164
|
+
existing_enum_collections = {item.collection for item in new_data_model.enum or []}
|
|
165
|
+
mapping_enums_by_collection: dict[ConceptEntity, list[PhysicalEnum]] = defaultdict(list)
|
|
154
166
|
for item in self.mapping.enum or []:
|
|
155
167
|
mapping_enums_by_collection[item.collection].append(item)
|
|
156
|
-
existing_containers = {container.container for container in
|
|
168
|
+
existing_containers = {container.container for container in new_data_model.containers or []}
|
|
157
169
|
mapping_containers_by_id = {container.container: container for container in self.mapping.containers or []}
|
|
158
170
|
for mapping_prop in self.mapping.properties:
|
|
159
171
|
if existing_prop := properties_by_view_property.get(
|
|
@@ -163,7 +175,10 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
163
175
|
if conflicts and self.data_type_conflict == "overwrite":
|
|
164
176
|
warnings.warn(
|
|
165
177
|
PropertyOverwritingWarning(
|
|
166
|
-
existing_prop.view.as_id(),
|
|
178
|
+
existing_prop.view.as_id(),
|
|
179
|
+
"view",
|
|
180
|
+
existing_prop.view_property,
|
|
181
|
+
tuple(conflicts),
|
|
167
182
|
),
|
|
168
183
|
stacklevel=2,
|
|
169
184
|
)
|
|
@@ -177,24 +192,24 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
177
192
|
existing_prop.container = mapping_prop.container
|
|
178
193
|
existing_prop.container_property = mapping_prop.container_property
|
|
179
194
|
elif isinstance(mapping_prop.value_type, ViewEntity):
|
|
180
|
-
# All connections must be included in the
|
|
195
|
+
# All connections must be included in the data model. This is to update the
|
|
181
196
|
# ValueTypes of the implemented views.
|
|
182
|
-
|
|
197
|
+
new_data_model.properties.append(mapping_prop)
|
|
183
198
|
elif "guid" in mapping_prop.view_property.casefold():
|
|
184
199
|
# All guid properties are included. Theses are necessary to get an appropriate
|
|
185
200
|
# filter on the resulting view.
|
|
186
|
-
|
|
201
|
+
new_data_model.properties.append(mapping_prop)
|
|
187
202
|
else:
|
|
188
|
-
# Skipping mapped properties that are not in the input
|
|
203
|
+
# Skipping mapped properties that are not in the input data model.
|
|
189
204
|
continue
|
|
190
205
|
|
|
191
206
|
if (
|
|
192
207
|
isinstance(mapping_prop.value_type, Enum)
|
|
193
208
|
and mapping_prop.value_type.collection not in existing_enum_collections
|
|
194
209
|
):
|
|
195
|
-
if not
|
|
196
|
-
|
|
197
|
-
|
|
210
|
+
if not new_data_model.enum:
|
|
211
|
+
new_data_model.enum = SheetList[PhysicalEnum]([])
|
|
212
|
+
new_data_model.enum.extend(mapping_enums_by_collection[mapping_prop.value_type.collection])
|
|
198
213
|
|
|
199
214
|
if (
|
|
200
215
|
mapping_prop.container
|
|
@@ -202,13 +217,15 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
202
217
|
and (new_container := mapping_containers_by_id.get(mapping_prop.container))
|
|
203
218
|
):
|
|
204
219
|
# Mapping can include new containers for GUID properties
|
|
205
|
-
if not
|
|
206
|
-
|
|
207
|
-
|
|
220
|
+
if not new_data_model.containers:
|
|
221
|
+
new_data_model.containers = SheetList[PhysicalContainer]([])
|
|
222
|
+
new_data_model.containers.append(new_container)
|
|
208
223
|
|
|
209
|
-
return
|
|
224
|
+
return new_data_model
|
|
210
225
|
|
|
211
|
-
def _find_overwrites(
|
|
226
|
+
def _find_overwrites(
|
|
227
|
+
self, prop: PhysicalProperty, mapping_prop: PhysicalProperty
|
|
228
|
+
) -> tuple[dict[str, Any], list[str]]:
|
|
212
229
|
"""Finds the properties that need to be overwritten and returns them.
|
|
213
230
|
|
|
214
231
|
In addition, conflicting properties are returned. Note that overwriting properties that are
|
|
@@ -240,7 +257,7 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
240
257
|
return f"Mapping to {self.mapping.metadata.as_data_model_id()!r}."
|
|
241
258
|
|
|
242
259
|
|
|
243
|
-
class AsParentPropertyId(
|
|
260
|
+
class AsParentPropertyId(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
244
261
|
"""Looks up all view properties that map to the same container property,
|
|
245
262
|
and changes the child view property id to match the parent property id.
|
|
246
263
|
"""
|
|
@@ -248,34 +265,36 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
248
265
|
def __init__(self, client: NeatClient | None = None) -> None:
|
|
249
266
|
self._client = client
|
|
250
267
|
|
|
251
|
-
def transform(self,
|
|
252
|
-
|
|
253
|
-
|
|
268
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
269
|
+
input_data_model = data_model
|
|
270
|
+
new_data_model = input_data_model.model_copy(deep=True)
|
|
254
271
|
|
|
255
|
-
path_by_view = self._inheritance_path_by_view(
|
|
256
|
-
view_by_container_property = self._view_by_container_properties(
|
|
272
|
+
path_by_view = self._inheritance_path_by_view(new_data_model)
|
|
273
|
+
view_by_container_property = self._view_by_container_properties(new_data_model)
|
|
257
274
|
|
|
258
275
|
parent_view_property_by_container_property = self._get_parent_view_property_by_container_property(
|
|
259
276
|
path_by_view, view_by_container_property
|
|
260
277
|
)
|
|
261
278
|
|
|
262
|
-
for prop in
|
|
279
|
+
for prop in new_data_model.properties:
|
|
263
280
|
if prop.container and prop.container_property:
|
|
264
281
|
if parent_name := parent_view_property_by_container_property.get(
|
|
265
282
|
(prop.container, prop.container_property)
|
|
266
283
|
):
|
|
267
284
|
prop.view_property = parent_name
|
|
268
285
|
|
|
269
|
-
return
|
|
286
|
+
return new_data_model
|
|
270
287
|
|
|
271
288
|
# Todo: Move into Probe class. Note this means that the Probe class must take a NeatClient as an argument.
|
|
272
|
-
def _inheritance_path_by_view(self,
|
|
273
|
-
parents_by_view: dict[ViewEntity, list[ViewEntity]] = {
|
|
289
|
+
def _inheritance_path_by_view(self, data_model: PhysicalDataModel) -> dict[ViewEntity, list[ViewEntity]]:
|
|
290
|
+
parents_by_view: dict[ViewEntity, list[ViewEntity]] = {
|
|
291
|
+
view.view: view.implements or [] for view in data_model.views
|
|
292
|
+
}
|
|
274
293
|
|
|
275
294
|
path_by_view: dict[ViewEntity, list[ViewEntity]] = {}
|
|
276
|
-
for view in
|
|
295
|
+
for view in data_model.views:
|
|
277
296
|
path_by_view[view.view] = self._get_inheritance_path(
|
|
278
|
-
view.view, parents_by_view,
|
|
297
|
+
view.view, parents_by_view, data_model.metadata.as_data_model_id()
|
|
279
298
|
)
|
|
280
299
|
return path_by_view
|
|
281
300
|
|
|
@@ -312,13 +331,13 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
312
331
|
return inheritance_path
|
|
313
332
|
|
|
314
333
|
def _view_by_container_properties(
|
|
315
|
-
self,
|
|
334
|
+
self, data_model: PhysicalDataModel
|
|
316
335
|
) -> dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]:
|
|
317
336
|
view_properties_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]] = (
|
|
318
337
|
defaultdict(list)
|
|
319
338
|
)
|
|
320
339
|
view_with_properties: set[ViewEntity] = set()
|
|
321
|
-
for prop in
|
|
340
|
+
for prop in data_model.properties:
|
|
322
341
|
if not prop.container or not prop.container_property:
|
|
323
342
|
continue
|
|
324
343
|
view_properties_by_container_properties[(prop.container, prop.container_property)].append(
|
|
@@ -327,7 +346,7 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
327
346
|
view_with_properties.add(prop.view)
|
|
328
347
|
|
|
329
348
|
# We need to look up all parent properties.
|
|
330
|
-
to_lookup = {view.view.as_id() for view in
|
|
349
|
+
to_lookup = {view.view.as_id() for view in data_model.views if view.view not in view_with_properties}
|
|
331
350
|
if to_lookup and self._client is None:
|
|
332
351
|
raise CDFMissingClientError(
|
|
333
352
|
f"Views {to_lookup} are not in the data model. Please provide a client to lookup the views."
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
from abc import ABC
|
|
2
|
+
from typing import cast
|
|
3
|
+
|
|
4
|
+
from cognite.neat.core._client import NeatClient
|
|
5
|
+
from cognite.neat.core._data_model._shared import (
|
|
6
|
+
ImportedDataModel,
|
|
7
|
+
T_ImportedUnverifiedDataModel,
|
|
8
|
+
T_VerifiedDataModel,
|
|
9
|
+
VerifiedDataModel,
|
|
10
|
+
)
|
|
11
|
+
from cognite.neat.core._data_model.models import (
|
|
12
|
+
ConceptualDataModel,
|
|
13
|
+
PhysicalDataModel,
|
|
14
|
+
UnverifiedConceptualDataModel,
|
|
15
|
+
UnverifiedPhysicalDataModel,
|
|
16
|
+
)
|
|
17
|
+
from cognite.neat.core._data_model.models.conceptual import ConceptualValidation
|
|
18
|
+
from cognite.neat.core._data_model.models.physical import PhysicalValidation
|
|
19
|
+
from cognite.neat.core._issues import MultiValueError, catch_issues
|
|
20
|
+
from cognite.neat.core._issues.errors import NeatTypeError, NeatValueError
|
|
21
|
+
|
|
22
|
+
from ._base import DataModelTransformer
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class VerificationTransformer(DataModelTransformer[T_ImportedUnverifiedDataModel, T_VerifiedDataModel], ABC):
|
|
26
|
+
"""Base class for all verification transformers."""
|
|
27
|
+
|
|
28
|
+
_data_model_cls: type[T_VerifiedDataModel]
|
|
29
|
+
_validation_cls: type
|
|
30
|
+
|
|
31
|
+
def __init__(self, validate: bool = True, client: NeatClient | None = None) -> None:
|
|
32
|
+
self.validate = validate
|
|
33
|
+
self._client = client
|
|
34
|
+
|
|
35
|
+
def transform(self, data_model: T_ImportedUnverifiedDataModel) -> T_VerifiedDataModel:
|
|
36
|
+
in_ = data_model.unverified_data_model
|
|
37
|
+
if in_ is None:
|
|
38
|
+
raise NeatValueError("Cannot verify rules. The reading of the rules failed.")
|
|
39
|
+
verified_data_model: T_VerifiedDataModel | None = None
|
|
40
|
+
# We need to catch issues as we use the error args to provide extra context for the errors/warnings
|
|
41
|
+
# For example, which row in the spreadsheet the error occurred.
|
|
42
|
+
with catch_issues(data_model.context) as issues:
|
|
43
|
+
data_model_cls = self._get_data_model_cls(data_model)
|
|
44
|
+
dumped = in_.dump()
|
|
45
|
+
verified_data_model = data_model_cls.model_validate(dumped) # type: ignore[assignment]
|
|
46
|
+
if self.validate:
|
|
47
|
+
validation_cls = self._get_validation_cls(verified_data_model) # type: ignore[arg-type]
|
|
48
|
+
if issubclass(validation_cls, PhysicalValidation):
|
|
49
|
+
validation_issues = PhysicalValidation(
|
|
50
|
+
cast(PhysicalDataModel, verified_data_model),
|
|
51
|
+
self._client,
|
|
52
|
+
data_model.context,
|
|
53
|
+
).validate() # type: ignore[arg-type]
|
|
54
|
+
elif issubclass(validation_cls, ConceptualValidation):
|
|
55
|
+
validation_issues = ConceptualValidation(verified_data_model, data_model.context).validate() # type: ignore[arg-type]
|
|
56
|
+
else:
|
|
57
|
+
raise NeatValueError("Unsupported rule type")
|
|
58
|
+
issues.extend(validation_issues)
|
|
59
|
+
|
|
60
|
+
# Raise issues which is expected to be handled outside of this method
|
|
61
|
+
issues.trigger_warnings()
|
|
62
|
+
if issues.has_errors:
|
|
63
|
+
raise MultiValueError(issues.errors)
|
|
64
|
+
if verified_data_model is None:
|
|
65
|
+
raise NeatValueError("Data model was not verified")
|
|
66
|
+
return verified_data_model
|
|
67
|
+
|
|
68
|
+
def _get_data_model_cls(self, in_: T_ImportedUnverifiedDataModel) -> type[T_VerifiedDataModel]:
|
|
69
|
+
return self._data_model_cls
|
|
70
|
+
|
|
71
|
+
def _get_validation_cls(self, data_model: T_VerifiedDataModel) -> type:
|
|
72
|
+
return self._validation_cls
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def description(self) -> str:
|
|
76
|
+
return "Verify data model"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class VerifyPhysicalDataModel(
|
|
80
|
+
VerificationTransformer[ImportedDataModel[UnverifiedPhysicalDataModel], PhysicalDataModel]
|
|
81
|
+
):
|
|
82
|
+
"""Class to verify physical data model."""
|
|
83
|
+
|
|
84
|
+
_data_model_cls = PhysicalDataModel
|
|
85
|
+
_validation_cls = PhysicalValidation
|
|
86
|
+
|
|
87
|
+
def transform(self, data_model: ImportedDataModel[UnverifiedPhysicalDataModel]) -> PhysicalDataModel:
|
|
88
|
+
return super().transform(data_model)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class VerifyConceptualDataModel(
|
|
92
|
+
VerificationTransformer[ImportedDataModel[UnverifiedConceptualDataModel], ConceptualDataModel]
|
|
93
|
+
):
|
|
94
|
+
"""Class to verify conceptual data model."""
|
|
95
|
+
|
|
96
|
+
_data_model_cls = ConceptualDataModel
|
|
97
|
+
_validation_cls = ConceptualValidation
|
|
98
|
+
|
|
99
|
+
def transform(self, data_model: ImportedDataModel[UnverifiedConceptualDataModel]) -> ConceptualDataModel:
|
|
100
|
+
return super().transform(data_model)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class VerifyAnyDataModel(VerificationTransformer[T_ImportedUnverifiedDataModel, VerifiedDataModel]):
|
|
104
|
+
"""Class to verify arbitrary data model"""
|
|
105
|
+
|
|
106
|
+
def _get_data_model_cls(self, in_: T_ImportedUnverifiedDataModel) -> type[VerifiedDataModel]:
|
|
107
|
+
if isinstance(in_.unverified_data_model, UnverifiedConceptualDataModel):
|
|
108
|
+
return ConceptualDataModel
|
|
109
|
+
elif isinstance(in_.unverified_data_model, UnverifiedPhysicalDataModel):
|
|
110
|
+
return PhysicalDataModel
|
|
111
|
+
else:
|
|
112
|
+
raise NeatTypeError(f"Unsupported data model type: {type(in_)}")
|
|
113
|
+
|
|
114
|
+
def _get_validation_cls(self, data_model: VerifiedDataModel) -> type:
|
|
115
|
+
if isinstance(data_model, ConceptualDataModel):
|
|
116
|
+
return ConceptualValidation
|
|
117
|
+
elif isinstance(data_model, PhysicalDataModel):
|
|
118
|
+
return PhysicalValidation
|
|
119
|
+
else:
|
|
120
|
+
raise NeatTypeError(f"Unsupported data model type: {type(data_model)}")
|
|
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
|
|
|
5
5
|
from rdflib import URIRef
|
|
6
6
|
|
|
7
7
|
from cognite.neat.core._constants import DEFAULT_NAMESPACE
|
|
8
|
-
from cognite.neat.core.
|
|
8
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel
|
|
9
9
|
from cognite.neat.core._shared import Triple
|
|
10
10
|
from cognite.neat.core._utils.auxiliary import class_html_doc
|
|
11
11
|
|
|
@@ -38,7 +38,7 @@ class KnowledgeGraphExtractor(BaseExtractor):
|
|
|
38
38
|
"""A knowledge graph extractor extracts triples with a schema"""
|
|
39
39
|
|
|
40
40
|
@abstractmethod
|
|
41
|
-
def
|
|
41
|
+
def get_conceptual_data_model(self) -> ConceptualDataModel:
|
|
42
42
|
"""Returns the information rules that the extractor uses."""
|
|
43
43
|
raise NotImplementedError()
|
|
44
44
|
|
|
@@ -18,7 +18,7 @@ from rdflib import RDF, XSD, Literal, Namespace, URIRef
|
|
|
18
18
|
from typing_extensions import Self
|
|
19
19
|
|
|
20
20
|
from cognite.neat.core._constants import DEFAULT_NAMESPACE
|
|
21
|
-
from cognite.neat.core.
|
|
21
|
+
from cognite.neat.core._instances.extractors._base import BaseExtractor
|
|
22
22
|
from cognite.neat.core._issues.errors import NeatValueError
|
|
23
23
|
from cognite.neat.core._issues.warnings import CDFAuthWarning, NeatValueWarning
|
|
24
24
|
from cognite.neat.core._shared import Triple
|
|
@@ -14,12 +14,15 @@ from cognite.neat.core._constants import (
|
|
|
14
14
|
DEFAULT_NAMESPACE,
|
|
15
15
|
get_default_prefixes_and_namespaces,
|
|
16
16
|
)
|
|
17
|
-
from cognite.neat.core.
|
|
17
|
+
from cognite.neat.core._data_model._shared import ImportedDataModel
|
|
18
|
+
from cognite.neat.core._data_model.catalog import classic_model
|
|
19
|
+
from cognite.neat.core._data_model.models import (
|
|
20
|
+
ConceptualDataModel,
|
|
21
|
+
UnverifiedConceptualDataModel,
|
|
22
|
+
)
|
|
23
|
+
from cognite.neat.core._instances.extractors._base import KnowledgeGraphExtractor
|
|
18
24
|
from cognite.neat.core._issues.errors import NeatValueError, ResourceNotFoundError
|
|
19
25
|
from cognite.neat.core._issues.warnings import CDFAuthWarning, NeatValueWarning
|
|
20
|
-
from cognite.neat.core._rules._shared import ReadRules
|
|
21
|
-
from cognite.neat.core._rules.catalog import classic_model
|
|
22
|
-
from cognite.neat.core._rules.models import InformationInputRules, InformationRules
|
|
23
26
|
from cognite.neat.core._shared import Triple
|
|
24
27
|
from cognite.neat.core._utils.collection_ import chunker, iterate_progress_bar
|
|
25
28
|
from cognite.neat.core._utils.rdf_ import remove_namespace_from_uri
|
|
@@ -205,15 +208,18 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
|
|
|
205
208
|
|
|
206
209
|
yield from self._extract_asset_parent_data_sets()
|
|
207
210
|
|
|
208
|
-
def
|
|
211
|
+
def get_conceptual_data_model(self) -> ConceptualDataModel:
|
|
209
212
|
# To avoid circular imports
|
|
210
|
-
from cognite.neat.core.
|
|
213
|
+
from cognite.neat.core._data_model.importers import ExcelImporter
|
|
211
214
|
|
|
212
|
-
unverified = cast(
|
|
213
|
-
|
|
215
|
+
unverified = cast(
|
|
216
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
217
|
+
ExcelImporter(classic_model).to_data_model(),
|
|
218
|
+
)
|
|
219
|
+
if unverified.unverified_data_model is None:
|
|
214
220
|
raise NeatValueError(f"Could not read the classic model rules from {classic_model}.")
|
|
215
221
|
|
|
216
|
-
verified = unverified.
|
|
222
|
+
verified = unverified.unverified_data_model.as_verified_data_model()
|
|
217
223
|
prefixes = get_default_prefixes_and_namespaces()
|
|
218
224
|
instance_prefix: str | None = next((k for k, v in prefixes.items() if v == self._namespace), None)
|
|
219
225
|
if instance_prefix is None:
|
|
@@ -228,8 +234,8 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
|
|
|
228
234
|
if is_snake_case:
|
|
229
235
|
prop_id = to_snake_case(prop_id)
|
|
230
236
|
prop.instance_source = [self._namespace[prop_id]]
|
|
231
|
-
for cls_ in verified.
|
|
232
|
-
cls_id = cls_.
|
|
237
|
+
for cls_ in verified.concepts:
|
|
238
|
+
cls_id = cls_.concept.suffix
|
|
233
239
|
if is_snake_case:
|
|
234
240
|
cls_id = to_snake_case(cls_id)
|
|
235
241
|
cls_.instance_source = self._namespace[cls_id]
|
|
@@ -7,18 +7,21 @@ from rdflib import Namespace, URIRef
|
|
|
7
7
|
|
|
8
8
|
from cognite.neat.core._client import NeatClient
|
|
9
9
|
from cognite.neat.core._constants import COGNITE_SPACES, DEFAULT_NAMESPACE
|
|
10
|
+
from cognite.neat.core._data_model.importers import DMSImporter
|
|
11
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
|
|
12
|
+
from cognite.neat.core._data_model.models.conceptual import ConceptualProperty
|
|
13
|
+
from cognite.neat.core._data_model.models.data_types import Json
|
|
14
|
+
from cognite.neat.core._data_model.models.entities import UnknownEntity
|
|
15
|
+
from cognite.neat.core._data_model.transformers import (
|
|
16
|
+
PhysicalToConceptual,
|
|
17
|
+
VerifyPhysicalDataModel,
|
|
18
|
+
)
|
|
10
19
|
from cognite.neat.core._issues import IssueList, NeatIssue, catch_warnings
|
|
11
20
|
from cognite.neat.core._issues.warnings import (
|
|
12
21
|
CDFAuthWarning,
|
|
13
22
|
ResourceNotFoundWarning,
|
|
14
23
|
ResourceRetrievalWarning,
|
|
15
24
|
)
|
|
16
|
-
from cognite.neat.core._rules.importers import DMSImporter
|
|
17
|
-
from cognite.neat.core._rules.models import DMSRules, InformationRules
|
|
18
|
-
from cognite.neat.core._rules.models.data_types import Json
|
|
19
|
-
from cognite.neat.core._rules.models.entities import UnknownEntity
|
|
20
|
-
from cognite.neat.core._rules.models.information import InformationProperty
|
|
21
|
-
from cognite.neat.core._rules.transformers import DMSToInformation, VerifyDMSRules
|
|
22
25
|
from cognite.neat.core._shared import Triple
|
|
23
26
|
|
|
24
27
|
from ._base import KnowledgeGraphExtractor
|
|
@@ -47,8 +50,8 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
47
50
|
self._str_to_ideal_type = str_to_ideal_type
|
|
48
51
|
|
|
49
52
|
self._views: list[dm.View] | None = None
|
|
50
|
-
self.
|
|
51
|
-
self.
|
|
53
|
+
self._conceptual_data_model: ConceptualDataModel | None = None
|
|
54
|
+
self._physical_data_model: PhysicalDataModel | None = None
|
|
52
55
|
|
|
53
56
|
@classmethod
|
|
54
57
|
def from_data_model_id(
|
|
@@ -164,32 +167,32 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
164
167
|
self._issues.append(ResourceNotFoundWarning(dm_view, "view", data_model_id, "data model"))
|
|
165
168
|
return views
|
|
166
169
|
|
|
167
|
-
def
|
|
168
|
-
"""Returns the
|
|
169
|
-
if self.
|
|
170
|
-
self.
|
|
171
|
-
return self.
|
|
170
|
+
def get_conceptual_data_model(self) -> ConceptualDataModel:
|
|
171
|
+
"""Returns the conceptual data model that the extractor uses."""
|
|
172
|
+
if self._conceptual_data_model is None:
|
|
173
|
+
self._conceptual_data_model, self._physical_data_model = self._create_data_models()
|
|
174
|
+
return self._conceptual_data_model
|
|
172
175
|
|
|
173
|
-
def
|
|
174
|
-
"""Returns the
|
|
175
|
-
if self.
|
|
176
|
-
self.
|
|
177
|
-
return self.
|
|
176
|
+
def get_physical_data_model(self) -> PhysicalDataModel:
|
|
177
|
+
"""Returns the physical data model that the extractor uses."""
|
|
178
|
+
if self._physical_data_model is None:
|
|
179
|
+
self._conceptual_data_model, self._physical_data_model = self._create_data_models()
|
|
180
|
+
return self._physical_data_model
|
|
178
181
|
|
|
179
182
|
def get_issues(self) -> IssueList:
|
|
180
183
|
"""Returns the issues that occurred during the extraction."""
|
|
181
184
|
return self._issues
|
|
182
185
|
|
|
183
|
-
def
|
|
184
|
-
# The
|
|
186
|
+
def _create_data_models(self) -> tuple[ConceptualDataModel, PhysicalDataModel]:
|
|
187
|
+
# The physical and conceptual data model must be created together to link them property.
|
|
185
188
|
importer = DMSImporter.from_data_model(self._client, self._data_model)
|
|
186
|
-
|
|
187
|
-
if self._unpack_json and (
|
|
188
|
-
# Drop the JSON properties from the
|
|
189
|
+
imported_physical_data_model = importer.to_data_model()
|
|
190
|
+
if self._unpack_json and (unverified_physical_data_model := imported_physical_data_model.unverified_data_model):
|
|
191
|
+
# Drop the JSON properties from the physical data model as these are no longer valid.
|
|
189
192
|
json_name = Json().name # To avoid instantiating Json multiple times.
|
|
190
|
-
|
|
193
|
+
unverified_physical_data_model.properties = [
|
|
191
194
|
prop
|
|
192
|
-
for prop in
|
|
195
|
+
for prop in unverified_physical_data_model.properties
|
|
193
196
|
if not (
|
|
194
197
|
(
|
|
195
198
|
isinstance(prop.value_type, Json)
|
|
@@ -202,23 +205,28 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
202
205
|
|
|
203
206
|
with catch_warnings() as issues:
|
|
204
207
|
# Any errors occur will be raised and caught outside the extractor.
|
|
205
|
-
|
|
206
|
-
|
|
208
|
+
verified_physical_data_model = VerifyPhysicalDataModel(client=self._client).transform(
|
|
209
|
+
imported_physical_data_model
|
|
210
|
+
)
|
|
211
|
+
verified_conceptual_data_model = PhysicalToConceptual(self._namespace).transform(
|
|
212
|
+
verified_physical_data_model
|
|
213
|
+
)
|
|
207
214
|
|
|
208
|
-
# We need to sync the metadata between the two
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
215
|
+
# We need to sync the metadata between the two data model, such that the
|
|
216
|
+
# `.sync_with_conceptual_data_model` method works.
|
|
217
|
+
verified_conceptual_data_model.metadata.physical = verified_physical_data_model.metadata.identifier
|
|
218
|
+
verified_physical_data_model.metadata.conceptual = verified_conceptual_data_model.metadata.identifier
|
|
219
|
+
verified_physical_data_model.sync_with_conceptual_data_model(verified_conceptual_data_model)
|
|
212
220
|
|
|
213
|
-
# Adding startNode and endNode to the
|
|
214
|
-
|
|
221
|
+
# Adding startNode and endNode to the conceptual data model for views that are used for edges.
|
|
222
|
+
concepts_by_prefix = {concept.concept.prefix: concept for concept in verified_conceptual_data_model.concepts}
|
|
215
223
|
for view in self._model_views:
|
|
216
|
-
if view.used_for == "edge" and view.external_id in
|
|
217
|
-
cls_ =
|
|
224
|
+
if view.used_for == "edge" and view.external_id in concepts_by_prefix:
|
|
225
|
+
cls_ = concepts_by_prefix[view.external_id]
|
|
218
226
|
for property_ in ("startNode", "endNode"):
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
227
|
+
verified_conceptual_data_model.properties.append(
|
|
228
|
+
ConceptualProperty(
|
|
229
|
+
concept=cls_.concept,
|
|
222
230
|
property_=property_,
|
|
223
231
|
value_type=UnknownEntity(),
|
|
224
232
|
min_count=0,
|
|
@@ -227,4 +235,4 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
227
235
|
)
|
|
228
236
|
|
|
229
237
|
self._issues.extend(issues)
|
|
230
|
-
return
|
|
238
|
+
return verified_conceptual_data_model, verified_physical_data_model
|