cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +2 -2
- cognite/neat/core/_data_model/exporters/__init__.py +6 -6
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +14 -14
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +8 -8
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +50 -25
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/__init__.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/{_imf2rules.py → _imf2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/{_inference2rules.py → _inference2rdata_model.py} +59 -59
- cognite/neat/core/_data_model/importers/_rdf/{_owl2rules.py → _owl2data_model.py} +17 -17
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +76 -19
- cognite/neat/core/_data_model/models/__init__.py +11 -9
- cognite/neat/core/_data_model/models/_base_unverified.py +12 -12
- cognite/neat/core/_data_model/models/_base_verified.py +9 -14
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +88 -78
- cognite/neat/core/_data_model/models/conceptual/_verified.py +54 -52
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +75 -55
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +17 -15
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +29 -25
- cognite/neat/core/_data_model/transformers/_base.py +27 -20
- cognite/neat/core/_data_model/transformers/_converters.py +707 -622
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +64 -55
- cognite/neat/core/_instances/extractors/_base.py +2 -2
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +9 -9
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +107 -103
- cognite/neat/core/_instances/loaders/_base.py +3 -3
- cognite/neat/core/_instances/loaders/_rdf2dms.py +22 -22
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_rdfpath.py +1 -1
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/_factory.py +1 -1
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_resources.py +1 -1
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +1 -1
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +42 -36
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_experimental.py +1 -1
- cognite/neat/session/_inspect.py +13 -13
- cognite/neat/session/_mapping.py +15 -9
- cognite/neat/session/_read.py +39 -37
- cognite/neat/session/_set.py +6 -6
- cognite/neat/session/_show.py +24 -21
- cognite/neat/session/_state/README.md +1 -1
- cognite/neat/session/_state.py +27 -27
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +23 -21
- cognite/neat/session/_to.py +42 -42
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/METADATA +14 -7
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/RECORD +102 -100
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import itertools
|
|
2
2
|
from collections import Counter, defaultdict
|
|
3
|
+
from collections.abc import Iterable
|
|
3
4
|
|
|
4
5
|
from cognite.neat.core._data_model._constants import PATTERNS, EntityTypes
|
|
5
|
-
from cognite.neat.core._data_model.models.entities import
|
|
6
|
+
from cognite.neat.core._data_model.models.entities import ConceptEntity, UnknownEntity
|
|
6
7
|
from cognite.neat.core._data_model.models.entities._multi_value import MultiValueTypeInfo
|
|
7
8
|
from cognite.neat.core._issues import IssueList
|
|
8
9
|
from cognite.neat.core._issues.errors import NeatValueError
|
|
@@ -10,7 +11,7 @@ from cognite.neat.core._issues.errors._resources import (
|
|
|
10
11
|
ResourceDuplicatedError,
|
|
11
12
|
ResourceNotDefinedError,
|
|
12
13
|
)
|
|
13
|
-
from cognite.neat.core._issues.warnings._models import
|
|
14
|
+
from cognite.neat.core._issues.warnings._models import UndefinedConceptWarning
|
|
14
15
|
from cognite.neat.core._issues.warnings._resources import (
|
|
15
16
|
ResourceNotDefinedWarning,
|
|
16
17
|
ResourceRegexViolationWarning,
|
|
@@ -18,23 +19,23 @@ from cognite.neat.core._issues.warnings._resources import (
|
|
|
18
19
|
from cognite.neat.core._utils.spreadsheet import SpreadsheetRead
|
|
19
20
|
from cognite.neat.core._utils.text import humanize_collection
|
|
20
21
|
|
|
21
|
-
from ._verified import ConceptualDataModel
|
|
22
|
+
from ._verified import ConceptualDataModel, ConceptualProperty
|
|
22
23
|
|
|
23
24
|
|
|
24
|
-
class
|
|
25
|
-
"""This class does all the validation of the
|
|
25
|
+
class ConceptualValidation:
|
|
26
|
+
"""This class does all the validation of the conceptual data model that have dependencies
|
|
26
27
|
between components."""
|
|
27
28
|
|
|
28
29
|
def __init__(
|
|
29
30
|
self,
|
|
30
|
-
|
|
31
|
+
data_model: ConceptualDataModel,
|
|
31
32
|
read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
|
|
32
33
|
):
|
|
33
|
-
self.
|
|
34
|
+
self.data_model = data_model
|
|
34
35
|
self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
|
|
35
|
-
self._metadata =
|
|
36
|
-
self._properties =
|
|
37
|
-
self.
|
|
36
|
+
self._metadata = data_model.metadata
|
|
37
|
+
self._properties = data_model.properties
|
|
38
|
+
self._concepts = data_model.concepts
|
|
38
39
|
self.issue_list = IssueList()
|
|
39
40
|
|
|
40
41
|
def validate(self) -> IssueList:
|
|
@@ -42,7 +43,7 @@ class InformationValidation:
|
|
|
42
43
|
self._namespaces_reassigned()
|
|
43
44
|
self._classes_without_properties()
|
|
44
45
|
self._undefined_classes()
|
|
45
|
-
self.
|
|
46
|
+
self._parent_concept_defined()
|
|
46
47
|
self._referenced_classes_exist()
|
|
47
48
|
self._referenced_value_types_exist()
|
|
48
49
|
self._regex_compliance_with_dms()
|
|
@@ -51,7 +52,7 @@ class InformationValidation:
|
|
|
51
52
|
|
|
52
53
|
def _duplicated_resources(self) -> None:
|
|
53
54
|
properties_sheet = self._read_info_by_spreadsheet.get("Properties")
|
|
54
|
-
|
|
55
|
+
concepts_sheet = self._read_info_by_spreadsheet.get("Concepts")
|
|
55
56
|
|
|
56
57
|
visited = defaultdict(list)
|
|
57
58
|
for row_no, property_ in enumerate(self._properties):
|
|
@@ -75,9 +76,9 @@ class InformationValidation:
|
|
|
75
76
|
)
|
|
76
77
|
|
|
77
78
|
visited = defaultdict(list)
|
|
78
|
-
for row_no,
|
|
79
|
-
visited[
|
|
80
|
-
|
|
79
|
+
for row_no, concept in enumerate(self._concepts):
|
|
80
|
+
visited[concept._identifier()].append(
|
|
81
|
+
concepts_sheet.adjusted_row_number(row_no) if concepts_sheet else row_no + 1
|
|
81
82
|
)
|
|
82
83
|
|
|
83
84
|
for identifier, rows in visited.items():
|
|
@@ -86,85 +87,85 @@ class InformationValidation:
|
|
|
86
87
|
self.issue_list.append(
|
|
87
88
|
ResourceDuplicatedError(
|
|
88
89
|
identifier[0],
|
|
89
|
-
"
|
|
90
|
+
"concept",
|
|
90
91
|
(f"the Classes sheet at row {humanize_collection(rows)} if data model is read from a spreadsheet."),
|
|
91
92
|
)
|
|
92
93
|
)
|
|
93
94
|
|
|
94
95
|
def _classes_without_properties(self) -> None:
|
|
95
|
-
|
|
96
|
-
referred_classes = {property_.
|
|
97
|
-
|
|
96
|
+
defined_concepts = {concept.concept for concept in self._concepts}
|
|
97
|
+
referred_classes = {property_.concept for property_ in self._properties}
|
|
98
|
+
concept_parent_pairs = self._concept_parent_pairs()
|
|
98
99
|
|
|
99
|
-
if
|
|
100
|
-
for
|
|
100
|
+
if concepts_without_properties := defined_concepts.difference(referred_classes):
|
|
101
|
+
for concept in concepts_without_properties:
|
|
101
102
|
# USE CASE: class has no direct properties and no parents with properties
|
|
102
103
|
# and it is a class in the prefix of data model, as long as it is in the
|
|
103
104
|
# same prefix, meaning same space
|
|
104
|
-
if not
|
|
105
|
+
if not concept_parent_pairs[concept] and concept.prefix == self._metadata.prefix:
|
|
105
106
|
self.issue_list.append(
|
|
106
107
|
ResourceNotDefinedWarning(
|
|
107
|
-
resource_type="
|
|
108
|
-
identifier=
|
|
108
|
+
resource_type="concept",
|
|
109
|
+
identifier=concept,
|
|
109
110
|
location="Properties sheet",
|
|
110
111
|
)
|
|
111
112
|
)
|
|
112
113
|
|
|
113
114
|
def _undefined_classes(self) -> None:
|
|
114
|
-
|
|
115
|
-
|
|
115
|
+
defined_concept = {concept.concept for concept in self._concepts}
|
|
116
|
+
referred_concepts = {property_.concept for property_ in self._properties}
|
|
116
117
|
|
|
117
|
-
if
|
|
118
|
-
for
|
|
118
|
+
if undefined_concepts := referred_concepts.difference(defined_concept):
|
|
119
|
+
for concept in undefined_concepts:
|
|
119
120
|
self.issue_list.append(
|
|
120
121
|
ResourceNotDefinedError(
|
|
121
|
-
identifier=
|
|
122
|
-
resource_type="
|
|
123
|
-
location="
|
|
122
|
+
identifier=concept,
|
|
123
|
+
resource_type="concept",
|
|
124
|
+
location="Concepts sheet",
|
|
124
125
|
)
|
|
125
126
|
)
|
|
126
127
|
|
|
127
|
-
def
|
|
128
|
-
"""This is a validation to check if the parent
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
parents = set(itertools.chain.from_iterable(
|
|
128
|
+
def _parent_concept_defined(self) -> None:
|
|
129
|
+
"""This is a validation to check if the parent concept is defined."""
|
|
130
|
+
concept_parent_pairs = self._concept_parent_pairs()
|
|
131
|
+
concepts = set(concept_parent_pairs.keys())
|
|
132
|
+
parents = set(itertools.chain.from_iterable(concept_parent_pairs.values()))
|
|
132
133
|
|
|
133
|
-
if undefined_parents := parents.difference(
|
|
134
|
+
if undefined_parents := parents.difference(concepts):
|
|
134
135
|
for parent in undefined_parents:
|
|
135
136
|
if parent.prefix != self._metadata.prefix:
|
|
136
|
-
self.issue_list.append(
|
|
137
|
+
self.issue_list.append(UndefinedConceptWarning(concept_id=str(parent)))
|
|
137
138
|
else:
|
|
138
139
|
self.issue_list.append(
|
|
139
140
|
ResourceNotDefinedWarning(
|
|
140
|
-
resource_type="
|
|
141
|
+
resource_type="concept",
|
|
141
142
|
identifier=parent,
|
|
142
|
-
location="
|
|
143
|
+
location="Concepts sheet",
|
|
143
144
|
)
|
|
144
145
|
)
|
|
145
146
|
|
|
146
147
|
def _referenced_classes_exist(self) -> None:
|
|
147
148
|
# needs to be complete for this validation to pass
|
|
148
|
-
|
|
149
|
-
classes_with_explicit_properties = {property_.
|
|
149
|
+
defined_concept = {concept.concept for concept in self._concepts}
|
|
150
|
+
classes_with_explicit_properties = {property_.concept for property_ in self._properties}
|
|
150
151
|
|
|
151
152
|
# USE CASE: models are complete
|
|
152
|
-
if missing_classes := classes_with_explicit_properties.difference(
|
|
153
|
-
for
|
|
153
|
+
if missing_classes := classes_with_explicit_properties.difference(defined_concept):
|
|
154
|
+
for concept in missing_classes:
|
|
154
155
|
self.issue_list.append(
|
|
155
156
|
ResourceNotDefinedWarning(
|
|
156
|
-
resource_type="
|
|
157
|
-
identifier=
|
|
158
|
-
location="
|
|
157
|
+
resource_type="concept",
|
|
158
|
+
identifier=concept,
|
|
159
|
+
location="Concepts sheet",
|
|
159
160
|
)
|
|
160
161
|
)
|
|
161
162
|
|
|
162
163
|
def _referenced_value_types_exist(self) -> None:
|
|
163
164
|
# adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
|
|
164
|
-
defined_classes = {
|
|
165
|
+
defined_classes = {concept.concept for concept in self._concepts} | {UnknownEntity()}
|
|
165
166
|
referred_object_types = {
|
|
166
167
|
property_.value_type
|
|
167
|
-
for property_ in self.
|
|
168
|
+
for property_ in self.data_model.properties
|
|
168
169
|
if property_.type_ == EntityTypes.object_property
|
|
169
170
|
}
|
|
170
171
|
|
|
@@ -173,9 +174,9 @@ class InformationValidation:
|
|
|
173
174
|
for missing in missing_value_types:
|
|
174
175
|
self.issue_list.append(
|
|
175
176
|
ResourceNotDefinedWarning(
|
|
176
|
-
resource_type="
|
|
177
|
+
resource_type="concept",
|
|
177
178
|
identifier=missing,
|
|
178
|
-
location="
|
|
179
|
+
location="Concepts sheet",
|
|
179
180
|
)
|
|
180
181
|
)
|
|
181
182
|
|
|
@@ -183,28 +184,28 @@ class InformationValidation:
|
|
|
183
184
|
"""Check regex compliance with DMS of properties, classes and value types."""
|
|
184
185
|
|
|
185
186
|
for prop_ in self._properties:
|
|
186
|
-
if not PATTERNS.
|
|
187
|
+
if not PATTERNS.physical_property_id_compliance.match(prop_.property_):
|
|
187
188
|
self.issue_list.append(
|
|
188
189
|
ResourceRegexViolationWarning(
|
|
189
190
|
prop_.property_,
|
|
190
191
|
"Property",
|
|
191
192
|
"Properties sheet, Property column",
|
|
192
|
-
PATTERNS.
|
|
193
|
+
PATTERNS.physical_property_id_compliance.pattern,
|
|
193
194
|
)
|
|
194
195
|
)
|
|
195
|
-
if not PATTERNS.view_id_compliance.match(prop_.
|
|
196
|
+
if not PATTERNS.view_id_compliance.match(prop_.concept.suffix):
|
|
196
197
|
self.issue_list.append(
|
|
197
198
|
ResourceRegexViolationWarning(
|
|
198
|
-
prop_.
|
|
199
|
-
"
|
|
200
|
-
"Properties sheet,
|
|
199
|
+
prop_.concept,
|
|
200
|
+
"Concept",
|
|
201
|
+
"Properties sheet, Concept column",
|
|
201
202
|
PATTERNS.view_id_compliance.pattern,
|
|
202
203
|
)
|
|
203
204
|
)
|
|
204
205
|
|
|
205
206
|
# Handling Value Type
|
|
206
207
|
if (
|
|
207
|
-
isinstance(prop_.value_type,
|
|
208
|
+
isinstance(prop_.value_type, ConceptEntity)
|
|
208
209
|
and prop_.value_type != UnknownEntity()
|
|
209
210
|
and not PATTERNS.view_id_compliance.match(prop_.value_type.suffix)
|
|
210
211
|
):
|
|
@@ -219,7 +220,7 @@ class InformationValidation:
|
|
|
219
220
|
if isinstance(prop_.value_type, MultiValueTypeInfo):
|
|
220
221
|
for value_type in prop_.value_type.types:
|
|
221
222
|
if (
|
|
222
|
-
isinstance(prop_.value_type,
|
|
223
|
+
isinstance(prop_.value_type, ConceptEntity)
|
|
223
224
|
and prop_.value_type != UnknownEntity()
|
|
224
225
|
and not PATTERNS.view_id_compliance.match(value_type.suffix)
|
|
225
226
|
):
|
|
@@ -232,44 +233,44 @@ class InformationValidation:
|
|
|
232
233
|
)
|
|
233
234
|
)
|
|
234
235
|
|
|
235
|
-
for
|
|
236
|
-
if not PATTERNS.view_id_compliance.match(
|
|
236
|
+
for concepts in self._concepts:
|
|
237
|
+
if not PATTERNS.view_id_compliance.match(concepts.concept.suffix):
|
|
237
238
|
self.issue_list.append(
|
|
238
239
|
ResourceRegexViolationWarning(
|
|
239
|
-
|
|
240
|
-
"
|
|
241
|
-
"
|
|
240
|
+
concepts.concept,
|
|
241
|
+
"Concept",
|
|
242
|
+
"Concepts sheet, Class column",
|
|
242
243
|
PATTERNS.view_id_compliance.pattern,
|
|
243
244
|
)
|
|
244
245
|
)
|
|
245
246
|
|
|
246
|
-
if
|
|
247
|
-
for parent in
|
|
247
|
+
if concepts.implements:
|
|
248
|
+
for parent in concepts.implements:
|
|
248
249
|
if not PATTERNS.view_id_compliance.match(parent.suffix):
|
|
249
250
|
self.issue_list.append(
|
|
250
251
|
ResourceRegexViolationWarning(
|
|
251
252
|
parent,
|
|
252
|
-
"
|
|
253
|
-
"
|
|
253
|
+
"Concept",
|
|
254
|
+
"Concepts sheet, Implements column",
|
|
254
255
|
PATTERNS.view_id_compliance.pattern,
|
|
255
256
|
)
|
|
256
257
|
)
|
|
257
258
|
|
|
258
|
-
def
|
|
259
|
-
|
|
260
|
-
|
|
259
|
+
def _concept_parent_pairs(self) -> dict[ConceptEntity, list[ConceptEntity]]:
|
|
260
|
+
concept_parent_pairs: dict[ConceptEntity, list[ConceptEntity]] = {}
|
|
261
|
+
concepts = self.data_model.model_copy(deep=True).concepts
|
|
261
262
|
|
|
262
|
-
for
|
|
263
|
-
|
|
264
|
-
if
|
|
263
|
+
for concept in concepts:
|
|
264
|
+
concept_parent_pairs[concept.concept] = []
|
|
265
|
+
if concept.implements is None:
|
|
265
266
|
continue
|
|
266
|
-
|
|
267
|
+
concept_parent_pairs[concept.concept].extend(concept.implements)
|
|
267
268
|
|
|
268
|
-
return
|
|
269
|
+
return concept_parent_pairs
|
|
269
270
|
|
|
270
271
|
def _namespaces_reassigned(self) -> None:
|
|
271
|
-
prefixes = self.
|
|
272
|
-
prefixes[self.
|
|
272
|
+
prefixes = self.data_model.prefixes.copy()
|
|
273
|
+
prefixes[self.data_model.metadata.namespace.prefix] = self.data_model.metadata.namespace
|
|
273
274
|
|
|
274
275
|
if len(set(prefixes.values())) != len(prefixes):
|
|
275
276
|
reused_namespaces = [value for value, count in Counter(prefixes.values()).items() if count > 1]
|
|
@@ -282,3 +283,12 @@ class InformationValidation:
|
|
|
282
283
|
"\nMake sure that each unique namespace is assigned to a unique prefix"
|
|
283
284
|
)
|
|
284
285
|
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def duplicated_properties(
|
|
289
|
+
properties: Iterable[ConceptualProperty],
|
|
290
|
+
) -> dict[tuple[ConceptEntity, str], list[tuple[int, ConceptualProperty]]]:
|
|
291
|
+
concept_properties_by_id: dict[tuple[ConceptEntity, str], list[tuple[int, ConceptualProperty]]] = defaultdict(list)
|
|
292
|
+
for prop_no, prop in enumerate(properties):
|
|
293
|
+
concept_properties_by_id[(prop.concept, prop.property_)].append((prop_no, prop))
|
|
294
|
+
return {k: v for k, v in concept_properties_by_id.items() if len(v) > 1}
|
|
@@ -18,7 +18,7 @@ from cognite.neat.core._data_model.models._base_verified import (
|
|
|
18
18
|
SheetRow,
|
|
19
19
|
)
|
|
20
20
|
from cognite.neat.core._data_model.models._types import (
|
|
21
|
-
|
|
21
|
+
ConceptEntityType,
|
|
22
22
|
ConceptualPropertyType,
|
|
23
23
|
MultiValueTypeType,
|
|
24
24
|
URIRefType,
|
|
@@ -27,15 +27,15 @@ from cognite.neat.core._data_model.models._types import (
|
|
|
27
27
|
# NeatIdType,
|
|
28
28
|
from cognite.neat.core._data_model.models.data_types import DataType
|
|
29
29
|
from cognite.neat.core._data_model.models.entities import (
|
|
30
|
-
ClassEntity,
|
|
31
30
|
ClassEntityList,
|
|
32
|
-
|
|
31
|
+
ConceptEntity,
|
|
32
|
+
ConceptualEntity,
|
|
33
33
|
UnknownEntity,
|
|
34
34
|
)
|
|
35
35
|
from cognite.neat.core._issues.errors import PropertyDefinitionError
|
|
36
36
|
|
|
37
37
|
if TYPE_CHECKING:
|
|
38
|
-
from cognite.neat.core._data_model.models import
|
|
38
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class ConceptualMetadata(BaseVerifiedMetadata):
|
|
@@ -43,7 +43,7 @@ class ConceptualMetadata(BaseVerifiedMetadata):
|
|
|
43
43
|
level: ClassVar[DataModelLevel] = DataModelLevel.conceptual
|
|
44
44
|
|
|
45
45
|
# Linking to Conceptual and Physical data model aspects
|
|
46
|
-
physical: URIRef | str | None = Field(None, description="Link to the physical data model
|
|
46
|
+
physical: URIRef | str | None = Field(None, description="Link to the physical data model level")
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
def _get_metadata(context: Any) -> ConceptualMetadata | None:
|
|
@@ -52,18 +52,19 @@ def _get_metadata(context: Any) -> ConceptualMetadata | None:
|
|
|
52
52
|
return None
|
|
53
53
|
|
|
54
54
|
|
|
55
|
-
class
|
|
55
|
+
class Concept(SheetRow):
|
|
56
56
|
"""
|
|
57
|
-
|
|
57
|
+
Concept is a category of things that share a common set of attributes and relationships.
|
|
58
58
|
|
|
59
59
|
Args:
|
|
60
|
-
|
|
60
|
+
concept: An ID of the concept.
|
|
61
61
|
description: A description of the class.
|
|
62
62
|
implements: Which classes the current class implements.
|
|
63
63
|
"""
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
alias="
|
|
65
|
+
concept: ConceptEntityType = Field(
|
|
66
|
+
alias="Concept",
|
|
67
|
+
description="Concept id being defined, use strongly advise `PascalCase` usage.",
|
|
67
68
|
)
|
|
68
69
|
name: str | None = Field(alias="Name", default=None, description="Human readable name of the class.")
|
|
69
70
|
description: str | None = Field(alias="Description", default=None, description="Short description of the class.")
|
|
@@ -83,11 +84,11 @@ class ConceptualClass(SheetRow):
|
|
|
83
84
|
)
|
|
84
85
|
|
|
85
86
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
86
|
-
return (self.
|
|
87
|
+
return (self.concept,)
|
|
87
88
|
|
|
88
|
-
@field_serializer("
|
|
89
|
+
@field_serializer("concept", when_used="unless-none")
|
|
89
90
|
def remove_default_prefix(self, value: Any, info: SerializationInfo) -> str:
|
|
90
|
-
if (metadata := _get_metadata(info.context)) and isinstance(value,
|
|
91
|
+
if (metadata := _get_metadata(info.context)) and isinstance(value, ConceptualEntity):
|
|
91
92
|
return value.dump(prefix=metadata.prefix, version=metadata.version)
|
|
92
93
|
return str(value)
|
|
93
94
|
|
|
@@ -96,22 +97,22 @@ class ConceptualClass(SheetRow):
|
|
|
96
97
|
if isinstance(value, list) and (metadata := _get_metadata(info.context)):
|
|
97
98
|
return ",".join(
|
|
98
99
|
(
|
|
99
|
-
|
|
100
|
-
if isinstance(
|
|
101
|
-
else str(
|
|
100
|
+
concept.dump(prefix=metadata.prefix, version=metadata.version)
|
|
101
|
+
if isinstance(concept, ConceptualEntity)
|
|
102
|
+
else str(concept)
|
|
102
103
|
)
|
|
103
|
-
for
|
|
104
|
+
for concept in value
|
|
104
105
|
)
|
|
105
106
|
return ",".join(str(value) for value in value)
|
|
106
107
|
|
|
107
108
|
|
|
108
109
|
class ConceptualProperty(SheetRow):
|
|
109
110
|
"""
|
|
110
|
-
A property is a characteristic of a
|
|
111
|
-
or a relationship to another
|
|
111
|
+
A property is a characteristic of a concept. It is a named attribute of a concept
|
|
112
|
+
that describes a range of values or a relationship to another concept.
|
|
112
113
|
|
|
113
114
|
Args:
|
|
114
|
-
|
|
115
|
+
concept: Concept ID to which property belongs
|
|
115
116
|
property_: Property ID of the property
|
|
116
117
|
name: Property name.
|
|
117
118
|
value_type: Type of value property will hold (data or link to another class)
|
|
@@ -122,8 +123,9 @@ class ConceptualProperty(SheetRow):
|
|
|
122
123
|
knowledge graph. Defaults to None (no transformation)
|
|
123
124
|
"""
|
|
124
125
|
|
|
125
|
-
|
|
126
|
-
alias="
|
|
126
|
+
concept: ConceptEntityType = Field(
|
|
127
|
+
alias="Concept",
|
|
128
|
+
description="Concept id that the property is defined for, strongly advise `PascalCase` usage.",
|
|
127
129
|
)
|
|
128
130
|
property_: ConceptualPropertyType = Field(
|
|
129
131
|
alias="Property",
|
|
@@ -131,7 +133,7 @@ class ConceptualProperty(SheetRow):
|
|
|
131
133
|
)
|
|
132
134
|
name: str | None = Field(alias="Name", default=None, description="Human readable name of the property.")
|
|
133
135
|
description: str | None = Field(alias="Description", default=None, description="Short description of the property.")
|
|
134
|
-
value_type: DataType |
|
|
136
|
+
value_type: DataType | ConceptEntityType | MultiValueTypeType | UnknownEntity = Field(
|
|
135
137
|
alias="Value Type",
|
|
136
138
|
union_mode="left_to_right",
|
|
137
139
|
description="Value type that the property can hold. It takes either subset of XSD type or a class defined.",
|
|
@@ -169,7 +171,7 @@ class ConceptualProperty(SheetRow):
|
|
|
169
171
|
)
|
|
170
172
|
|
|
171
173
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
172
|
-
return self.
|
|
174
|
+
return self.concept, self.property_
|
|
173
175
|
|
|
174
176
|
@field_validator("max_count", mode="before")
|
|
175
177
|
def parse_max_count(cls, value: int | float | None) -> int | float | None:
|
|
@@ -201,7 +203,7 @@ class ConceptualProperty(SheetRow):
|
|
|
201
203
|
# this value_type.python does not seems correct. Need to check this further
|
|
202
204
|
except Exception:
|
|
203
205
|
raise PropertyDefinitionError(
|
|
204
|
-
self.
|
|
206
|
+
self.concept,
|
|
205
207
|
"Class",
|
|
206
208
|
self.property_,
|
|
207
209
|
f"Default value {self.default} is not of type {self.value_type.python}", # type: ignore
|
|
@@ -220,9 +222,9 @@ class ConceptualProperty(SheetRow):
|
|
|
220
222
|
return None
|
|
221
223
|
return value
|
|
222
224
|
|
|
223
|
-
@field_serializer("
|
|
225
|
+
@field_serializer("concept", "value_type", when_used="unless-none")
|
|
224
226
|
def remove_default_prefix(self, value: Any, info: SerializationInfo) -> str:
|
|
225
|
-
if (metadata := _get_metadata(info.context)) and isinstance(value,
|
|
227
|
+
if (metadata := _get_metadata(info.context)) and isinstance(value, ConceptualEntity):
|
|
226
228
|
return value.dump(prefix=metadata.prefix, version=metadata.version)
|
|
227
229
|
return str(value)
|
|
228
230
|
|
|
@@ -231,7 +233,7 @@ class ConceptualProperty(SheetRow):
|
|
|
231
233
|
"""Type of property based on value type. Either data (attribute) or object (edge) property."""
|
|
232
234
|
if isinstance(self.value_type, DataType):
|
|
233
235
|
return EntityTypes.data_property
|
|
234
|
-
elif isinstance(self.value_type,
|
|
236
|
+
elif isinstance(self.value_type, ConceptEntity):
|
|
235
237
|
return EntityTypes.object_property
|
|
236
238
|
else:
|
|
237
239
|
return EntityTypes.undefined
|
|
@@ -240,7 +242,7 @@ class ConceptualProperty(SheetRow):
|
|
|
240
242
|
class ConceptualDataModel(BaseVerifiedDataModel):
|
|
241
243
|
metadata: ConceptualMetadata = Field(alias="Metadata", description="Metadata for the conceptual data model")
|
|
242
244
|
properties: SheetList[ConceptualProperty] = Field(alias="Properties", description="List of properties")
|
|
243
|
-
|
|
245
|
+
concepts: SheetList[Concept] = Field(alias="Concepts", description="List of concepts")
|
|
244
246
|
prefixes: dict[str, Namespace] = Field(
|
|
245
247
|
alias="Prefixes",
|
|
246
248
|
default_factory=get_default_prefixes_and_namespaces,
|
|
@@ -259,12 +261,12 @@ class ConceptualDataModel(BaseVerifiedDataModel):
|
|
|
259
261
|
def set_neat_id(self) -> "ConceptualDataModel":
|
|
260
262
|
namespace = self.metadata.namespace
|
|
261
263
|
|
|
262
|
-
for
|
|
263
|
-
if not
|
|
264
|
-
|
|
264
|
+
for concept in self.concepts:
|
|
265
|
+
if not concept.neatId:
|
|
266
|
+
concept.neatId = namespace[concept.concept.suffix]
|
|
265
267
|
for property_ in self.properties:
|
|
266
268
|
if not property_.neatId:
|
|
267
|
-
property_.neatId = namespace[f"{property_.
|
|
269
|
+
property_.neatId = namespace[f"{property_.concept.suffix}/{property_.property_}"]
|
|
268
270
|
|
|
269
271
|
return self
|
|
270
272
|
|
|
@@ -273,37 +275,37 @@ class ConceptualDataModel(BaseVerifiedDataModel):
|
|
|
273
275
|
|
|
274
276
|
namespace = self.metadata.namespace
|
|
275
277
|
|
|
276
|
-
for
|
|
277
|
-
|
|
278
|
+
for concept in self.concepts:
|
|
279
|
+
concept.neatId = namespace[concept.concept.suffix]
|
|
278
280
|
for property_ in self.properties:
|
|
279
|
-
property_.neatId = namespace[f"{property_.
|
|
281
|
+
property_.neatId = namespace[f"{property_.concept.suffix}/{property_.property_}"]
|
|
280
282
|
|
|
281
|
-
def sync_with_physical_data_model(self,
|
|
283
|
+
def sync_with_physical_data_model(self, physical_data_model: "PhysicalDataModel") -> None:
|
|
282
284
|
# Sync at the metadata level
|
|
283
|
-
if
|
|
284
|
-
self.metadata.physical =
|
|
285
|
+
if physical_data_model.metadata.conceptual == self.metadata.identifier:
|
|
286
|
+
self.metadata.physical = physical_data_model.metadata.identifier
|
|
285
287
|
else:
|
|
286
288
|
# if models are not linked to start with, we skip
|
|
287
289
|
return None
|
|
288
290
|
|
|
289
291
|
conceptual_properties_by_neat_id = {prop.neatId: prop for prop in self.properties}
|
|
290
|
-
physical_properties_by_neat_id = {prop.neatId: prop for prop in
|
|
292
|
+
physical_properties_by_neat_id = {prop.neatId: prop for prop in physical_data_model.properties}
|
|
291
293
|
for neat_id, prop in physical_properties_by_neat_id.items():
|
|
292
|
-
if prop.
|
|
293
|
-
conceptual_properties_by_neat_id[prop.
|
|
294
|
+
if prop.conceptual in conceptual_properties_by_neat_id:
|
|
295
|
+
conceptual_properties_by_neat_id[prop.conceptual].physical = neat_id
|
|
294
296
|
|
|
295
|
-
classes_by_neat_id = {cls.neatId: cls for cls in self.
|
|
296
|
-
views_by_neat_id = {view.neatId: view for view in
|
|
297
|
+
classes_by_neat_id = {cls.neatId: cls for cls in self.concepts}
|
|
298
|
+
views_by_neat_id = {view.neatId: view for view in physical_data_model.views}
|
|
297
299
|
for neat_id, view in views_by_neat_id.items():
|
|
298
|
-
if view.
|
|
299
|
-
classes_by_neat_id[view.
|
|
300
|
+
if view.conceptual in classes_by_neat_id:
|
|
301
|
+
classes_by_neat_id[view.conceptual].physical = neat_id
|
|
300
302
|
|
|
301
|
-
def
|
|
303
|
+
def as_physical_data_model(self) -> "PhysicalDataModel":
|
|
302
304
|
from cognite.neat.core._data_model.transformers._converters import (
|
|
303
|
-
|
|
305
|
+
_ConceptualDataModelConverter,
|
|
304
306
|
)
|
|
305
307
|
|
|
306
|
-
return
|
|
308
|
+
return _ConceptualDataModelConverter(self).as_physical_data_model()
|
|
307
309
|
|
|
308
310
|
@classmethod
|
|
309
311
|
def display_type_name(cls) -> str:
|
|
@@ -311,12 +313,12 @@ class ConceptualDataModel(BaseVerifiedDataModel):
|
|
|
311
313
|
|
|
312
314
|
def _repr_html_(self) -> str:
|
|
313
315
|
summary = {
|
|
314
|
-
"
|
|
315
|
-
"intended for": "Information Architect",
|
|
316
|
+
"level": self.metadata.level,
|
|
317
|
+
"intended for": "Domain Expert and/or Information Architect",
|
|
316
318
|
"name": self.metadata.name,
|
|
317
319
|
"external_id": self.metadata.external_id,
|
|
318
320
|
"version": self.metadata.version,
|
|
319
|
-
"
|
|
321
|
+
"concepts": len(self.concepts),
|
|
320
322
|
"properties": len(self.properties),
|
|
321
323
|
}
|
|
322
324
|
|
|
@@ -16,7 +16,7 @@ from cognite.neat.core._data_model._constants import (
|
|
|
16
16
|
SPLIT_ON_EQUAL_PATTERN,
|
|
17
17
|
)
|
|
18
18
|
from cognite.neat.core._data_model.models.entities._single_value import (
|
|
19
|
-
|
|
19
|
+
ConceptEntity,
|
|
20
20
|
UnitEntity,
|
|
21
21
|
)
|
|
22
22
|
|
|
@@ -397,7 +397,7 @@ class Enum(DataType):
|
|
|
397
397
|
|
|
398
398
|
name: typing.Literal["enum"] = "enum"
|
|
399
399
|
|
|
400
|
-
collection:
|
|
400
|
+
collection: ConceptEntity
|
|
401
401
|
unknown_value: str | None = Field(None, alias="unknownValue")
|
|
402
402
|
|
|
403
403
|
|
|
@@ -4,15 +4,15 @@ from ._multi_value import MultiValueTypeInfo
|
|
|
4
4
|
from ._single_value import (
|
|
5
5
|
AssetEntity,
|
|
6
6
|
AssetFields,
|
|
7
|
-
|
|
7
|
+
ConceptEntity,
|
|
8
|
+
ConceptualEntity,
|
|
8
9
|
ContainerEntity,
|
|
9
10
|
DataModelEntity,
|
|
10
|
-
DMSEntity,
|
|
11
11
|
DMSNodeEntity,
|
|
12
|
-
DMSUnknownEntity,
|
|
13
12
|
DMSVersionedEntity,
|
|
14
13
|
EdgeEntity,
|
|
15
|
-
|
|
14
|
+
PhysicalEntity,
|
|
15
|
+
PhysicalUnknownEntity,
|
|
16
16
|
ReferenceEntity,
|
|
17
17
|
RelationshipEntity,
|
|
18
18
|
ReverseConnectionEntity,
|
|
@@ -28,21 +28,21 @@ __all__ = [
|
|
|
28
28
|
"AssetEntity",
|
|
29
29
|
"AssetFields",
|
|
30
30
|
"CdfResourceEntityList",
|
|
31
|
-
"ClassEntity",
|
|
32
31
|
"ClassEntityList",
|
|
32
|
+
"ConceptEntity",
|
|
33
|
+
"ConceptualEntity",
|
|
33
34
|
"ContainerEntity",
|
|
34
35
|
"ContainerEntityList",
|
|
35
|
-
"DMSEntity",
|
|
36
36
|
"DMSFilter",
|
|
37
37
|
"DMSNodeEntity",
|
|
38
|
-
"DMSUnknownEntity",
|
|
39
38
|
"DMSVersionedEntity",
|
|
40
39
|
"DataModelEntity",
|
|
41
40
|
"EdgeEntity",
|
|
42
|
-
"Entity",
|
|
43
41
|
"HasDataFilter",
|
|
44
42
|
"MultiValueTypeInfo",
|
|
45
43
|
"NodeTypeFilter",
|
|
44
|
+
"PhysicalEntity",
|
|
45
|
+
"PhysicalUnknownEntity",
|
|
46
46
|
"RawFilter",
|
|
47
47
|
"ReferenceEntity",
|
|
48
48
|
"RelationshipEntity",
|