cognite-neat 0.123.14__py3-none-any.whl → 0.123.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_data_model/exporters/_data_model2excel.py +1 -1
- cognite/neat/core/_data_model/models/conceptual/_validation.py +82 -97
- cognite/neat/core/_data_model/transformers/__init__.py +2 -0
- cognite/neat/core/_data_model/transformers/_union_conceptual.py +208 -0
- cognite/neat/core/_issues/_base.py +5 -0
- cognite/neat/core/_issues/warnings/_models.py +2 -2
- {cognite_neat-0.123.14.dist-info → cognite_neat-0.123.16.dist-info}/METADATA +1 -1
- {cognite_neat-0.123.14.dist-info → cognite_neat-0.123.16.dist-info}/RECORD +11 -10
- {cognite_neat-0.123.14.dist-info → cognite_neat-0.123.16.dist-info}/WHEEL +0 -0
- {cognite_neat-0.123.14.dist-info → cognite_neat-0.123.16.dist-info}/licenses/LICENSE +0 -0
cognite/neat/_version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "0.123.
|
|
1
|
+
__version__ = "0.123.16"
|
|
2
2
|
__engine__ = "^2.0.4"
|
|
@@ -344,7 +344,7 @@ class ExcelExporter(BaseExporter[VerifiedDataModel, Workbook]):
|
|
|
344
344
|
for value_type_counter, value_type in enumerate(_DATA_TYPE_BY_DMS_TYPE.values()):
|
|
345
345
|
value_type_as_str = value_type.dms._type.casefold() if role == RoleTypes.dms else value_type.xsd
|
|
346
346
|
# skip types which require special handling or are surpassed by CDM
|
|
347
|
-
if value_type_as_str in ["enum", "timeseries", "sequence", "file"
|
|
347
|
+
if value_type_as_str in ["enum", "timeseries", "sequence", "file"]:
|
|
348
348
|
continue
|
|
349
349
|
workbook[self._helper_sheet_name].cell(
|
|
350
350
|
row=value_type_counter + 1,
|
|
@@ -2,6 +2,7 @@ import itertools
|
|
|
2
2
|
from collections import Counter, defaultdict
|
|
3
3
|
from collections.abc import Iterable
|
|
4
4
|
|
|
5
|
+
from cognite.neat.core._constants import get_base_concepts
|
|
5
6
|
from cognite.neat.core._data_model._constants import PATTERNS, EntityTypes
|
|
6
7
|
from cognite.neat.core._data_model.models.entities import ConceptEntity, UnknownEntity
|
|
7
8
|
from cognite.neat.core._data_model.models.entities._multi_value import MultiValueTypeInfo
|
|
@@ -35,23 +36,31 @@ class ConceptualValidation:
|
|
|
35
36
|
data_model: ConceptualDataModel,
|
|
36
37
|
read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
|
|
37
38
|
):
|
|
39
|
+
# import here to avoid circular import issues
|
|
40
|
+
from cognite.neat.core._data_model.analysis._base import DataModelAnalysis
|
|
41
|
+
|
|
38
42
|
self.data_model = data_model
|
|
43
|
+
self.analysis = DataModelAnalysis(self.data_model)
|
|
39
44
|
self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
|
|
40
45
|
self._metadata = data_model.metadata
|
|
41
46
|
self._properties = data_model.properties
|
|
42
47
|
self._concepts = data_model.concepts
|
|
48
|
+
self._cdf_concepts = {
|
|
49
|
+
ConceptEntity.load(concept_as_string) for concept_as_string in get_base_concepts(base_model="CogniteCore")
|
|
50
|
+
}
|
|
43
51
|
self.issue_list = IssueList()
|
|
44
52
|
|
|
45
53
|
def validate(self) -> IssueList:
|
|
46
54
|
self._duplicated_resources()
|
|
47
55
|
self._namespaces_reassigned()
|
|
48
|
-
self.
|
|
49
|
-
self.
|
|
50
|
-
self.
|
|
51
|
-
|
|
52
|
-
self.
|
|
53
|
-
self._concept_only_data_model()
|
|
56
|
+
self._concepts_without_properties_exist()
|
|
57
|
+
self._concepts_with_properties_defined()
|
|
58
|
+
self._ancestors_defined()
|
|
59
|
+
|
|
60
|
+
self._object_properties_use_defined_concepts()
|
|
54
61
|
self._dangling_properties()
|
|
62
|
+
|
|
63
|
+
self._concept_only_data_model()
|
|
55
64
|
self._regex_compliance_with_physical_data_model()
|
|
56
65
|
|
|
57
66
|
return self.issue_list
|
|
@@ -59,23 +68,23 @@ class ConceptualValidation:
|
|
|
59
68
|
def _concept_only_data_model(self) -> None:
|
|
60
69
|
"""Check if the data model only consists of concepts without any properties."""
|
|
61
70
|
if not self._properties:
|
|
62
|
-
self.issue_list.
|
|
71
|
+
self.issue_list.append_if_not_exist(ConceptOnlyDataModelWarning())
|
|
63
72
|
|
|
64
73
|
def _dangling_properties(self) -> None:
|
|
65
74
|
"""Check if there are properties that do not reference any concept."""
|
|
66
75
|
dangling_properties = [prop for prop in self._properties if prop.concept == UnknownEntity()]
|
|
67
76
|
if dangling_properties:
|
|
68
77
|
for prop in dangling_properties:
|
|
69
|
-
self.issue_list.
|
|
78
|
+
self.issue_list.append_if_not_exist(DanglingPropertyWarning(property_id=prop.property_))
|
|
70
79
|
|
|
71
80
|
def _duplicated_resources(self) -> None:
|
|
72
|
-
|
|
73
|
-
|
|
81
|
+
properties_sheet_info = self._read_info_by_spreadsheet.get("Properties")
|
|
82
|
+
concepts_sheet_info = self._read_info_by_spreadsheet.get("Concepts")
|
|
74
83
|
|
|
75
84
|
visited = defaultdict(list)
|
|
76
85
|
for row_no, property_ in enumerate(self._properties):
|
|
77
86
|
visited[property_._identifier()].append(
|
|
78
|
-
|
|
87
|
+
properties_sheet_info.adjusted_row_number(row_no) if properties_sheet_info else row_no + 1
|
|
79
88
|
)
|
|
80
89
|
|
|
81
90
|
for identifier, rows in visited.items():
|
|
@@ -96,46 +105,47 @@ class ConceptualValidation:
|
|
|
96
105
|
visited = defaultdict(list)
|
|
97
106
|
for row_no, concept in enumerate(self._concepts):
|
|
98
107
|
visited[concept._identifier()].append(
|
|
99
|
-
|
|
108
|
+
concepts_sheet_info.adjusted_row_number(row_no) if concepts_sheet_info else row_no + 1
|
|
100
109
|
)
|
|
101
110
|
|
|
102
111
|
for identifier, rows in visited.items():
|
|
103
112
|
if len(rows) == 1:
|
|
104
113
|
continue
|
|
105
|
-
self.issue_list.
|
|
114
|
+
self.issue_list.append_if_not_exist(
|
|
106
115
|
ResourceDuplicatedError(
|
|
107
116
|
identifier[0],
|
|
108
117
|
"concept",
|
|
109
|
-
(
|
|
118
|
+
(
|
|
119
|
+
f"the Concepts sheet at row {humanize_collection(rows)}"
|
|
120
|
+
" if data model is read from a spreadsheet."
|
|
121
|
+
),
|
|
110
122
|
)
|
|
111
123
|
)
|
|
112
124
|
|
|
113
|
-
def
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
#
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
for concept in undefined_concepts:
|
|
138
|
-
self.issue_list.append(
|
|
125
|
+
def _concepts_without_properties_exist(self) -> None:
|
|
126
|
+
"""This validation checks if concepts have properties defined or inherit properties from other concepts."""
|
|
127
|
+
concepts = {concept.concept for concept in self._concepts}
|
|
128
|
+
ancestors_by_concept = self.analysis.parents_by_concept(include_ancestors=True, include_different_space=True)
|
|
129
|
+
concepts_with_properties = self.analysis.defined_concepts().union(self._cdf_concepts)
|
|
130
|
+
|
|
131
|
+
if candidate_concepts := concepts.difference(concepts_with_properties):
|
|
132
|
+
for concept in candidate_concepts:
|
|
133
|
+
# Here we check if at least one of the ancestors of the concept has properties
|
|
134
|
+
if (ancestors := ancestors_by_concept.get(concept)) and ancestors.intersection(
|
|
135
|
+
concepts_with_properties
|
|
136
|
+
):
|
|
137
|
+
continue
|
|
138
|
+
|
|
139
|
+
self.issue_list.append_if_not_exist(UndefinedConceptWarning(concept_id=str(concept)))
|
|
140
|
+
|
|
141
|
+
def _concepts_with_properties_defined(self) -> None:
|
|
142
|
+
"""This validation checks if concepts to which properties are attached are defined."""
|
|
143
|
+
concepts = {concept.concept for concept in self._concepts}
|
|
144
|
+
concepts_with_properties = {property_.concept for property_ in self._properties} - {UnknownEntity()}
|
|
145
|
+
|
|
146
|
+
if undefined_concepts_with_properties := concepts_with_properties.difference(concepts):
|
|
147
|
+
for concept in undefined_concepts_with_properties:
|
|
148
|
+
self.issue_list.append_if_not_exist(
|
|
139
149
|
ResourceNotDefinedError(
|
|
140
150
|
identifier=concept,
|
|
141
151
|
resource_type="concept",
|
|
@@ -143,57 +153,44 @@ class ConceptualValidation:
|
|
|
143
153
|
)
|
|
144
154
|
)
|
|
145
155
|
|
|
146
|
-
def
|
|
147
|
-
"""This is a validation to check if the
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
if parent.prefix != self._metadata.prefix:
|
|
155
|
-
self.issue_list.append(UndefinedConceptWarning(concept_id=str(parent)))
|
|
156
|
-
else:
|
|
157
|
-
self.issue_list.append(
|
|
158
|
-
ResourceNotDefinedWarning(
|
|
159
|
-
resource_type="concept",
|
|
160
|
-
identifier=parent,
|
|
161
|
-
location="Concepts sheet",
|
|
162
|
-
)
|
|
163
|
-
)
|
|
164
|
-
|
|
165
|
-
def _referenced_classes_exist(self) -> None:
|
|
166
|
-
# needs to be complete for this validation to pass
|
|
167
|
-
defined_concept = {concept.concept for concept in self._concepts}
|
|
168
|
-
classes_with_explicit_properties = {property_.concept for property_ in self._properties} - {UnknownEntity()}
|
|
156
|
+
def _ancestors_defined(self) -> None:
|
|
157
|
+
"""This is a validation to check if the ancestor concepts (e.g. parents) are defined."""
|
|
158
|
+
concepts = {concept.concept for concept in self._concepts}.union(self._cdf_concepts)
|
|
159
|
+
ancestors = set(
|
|
160
|
+
itertools.chain.from_iterable(
|
|
161
|
+
self.analysis.parents_by_concept(include_ancestors=True, include_different_space=True).values()
|
|
162
|
+
)
|
|
163
|
+
).difference(self._cdf_concepts)
|
|
169
164
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
self.issue_list.append(
|
|
165
|
+
if undefined_ancestor := ancestors.difference(concepts):
|
|
166
|
+
for ancestor in undefined_ancestor:
|
|
167
|
+
self.issue_list.append_if_not_exist(
|
|
174
168
|
ResourceNotDefinedWarning(
|
|
175
169
|
resource_type="concept",
|
|
176
|
-
identifier=
|
|
170
|
+
identifier=ancestor,
|
|
177
171
|
location="Concepts sheet",
|
|
178
172
|
)
|
|
179
173
|
)
|
|
180
174
|
|
|
181
|
-
def
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
175
|
+
def _object_properties_use_defined_concepts(self) -> None:
|
|
176
|
+
"""Check if the value types of object properties are defined as concepts."""
|
|
177
|
+
|
|
178
|
+
concepts = {concept.concept for concept in self._concepts}
|
|
179
|
+
|
|
180
|
+
# We remove UnknownEntity from the concepts to avoid false positives
|
|
181
|
+
# as `UnknownEntity` is used as a placeholder when the value type is not defined.
|
|
182
|
+
value_types = {
|
|
185
183
|
property_.value_type
|
|
186
184
|
for property_ in self.data_model.properties
|
|
187
185
|
if property_.type_ == EntityTypes.object_property
|
|
188
|
-
}
|
|
186
|
+
}.difference({UnknownEntity()})
|
|
189
187
|
|
|
190
|
-
if
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
self.issue_list.append(
|
|
188
|
+
if undefined_value_types := value_types.difference(concepts):
|
|
189
|
+
for value_type in undefined_value_types:
|
|
190
|
+
self.issue_list.append_if_not_exist(
|
|
194
191
|
ResourceNotDefinedWarning(
|
|
195
192
|
resource_type="concept",
|
|
196
|
-
identifier=
|
|
193
|
+
identifier=value_type,
|
|
197
194
|
location="Concepts sheet",
|
|
198
195
|
)
|
|
199
196
|
)
|
|
@@ -203,7 +200,7 @@ class ConceptualValidation:
|
|
|
203
200
|
|
|
204
201
|
for prop_ in self._properties:
|
|
205
202
|
if not PATTERNS.physical_property_id_compliance.match(prop_.property_):
|
|
206
|
-
self.issue_list.
|
|
203
|
+
self.issue_list.append_if_not_exist(
|
|
207
204
|
ResourceRegexViolationWarning(
|
|
208
205
|
prop_.property_,
|
|
209
206
|
"Property",
|
|
@@ -212,7 +209,7 @@ class ConceptualValidation:
|
|
|
212
209
|
)
|
|
213
210
|
)
|
|
214
211
|
if prop_.concept != UnknownEntity() and not PATTERNS.view_id_compliance.match(prop_.concept.suffix):
|
|
215
|
-
self.issue_list.
|
|
212
|
+
self.issue_list.append_if_not_exist(
|
|
216
213
|
ResourceRegexViolationWarning(
|
|
217
214
|
prop_.concept,
|
|
218
215
|
"Concept",
|
|
@@ -227,7 +224,7 @@ class ConceptualValidation:
|
|
|
227
224
|
and prop_.value_type != UnknownEntity()
|
|
228
225
|
and not PATTERNS.view_id_compliance.match(prop_.value_type.suffix)
|
|
229
226
|
):
|
|
230
|
-
self.issue_list.
|
|
227
|
+
self.issue_list.append_if_not_exist(
|
|
231
228
|
ResourceRegexViolationWarning(
|
|
232
229
|
prop_.value_type,
|
|
233
230
|
"Value Type",
|
|
@@ -242,7 +239,7 @@ class ConceptualValidation:
|
|
|
242
239
|
and prop_.value_type != UnknownEntity()
|
|
243
240
|
and not PATTERNS.view_id_compliance.match(value_type.suffix)
|
|
244
241
|
):
|
|
245
|
-
self.issue_list.
|
|
242
|
+
self.issue_list.append_if_not_exist(
|
|
246
243
|
ResourceRegexViolationWarning(
|
|
247
244
|
value_type,
|
|
248
245
|
"Value Type",
|
|
@@ -253,7 +250,7 @@ class ConceptualValidation:
|
|
|
253
250
|
|
|
254
251
|
for concepts in self._concepts:
|
|
255
252
|
if not PATTERNS.view_id_compliance.match(concepts.concept.suffix):
|
|
256
|
-
self.issue_list.
|
|
253
|
+
self.issue_list.append_if_not_exist(
|
|
257
254
|
ResourceRegexViolationWarning(
|
|
258
255
|
concepts.concept,
|
|
259
256
|
"Concept",
|
|
@@ -265,7 +262,7 @@ class ConceptualValidation:
|
|
|
265
262
|
if concepts.implements:
|
|
266
263
|
for parent in concepts.implements:
|
|
267
264
|
if not PATTERNS.view_id_compliance.match(parent.suffix):
|
|
268
|
-
self.issue_list.
|
|
265
|
+
self.issue_list.append_if_not_exist(
|
|
269
266
|
ResourceRegexViolationWarning(
|
|
270
267
|
parent,
|
|
271
268
|
"Concept",
|
|
@@ -274,18 +271,6 @@ class ConceptualValidation:
|
|
|
274
271
|
)
|
|
275
272
|
)
|
|
276
273
|
|
|
277
|
-
def _concept_parent_pairs(self) -> dict[ConceptEntity, list[ConceptEntity]]:
|
|
278
|
-
concept_parent_pairs: dict[ConceptEntity, list[ConceptEntity]] = {}
|
|
279
|
-
concepts = self.data_model.model_copy(deep=True).concepts
|
|
280
|
-
|
|
281
|
-
for concept in concepts:
|
|
282
|
-
concept_parent_pairs[concept.concept] = []
|
|
283
|
-
if concept.implements is None:
|
|
284
|
-
continue
|
|
285
|
-
concept_parent_pairs[concept.concept].extend(concept.implements)
|
|
286
|
-
|
|
287
|
-
return concept_parent_pairs
|
|
288
|
-
|
|
289
274
|
def _namespaces_reassigned(self) -> None:
|
|
290
275
|
prefixes = self.data_model.prefixes.copy()
|
|
291
276
|
prefixes[self.data_model.metadata.namespace.prefix] = self.data_model.metadata.namespace
|
|
@@ -293,7 +278,7 @@ class ConceptualValidation:
|
|
|
293
278
|
if len(set(prefixes.values())) != len(prefixes):
|
|
294
279
|
reused_namespaces = [value for value, count in Counter(prefixes.values()).items() if count > 1]
|
|
295
280
|
impacted_prefixes = [key for key, value in prefixes.items() if value in reused_namespaces]
|
|
296
|
-
self.issue_list.
|
|
281
|
+
self.issue_list.append_if_not_exist(
|
|
297
282
|
NeatValueError(
|
|
298
283
|
"Namespace collision detected. The following prefixes "
|
|
299
284
|
f"are assigned to the same namespace: {impacted_prefixes}"
|
|
@@ -26,6 +26,7 @@ from ._converters import (
|
|
|
26
26
|
ToSolutionModel,
|
|
27
27
|
)
|
|
28
28
|
from ._mapping import AsParentPropertyId, MapOneToOne, PhysicalDataModelMapper
|
|
29
|
+
from ._union_conceptual import UnionConceptualDataModel
|
|
29
30
|
from ._verification import (
|
|
30
31
|
VerifyAnyDataModel,
|
|
31
32
|
VerifyConceptualDataModel,
|
|
@@ -61,6 +62,7 @@ __all__ = [
|
|
|
61
62
|
"ToEnterpriseModel",
|
|
62
63
|
"ToExtensionModel",
|
|
63
64
|
"ToSolutionModel",
|
|
65
|
+
"UnionConceptualDataModel",
|
|
64
66
|
"VerifiedDataModelTransformer",
|
|
65
67
|
"VerifyAnyDataModel",
|
|
66
68
|
"VerifyConceptualDataModel",
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
from collections.abc import Iterable, Set
|
|
2
|
+
from typing import Literal
|
|
3
|
+
|
|
4
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, SheetList
|
|
5
|
+
from cognite.neat.core._data_model.models.conceptual import Concept, ConceptualProperty
|
|
6
|
+
from cognite.neat.core._data_model.models.data_types import DataType
|
|
7
|
+
from cognite.neat.core._data_model.models.entities import (
|
|
8
|
+
ConceptEntity,
|
|
9
|
+
MultiValueTypeInfo,
|
|
10
|
+
UnknownEntity,
|
|
11
|
+
)
|
|
12
|
+
from cognite.neat.core._data_model.transformers import VerifiedDataModelTransformer
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class UnionConceptualDataModel(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
16
|
+
"""Takes the union two conceptual models.
|
|
17
|
+
Args:
|
|
18
|
+
primary: The primary model to merge with the secondary model given in the transform method.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, primary: ConceptualDataModel) -> None:
|
|
22
|
+
self.primary = primary
|
|
23
|
+
|
|
24
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
25
|
+
primary_model = self.primary
|
|
26
|
+
secondary_model = data_model
|
|
27
|
+
|
|
28
|
+
output = primary_model.model_copy(deep=True)
|
|
29
|
+
secondary_concepts = {cls.concept: cls for cls in secondary_model.concepts}
|
|
30
|
+
secondary_properties = {(prop.concept, prop.property_): prop for prop in secondary_model.properties}
|
|
31
|
+
|
|
32
|
+
union_concepts_by_id = self._union_concepts(output.concepts, secondary_concepts)
|
|
33
|
+
output.concepts = SheetList[Concept](union_concepts_by_id.values())
|
|
34
|
+
|
|
35
|
+
union_properties = self._union_properties(
|
|
36
|
+
output.properties, secondary_properties, set(union_concepts_by_id.keys())
|
|
37
|
+
)
|
|
38
|
+
output.properties = SheetList[ConceptualProperty](union_properties.values())
|
|
39
|
+
|
|
40
|
+
return output
|
|
41
|
+
|
|
42
|
+
def _union_concepts(
|
|
43
|
+
self, primary_concepts: Iterable[Concept], new_concepts: dict[ConceptEntity, Concept]
|
|
44
|
+
) -> dict[ConceptEntity, Concept]:
|
|
45
|
+
union_concepts = {cls.concept: cls for cls in primary_concepts}
|
|
46
|
+
for concept, primary_concept in union_concepts.items():
|
|
47
|
+
if concept not in new_concepts:
|
|
48
|
+
continue
|
|
49
|
+
secondary_concept = new_concepts[concept]
|
|
50
|
+
union_concepts[concept] = self.union_concepts(
|
|
51
|
+
primary=primary_concept,
|
|
52
|
+
secondary=secondary_concept,
|
|
53
|
+
conflict_resolution="combined",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
for concept, secondary_concept in new_concepts.items():
|
|
57
|
+
if concept not in union_concepts:
|
|
58
|
+
union_concepts[concept] = secondary_concept
|
|
59
|
+
return union_concepts
|
|
60
|
+
|
|
61
|
+
def _union_properties(
|
|
62
|
+
self,
|
|
63
|
+
primary_properties: Iterable[ConceptualProperty],
|
|
64
|
+
secondary_properties: dict[tuple[ConceptEntity, str], ConceptualProperty],
|
|
65
|
+
used_concepts: Set[ConceptEntity],
|
|
66
|
+
) -> dict[tuple[ConceptEntity, str], ConceptualProperty]:
|
|
67
|
+
union_properties = {(prop.concept, prop.property_): prop for prop in primary_properties}
|
|
68
|
+
for (concept, prop_id), primary_property in union_properties.items():
|
|
69
|
+
if (concept not in used_concepts) or (concept, prop_id) not in secondary_properties:
|
|
70
|
+
continue
|
|
71
|
+
secondary_property = secondary_properties[(concept, prop_id)]
|
|
72
|
+
union_properties[(concept, prop_id)] = self.union_properties(
|
|
73
|
+
primary=primary_property,
|
|
74
|
+
secondary=secondary_property,
|
|
75
|
+
conflict_resolution="combined",
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
for (concept, prop_id), prop in secondary_properties.items():
|
|
79
|
+
if (concept, prop_id) not in union_properties and concept in used_concepts:
|
|
80
|
+
union_properties[(concept, prop_id)] = prop
|
|
81
|
+
return union_properties
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
def union_concepts(
|
|
85
|
+
cls,
|
|
86
|
+
primary: Concept,
|
|
87
|
+
secondary: Concept,
|
|
88
|
+
conflict_resolution: Literal["priority", "combined"] = "priority",
|
|
89
|
+
) -> Concept:
|
|
90
|
+
"""Union two concepts.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
primary (Concept): The primary concept.
|
|
94
|
+
secondary (Concept): The secondary concept.
|
|
95
|
+
conflict_resolution (Literal["priority", "combined"]): How to resolve conflicts:
|
|
96
|
+
- "priority": Keep the primary concept with fallback to the secondary.
|
|
97
|
+
- "combined": Merge implements from both concepts. (only applies to implements)
|
|
98
|
+
Returns:
|
|
99
|
+
Concept: The union of the two concepts.
|
|
100
|
+
|
|
101
|
+
"""
|
|
102
|
+
if conflict_resolution == "combined":
|
|
103
|
+
all_implements = (primary.implements or []) + (secondary.implements or [])
|
|
104
|
+
implements = list(dict.fromkeys(all_implements))
|
|
105
|
+
else:
|
|
106
|
+
implements = (primary.implements or secondary.implements or []).copy()
|
|
107
|
+
|
|
108
|
+
return Concept(
|
|
109
|
+
neatId=primary.neatId,
|
|
110
|
+
concept=primary.concept,
|
|
111
|
+
name=primary.name or secondary.name,
|
|
112
|
+
description=primary.description or secondary.description,
|
|
113
|
+
implements=implements,
|
|
114
|
+
instance_source=primary.instance_source or secondary.instance_source,
|
|
115
|
+
physical=primary.physical,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
@classmethod
|
|
119
|
+
def union_properties(
|
|
120
|
+
cls,
|
|
121
|
+
primary: ConceptualProperty,
|
|
122
|
+
secondary: ConceptualProperty,
|
|
123
|
+
conflict_resolution: Literal["priority", "combined"] = "priority",
|
|
124
|
+
) -> ConceptualProperty:
|
|
125
|
+
"""Union two conceptual properties.
|
|
126
|
+
|
|
127
|
+
Args:
|
|
128
|
+
primary (ConceptualProperty): The primary property.
|
|
129
|
+
secondary (ConceptualProperty): The secondary property.
|
|
130
|
+
conflict_resolution (Literal["priority", "combined"]): How to resolve conflicts:
|
|
131
|
+
- "priority": Keep the primary property with fallback to the secondary.
|
|
132
|
+
- "combined": Merge value types and instance sources.
|
|
133
|
+
Returns:
|
|
134
|
+
ConceptualProperty: The union of the two properties.
|
|
135
|
+
|
|
136
|
+
"""
|
|
137
|
+
if conflict_resolution == "combined":
|
|
138
|
+
all_sources = (primary.instance_source or []) + (secondary.instance_source or [])
|
|
139
|
+
instance_source = list(dict.fromkeys(all_sources))
|
|
140
|
+
else:
|
|
141
|
+
instance_source = (primary.instance_source or secondary.instance_source or []).copy()
|
|
142
|
+
|
|
143
|
+
use_primary = conflict_resolution == "priority"
|
|
144
|
+
return ConceptualProperty(
|
|
145
|
+
neatId=primary.neatId,
|
|
146
|
+
concept=primary.concept,
|
|
147
|
+
property_=primary.property_,
|
|
148
|
+
name=primary.name or secondary.name,
|
|
149
|
+
description=primary.description or secondary.description,
|
|
150
|
+
min_count=primary.min_count
|
|
151
|
+
if use_primary
|
|
152
|
+
else cls._union_min_count(primary.min_count, secondary.min_count),
|
|
153
|
+
max_count=primary.max_count
|
|
154
|
+
if use_primary
|
|
155
|
+
else cls._union_max_count(primary.max_count, secondary.max_count),
|
|
156
|
+
default=primary.default or secondary.default,
|
|
157
|
+
value_type=primary.value_type
|
|
158
|
+
if use_primary
|
|
159
|
+
else cls.union_value_type(primary.value_type, secondary.value_type),
|
|
160
|
+
instance_source=instance_source,
|
|
161
|
+
inherited=primary.inherited,
|
|
162
|
+
physical=primary.physical,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
@staticmethod
|
|
166
|
+
def _union_min_count(primary: int | None, secondary: int | None) -> int | None:
|
|
167
|
+
if primary is None:
|
|
168
|
+
return secondary
|
|
169
|
+
if secondary is None:
|
|
170
|
+
return primary
|
|
171
|
+
return min(primary, secondary)
|
|
172
|
+
|
|
173
|
+
@staticmethod
|
|
174
|
+
def _union_max_count(primary: int | float | None, secondary: int | float | None) -> int | float | None:
|
|
175
|
+
if primary is None:
|
|
176
|
+
return secondary
|
|
177
|
+
if secondary is None:
|
|
178
|
+
return primary
|
|
179
|
+
output = max(primary, secondary)
|
|
180
|
+
try:
|
|
181
|
+
return int(output)
|
|
182
|
+
except (OverflowError, ValueError):
|
|
183
|
+
# The value is float('inf') or float('-inf')
|
|
184
|
+
return output
|
|
185
|
+
|
|
186
|
+
@staticmethod
|
|
187
|
+
def union_value_type(
|
|
188
|
+
primary: DataType | ConceptEntity | MultiValueTypeInfo | UnknownEntity,
|
|
189
|
+
secondary: DataType | ConceptEntity | MultiValueTypeInfo | UnknownEntity,
|
|
190
|
+
) -> DataType | ConceptEntity | MultiValueTypeInfo | UnknownEntity:
|
|
191
|
+
all_types: list[DataType | ConceptEntity] = []
|
|
192
|
+
for type_ in (primary, secondary):
|
|
193
|
+
if isinstance(type_, MultiValueTypeInfo):
|
|
194
|
+
all_types.extend(type_.types)
|
|
195
|
+
elif isinstance(type_, UnknownEntity):
|
|
196
|
+
continue
|
|
197
|
+
elif isinstance(type_, DataType | ConceptEntity):
|
|
198
|
+
all_types.append(type_)
|
|
199
|
+
else:
|
|
200
|
+
raise NotImplementedError(f"Unsupported type: {type_}")
|
|
201
|
+
|
|
202
|
+
ordered_types = list(dict.fromkeys(all_types))
|
|
203
|
+
if len(ordered_types) == 0:
|
|
204
|
+
return UnknownEntity()
|
|
205
|
+
if len(ordered_types) == 1:
|
|
206
|
+
return ordered_types[0]
|
|
207
|
+
else: # len(ordered_types) > 1:
|
|
208
|
+
return MultiValueTypeInfo(types=ordered_types)
|
|
@@ -255,6 +255,11 @@ class IssueList(list, Sequence[NeatIssue]):
|
|
|
255
255
|
self.action = action
|
|
256
256
|
self.hint = hint
|
|
257
257
|
|
|
258
|
+
def append_if_not_exist(self, issue: NeatIssue) -> None:
|
|
259
|
+
"""Append an issue to the list if it does not already exist."""
|
|
260
|
+
if issue not in self:
|
|
261
|
+
self.append(issue)
|
|
262
|
+
|
|
258
263
|
@property
|
|
259
264
|
def errors(self) -> Self:
|
|
260
265
|
"""Return all the errors in this list."""
|
|
@@ -94,9 +94,9 @@ class NotSupportedHasDataFilterLimitWarning(CDFNotSupportedWarning):
|
|
|
94
94
|
|
|
95
95
|
@dataclass(unsafe_hash=True)
|
|
96
96
|
class UndefinedConceptWarning(UserModelingWarning):
|
|
97
|
-
"""
|
|
97
|
+
"""Concept {concept_id} has no explicit properties neither implements concepts that have properties."""
|
|
98
98
|
|
|
99
|
-
fix = "Define properties for concept or inherit properties by implementing
|
|
99
|
+
fix = "Define properties for concept or inherit properties by implementing concept(s) that has properties."
|
|
100
100
|
|
|
101
101
|
concept_id: str
|
|
102
102
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cognite-neat
|
|
3
|
-
Version: 0.123.
|
|
3
|
+
Version: 0.123.16
|
|
4
4
|
Summary: Knowledge graph transformation
|
|
5
5
|
Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
|
|
6
6
|
Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
cognite/neat/__init__.py,sha256=12StS1dzH9_MElqxGvLWrNsxCJl9Hv8A2a9D0E5OD_U,193
|
|
2
|
-
cognite/neat/_version.py,sha256=
|
|
2
|
+
cognite/neat/_version.py,sha256=BlmwKmiATKYRKR7O0PuP9r182NuNd4-ke2_e5KZ6arM,47
|
|
3
3
|
cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
cognite/neat/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
cognite/neat/core/_config.py,sha256=WT1BS8uADcFvGoUYOOfwFOVq_VBl472TisdoA3wLick,280
|
|
@@ -30,7 +30,7 @@ cognite/neat/core/_data_model/catalog/hello_world_pump.xlsx,sha256=E63t5U1PQLIoU
|
|
|
30
30
|
cognite/neat/core/_data_model/exporters/__init__.py,sha256=6UbiK-dzVUCtYo09s8ICe36BbqPCur6OWB71Lwiu50U,1207
|
|
31
31
|
cognite/neat/core/_data_model/exporters/_base.py,sha256=PHKTUiio4PmiEjWP9E9tJiOkfh_Po1JvcutwP_84-4A,2391
|
|
32
32
|
cognite/neat/core/_data_model/exporters/_data_model2dms.py,sha256=gFSWvYV71LSzDVYsGlJ_mMxhLuKK0nwBvBiwRJhbvTE,19891
|
|
33
|
-
cognite/neat/core/_data_model/exporters/_data_model2excel.py,sha256=
|
|
33
|
+
cognite/neat/core/_data_model/exporters/_data_model2excel.py,sha256=88ReyrsRqoIRJcF7ezZGtsZ_0FBL0Wq7UsGI1uCXgJ4,25457
|
|
34
34
|
cognite/neat/core/_data_model/exporters/_data_model2instance_template.py,sha256=9k8A70b1paeOHjvJRtbl6Xror1GD8AIMdo3cCx5aejE,6103
|
|
35
35
|
cognite/neat/core/_data_model/exporters/_data_model2ontology.py,sha256=YrLTwPAvOOyLFHFJaNs4I82HCp1llJnkF1BRdoIQMck,23409
|
|
36
36
|
cognite/neat/core/_data_model/exporters/_data_model2yaml.py,sha256=1dlb-v4sV8BArnX_6J4wpjQT7r-FinFAvoPDoMNkHYw,3284
|
|
@@ -52,7 +52,7 @@ cognite/neat/core/_data_model/models/_types.py,sha256=70E8fiLdZkVF2sDUGPuDhzXNA5
|
|
|
52
52
|
cognite/neat/core/_data_model/models/data_types.py,sha256=uQ_u9KxCetLjxo-VtFzOXSxQuuf97Kg-9lfTTGzY6hc,10150
|
|
53
53
|
cognite/neat/core/_data_model/models/conceptual/__init__.py,sha256=9A6myEV8s0-LqdXejaljqPj8S0pIpUL75rNdRDZzyR8,585
|
|
54
54
|
cognite/neat/core/_data_model/models/conceptual/_unverified.py,sha256=VswgnTSjSCRzBX3z5HvintBGaWBPexxIs-7z7S4J57c,6298
|
|
55
|
-
cognite/neat/core/_data_model/models/conceptual/_validation.py,sha256=
|
|
55
|
+
cognite/neat/core/_data_model/models/conceptual/_validation.py,sha256=AeTNwoby-lHVf_-7XBMPS7I_NuR3iyCCXv5wcALaNsE,13425
|
|
56
56
|
cognite/neat/core/_data_model/models/conceptual/_verified.py,sha256=BUB4Ur4kpBoWiwTf57tjxJ2l0tDTSbY7zGrg1g0yVNQ,13716
|
|
57
57
|
cognite/neat/core/_data_model/models/entities/__init__.py,sha256=UsW-_6fwd-TW0WcnShPKf40h75l1elVn80VurUwRAic,1567
|
|
58
58
|
cognite/neat/core/_data_model/models/entities/_constants.py,sha256=GXRzVfArwxF3C67VCkzy0JWTZRkRJUYXBQaaecrqcWc,351
|
|
@@ -69,10 +69,11 @@ cognite/neat/core/_data_model/models/physical/_exporter.py,sha256=DPOytV-sIzpGJt
|
|
|
69
69
|
cognite/neat/core/_data_model/models/physical/_unverified.py,sha256=VyI-JULAu6kHJygUclDPH1JYjhf_XcO58tI9BkXORC0,18430
|
|
70
70
|
cognite/neat/core/_data_model/models/physical/_validation.py,sha256=icgNNmvc60lIxI91NGGL5Bs7rR9evNtEubYYMMeKBVg,39529
|
|
71
71
|
cognite/neat/core/_data_model/models/physical/_verified.py,sha256=4_7XUj6-x74DhL8qe-duXhlNnq6ANmShB7UpICjbQW4,26783
|
|
72
|
-
cognite/neat/core/_data_model/transformers/__init__.py,sha256=
|
|
72
|
+
cognite/neat/core/_data_model/transformers/__init__.py,sha256=N6yRBplAkrwwxoTAre_1BE_fdSZL5jihr7xTQjW3KnM,1876
|
|
73
73
|
cognite/neat/core/_data_model/transformers/_base.py,sha256=7adUBJgDkXgRq_h7l1q2VsLQo3lE7-xmzmHdcF4QHq8,3133
|
|
74
74
|
cognite/neat/core/_data_model/transformers/_converters.py,sha256=OazYC7DgAXXEvxdiaPfJSe2ZNkYn2mRqWhtvtvWK59g,111575
|
|
75
75
|
cognite/neat/core/_data_model/transformers/_mapping.py,sha256=GwmTRnhiUPIG37CgUSIbjT7ZpWOwdWuBZ_HAIIBiKYY,19024
|
|
76
|
+
cognite/neat/core/_data_model/transformers/_union_conceptual.py,sha256=sg-VGjrK7PwZS1U18ov-8Or00uR3pFLOaPOYT1edD8Q,8852
|
|
76
77
|
cognite/neat/core/_data_model/transformers/_verification.py,sha256=yyPK6irhMGjVtwKxRIElSsPLUvLLVfk1lBAGny6jN5w,5193
|
|
77
78
|
cognite/neat/core/_instances/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
78
79
|
cognite/neat/core/_instances/_shared.py,sha256=6avH6mtjxjHI7JDLkXwICxGvZwooGBr6APs1_w1To-A,940
|
|
@@ -118,7 +119,7 @@ cognite/neat/core/_instances/transformers/_prune_graph.py,sha256=fWE73BndkEB7qfM
|
|
|
118
119
|
cognite/neat/core/_instances/transformers/_rdfpath.py,sha256=4PIVpjlng59oTjoToS683XU0WgtKdEOf8zEhXCD94-I,3161
|
|
119
120
|
cognite/neat/core/_instances/transformers/_value_type.py,sha256=-d18yefiGrx8CaVNLgJe0dF0zsMxtCQxlD2q2ZFGJ8U,15820
|
|
120
121
|
cognite/neat/core/_issues/__init__.py,sha256=NQ-PN3fqp-hBPlpG2AZEND4cDn3_3UXAPfhLNtF5mtc,457
|
|
121
|
-
cognite/neat/core/_issues/_base.py,sha256=
|
|
122
|
+
cognite/neat/core/_issues/_base.py,sha256=LZhTF_QOQvCxFrGUnWmF9gQnuyr7nM3MqXf_r8KG7GY,12003
|
|
122
123
|
cognite/neat/core/_issues/_contextmanagers.py,sha256=5-QXVmfplt4S_k2csrQ2xuezOOuE5_FxSA9GVGVG1s4,1582
|
|
123
124
|
cognite/neat/core/_issues/_factory.py,sha256=ifEzHZcvPyO0ZGJo8T8CE20F5L4yRzrrGPxl9d87oIs,2829
|
|
124
125
|
cognite/neat/core/_issues/formatters.py,sha256=k2h_6wHW0ve52gXeuRoEcGwrxqqSe5sYFa_HycPiqW8,3323
|
|
@@ -131,7 +132,7 @@ cognite/neat/core/_issues/errors/_wrapper.py,sha256=clhuSwUuHy-FQXQopFIQRY8c_NZM
|
|
|
131
132
|
cognite/neat/core/_issues/warnings/__init__.py,sha256=lzNZrguzwXyifehsCilAXa5UL94DWHIeO-slyC-EYZc,3165
|
|
132
133
|
cognite/neat/core/_issues/warnings/_external.py,sha256=w-1R7ea6DXTIWqwlwMMjY0YxKDMSJ8gKAbp_nIIM1AI,1324
|
|
133
134
|
cognite/neat/core/_issues/warnings/_general.py,sha256=_6dAFaMz-LIv7GsBBIBq2d-kmbuxVXKvU4jZeb7tjAo,972
|
|
134
|
-
cognite/neat/core/_issues/warnings/_models.py,sha256=
|
|
135
|
+
cognite/neat/core/_issues/warnings/_models.py,sha256=u9GH2i1vWet5fofB1ObDsiMxkLF7ZKj2s2a3J-oeiF4,5111
|
|
135
136
|
cognite/neat/core/_issues/warnings/_properties.py,sha256=I3vqc1aL-ce_FRQNgQQy34RW7kQxcjbwhZIIVtGVmg8,3807
|
|
136
137
|
cognite/neat/core/_issues/warnings/_resources.py,sha256=_iPRq0pRMmRu3LFjqZTaG3OqOzw4f8-Vc9G4Im__FHc,3578
|
|
137
138
|
cognite/neat/core/_issues/warnings/user_modeling.py,sha256=Qn_S8TLw7MMYQaJcZBScJA48kz_PrTWz0NaepSR70Fk,4144
|
|
@@ -186,7 +187,7 @@ cognite/neat/session/engine/__init__.py,sha256=D3MxUorEs6-NtgoICqtZ8PISQrjrr4dvc
|
|
|
186
187
|
cognite/neat/session/engine/_import.py,sha256=1QxA2_EK613lXYAHKQbZyw2yjo5P9XuiX4Z6_6-WMNQ,169
|
|
187
188
|
cognite/neat/session/engine/_interface.py,sha256=3W-cYr493c_mW3P5O6MKN1xEQg3cA7NHR_ev3zdF9Vk,533
|
|
188
189
|
cognite/neat/session/engine/_load.py,sha256=g52uYakQM03VqHt_RDHtpHso1-mFFifH5M4T2ScuH8A,5198
|
|
189
|
-
cognite_neat-0.123.
|
|
190
|
-
cognite_neat-0.123.
|
|
191
|
-
cognite_neat-0.123.
|
|
192
|
-
cognite_neat-0.123.
|
|
190
|
+
cognite_neat-0.123.16.dist-info/METADATA,sha256=M0gosIy1EnoQHqzCfO7Wcm_YGDpNWf0inn1iFK754Yg,9172
|
|
191
|
+
cognite_neat-0.123.16.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
192
|
+
cognite_neat-0.123.16.dist-info/licenses/LICENSE,sha256=W8VmvFia4WHa3Gqxq1Ygrq85McUNqIGDVgtdvzT-XqA,11351
|
|
193
|
+
cognite_neat-0.123.16.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|