cognite-neat 0.121.0__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/{_rules → _data_model}/_constants.py +25 -18
- cognite/neat/core/_data_model/_shared.py +59 -0
- cognite/neat/core/_data_model/analysis/__init__.py +3 -0
- cognite/neat/core/{_rules → _data_model}/analysis/_base.py +202 -195
- cognite/neat/core/{_rules → _data_model}/catalog/__init__.py +1 -1
- cognite/neat/core/{_rules → _data_model}/exporters/__init__.py +5 -5
- cognite/neat/core/{_rules → _data_model}/exporters/_base.py +10 -8
- cognite/neat/core/{_rules/exporters/_rules2dms.py → _data_model/exporters/_data_model2dms.py} +22 -18
- cognite/neat/core/{_rules/exporters/_rules2excel.py → _data_model/exporters/_data_model2excel.py} +61 -56
- cognite/neat/core/{_rules/exporters/_rules2instance_template.py → _data_model/exporters/_data_model2instance_template.py} +11 -9
- cognite/neat/core/{_rules/exporters/_rules2ontology.py → _data_model/exporters/_data_model2ontology.py} +64 -61
- cognite/neat/core/{_rules/exporters/_rules2yaml.py → _data_model/exporters/_data_model2yaml.py} +21 -18
- cognite/neat/core/{_rules → _data_model}/importers/__init__.py +6 -8
- cognite/neat/core/{_rules → _data_model}/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/{_rules/importers/_yaml2rules.py → _data_model/importers/_dict2data_model.py} +41 -21
- cognite/neat/core/{_rules/importers/_dms2rules.py → _data_model/importers/_dms2data_model.py} +79 -66
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_converter.py +41 -41
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_importer.py +16 -16
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_base.py +18 -16
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_imf2rules.py +17 -17
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_inference2rules.py +50 -50
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_owl2rules.py +14 -14
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/{_rules/importers/_spreadsheet2rules.py → _data_model/importers/_spreadsheet2data_model.py} +69 -38
- cognite/neat/core/_data_model/models/__init__.py +36 -0
- cognite/neat/core/{_rules/models/_base_input.py → _data_model/models/_base_unverified.py} +12 -12
- cognite/neat/core/{_rules/models/_base_rules.py → _data_model/models/_base_verified.py} +13 -13
- cognite/neat/core/{_rules → _data_model}/models/_types.py +13 -13
- cognite/neat/core/_data_model/models/conceptual/__init__.py +25 -0
- cognite/neat/core/{_rules/models/information/_rules_input.py → _data_model/models/conceptual/_unverified.py} +46 -43
- cognite/neat/core/{_rules/models/information → _data_model/models/conceptual}/_validation.py +93 -79
- cognite/neat/core/{_rules/models/information/_rules.py → _data_model/models/conceptual/_verified.py} +83 -83
- cognite/neat/core/{_rules → _data_model}/models/data_types.py +4 -4
- cognite/neat/core/{_rules → _data_model}/models/entities/__init__.py +8 -8
- cognite/neat/core/{_rules → _data_model}/models/entities/_loaders.py +12 -11
- cognite/neat/core/{_rules → _data_model}/models/entities/_multi_value.py +7 -7
- cognite/neat/core/{_rules → _data_model}/models/entities/_single_value.py +45 -39
- cognite/neat/core/{_rules → _data_model}/models/entities/_types.py +9 -3
- cognite/neat/core/{_rules → _data_model}/models/entities/_wrapped.py +3 -3
- cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_exporter.py +83 -64
- cognite/neat/core/{_rules/models/dms/_rules_input.py → _data_model/models/physical/_unverified.py} +56 -44
- cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_validation.py +20 -17
- cognite/neat/core/{_rules/models/dms/_rules.py → _data_model/models/physical/_verified.py} +79 -71
- cognite/neat/core/{_rules → _data_model}/transformers/__init__.py +27 -23
- cognite/neat/core/{_rules → _data_model}/transformers/_base.py +29 -19
- cognite/neat/core/{_rules → _data_model}/transformers/_converters.py +758 -659
- cognite/neat/core/{_rules → _data_model}/transformers/_mapping.py +79 -60
- cognite/neat/core/_data_model/transformers/_verification.py +120 -0
- cognite/neat/core/{_graph → _instances}/extractors/_base.py +2 -2
- cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_base.py +1 -1
- cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_classic.py +17 -11
- cognite/neat/core/{_graph → _instances}/extractors/_dms_graph.py +47 -39
- cognite/neat/core/{_graph → _instances}/extractors/_mock_graph_generator.py +102 -99
- cognite/neat/core/{_graph → _instances}/extractors/_rdf_file.py +2 -2
- cognite/neat/core/{_graph → _instances}/loaders/_base.py +2 -2
- cognite/neat/core/{_graph → _instances}/loaders/_rdf2dms.py +16 -14
- cognite/neat/core/{_graph → _instances}/transformers/_base.py +7 -4
- cognite/neat/core/{_graph → _instances}/transformers/_classic_cdf.py +1 -1
- cognite/neat/core/{_graph → _instances}/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/__init__.py +2 -0
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +7 -0
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +128 -121
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +7 -8
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +4 -4
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/spreadsheet.py +1 -1
- cognite/neat/core/_utils/text.py +2 -2
- cognite/neat/session/_base.py +29 -25
- cognite/neat/session/_drop.py +3 -3
- cognite/neat/session/_fix.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +11 -9
- cognite/neat/session/_prepare.py +4 -4
- cognite/neat/session/_read.py +15 -15
- cognite/neat/session/_set.py +5 -5
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +17 -17
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +19 -19
- cognite/neat/session/_to.py +21 -21
- cognite/neat/session/_wizard.py +1 -1
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- cognite_neat-0.121.2.dist-info/RECORD +189 -0
- cognite/neat/core/_rules/_shared.py +0 -43
- cognite/neat/core/_rules/analysis/__init__.py +0 -3
- cognite/neat/core/_rules/exporters/_validation.py +0 -14
- cognite/neat/core/_rules/models/__init__.py +0 -34
- cognite/neat/core/_rules/models/dms/__init__.py +0 -32
- cognite/neat/core/_rules/models/information/__init__.py +0 -20
- cognite/neat/core/_rules/transformers/_verification.py +0 -111
- cognite_neat-0.121.0.dist-info/RECORD +0 -187
- /cognite/neat/core/{_graph → _data_model}/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/catalog/classic_model.xlsx +0 -0
- /cognite/neat/core/{_rules/catalog/info-rules-imf.xlsx → _data_model/catalog/conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/{_rules → _data_model}/catalog/hello_world_pump.xlsx +0 -0
- /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/_unit_lookup.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/importers/_rdf/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/entities/_constants.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/mapping/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.yaml +0 -0
- /cognite/neat/core/{_graph/extractors/_classic_cdf → _instances}/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_shared.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/base.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/log.py +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44.xml +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/__init__.py +0 -0
- /cognite/neat/core/{_rules → _instances/extractors/_classic_cdf}/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_assets.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_data_sets.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_events.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_files.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_labels.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_relationships.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_sequences.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_timeseries.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_dict.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_dms.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_raw.py +0 -0
- /cognite/neat/core/{_graph → _instances}/loaders/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_base.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_queries.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_select.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_update.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/_prune_graph.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/_rdfpath.py +0 -0
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
cognite/neat/core/{_rules/models/information → _data_model/models/conceptual}/_validation.py
RENAMED
|
@@ -1,36 +1,41 @@
|
|
|
1
1
|
import itertools
|
|
2
2
|
from collections import Counter, defaultdict
|
|
3
|
+
from collections.abc import Iterable
|
|
3
4
|
|
|
5
|
+
from cognite.neat.core._data_model._constants import PATTERNS, EntityTypes
|
|
6
|
+
from cognite.neat.core._data_model.models.entities import ConceptEntity, UnknownEntity
|
|
7
|
+
from cognite.neat.core._data_model.models.entities._multi_value import MultiValueTypeInfo
|
|
4
8
|
from cognite.neat.core._issues import IssueList
|
|
5
9
|
from cognite.neat.core._issues.errors import NeatValueError
|
|
6
10
|
from cognite.neat.core._issues.errors._resources import (
|
|
7
11
|
ResourceDuplicatedError,
|
|
8
12
|
ResourceNotDefinedError,
|
|
9
13
|
)
|
|
10
|
-
from cognite.neat.core._issues.warnings._models import
|
|
14
|
+
from cognite.neat.core._issues.warnings._models import UndefinedConceptWarning
|
|
11
15
|
from cognite.neat.core._issues.warnings._resources import (
|
|
12
16
|
ResourceNotDefinedWarning,
|
|
13
17
|
ResourceRegexViolationWarning,
|
|
14
18
|
)
|
|
15
|
-
from cognite.neat.core._rules._constants import PATTERNS, EntityTypes
|
|
16
|
-
from cognite.neat.core._rules.models.entities import ClassEntity, UnknownEntity
|
|
17
|
-
from cognite.neat.core._rules.models.entities._multi_value import MultiValueTypeInfo
|
|
18
19
|
from cognite.neat.core._utils.spreadsheet import SpreadsheetRead
|
|
19
20
|
from cognite.neat.core._utils.text import humanize_collection
|
|
20
21
|
|
|
21
|
-
from .
|
|
22
|
+
from ._verified import ConceptualDataModel, ConceptualProperty
|
|
22
23
|
|
|
23
24
|
|
|
24
|
-
class
|
|
25
|
+
class ConceptualValidation:
|
|
25
26
|
"""This class does all the validation of the Information rules that have dependencies
|
|
26
27
|
between components."""
|
|
27
28
|
|
|
28
|
-
def __init__(
|
|
29
|
-
self
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
data_model: ConceptualDataModel,
|
|
32
|
+
read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
|
|
33
|
+
):
|
|
34
|
+
self.data_model = data_model
|
|
30
35
|
self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
|
|
31
|
-
self._metadata =
|
|
32
|
-
self._properties =
|
|
33
|
-
self.
|
|
36
|
+
self._metadata = data_model.metadata
|
|
37
|
+
self._properties = data_model.properties
|
|
38
|
+
self._concepts = data_model.concepts
|
|
34
39
|
self.issue_list = IssueList()
|
|
35
40
|
|
|
36
41
|
def validate(self) -> IssueList:
|
|
@@ -38,7 +43,7 @@ class InformationValidation:
|
|
|
38
43
|
self._namespaces_reassigned()
|
|
39
44
|
self._classes_without_properties()
|
|
40
45
|
self._undefined_classes()
|
|
41
|
-
self.
|
|
46
|
+
self._parent_concept_defined()
|
|
42
47
|
self._referenced_classes_exist()
|
|
43
48
|
self._referenced_value_types_exist()
|
|
44
49
|
self._regex_compliance_with_dms()
|
|
@@ -47,7 +52,7 @@ class InformationValidation:
|
|
|
47
52
|
|
|
48
53
|
def _duplicated_resources(self) -> None:
|
|
49
54
|
properties_sheet = self._read_info_by_spreadsheet.get("Properties")
|
|
50
|
-
|
|
55
|
+
concepts_sheet = self._read_info_by_spreadsheet.get("Concepts")
|
|
51
56
|
|
|
52
57
|
visited = defaultdict(list)
|
|
53
58
|
for row_no, property_ in enumerate(self._properties):
|
|
@@ -71,9 +76,9 @@ class InformationValidation:
|
|
|
71
76
|
)
|
|
72
77
|
|
|
73
78
|
visited = defaultdict(list)
|
|
74
|
-
for row_no,
|
|
75
|
-
visited[
|
|
76
|
-
|
|
79
|
+
for row_no, concept in enumerate(self._concepts):
|
|
80
|
+
visited[concept._identifier()].append(
|
|
81
|
+
concepts_sheet.adjusted_row_number(row_no) if concepts_sheet else row_no + 1
|
|
77
82
|
)
|
|
78
83
|
|
|
79
84
|
for identifier, rows in visited.items():
|
|
@@ -82,85 +87,85 @@ class InformationValidation:
|
|
|
82
87
|
self.issue_list.append(
|
|
83
88
|
ResourceDuplicatedError(
|
|
84
89
|
identifier[0],
|
|
85
|
-
"
|
|
90
|
+
"concept",
|
|
86
91
|
(f"the Classes sheet at row {humanize_collection(rows)} if data model is read from a spreadsheet."),
|
|
87
92
|
)
|
|
88
93
|
)
|
|
89
94
|
|
|
90
95
|
def _classes_without_properties(self) -> None:
|
|
91
|
-
|
|
92
|
-
referred_classes = {property_.
|
|
93
|
-
|
|
96
|
+
defined_concepts = {concept.concept for concept in self._concepts}
|
|
97
|
+
referred_classes = {property_.concept for property_ in self._properties}
|
|
98
|
+
concept_parent_pairs = self._concept_parent_pairs()
|
|
94
99
|
|
|
95
|
-
if
|
|
96
|
-
for
|
|
100
|
+
if concepts_without_properties := defined_concepts.difference(referred_classes):
|
|
101
|
+
for concept in concepts_without_properties:
|
|
97
102
|
# USE CASE: class has no direct properties and no parents with properties
|
|
98
103
|
# and it is a class in the prefix of data model, as long as it is in the
|
|
99
104
|
# same prefix, meaning same space
|
|
100
|
-
if not
|
|
105
|
+
if not concept_parent_pairs[concept] and concept.prefix == self._metadata.prefix:
|
|
101
106
|
self.issue_list.append(
|
|
102
107
|
ResourceNotDefinedWarning(
|
|
103
|
-
resource_type="
|
|
104
|
-
identifier=
|
|
108
|
+
resource_type="concept",
|
|
109
|
+
identifier=concept,
|
|
105
110
|
location="Properties sheet",
|
|
106
111
|
)
|
|
107
112
|
)
|
|
108
113
|
|
|
109
114
|
def _undefined_classes(self) -> None:
|
|
110
|
-
|
|
111
|
-
|
|
115
|
+
defined_concept = {concept.concept for concept in self._concepts}
|
|
116
|
+
referred_concepts = {property_.concept for property_ in self._properties}
|
|
112
117
|
|
|
113
|
-
if
|
|
114
|
-
for
|
|
118
|
+
if undefined_concepts := referred_concepts.difference(defined_concept):
|
|
119
|
+
for concept in undefined_concepts:
|
|
115
120
|
self.issue_list.append(
|
|
116
121
|
ResourceNotDefinedError(
|
|
117
|
-
identifier=
|
|
118
|
-
resource_type="
|
|
119
|
-
location="
|
|
122
|
+
identifier=concept,
|
|
123
|
+
resource_type="concept",
|
|
124
|
+
location="Concepts sheet",
|
|
120
125
|
)
|
|
121
126
|
)
|
|
122
127
|
|
|
123
|
-
def
|
|
124
|
-
"""This is a validation to check if the parent
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
parents = set(itertools.chain.from_iterable(
|
|
128
|
+
def _parent_concept_defined(self) -> None:
|
|
129
|
+
"""This is a validation to check if the parent concept is defined."""
|
|
130
|
+
concept_parent_pairs = self._concept_parent_pairs()
|
|
131
|
+
concepts = set(concept_parent_pairs.keys())
|
|
132
|
+
parents = set(itertools.chain.from_iterable(concept_parent_pairs.values()))
|
|
128
133
|
|
|
129
|
-
if undefined_parents := parents.difference(
|
|
134
|
+
if undefined_parents := parents.difference(concepts):
|
|
130
135
|
for parent in undefined_parents:
|
|
131
136
|
if parent.prefix != self._metadata.prefix:
|
|
132
|
-
self.issue_list.append(
|
|
137
|
+
self.issue_list.append(UndefinedConceptWarning(concept_id=str(parent)))
|
|
133
138
|
else:
|
|
134
139
|
self.issue_list.append(
|
|
135
140
|
ResourceNotDefinedWarning(
|
|
136
|
-
resource_type="
|
|
141
|
+
resource_type="concept",
|
|
137
142
|
identifier=parent,
|
|
138
|
-
location="
|
|
143
|
+
location="Concepts sheet",
|
|
139
144
|
)
|
|
140
145
|
)
|
|
141
146
|
|
|
142
147
|
def _referenced_classes_exist(self) -> None:
|
|
143
148
|
# needs to be complete for this validation to pass
|
|
144
|
-
|
|
145
|
-
classes_with_explicit_properties = {property_.
|
|
149
|
+
defined_concept = {concept.concept for concept in self._concepts}
|
|
150
|
+
classes_with_explicit_properties = {property_.concept for property_ in self._properties}
|
|
146
151
|
|
|
147
152
|
# USE CASE: models are complete
|
|
148
|
-
if missing_classes := classes_with_explicit_properties.difference(
|
|
149
|
-
for
|
|
153
|
+
if missing_classes := classes_with_explicit_properties.difference(defined_concept):
|
|
154
|
+
for concept in missing_classes:
|
|
150
155
|
self.issue_list.append(
|
|
151
156
|
ResourceNotDefinedWarning(
|
|
152
|
-
resource_type="
|
|
153
|
-
identifier=
|
|
154
|
-
location="
|
|
157
|
+
resource_type="concept",
|
|
158
|
+
identifier=concept,
|
|
159
|
+
location="Concepts sheet",
|
|
155
160
|
)
|
|
156
161
|
)
|
|
157
162
|
|
|
158
163
|
def _referenced_value_types_exist(self) -> None:
|
|
159
164
|
# adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
|
|
160
|
-
defined_classes = {
|
|
165
|
+
defined_classes = {concept.concept for concept in self._concepts} | {UnknownEntity()}
|
|
161
166
|
referred_object_types = {
|
|
162
167
|
property_.value_type
|
|
163
|
-
for property_ in self.
|
|
168
|
+
for property_ in self.data_model.properties
|
|
164
169
|
if property_.type_ == EntityTypes.object_property
|
|
165
170
|
}
|
|
166
171
|
|
|
@@ -169,9 +174,9 @@ class InformationValidation:
|
|
|
169
174
|
for missing in missing_value_types:
|
|
170
175
|
self.issue_list.append(
|
|
171
176
|
ResourceNotDefinedWarning(
|
|
172
|
-
resource_type="
|
|
177
|
+
resource_type="concept",
|
|
173
178
|
identifier=missing,
|
|
174
|
-
location="
|
|
179
|
+
location="Concepts sheet",
|
|
175
180
|
)
|
|
176
181
|
)
|
|
177
182
|
|
|
@@ -179,28 +184,28 @@ class InformationValidation:
|
|
|
179
184
|
"""Check regex compliance with DMS of properties, classes and value types."""
|
|
180
185
|
|
|
181
186
|
for prop_ in self._properties:
|
|
182
|
-
if not PATTERNS.
|
|
187
|
+
if not PATTERNS.physical_property_id_compliance.match(prop_.property_):
|
|
183
188
|
self.issue_list.append(
|
|
184
189
|
ResourceRegexViolationWarning(
|
|
185
190
|
prop_.property_,
|
|
186
191
|
"Property",
|
|
187
192
|
"Properties sheet, Property column",
|
|
188
|
-
PATTERNS.
|
|
193
|
+
PATTERNS.physical_property_id_compliance.pattern,
|
|
189
194
|
)
|
|
190
195
|
)
|
|
191
|
-
if not PATTERNS.view_id_compliance.match(prop_.
|
|
196
|
+
if not PATTERNS.view_id_compliance.match(prop_.concept.suffix):
|
|
192
197
|
self.issue_list.append(
|
|
193
198
|
ResourceRegexViolationWarning(
|
|
194
|
-
prop_.
|
|
195
|
-
"
|
|
196
|
-
"Properties sheet,
|
|
199
|
+
prop_.concept,
|
|
200
|
+
"Concept",
|
|
201
|
+
"Properties sheet, Concept column",
|
|
197
202
|
PATTERNS.view_id_compliance.pattern,
|
|
198
203
|
)
|
|
199
204
|
)
|
|
200
205
|
|
|
201
206
|
# Handling Value Type
|
|
202
207
|
if (
|
|
203
|
-
isinstance(prop_.value_type,
|
|
208
|
+
isinstance(prop_.value_type, ConceptEntity)
|
|
204
209
|
and prop_.value_type != UnknownEntity()
|
|
205
210
|
and not PATTERNS.view_id_compliance.match(prop_.value_type.suffix)
|
|
206
211
|
):
|
|
@@ -215,7 +220,7 @@ class InformationValidation:
|
|
|
215
220
|
if isinstance(prop_.value_type, MultiValueTypeInfo):
|
|
216
221
|
for value_type in prop_.value_type.types:
|
|
217
222
|
if (
|
|
218
|
-
isinstance(prop_.value_type,
|
|
223
|
+
isinstance(prop_.value_type, ConceptEntity)
|
|
219
224
|
and prop_.value_type != UnknownEntity()
|
|
220
225
|
and not PATTERNS.view_id_compliance.match(value_type.suffix)
|
|
221
226
|
):
|
|
@@ -228,44 +233,44 @@ class InformationValidation:
|
|
|
228
233
|
)
|
|
229
234
|
)
|
|
230
235
|
|
|
231
|
-
for
|
|
232
|
-
if not PATTERNS.view_id_compliance.match(
|
|
236
|
+
for concepts in self._concepts:
|
|
237
|
+
if not PATTERNS.view_id_compliance.match(concepts.concept.suffix):
|
|
233
238
|
self.issue_list.append(
|
|
234
239
|
ResourceRegexViolationWarning(
|
|
235
|
-
|
|
236
|
-
"
|
|
237
|
-
"
|
|
240
|
+
concepts.concept,
|
|
241
|
+
"Concept",
|
|
242
|
+
"Concepts sheet, Class column",
|
|
238
243
|
PATTERNS.view_id_compliance.pattern,
|
|
239
244
|
)
|
|
240
245
|
)
|
|
241
246
|
|
|
242
|
-
if
|
|
243
|
-
for parent in
|
|
247
|
+
if concepts.implements:
|
|
248
|
+
for parent in concepts.implements:
|
|
244
249
|
if not PATTERNS.view_id_compliance.match(parent.suffix):
|
|
245
250
|
self.issue_list.append(
|
|
246
251
|
ResourceRegexViolationWarning(
|
|
247
252
|
parent,
|
|
248
|
-
"
|
|
249
|
-
"
|
|
253
|
+
"Concept",
|
|
254
|
+
"Concepts sheet, Implements column",
|
|
250
255
|
PATTERNS.view_id_compliance.pattern,
|
|
251
256
|
)
|
|
252
257
|
)
|
|
253
258
|
|
|
254
|
-
def
|
|
255
|
-
|
|
256
|
-
|
|
259
|
+
def _concept_parent_pairs(self) -> dict[ConceptEntity, list[ConceptEntity]]:
|
|
260
|
+
concept_parent_pairs: dict[ConceptEntity, list[ConceptEntity]] = {}
|
|
261
|
+
concepts = self.data_model.model_copy(deep=True).concepts
|
|
257
262
|
|
|
258
|
-
for
|
|
259
|
-
|
|
260
|
-
if
|
|
263
|
+
for concept in concepts:
|
|
264
|
+
concept_parent_pairs[concept.concept] = []
|
|
265
|
+
if concept.implements is None:
|
|
261
266
|
continue
|
|
262
|
-
|
|
267
|
+
concept_parent_pairs[concept.concept].extend(concept.implements)
|
|
263
268
|
|
|
264
|
-
return
|
|
269
|
+
return concept_parent_pairs
|
|
265
270
|
|
|
266
271
|
def _namespaces_reassigned(self) -> None:
|
|
267
|
-
prefixes = self.
|
|
268
|
-
prefixes[self.
|
|
272
|
+
prefixes = self.data_model.prefixes.copy()
|
|
273
|
+
prefixes[self.data_model.metadata.namespace.prefix] = self.data_model.metadata.namespace
|
|
269
274
|
|
|
270
275
|
if len(set(prefixes.values())) != len(prefixes):
|
|
271
276
|
reused_namespaces = [value for value, count in Counter(prefixes.values()).items() if count > 1]
|
|
@@ -278,3 +283,12 @@ class InformationValidation:
|
|
|
278
283
|
"\nMake sure that each unique namespace is assigned to a unique prefix"
|
|
279
284
|
)
|
|
280
285
|
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def duplicated_properties(
|
|
289
|
+
properties: Iterable[ConceptualProperty],
|
|
290
|
+
) -> dict[tuple[ConceptEntity, str], list[tuple[int, ConceptualProperty]]]:
|
|
291
|
+
concept_properties_by_id: dict[tuple[ConceptEntity, str], list[tuple[int, ConceptualProperty]]] = defaultdict(list)
|
|
292
|
+
for prop_no, prop in enumerate(properties):
|
|
293
|
+
concept_properties_by_id[(prop.concept, prop.property_)].append((prop_no, prop))
|
|
294
|
+
return {k: v for k, v in concept_properties_by_id.items() if len(v) > 1}
|
cognite/neat/core/{_rules/models/information/_rules.py → _data_model/models/conceptual/_verified.py}
RENAMED
|
@@ -8,63 +8,63 @@ from pydantic_core.core_schema import SerializationInfo
|
|
|
8
8
|
from rdflib import Namespace, URIRef
|
|
9
9
|
|
|
10
10
|
from cognite.neat.core._constants import get_default_prefixes_and_namespaces
|
|
11
|
-
from cognite.neat.core.
|
|
12
|
-
from cognite.neat.core.
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
DataModelAspect,
|
|
11
|
+
from cognite.neat.core._data_model._constants import EntityTypes
|
|
12
|
+
from cognite.neat.core._data_model.models._base_verified import (
|
|
13
|
+
BaseVerifiedDataModel,
|
|
14
|
+
BaseVerifiedMetadata,
|
|
15
|
+
DataModelLevel,
|
|
17
16
|
RoleTypes,
|
|
18
17
|
SheetList,
|
|
19
18
|
SheetRow,
|
|
20
19
|
)
|
|
21
|
-
from cognite.neat.core.
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
from cognite.neat.core._data_model.models._types import (
|
|
21
|
+
ConceptEntityType,
|
|
22
|
+
ConceptualPropertyType,
|
|
24
23
|
MultiValueTypeType,
|
|
25
24
|
URIRefType,
|
|
26
25
|
)
|
|
27
26
|
|
|
28
27
|
# NeatIdType,
|
|
29
|
-
from cognite.neat.core.
|
|
30
|
-
from cognite.neat.core.
|
|
31
|
-
ClassEntity,
|
|
28
|
+
from cognite.neat.core._data_model.models.data_types import DataType
|
|
29
|
+
from cognite.neat.core._data_model.models.entities import (
|
|
32
30
|
ClassEntityList,
|
|
33
|
-
|
|
31
|
+
ConceptEntity,
|
|
32
|
+
ConceptualEntity,
|
|
34
33
|
UnknownEntity,
|
|
35
34
|
)
|
|
35
|
+
from cognite.neat.core._issues.errors import PropertyDefinitionError
|
|
36
36
|
|
|
37
37
|
if TYPE_CHECKING:
|
|
38
|
-
from cognite.neat.core.
|
|
38
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
class
|
|
41
|
+
class ConceptualMetadata(BaseVerifiedMetadata):
|
|
42
42
|
role: ClassVar[RoleTypes] = RoleTypes.information
|
|
43
|
-
|
|
43
|
+
level: ClassVar[DataModelLevel] = DataModelLevel.conceptual
|
|
44
44
|
|
|
45
45
|
# Linking to Conceptual and Physical data model aspects
|
|
46
46
|
physical: URIRef | str | None = Field(None, description="Link to the physical data model aspect")
|
|
47
|
-
conceptual: URIRef | str | None = Field(None, description="Link to the conceptual data model aspect")
|
|
48
47
|
|
|
49
48
|
|
|
50
|
-
def _get_metadata(context: Any) ->
|
|
51
|
-
if isinstance(context, dict) and isinstance(context.get("metadata"),
|
|
49
|
+
def _get_metadata(context: Any) -> ConceptualMetadata | None:
|
|
50
|
+
if isinstance(context, dict) and isinstance(context.get("metadata"), ConceptualMetadata):
|
|
52
51
|
return context["metadata"]
|
|
53
52
|
return None
|
|
54
53
|
|
|
55
54
|
|
|
56
|
-
class
|
|
55
|
+
class Concept(SheetRow):
|
|
57
56
|
"""
|
|
58
|
-
|
|
57
|
+
Concept is a category of things that share a common set of attributes and relationships.
|
|
59
58
|
|
|
60
59
|
Args:
|
|
61
|
-
|
|
60
|
+
concept: An ID of the concept.
|
|
62
61
|
description: A description of the class.
|
|
63
62
|
implements: Which classes the current class implements.
|
|
64
63
|
"""
|
|
65
64
|
|
|
66
|
-
|
|
67
|
-
alias="
|
|
65
|
+
concept: ConceptEntityType = Field(
|
|
66
|
+
alias="Concept",
|
|
67
|
+
description="Concept id being defined, use strongly advise `PascalCase` usage.",
|
|
68
68
|
)
|
|
69
69
|
name: str | None = Field(alias="Name", default=None, description="Human readable name of the class.")
|
|
70
70
|
description: str | None = Field(alias="Description", default=None, description="Short description of the class.")
|
|
@@ -82,14 +82,13 @@ class InformationClass(SheetRow):
|
|
|
82
82
|
None,
|
|
83
83
|
description="Link to the class representation in the physical data model aspect",
|
|
84
84
|
)
|
|
85
|
-
conceptual: URIRefType | None = Field(None, description="Link to the conceptual data model aspect")
|
|
86
85
|
|
|
87
86
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
88
|
-
return (self.
|
|
87
|
+
return (self.concept,)
|
|
89
88
|
|
|
90
|
-
@field_serializer("
|
|
89
|
+
@field_serializer("concept", when_used="unless-none")
|
|
91
90
|
def remove_default_prefix(self, value: Any, info: SerializationInfo) -> str:
|
|
92
|
-
if (metadata := _get_metadata(info.context)) and isinstance(value,
|
|
91
|
+
if (metadata := _get_metadata(info.context)) and isinstance(value, ConceptualEntity):
|
|
93
92
|
return value.dump(prefix=metadata.prefix, version=metadata.version)
|
|
94
93
|
return str(value)
|
|
95
94
|
|
|
@@ -98,22 +97,22 @@ class InformationClass(SheetRow):
|
|
|
98
97
|
if isinstance(value, list) and (metadata := _get_metadata(info.context)):
|
|
99
98
|
return ",".join(
|
|
100
99
|
(
|
|
101
|
-
|
|
102
|
-
if isinstance(
|
|
103
|
-
else str(
|
|
100
|
+
concept.dump(prefix=metadata.prefix, version=metadata.version)
|
|
101
|
+
if isinstance(concept, ConceptualEntity)
|
|
102
|
+
else str(concept)
|
|
104
103
|
)
|
|
105
|
-
for
|
|
104
|
+
for concept in value
|
|
106
105
|
)
|
|
107
106
|
return ",".join(str(value) for value in value)
|
|
108
107
|
|
|
109
108
|
|
|
110
|
-
class
|
|
109
|
+
class ConceptualProperty(SheetRow):
|
|
111
110
|
"""
|
|
112
|
-
A property is a characteristic of a
|
|
113
|
-
or a relationship to another
|
|
111
|
+
A property is a characteristic of a concept. It is a named attribute of a concept
|
|
112
|
+
that describes a range of values or a relationship to another concept.
|
|
114
113
|
|
|
115
114
|
Args:
|
|
116
|
-
|
|
115
|
+
concept: Concept ID to which property belongs
|
|
117
116
|
property_: Property ID of the property
|
|
118
117
|
name: Property name.
|
|
119
118
|
value_type: Type of value property will hold (data or link to another class)
|
|
@@ -124,15 +123,17 @@ class InformationProperty(SheetRow):
|
|
|
124
123
|
knowledge graph. Defaults to None (no transformation)
|
|
125
124
|
"""
|
|
126
125
|
|
|
127
|
-
|
|
128
|
-
alias="
|
|
126
|
+
concept: ConceptEntityType = Field(
|
|
127
|
+
alias="Concept",
|
|
128
|
+
description="Concept id that the property is defined for, strongly advise `PascalCase` usage.",
|
|
129
129
|
)
|
|
130
|
-
property_:
|
|
131
|
-
alias="Property",
|
|
130
|
+
property_: ConceptualPropertyType = Field(
|
|
131
|
+
alias="Property",
|
|
132
|
+
description="Property id, strongly advised to `camelCase` usage.",
|
|
132
133
|
)
|
|
133
134
|
name: str | None = Field(alias="Name", default=None, description="Human readable name of the property.")
|
|
134
135
|
description: str | None = Field(alias="Description", default=None, description="Short description of the property.")
|
|
135
|
-
value_type: DataType |
|
|
136
|
+
value_type: DataType | ConceptEntityType | MultiValueTypeType | UnknownEntity = Field(
|
|
136
137
|
alias="Value Type",
|
|
137
138
|
union_mode="left_to_right",
|
|
138
139
|
description="Value type that the property can hold. It takes either subset of XSD type or a class defined.",
|
|
@@ -168,10 +169,9 @@ class InformationProperty(SheetRow):
|
|
|
168
169
|
None,
|
|
169
170
|
description="Link to the class representation in the physical data model aspect",
|
|
170
171
|
)
|
|
171
|
-
conceptual: URIRefType | None = Field(None, description="Link to the conceptual data model aspect")
|
|
172
172
|
|
|
173
173
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
174
|
-
return self.
|
|
174
|
+
return self.concept, self.property_
|
|
175
175
|
|
|
176
176
|
@field_validator("max_count", mode="before")
|
|
177
177
|
def parse_max_count(cls, value: int | float | None) -> int | float | None:
|
|
@@ -203,7 +203,7 @@ class InformationProperty(SheetRow):
|
|
|
203
203
|
# this value_type.python does not seems correct. Need to check this further
|
|
204
204
|
except Exception:
|
|
205
205
|
raise PropertyDefinitionError(
|
|
206
|
-
self.
|
|
206
|
+
self.concept,
|
|
207
207
|
"Class",
|
|
208
208
|
self.property_,
|
|
209
209
|
f"Default value {self.default} is not of type {self.value_type.python}", # type: ignore
|
|
@@ -222,9 +222,9 @@ class InformationProperty(SheetRow):
|
|
|
222
222
|
return None
|
|
223
223
|
return value
|
|
224
224
|
|
|
225
|
-
@field_serializer("
|
|
225
|
+
@field_serializer("concept", "value_type", when_used="unless-none")
|
|
226
226
|
def remove_default_prefix(self, value: Any, info: SerializationInfo) -> str:
|
|
227
|
-
if (metadata := _get_metadata(info.context)) and isinstance(value,
|
|
227
|
+
if (metadata := _get_metadata(info.context)) and isinstance(value, ConceptualEntity):
|
|
228
228
|
return value.dump(prefix=metadata.prefix, version=metadata.version)
|
|
229
229
|
return str(value)
|
|
230
230
|
|
|
@@ -233,20 +233,20 @@ class InformationProperty(SheetRow):
|
|
|
233
233
|
"""Type of property based on value type. Either data (attribute) or object (edge) property."""
|
|
234
234
|
if isinstance(self.value_type, DataType):
|
|
235
235
|
return EntityTypes.data_property
|
|
236
|
-
elif isinstance(self.value_type,
|
|
236
|
+
elif isinstance(self.value_type, ConceptEntity):
|
|
237
237
|
return EntityTypes.object_property
|
|
238
238
|
else:
|
|
239
239
|
return EntityTypes.undefined
|
|
240
240
|
|
|
241
241
|
|
|
242
|
-
class
|
|
243
|
-
metadata:
|
|
244
|
-
properties: SheetList[
|
|
245
|
-
|
|
242
|
+
class ConceptualDataModel(BaseVerifiedDataModel):
|
|
243
|
+
metadata: ConceptualMetadata = Field(alias="Metadata", description="Metadata for the conceptual data model")
|
|
244
|
+
properties: SheetList[ConceptualProperty] = Field(alias="Properties", description="List of properties")
|
|
245
|
+
concepts: SheetList[Concept] = Field(alias="Concepts", description="List of concepts")
|
|
246
246
|
prefixes: dict[str, Namespace] = Field(
|
|
247
247
|
alias="Prefixes",
|
|
248
248
|
default_factory=get_default_prefixes_and_namespaces,
|
|
249
|
-
description="the definition of the prefixes that are used in the
|
|
249
|
+
description="the definition of the prefixes that are used in the conceptual data model",
|
|
250
250
|
)
|
|
251
251
|
|
|
252
252
|
@field_validator("prefixes", mode="before")
|
|
@@ -258,15 +258,15 @@ class InformationRules(BaseRules):
|
|
|
258
258
|
return values
|
|
259
259
|
|
|
260
260
|
@model_validator(mode="after")
|
|
261
|
-
def set_neat_id(self) -> "
|
|
261
|
+
def set_neat_id(self) -> "ConceptualDataModel":
|
|
262
262
|
namespace = self.metadata.namespace
|
|
263
263
|
|
|
264
|
-
for
|
|
265
|
-
if not
|
|
266
|
-
|
|
264
|
+
for concept in self.concepts:
|
|
265
|
+
if not concept.neatId:
|
|
266
|
+
concept.neatId = namespace[concept.concept.suffix]
|
|
267
267
|
for property_ in self.properties:
|
|
268
268
|
if not property_.neatId:
|
|
269
|
-
property_.neatId = namespace[f"{property_.
|
|
269
|
+
property_.neatId = namespace[f"{property_.concept.suffix}/{property_.property_}"]
|
|
270
270
|
|
|
271
271
|
return self
|
|
272
272
|
|
|
@@ -275,50 +275,50 @@ class InformationRules(BaseRules):
|
|
|
275
275
|
|
|
276
276
|
namespace = self.metadata.namespace
|
|
277
277
|
|
|
278
|
-
for
|
|
279
|
-
|
|
278
|
+
for concept in self.concepts:
|
|
279
|
+
concept.neatId = namespace[concept.concept.suffix]
|
|
280
280
|
for property_ in self.properties:
|
|
281
|
-
property_.neatId = namespace[f"{property_.
|
|
281
|
+
property_.neatId = namespace[f"{property_.concept.suffix}/{property_.property_}"]
|
|
282
282
|
|
|
283
|
-
def
|
|
283
|
+
def sync_with_physical_data_model(self, physical_data_model: "PhysicalDataModel") -> None:
|
|
284
284
|
# Sync at the metadata level
|
|
285
|
-
if
|
|
286
|
-
self.metadata.physical =
|
|
285
|
+
if physical_data_model.metadata.conceptual == self.metadata.identifier:
|
|
286
|
+
self.metadata.physical = physical_data_model.metadata.identifier
|
|
287
287
|
else:
|
|
288
288
|
# if models are not linked to start with, we skip
|
|
289
289
|
return None
|
|
290
290
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
for neat_id, prop in
|
|
294
|
-
if prop.
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
for neat_id, view in
|
|
300
|
-
if view.
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
def as_dms_rules(self) -> "
|
|
304
|
-
from cognite.neat.core.
|
|
305
|
-
|
|
291
|
+
conceptual_properties_by_neat_id = {prop.neatId: prop for prop in self.properties}
|
|
292
|
+
physical_properties_by_neat_id = {prop.neatId: prop for prop in physical_data_model.properties}
|
|
293
|
+
for neat_id, prop in physical_properties_by_neat_id.items():
|
|
294
|
+
if prop.conceptual in conceptual_properties_by_neat_id:
|
|
295
|
+
conceptual_properties_by_neat_id[prop.conceptual].physical = neat_id
|
|
296
|
+
|
|
297
|
+
classes_by_neat_id = {cls.neatId: cls for cls in self.concepts}
|
|
298
|
+
views_by_neat_id = {view.neatId: view for view in physical_data_model.views}
|
|
299
|
+
for neat_id, view in views_by_neat_id.items():
|
|
300
|
+
if view.conceptual in classes_by_neat_id:
|
|
301
|
+
classes_by_neat_id[view.conceptual].physical = neat_id
|
|
302
|
+
|
|
303
|
+
def as_dms_rules(self) -> "PhysicalDataModel":
|
|
304
|
+
from cognite.neat.core._data_model.transformers._converters import (
|
|
305
|
+
_ConceptualDataModelConverter,
|
|
306
306
|
)
|
|
307
307
|
|
|
308
|
-
return
|
|
308
|
+
return _ConceptualDataModelConverter(self).as_physical_data_model()
|
|
309
309
|
|
|
310
310
|
@classmethod
|
|
311
311
|
def display_type_name(cls) -> str:
|
|
312
|
-
return "
|
|
312
|
+
return "VerifiedConceptualDataModel"
|
|
313
313
|
|
|
314
314
|
def _repr_html_(self) -> str:
|
|
315
315
|
summary = {
|
|
316
|
-
"
|
|
317
|
-
"intended for": "Information Architect",
|
|
316
|
+
"level": self.metadata.level,
|
|
317
|
+
"intended for": "Domain Expert and/or Information Architect",
|
|
318
318
|
"name": self.metadata.name,
|
|
319
319
|
"external_id": self.metadata.external_id,
|
|
320
320
|
"version": self.metadata.version,
|
|
321
|
-
"
|
|
321
|
+
"concepts": len(self.concepts),
|
|
322
322
|
"properties": len(self.properties),
|
|
323
323
|
}
|
|
324
324
|
|