cognite-neat 0.109.4__py3-none-any.whl → 0.110.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_alpha.py +2 -0
- cognite/neat/_client/_api/schema.py +17 -1
- cognite/neat/_client/data_classes/schema.py +3 -3
- cognite/neat/_constants.py +11 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +9 -10
- cognite/neat/_graph/extractors/_iodd.py +3 -3
- cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
- cognite/neat/_graph/loaders/_rdf2dms.py +285 -346
- cognite/neat/_graph/queries/_base.py +28 -92
- cognite/neat/_graph/transformers/__init__.py +1 -3
- cognite/neat/_graph/transformers/_rdfpath.py +2 -49
- cognite/neat/_issues/__init__.py +1 -6
- cognite/neat/_issues/_base.py +21 -252
- cognite/neat/_issues/_contextmanagers.py +46 -0
- cognite/neat/_issues/_factory.py +61 -0
- cognite/neat/_issues/errors/__init__.py +18 -4
- cognite/neat/_issues/errors/_wrapper.py +81 -3
- cognite/neat/_issues/formatters.py +4 -4
- cognite/neat/_issues/warnings/__init__.py +3 -2
- cognite/neat/_issues/warnings/_properties.py +8 -0
- cognite/neat/_rules/_constants.py +9 -0
- cognite/neat/_rules/_shared.py +3 -2
- cognite/neat/_rules/analysis/__init__.py +2 -3
- cognite/neat/_rules/analysis/_base.py +450 -258
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +2 -8
- cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
- cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
- cognite/neat/_rules/importers/_base.py +2 -47
- cognite/neat/_rules/importers/_dms2rules.py +7 -10
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +59 -25
- cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
- cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
- cognite/neat/_rules/models/dms/_rules.py +3 -1
- cognite/neat/_rules/models/dms/_rules_input.py +4 -0
- cognite/neat/_rules/models/dms/_validation.py +14 -4
- cognite/neat/_rules/models/entities/_loaders.py +1 -1
- cognite/neat/_rules/models/entities/_multi_value.py +2 -2
- cognite/neat/_rules/models/information/_rules.py +18 -17
- cognite/neat/_rules/models/information/_rules_input.py +2 -1
- cognite/neat/_rules/models/information/_validation.py +3 -1
- cognite/neat/_rules/transformers/__init__.py +8 -2
- cognite/neat/_rules/transformers/_converters.py +228 -43
- cognite/neat/_rules/transformers/_verification.py +5 -10
- cognite/neat/_session/_base.py +4 -4
- cognite/neat/_session/_prepare.py +12 -0
- cognite/neat/_session/_read.py +21 -17
- cognite/neat/_session/_show.py +11 -123
- cognite/neat/_session/_state.py +0 -2
- cognite/neat/_session/_subset.py +64 -0
- cognite/neat/_session/_to.py +63 -12
- cognite/neat/_store/_graph_store.py +5 -246
- cognite/neat/_utils/rdf_.py +2 -2
- cognite/neat/_utils/spreadsheet.py +44 -1
- cognite/neat/_utils/text.py +51 -32
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.110.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.110.0.dist-info}/RECORD +62 -64
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.110.0.dist-info}/WHEEL +1 -1
- cognite/neat/_graph/queries/_construct.py +0 -187
- cognite/neat/_graph/queries/_shared.py +0 -173
- cognite/neat/_rules/analysis/_dms.py +0 -57
- cognite/neat/_rules/analysis/_information.py +0 -249
- cognite/neat/_rules/models/_rdfpath.py +0 -372
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.110.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.110.0.dist-info}/entry_points.txt +0 -0
|
@@ -12,6 +12,7 @@ from cognite.neat._issues.warnings._resources import (
|
|
|
12
12
|
from cognite.neat._rules._constants import PATTERNS, EntityTypes
|
|
13
13
|
from cognite.neat._rules.models.entities import ClassEntity, UnknownEntity
|
|
14
14
|
from cognite.neat._rules.models.entities._multi_value import MultiValueTypeInfo
|
|
15
|
+
from cognite.neat._utils.spreadsheet import SpreadsheetRead
|
|
15
16
|
|
|
16
17
|
from ._rules import InformationRules
|
|
17
18
|
|
|
@@ -20,8 +21,9 @@ class InformationValidation:
|
|
|
20
21
|
"""This class does all the validation of the Information rules that have dependencies
|
|
21
22
|
between components."""
|
|
22
23
|
|
|
23
|
-
def __init__(self, rules: InformationRules):
|
|
24
|
+
def __init__(self, rules: InformationRules, read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None):
|
|
24
25
|
self.rules = rules
|
|
26
|
+
self.read_info_by_spreadsheet = read_info_by_spreadsheet
|
|
25
27
|
self.metadata = rules.metadata
|
|
26
28
|
self.properties = rules.properties
|
|
27
29
|
self.classes = rules.classes
|
|
@@ -14,11 +14,14 @@ from ._converters import (
|
|
|
14
14
|
PrefixEntities,
|
|
15
15
|
SetIDDMSModel,
|
|
16
16
|
StandardizeNaming,
|
|
17
|
+
StandardizeSpaceAndVersion,
|
|
18
|
+
SubsetDMSRules,
|
|
19
|
+
SubsetInformationRules,
|
|
17
20
|
ToCompliantEntities,
|
|
18
21
|
ToDataProductModel,
|
|
22
|
+
ToDMSCompliantEntities,
|
|
19
23
|
ToEnterpriseModel,
|
|
20
24
|
ToExtensionModel,
|
|
21
|
-
ToInformationCompliantEntities,
|
|
22
25
|
ToSolutionModel,
|
|
23
26
|
)
|
|
24
27
|
from ._mapping import AsParentPropertyId, MapOneToOne, RuleMapper
|
|
@@ -43,11 +46,14 @@ __all__ = [
|
|
|
43
46
|
"RulesTransformer",
|
|
44
47
|
"SetIDDMSModel",
|
|
45
48
|
"StandardizeNaming",
|
|
49
|
+
"StandardizeSpaceAndVersion",
|
|
50
|
+
"SubsetDMSRules",
|
|
51
|
+
"SubsetInformationRules",
|
|
46
52
|
"ToCompliantEntities",
|
|
53
|
+
"ToDMSCompliantEntities",
|
|
47
54
|
"ToDataProductModel",
|
|
48
55
|
"ToEnterpriseModel",
|
|
49
56
|
"ToExtensionModel",
|
|
50
|
-
"ToInformationCompliantEntities",
|
|
51
57
|
"ToSolutionModel",
|
|
52
58
|
"VerifiedRulesTransformer",
|
|
53
59
|
"VerifyAnyRules",
|
|
@@ -6,11 +6,12 @@ from collections import Counter, defaultdict
|
|
|
6
6
|
from collections.abc import Collection, Mapping
|
|
7
7
|
from datetime import date, datetime
|
|
8
8
|
from functools import cached_property
|
|
9
|
-
from typing import ClassVar, Literal, TypeVar, cast, overload
|
|
9
|
+
from typing import Any, ClassVar, Literal, TypeVar, cast, overload
|
|
10
10
|
|
|
11
11
|
from cognite.client.data_classes import data_modeling as dms
|
|
12
12
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
|
|
13
13
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
14
|
+
from pydantic import ValidationError
|
|
14
15
|
from rdflib import Namespace
|
|
15
16
|
|
|
16
17
|
from cognite.neat._client import NeatClient
|
|
@@ -33,7 +34,7 @@ from cognite.neat._rules._shared import (
|
|
|
33
34
|
ReadRules,
|
|
34
35
|
VerifiedRules,
|
|
35
36
|
)
|
|
36
|
-
from cognite.neat._rules.analysis import
|
|
37
|
+
from cognite.neat._rules.analysis import RulesAnalysis
|
|
37
38
|
from cognite.neat._rules.importers import DMSImporter
|
|
38
39
|
from cognite.neat._rules.models import (
|
|
39
40
|
DMSInputRules,
|
|
@@ -43,8 +44,6 @@ from cognite.neat._rules.models import (
|
|
|
43
44
|
SheetList,
|
|
44
45
|
data_types,
|
|
45
46
|
)
|
|
46
|
-
from cognite.neat._rules.models._rdfpath import Entity as RDFPathEntity
|
|
47
|
-
from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
|
|
48
47
|
from cognite.neat._rules.models.data_types import AnyURI, DataType, Enum, File, String, Timeseries
|
|
49
48
|
from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSValidation, DMSView
|
|
50
49
|
from cognite.neat._rules.models.dms._rules import DMSContainer, DMSEnum, DMSNode
|
|
@@ -60,7 +59,8 @@ from cognite.neat._rules.models.entities import (
|
|
|
60
59
|
ViewEntity,
|
|
61
60
|
)
|
|
62
61
|
from cognite.neat._rules.models.information import InformationClass, InformationMetadata, InformationProperty
|
|
63
|
-
from cognite.neat._utils.
|
|
62
|
+
from cognite.neat._utils.rdf_ import get_inheritance_path
|
|
63
|
+
from cognite.neat._utils.text import NamingStandardization, to_camel_case
|
|
64
64
|
|
|
65
65
|
from ._base import RulesTransformer, T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
|
|
66
66
|
from ._verification import VerifyDMSRules
|
|
@@ -75,22 +75,20 @@ class ConversionTransformer(VerifiedRulesTransformer[T_VerifiedIn, T_VerifiedOut
|
|
|
75
75
|
...
|
|
76
76
|
|
|
77
77
|
|
|
78
|
-
class
|
|
79
|
-
RulesTransformer[ReadRules[InformationInputRules], ReadRules[InformationInputRules]]
|
|
80
|
-
):
|
|
78
|
+
class ToDMSCompliantEntities(RulesTransformer[ReadRules[InformationInputRules], ReadRules[InformationInputRules]]):
|
|
81
79
|
"""Converts input rules to rules that is compliant with the Information Model.
|
|
82
80
|
|
|
83
81
|
This is typically used with importers from arbitrary sources to ensure that classes and properties have valid
|
|
84
82
|
names.
|
|
85
83
|
|
|
86
84
|
Args:
|
|
87
|
-
|
|
88
|
-
- "
|
|
85
|
+
rename_warning: How to handle renaming of entities that are not compliant with the Information Model.
|
|
86
|
+
- "raise": Raises a warning and renames the entity.
|
|
89
87
|
- "skip": Renames the entity without raising a warning.
|
|
90
88
|
"""
|
|
91
89
|
|
|
92
|
-
def __init__(self,
|
|
93
|
-
self._renaming =
|
|
90
|
+
def __init__(self, rename_warning: Literal["raise", "skip"] = "skip") -> None:
|
|
91
|
+
self._renaming = rename_warning
|
|
94
92
|
|
|
95
93
|
@property
|
|
96
94
|
def description(self) -> str:
|
|
@@ -107,9 +105,9 @@ class ToInformationCompliantEntities(
|
|
|
107
105
|
new_by_old_class_suffix: dict[str, str] = {}
|
|
108
106
|
for cls in copy.classes:
|
|
109
107
|
cls_entity = cast(ClassEntity, cls.class_) # Safe due to the dump above
|
|
110
|
-
if not PATTERNS.
|
|
108
|
+
if not PATTERNS.view_id_compliance.match(cls_entity.suffix):
|
|
111
109
|
new_suffix = self._fix_cls_suffix(cls_entity.suffix)
|
|
112
|
-
if self._renaming == "
|
|
110
|
+
if self._renaming == "raise":
|
|
113
111
|
warnings.warn(
|
|
114
112
|
NeatValueWarning(f"Invalid class name {cls_entity.suffix!r}.Renaming to {new_suffix}"),
|
|
115
113
|
stacklevel=2,
|
|
@@ -123,7 +121,7 @@ class ToInformationCompliantEntities(
|
|
|
123
121
|
cls_.implements[i].suffix = new_by_old_class_suffix[parent.suffix] # type: ignore[union-attr]
|
|
124
122
|
|
|
125
123
|
for prop in copy.properties:
|
|
126
|
-
if not PATTERNS.
|
|
124
|
+
if not PATTERNS.dms_property_id_compliance.match(prop.property_):
|
|
127
125
|
new_property = self._fix_property(prop.property_)
|
|
128
126
|
if self._renaming == "warning":
|
|
129
127
|
warnings.warn(
|
|
@@ -174,6 +172,65 @@ class ToInformationCompliantEntities(
|
|
|
174
172
|
return property_
|
|
175
173
|
|
|
176
174
|
|
|
175
|
+
class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]): # type: ignore[misc]
|
|
176
|
+
"""This transformer standardizes the space and version of the DMSRules.
|
|
177
|
+
|
|
178
|
+
typically used to ensure all the views are moved to the same version as the data model.
|
|
179
|
+
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def description(self) -> str:
|
|
184
|
+
return "Ensures uniform version and space of the views belonging to the data model."
|
|
185
|
+
|
|
186
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
187
|
+
copy = rules.model_copy(deep=True)
|
|
188
|
+
|
|
189
|
+
space = copy.metadata.space
|
|
190
|
+
version = copy.metadata.version
|
|
191
|
+
|
|
192
|
+
copy.views = self._standardize_views(copy.views, space, version)
|
|
193
|
+
copy.properties = self._standardize_properties(copy.properties, space, version)
|
|
194
|
+
return copy
|
|
195
|
+
|
|
196
|
+
def _standardize_views(self, views: SheetList[DMSView], space: str, version: str) -> SheetList[DMSView]:
|
|
197
|
+
for view in views:
|
|
198
|
+
if view.view.space not in COGNITE_SPACES:
|
|
199
|
+
view.view.version = version
|
|
200
|
+
view.view.prefix = space
|
|
201
|
+
|
|
202
|
+
if view.implements:
|
|
203
|
+
for i, parent in enumerate(view.implements):
|
|
204
|
+
if parent.space not in COGNITE_SPACES:
|
|
205
|
+
view.implements[i].version = version
|
|
206
|
+
view.implements[i].prefix = space
|
|
207
|
+
return views
|
|
208
|
+
|
|
209
|
+
def _standardize_properties(
|
|
210
|
+
self, properties: SheetList[DMSProperty], space: str, version: str
|
|
211
|
+
) -> SheetList[DMSProperty]:
|
|
212
|
+
for property_ in properties:
|
|
213
|
+
if property_.view.space not in COGNITE_SPACES:
|
|
214
|
+
property_.view.version = version
|
|
215
|
+
property_.view.prefix = space
|
|
216
|
+
|
|
217
|
+
if isinstance(property_.value_type, ViewEntity) and property_.value_type.space not in COGNITE_SPACES:
|
|
218
|
+
property_.value_type.version = version
|
|
219
|
+
property_.value_type.prefix = space
|
|
220
|
+
|
|
221
|
+
# for edge connection
|
|
222
|
+
if (
|
|
223
|
+
property_.connection
|
|
224
|
+
and isinstance(property_.connection, EdgeEntity)
|
|
225
|
+
and property_.connection.properties
|
|
226
|
+
):
|
|
227
|
+
if property_.connection.properties.space not in COGNITE_SPACES:
|
|
228
|
+
property_.connection.properties.version = version
|
|
229
|
+
property_.connection.properties.prefix = space
|
|
230
|
+
|
|
231
|
+
return properties
|
|
232
|
+
|
|
233
|
+
|
|
177
234
|
class ToCompliantEntities(VerifiedRulesTransformer[InformationRules, InformationRules]): # type: ignore[misc]
|
|
178
235
|
"""Converts input rules to rules with compliant entity IDs that match regex patters used
|
|
179
236
|
by DMS schema components."""
|
|
@@ -649,7 +706,7 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
649
706
|
|
|
650
707
|
container = DMSContainer(container=container_entity)
|
|
651
708
|
|
|
652
|
-
property_id = f"{
|
|
709
|
+
property_id = f"{to_camel_case(view_entity.suffix)}{self.dummy_property}"
|
|
653
710
|
property_ = DMSProperty(
|
|
654
711
|
view=view_entity,
|
|
655
712
|
view_property=property_id,
|
|
@@ -759,13 +816,16 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
759
816
|
|
|
760
817
|
@staticmethod
|
|
761
818
|
def _expand_properties(rules: DMSRules) -> DMSRules:
|
|
762
|
-
probe =
|
|
763
|
-
ancestor_properties_by_view = probe.
|
|
764
|
-
|
|
819
|
+
probe = RulesAnalysis(dms=rules)
|
|
820
|
+
ancestor_properties_by_view = probe.properties_by_view(
|
|
821
|
+
include_ancestors=True,
|
|
822
|
+
include_different_space=True,
|
|
765
823
|
)
|
|
766
824
|
property_ids_by_view = {
|
|
767
825
|
view: {prop.view_property for prop in properties}
|
|
768
|
-
for view, properties in probe.
|
|
826
|
+
for view, properties in probe.properties_by_view(
|
|
827
|
+
include_ancestors=False, include_different_space=True
|
|
828
|
+
).items()
|
|
769
829
|
}
|
|
770
830
|
for view, property_ids in property_ids_by_view.items():
|
|
771
831
|
ancestor_properties = ancestor_properties_by_view.get(view, [])
|
|
@@ -852,7 +912,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
852
912
|
if view.view in read_view_by_new_view:
|
|
853
913
|
read_view = read_view_by_new_view[view.view]
|
|
854
914
|
container_entity = ContainerEntity(space=self.new_model_id.space, externalId=view.view.external_id)
|
|
855
|
-
prefix =
|
|
915
|
+
prefix = to_camel_case(view.view.suffix)
|
|
856
916
|
if self.properties == "repeat" and self.dummy_property:
|
|
857
917
|
property_ = DMSProperty(
|
|
858
918
|
view=view.view,
|
|
@@ -937,7 +997,7 @@ class ToDataProductModel(ToSolutionModel):
|
|
|
937
997
|
self.include = include
|
|
938
998
|
|
|
939
999
|
def transform(self, rules: DMSRules) -> DMSRules:
|
|
940
|
-
# Overwrite
|
|
1000
|
+
# Overwrite transform to avoid the warning.
|
|
941
1001
|
return self._to_solution(rules)
|
|
942
1002
|
|
|
943
1003
|
|
|
@@ -1011,7 +1071,7 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1011
1071
|
}
|
|
1012
1072
|
new_model = rules.model_copy(deep=True)
|
|
1013
1073
|
|
|
1014
|
-
properties_by_view =
|
|
1074
|
+
properties_by_view = RulesAnalysis(dms=new_model).properties_by_view(include_ancestors=True)
|
|
1015
1075
|
|
|
1016
1076
|
new_model.views = SheetList[DMSView]([view for view in new_model.views if view.view not in exclude_views])
|
|
1017
1077
|
new_properties = SheetList[DMSProperty]()
|
|
@@ -1155,6 +1215,7 @@ class ClassicPrepareCore(VerifiedRulesTransformer[InformationRules, InformationR
|
|
|
1155
1215
|
class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
1156
1216
|
description="A source system that provides data to the data model.",
|
|
1157
1217
|
neatId=namespace["ClassicSourceSystem"],
|
|
1218
|
+
instance_source=self.instance_namespace["ClassicSourceSystem"],
|
|
1158
1219
|
)
|
|
1159
1220
|
output.classes.append(source_system_class)
|
|
1160
1221
|
for prop in output.properties:
|
|
@@ -1183,15 +1244,7 @@ class ClassicPrepareCore(VerifiedRulesTransformer[InformationRules, InformationR
|
|
|
1183
1244
|
value_type=String(),
|
|
1184
1245
|
class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
1185
1246
|
max_count=1,
|
|
1186
|
-
instance_source=
|
|
1187
|
-
traversal=SingleProperty(
|
|
1188
|
-
class_=RDFPathEntity(
|
|
1189
|
-
prefix=instance_prefix,
|
|
1190
|
-
suffix="ClassicSourceSystem",
|
|
1191
|
-
),
|
|
1192
|
-
property=RDFPathEntity(prefix=instance_prefix, suffix="name"),
|
|
1193
|
-
),
|
|
1194
|
-
),
|
|
1247
|
+
instance_source=[self.instance_namespace["name"]],
|
|
1195
1248
|
)
|
|
1196
1249
|
)
|
|
1197
1250
|
return output
|
|
@@ -1673,7 +1726,6 @@ class _DMSRulesConverter:
|
|
|
1673
1726
|
classes.append(info_class)
|
|
1674
1727
|
|
|
1675
1728
|
prefixes = get_default_prefixes_and_namespaces()
|
|
1676
|
-
instance_prefix: str | None = None
|
|
1677
1729
|
if self.instance_namespace:
|
|
1678
1730
|
instance_prefix = next((k for k, v in prefixes.items() if v == self.instance_namespace), None)
|
|
1679
1731
|
if instance_prefix is None:
|
|
@@ -1696,15 +1748,6 @@ class _DMSRulesConverter:
|
|
|
1696
1748
|
else:
|
|
1697
1749
|
raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
|
|
1698
1750
|
|
|
1699
|
-
transformation: RDFPath | None = None
|
|
1700
|
-
if instance_prefix is not None:
|
|
1701
|
-
transformation = RDFPath(
|
|
1702
|
-
traversal=SingleProperty(
|
|
1703
|
-
class_=RDFPathEntity(prefix=instance_prefix, suffix=property_.view.external_id),
|
|
1704
|
-
property=RDFPathEntity(prefix=instance_prefix, suffix=property_.view_property),
|
|
1705
|
-
)
|
|
1706
|
-
)
|
|
1707
|
-
|
|
1708
1751
|
info_property = InformationProperty(
|
|
1709
1752
|
# Removing version
|
|
1710
1753
|
class_=ClassEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
|
|
@@ -1713,7 +1756,6 @@ class _DMSRulesConverter:
|
|
|
1713
1756
|
description=property_.description,
|
|
1714
1757
|
min_count=(0 if property_.nullable or property_.nullable is None else 1),
|
|
1715
1758
|
max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
|
|
1716
|
-
instance_source=transformation,
|
|
1717
1759
|
)
|
|
1718
1760
|
|
|
1719
1761
|
# Linking
|
|
@@ -1746,3 +1788,146 @@ class _DMSRulesConverter:
|
|
|
1746
1788
|
created=metadata.created,
|
|
1747
1789
|
updated=metadata.updated,
|
|
1748
1790
|
)
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
class SubsetDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
1794
|
+
"""Subsets DMSRules to only include the specified views."""
|
|
1795
|
+
|
|
1796
|
+
def __init__(self, views: set[ViewEntity]):
|
|
1797
|
+
self._views = views
|
|
1798
|
+
|
|
1799
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
1800
|
+
analysis = RulesAnalysis(dms=rules)
|
|
1801
|
+
|
|
1802
|
+
views_by_view = analysis.view_by_view_entity
|
|
1803
|
+
implements_by_view = analysis.implements_by_view()
|
|
1804
|
+
|
|
1805
|
+
available = analysis.defined_views(include_ancestors=True)
|
|
1806
|
+
subset = available.intersection(self._views)
|
|
1807
|
+
|
|
1808
|
+
ancestors: set[ViewEntity] = set()
|
|
1809
|
+
for view in subset:
|
|
1810
|
+
ancestors = ancestors.union({ancestor for ancestor in get_inheritance_path(view, implements_by_view)})
|
|
1811
|
+
subset = subset.union(ancestors)
|
|
1812
|
+
|
|
1813
|
+
if not subset:
|
|
1814
|
+
raise NeatValueError("None of the requested views are defined in the rules!")
|
|
1815
|
+
|
|
1816
|
+
if nonexisting := self._views - subset:
|
|
1817
|
+
raise NeatValueError(
|
|
1818
|
+
"Following requested views do not exist"
|
|
1819
|
+
f" in the rules: [{','.join([view.external_id for view in nonexisting])}]. Aborting."
|
|
1820
|
+
)
|
|
1821
|
+
|
|
1822
|
+
subsetted_rules: dict[str, Any] = {
|
|
1823
|
+
"metadata": rules.metadata.model_copy(),
|
|
1824
|
+
"views": SheetList[DMSView](),
|
|
1825
|
+
"properties": SheetList[DMSProperty](),
|
|
1826
|
+
"containers": SheetList[DMSContainer](),
|
|
1827
|
+
"enum": rules.enum,
|
|
1828
|
+
"nodes": rules.nodes,
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
# add views
|
|
1832
|
+
for view in subset:
|
|
1833
|
+
subsetted_rules["views"].append(views_by_view[view])
|
|
1834
|
+
|
|
1835
|
+
used_containers = set()
|
|
1836
|
+
|
|
1837
|
+
# add properties
|
|
1838
|
+
for view, properties in analysis.properties_by_view(include_ancestors=False).items():
|
|
1839
|
+
if view not in subset:
|
|
1840
|
+
continue
|
|
1841
|
+
|
|
1842
|
+
for property_ in properties:
|
|
1843
|
+
if (
|
|
1844
|
+
isinstance(property_.value_type, DataType)
|
|
1845
|
+
or isinstance(property_.value_type, DMSUnknownEntity)
|
|
1846
|
+
or (isinstance(property_.value_type, ViewEntity) and property_.value_type in subset)
|
|
1847
|
+
):
|
|
1848
|
+
subsetted_rules["properties"].append(property_)
|
|
1849
|
+
|
|
1850
|
+
if property_.container:
|
|
1851
|
+
used_containers.add(property_.container)
|
|
1852
|
+
|
|
1853
|
+
# add containers
|
|
1854
|
+
if rules.containers:
|
|
1855
|
+
for container in rules.containers:
|
|
1856
|
+
if container.container in used_containers:
|
|
1857
|
+
subsetted_rules["containers"].append(container)
|
|
1858
|
+
|
|
1859
|
+
try:
|
|
1860
|
+
return DMSRules.model_validate(subsetted_rules)
|
|
1861
|
+
except ValidationError as e:
|
|
1862
|
+
raise NeatValueError(f"Cannot subset rules: {e}") from e
|
|
1863
|
+
|
|
1864
|
+
|
|
1865
|
+
class SubsetInformationRules(VerifiedRulesTransformer[InformationRules, InformationRules]):
|
|
1866
|
+
"""Subsets InformationRules to only include the specified classes."""
|
|
1867
|
+
|
|
1868
|
+
def __init__(self, classes: set[ClassEntity]):
|
|
1869
|
+
self._classes = classes
|
|
1870
|
+
|
|
1871
|
+
def transform(self, rules: InformationRules) -> InformationRules:
|
|
1872
|
+
analysis = RulesAnalysis(information=rules)
|
|
1873
|
+
|
|
1874
|
+
class_by_class_entity = analysis.class_by_class_entity
|
|
1875
|
+
parent_entity_by_class_entity = analysis.parents_by_class()
|
|
1876
|
+
|
|
1877
|
+
available = analysis.defined_classes(include_ancestors=True)
|
|
1878
|
+
subset = available.intersection(self._classes)
|
|
1879
|
+
|
|
1880
|
+
# need to add all the parent classes of the desired classes to the possible classes
|
|
1881
|
+
ancestors: set[ClassEntity] = set()
|
|
1882
|
+
for class_ in subset:
|
|
1883
|
+
ancestors = ancestors.union(
|
|
1884
|
+
{ancestor for ancestor in get_inheritance_path(class_, parent_entity_by_class_entity)}
|
|
1885
|
+
)
|
|
1886
|
+
subset = subset.union(ancestors)
|
|
1887
|
+
|
|
1888
|
+
if not subset:
|
|
1889
|
+
raise NeatValueError("None of the requested classes are defined in the rules!")
|
|
1890
|
+
|
|
1891
|
+
if nonexisting := self._classes - subset:
|
|
1892
|
+
raise NeatValueError(
|
|
1893
|
+
"Following requested classes do not exist"
|
|
1894
|
+
f" in the rules: [{','.join([class_.suffix for class_ in nonexisting])}]"
|
|
1895
|
+
". Aborting."
|
|
1896
|
+
)
|
|
1897
|
+
|
|
1898
|
+
subsetted_rules: dict[str, Any] = {
|
|
1899
|
+
"metadata": rules.metadata.model_copy(),
|
|
1900
|
+
"prefixes": (rules.prefixes or {}).copy(),
|
|
1901
|
+
"classes": SheetList[InformationClass](),
|
|
1902
|
+
"properties": SheetList[InformationProperty](),
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
for class_ in subset:
|
|
1906
|
+
subsetted_rules["classes"].append(class_by_class_entity[class_])
|
|
1907
|
+
|
|
1908
|
+
for class_, properties in analysis.properties_by_class(include_ancestors=False).items():
|
|
1909
|
+
if class_ not in subset:
|
|
1910
|
+
continue
|
|
1911
|
+
for property_ in properties:
|
|
1912
|
+
# datatype property can be added directly
|
|
1913
|
+
if (
|
|
1914
|
+
isinstance(property_.value_type, DataType)
|
|
1915
|
+
or (isinstance(property_.value_type, ClassEntity) and property_.value_type in subset)
|
|
1916
|
+
or isinstance(property_.value_type, UnknownEntity)
|
|
1917
|
+
):
|
|
1918
|
+
subsetted_rules["properties"].append(property_)
|
|
1919
|
+
# object property can be added if the value type is in the subset
|
|
1920
|
+
elif isinstance(property_.value_type, MultiValueTypeInfo):
|
|
1921
|
+
allowed = [t for t in property_.value_type.types if t in subset or isinstance(t, DataType)]
|
|
1922
|
+
if allowed:
|
|
1923
|
+
subsetted_rules["properties"].append(
|
|
1924
|
+
property_.model_copy(
|
|
1925
|
+
deep=True,
|
|
1926
|
+
update={"value_type": MultiValueTypeInfo(types=allowed)},
|
|
1927
|
+
)
|
|
1928
|
+
)
|
|
1929
|
+
|
|
1930
|
+
try:
|
|
1931
|
+
return InformationRules.model_validate(subsetted_rules)
|
|
1932
|
+
except ValidationError as e:
|
|
1933
|
+
raise NeatValueError(f"Cannot subset rules: {e}") from e
|
|
@@ -35,27 +35,22 @@ class VerificationTransformer(RulesTransformer[T_ReadInputRules, T_VerifiedRules
|
|
|
35
35
|
in_ = rules.rules
|
|
36
36
|
if in_ is None:
|
|
37
37
|
raise NeatValueError("Cannot verify rules. The reading of the rules failed.")
|
|
38
|
-
error_args = rules.read_context
|
|
39
38
|
verified_rules: T_VerifiedRules | None = None
|
|
40
39
|
# We need to catch issues as we use the error args to provide extra context for the errors/warnings
|
|
41
|
-
# For example, which row in the spreadsheet the error occurred
|
|
42
|
-
with catch_issues(
|
|
40
|
+
# For example, which row in the spreadsheet the error occurred.
|
|
41
|
+
with catch_issues(rules.read_context) as issues:
|
|
43
42
|
rules_cls = self._get_rules_cls(rules)
|
|
44
43
|
dumped = in_.dump()
|
|
45
44
|
verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
|
|
46
45
|
if self.validate:
|
|
47
46
|
validation_cls = self._get_validation_cls(verified_rules) # type: ignore[arg-type]
|
|
48
47
|
if issubclass(validation_cls, DMSValidation):
|
|
49
|
-
validation_issues = DMSValidation(verified_rules, self._client).validate() # type: ignore[arg-type]
|
|
48
|
+
validation_issues = DMSValidation(verified_rules, self._client, rules.read_context).validate() # type: ignore[arg-type]
|
|
50
49
|
elif issubclass(validation_cls, InformationValidation):
|
|
51
|
-
validation_issues = InformationValidation(verified_rules).validate() # type: ignore[arg-type]
|
|
50
|
+
validation_issues = InformationValidation(verified_rules, rules.read_context).validate() # type: ignore[arg-type]
|
|
52
51
|
else:
|
|
53
52
|
raise NeatValueError("Unsupported rule type")
|
|
54
|
-
|
|
55
|
-
# Need to trigger and raise such that the catch_issues can add the extra context
|
|
56
|
-
validation_issues.trigger_warnings()
|
|
57
|
-
if validation_issues.has_errors:
|
|
58
|
-
raise MultiValueError(validation_issues.errors)
|
|
53
|
+
issues.extend(validation_issues)
|
|
59
54
|
|
|
60
55
|
# Raise issues which is expected to be handled outside of this method
|
|
61
56
|
issues.trigger_warnings()
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -16,7 +16,7 @@ from cognite.neat._rules.transformers import (
|
|
|
16
16
|
InformationToDMS,
|
|
17
17
|
MergeDMSRules,
|
|
18
18
|
MergeInformationRules,
|
|
19
|
-
|
|
19
|
+
ToDMSCompliantEntities,
|
|
20
20
|
VerifyInformationRules,
|
|
21
21
|
)
|
|
22
22
|
from cognite.neat._store._rules_store import RulesEntity
|
|
@@ -33,6 +33,7 @@ from ._read import ReadAPI
|
|
|
33
33
|
from ._set import SetAPI
|
|
34
34
|
from ._show import ShowAPI
|
|
35
35
|
from ._state import SessionState
|
|
36
|
+
from ._subset import SubsetAPI
|
|
36
37
|
from ._to import ToAPI
|
|
37
38
|
from .engine import load_neat_engine
|
|
38
39
|
from .exceptions import session_class_wrapper
|
|
@@ -101,6 +102,7 @@ class NeatSession:
|
|
|
101
102
|
self.inspect = InspectAPI(self._state)
|
|
102
103
|
self.mapping = MappingAPI(self._state)
|
|
103
104
|
self.drop = DropAPI(self._state)
|
|
105
|
+
self.subset = SubsetAPI(self._state)
|
|
104
106
|
self.create = CreateAPI(self._state)
|
|
105
107
|
self.opt = OptAPI()
|
|
106
108
|
self.opt._display()
|
|
@@ -236,9 +238,7 @@ class NeatSession:
|
|
|
236
238
|
|
|
237
239
|
def action() -> tuple[InformationRules, DMSRules | None]:
|
|
238
240
|
unverified_information = importer.to_rules()
|
|
239
|
-
unverified_information =
|
|
240
|
-
unverified_information
|
|
241
|
-
)
|
|
241
|
+
unverified_information = ToDMSCompliantEntities(rename_warning="raise").transform(unverified_information)
|
|
242
242
|
|
|
243
243
|
extra_info = VerifyInformationRules().transform(unverified_information)
|
|
244
244
|
if not last_entity:
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from collections.abc import Callable
|
|
2
3
|
from typing import Any
|
|
3
4
|
|
|
@@ -14,6 +15,7 @@ from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
|
|
|
14
15
|
from cognite.neat._issues import IssueList
|
|
15
16
|
from cognite.neat._issues.errors import NeatValueError
|
|
16
17
|
from cognite.neat._rules.transformers import PrefixEntities, StandardizeNaming
|
|
18
|
+
from cognite.neat._rules.transformers._converters import StandardizeSpaceAndVersion
|
|
17
19
|
from cognite.neat._utils.text import humanize_collection
|
|
18
20
|
|
|
19
21
|
from ._state import SessionState
|
|
@@ -270,5 +272,15 @@ class DataModelPrepareAPI:
|
|
|
270
272
|
For classes/views/containers, the naming will be standardized to PascalCase.
|
|
271
273
|
For properties, the naming will be standardized to camelCase.
|
|
272
274
|
"""
|
|
275
|
+
warnings.filterwarnings("default")
|
|
273
276
|
AlphaFlags.standardize_naming.warn()
|
|
274
277
|
return self._state.rule_transform(StandardizeNaming())
|
|
278
|
+
|
|
279
|
+
def standardize_space_and_version(self) -> IssueList:
|
|
280
|
+
"""Standardize space and version in the data model.
|
|
281
|
+
|
|
282
|
+
This method will standardize the space and version in the data model to the Cognite standard.
|
|
283
|
+
"""
|
|
284
|
+
warnings.filterwarnings("default")
|
|
285
|
+
AlphaFlags.standardize_space_and_version.warn()
|
|
286
|
+
return self._state.rule_transform(StandardizeSpaceAndVersion())
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -50,6 +50,7 @@ class ReadAPI:
|
|
|
50
50
|
self.csv = CSVReadAPI(state, verbose)
|
|
51
51
|
self.yaml = YamlReadAPI(state, verbose)
|
|
52
52
|
self.xml = XMLReadAPI(state, verbose)
|
|
53
|
+
self.examples = Examples(state)
|
|
53
54
|
|
|
54
55
|
def session(self, io: Any) -> None:
|
|
55
56
|
"""Reads a Neat Session from a zip file.
|
|
@@ -256,9 +257,6 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
256
257
|
prepare_issues = self._state.rule_store.transform(
|
|
257
258
|
ClassicPrepareCore(namespace, reference_timeseries, reference_files)
|
|
258
259
|
)
|
|
259
|
-
# Update the instance store with the latest rules
|
|
260
|
-
information_rules = self._state.rule_store.last_verified_information_rules
|
|
261
|
-
self._state.instances.store.rules[self._state.instances.store.default_named_graph] = information_rules
|
|
262
260
|
|
|
263
261
|
all_issues = IssueList(extract_issues + prepare_issues)
|
|
264
262
|
# Update the provenance with all issue.
|
|
@@ -286,7 +284,6 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
286
284
|
|
|
287
285
|
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
288
286
|
super().__init__(state, verbose)
|
|
289
|
-
self.examples = ExcelExampleAPI(state, verbose)
|
|
290
287
|
|
|
291
288
|
def __call__(self, io: Any, enable_manual_edit: bool = False) -> IssueList:
|
|
292
289
|
"""Reads a Neat Excel Rules sheet to the graph store. The rules sheet may stem from an Information architect,
|
|
@@ -310,16 +307,6 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
310
307
|
return self._state.rule_import(importers.ExcelImporter(path), enable_manual_edit)
|
|
311
308
|
|
|
312
309
|
|
|
313
|
-
@session_class_wrapper
|
|
314
|
-
class ExcelExampleAPI(BaseReadAPI):
|
|
315
|
-
"""Used as example for reading some data model into the NeatSession."""
|
|
316
|
-
|
|
317
|
-
def pump_example(self) -> IssueList:
|
|
318
|
-
"""Reads the Hello World pump example into the NeatSession."""
|
|
319
|
-
importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)
|
|
320
|
-
return self._state.rule_import(importer)
|
|
321
|
-
|
|
322
|
-
|
|
323
310
|
@session_class_wrapper
|
|
324
311
|
class YamlReadAPI(BaseReadAPI):
|
|
325
312
|
def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
|
|
@@ -519,7 +506,6 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
519
506
|
|
|
520
507
|
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
521
508
|
super().__init__(state, verbose)
|
|
522
|
-
self.examples = RDFExamples(state)
|
|
523
509
|
|
|
524
510
|
def ontology(self, io: Any) -> IssueList:
|
|
525
511
|
"""Reads an OWL ontology source into NeatSession.
|
|
@@ -582,13 +568,31 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
582
568
|
|
|
583
569
|
|
|
584
570
|
@session_class_wrapper
|
|
585
|
-
class
|
|
586
|
-
"""Used as example for reading
|
|
571
|
+
class Examples:
|
|
572
|
+
"""Used as example for reading various sources into NeatSession."""
|
|
587
573
|
|
|
588
574
|
def __init__(self, state: SessionState) -> None:
|
|
589
575
|
self._state = state
|
|
590
576
|
|
|
577
|
+
@property
|
|
578
|
+
def _get_client(self) -> NeatClient:
|
|
579
|
+
if self._state.client is None:
|
|
580
|
+
raise NeatValueError("No client provided. Please provide a client to read a data model.")
|
|
581
|
+
return self._state.client
|
|
582
|
+
|
|
591
583
|
def nordic44(self) -> IssueList:
|
|
592
584
|
"""Reads the Nordic 44 knowledge graph into the NeatSession graph store."""
|
|
593
585
|
self._state.instances.store.write(extractors.RdfFileExtractor(instances_examples.nordic44_knowledge_graph))
|
|
594
586
|
return IssueList()
|
|
587
|
+
|
|
588
|
+
def pump_example(self) -> IssueList:
|
|
589
|
+
"""Reads the Hello World pump example into the NeatSession."""
|
|
590
|
+
importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)
|
|
591
|
+
return self._state.rule_import(importer)
|
|
592
|
+
|
|
593
|
+
def core_data_model(self) -> IssueList:
|
|
594
|
+
"""Reads the core data model example into the NeatSession."""
|
|
595
|
+
|
|
596
|
+
cdm_v1 = DataModelId.load(("cdf_cdm", "CogniteCore", "v1"))
|
|
597
|
+
importer: importers.DMSImporter = importers.DMSImporter.from_data_model_id(self._get_client, cdm_v1)
|
|
598
|
+
return self._state.rule_import(importer)
|