cognite-neat 0.109.3__py3-none-any.whl → 0.110.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_alpha.py +2 -0
- cognite/neat/_client/_api/schema.py +17 -1
- cognite/neat/_client/data_classes/schema.py +3 -3
- cognite/neat/_constants.py +11 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +9 -10
- cognite/neat/_graph/extractors/_iodd.py +3 -3
- cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
- cognite/neat/_graph/loaders/_rdf2dms.py +285 -346
- cognite/neat/_graph/queries/_base.py +28 -92
- cognite/neat/_graph/transformers/__init__.py +1 -3
- cognite/neat/_graph/transformers/_rdfpath.py +2 -49
- cognite/neat/_issues/__init__.py +1 -6
- cognite/neat/_issues/_base.py +21 -252
- cognite/neat/_issues/_contextmanagers.py +46 -0
- cognite/neat/_issues/_factory.py +61 -0
- cognite/neat/_issues/errors/__init__.py +18 -4
- cognite/neat/_issues/errors/_wrapper.py +81 -3
- cognite/neat/_issues/formatters.py +4 -4
- cognite/neat/_issues/warnings/__init__.py +3 -2
- cognite/neat/_issues/warnings/_properties.py +8 -0
- cognite/neat/_rules/_constants.py +9 -0
- cognite/neat/_rules/_shared.py +3 -2
- cognite/neat/_rules/analysis/__init__.py +2 -3
- cognite/neat/_rules/analysis/_base.py +450 -258
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +2 -8
- cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
- cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
- cognite/neat/_rules/importers/_base.py +2 -47
- cognite/neat/_rules/importers/_dms2rules.py +7 -10
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +59 -25
- cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
- cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
- cognite/neat/_rules/models/dms/_rules.py +3 -1
- cognite/neat/_rules/models/dms/_rules_input.py +4 -0
- cognite/neat/_rules/models/dms/_validation.py +14 -4
- cognite/neat/_rules/models/entities/_loaders.py +1 -1
- cognite/neat/_rules/models/entities/_multi_value.py +2 -2
- cognite/neat/_rules/models/information/_rules.py +18 -17
- cognite/neat/_rules/models/information/_rules_input.py +2 -1
- cognite/neat/_rules/models/information/_validation.py +3 -1
- cognite/neat/_rules/transformers/__init__.py +8 -2
- cognite/neat/_rules/transformers/_converters.py +242 -43
- cognite/neat/_rules/transformers/_verification.py +5 -10
- cognite/neat/_session/_base.py +4 -4
- cognite/neat/_session/_prepare.py +12 -0
- cognite/neat/_session/_read.py +21 -17
- cognite/neat/_session/_show.py +11 -123
- cognite/neat/_session/_state.py +0 -2
- cognite/neat/_session/_subset.py +64 -0
- cognite/neat/_session/_to.py +63 -12
- cognite/neat/_store/_graph_store.py +5 -246
- cognite/neat/_utils/rdf_.py +2 -2
- cognite/neat/_utils/spreadsheet.py +44 -1
- cognite/neat/_utils/text.py +51 -32
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/RECORD +62 -64
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/WHEEL +1 -1
- cognite/neat/_graph/queries/_construct.py +0 -187
- cognite/neat/_graph/queries/_shared.py +0 -173
- cognite/neat/_rules/analysis/_dms.py +0 -57
- cognite/neat/_rules/analysis/_information.py +0 -249
- cognite/neat/_rules/models/_rdfpath.py +0 -372
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/entry_points.txt +0 -0
|
@@ -12,6 +12,7 @@ from cognite.neat._issues.warnings._resources import (
|
|
|
12
12
|
from cognite.neat._rules._constants import PATTERNS, EntityTypes
|
|
13
13
|
from cognite.neat._rules.models.entities import ClassEntity, UnknownEntity
|
|
14
14
|
from cognite.neat._rules.models.entities._multi_value import MultiValueTypeInfo
|
|
15
|
+
from cognite.neat._utils.spreadsheet import SpreadsheetRead
|
|
15
16
|
|
|
16
17
|
from ._rules import InformationRules
|
|
17
18
|
|
|
@@ -20,8 +21,9 @@ class InformationValidation:
|
|
|
20
21
|
"""This class does all the validation of the Information rules that have dependencies
|
|
21
22
|
between components."""
|
|
22
23
|
|
|
23
|
-
def __init__(self, rules: InformationRules):
|
|
24
|
+
def __init__(self, rules: InformationRules, read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None):
|
|
24
25
|
self.rules = rules
|
|
26
|
+
self.read_info_by_spreadsheet = read_info_by_spreadsheet
|
|
25
27
|
self.metadata = rules.metadata
|
|
26
28
|
self.properties = rules.properties
|
|
27
29
|
self.classes = rules.classes
|
|
@@ -14,11 +14,14 @@ from ._converters import (
|
|
|
14
14
|
PrefixEntities,
|
|
15
15
|
SetIDDMSModel,
|
|
16
16
|
StandardizeNaming,
|
|
17
|
+
StandardizeSpaceAndVersion,
|
|
18
|
+
SubsetDMSRules,
|
|
19
|
+
SubsetInformationRules,
|
|
17
20
|
ToCompliantEntities,
|
|
18
21
|
ToDataProductModel,
|
|
22
|
+
ToDMSCompliantEntities,
|
|
19
23
|
ToEnterpriseModel,
|
|
20
24
|
ToExtensionModel,
|
|
21
|
-
ToInformationCompliantEntities,
|
|
22
25
|
ToSolutionModel,
|
|
23
26
|
)
|
|
24
27
|
from ._mapping import AsParentPropertyId, MapOneToOne, RuleMapper
|
|
@@ -43,11 +46,14 @@ __all__ = [
|
|
|
43
46
|
"RulesTransformer",
|
|
44
47
|
"SetIDDMSModel",
|
|
45
48
|
"StandardizeNaming",
|
|
49
|
+
"StandardizeSpaceAndVersion",
|
|
50
|
+
"SubsetDMSRules",
|
|
51
|
+
"SubsetInformationRules",
|
|
46
52
|
"ToCompliantEntities",
|
|
53
|
+
"ToDMSCompliantEntities",
|
|
47
54
|
"ToDataProductModel",
|
|
48
55
|
"ToEnterpriseModel",
|
|
49
56
|
"ToExtensionModel",
|
|
50
|
-
"ToInformationCompliantEntities",
|
|
51
57
|
"ToSolutionModel",
|
|
52
58
|
"VerifiedRulesTransformer",
|
|
53
59
|
"VerifyAnyRules",
|
|
@@ -6,11 +6,12 @@ from collections import Counter, defaultdict
|
|
|
6
6
|
from collections.abc import Collection, Mapping
|
|
7
7
|
from datetime import date, datetime
|
|
8
8
|
from functools import cached_property
|
|
9
|
-
from typing import ClassVar, Literal, TypeVar, cast, overload
|
|
9
|
+
from typing import Any, ClassVar, Literal, TypeVar, cast, overload
|
|
10
10
|
|
|
11
11
|
from cognite.client.data_classes import data_modeling as dms
|
|
12
12
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
|
|
13
13
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
14
|
+
from pydantic import ValidationError
|
|
14
15
|
from rdflib import Namespace
|
|
15
16
|
|
|
16
17
|
from cognite.neat._client import NeatClient
|
|
@@ -33,7 +34,7 @@ from cognite.neat._rules._shared import (
|
|
|
33
34
|
ReadRules,
|
|
34
35
|
VerifiedRules,
|
|
35
36
|
)
|
|
36
|
-
from cognite.neat._rules.analysis import
|
|
37
|
+
from cognite.neat._rules.analysis import RulesAnalysis
|
|
37
38
|
from cognite.neat._rules.importers import DMSImporter
|
|
38
39
|
from cognite.neat._rules.models import (
|
|
39
40
|
DMSInputRules,
|
|
@@ -43,8 +44,6 @@ from cognite.neat._rules.models import (
|
|
|
43
44
|
SheetList,
|
|
44
45
|
data_types,
|
|
45
46
|
)
|
|
46
|
-
from cognite.neat._rules.models._rdfpath import Entity as RDFPathEntity
|
|
47
|
-
from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
|
|
48
47
|
from cognite.neat._rules.models.data_types import AnyURI, DataType, Enum, File, String, Timeseries
|
|
49
48
|
from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSValidation, DMSView
|
|
50
49
|
from cognite.neat._rules.models.dms._rules import DMSContainer, DMSEnum, DMSNode
|
|
@@ -60,7 +59,8 @@ from cognite.neat._rules.models.entities import (
|
|
|
60
59
|
ViewEntity,
|
|
61
60
|
)
|
|
62
61
|
from cognite.neat._rules.models.information import InformationClass, InformationMetadata, InformationProperty
|
|
63
|
-
from cognite.neat._utils.
|
|
62
|
+
from cognite.neat._utils.rdf_ import get_inheritance_path
|
|
63
|
+
from cognite.neat._utils.text import NamingStandardization, to_camel_case
|
|
64
64
|
|
|
65
65
|
from ._base import RulesTransformer, T_VerifiedIn, T_VerifiedOut, VerifiedRulesTransformer
|
|
66
66
|
from ._verification import VerifyDMSRules
|
|
@@ -75,22 +75,20 @@ class ConversionTransformer(VerifiedRulesTransformer[T_VerifiedIn, T_VerifiedOut
|
|
|
75
75
|
...
|
|
76
76
|
|
|
77
77
|
|
|
78
|
-
class
|
|
79
|
-
RulesTransformer[ReadRules[InformationInputRules], ReadRules[InformationInputRules]]
|
|
80
|
-
):
|
|
78
|
+
class ToDMSCompliantEntities(RulesTransformer[ReadRules[InformationInputRules], ReadRules[InformationInputRules]]):
|
|
81
79
|
"""Converts input rules to rules that is compliant with the Information Model.
|
|
82
80
|
|
|
83
81
|
This is typically used with importers from arbitrary sources to ensure that classes and properties have valid
|
|
84
82
|
names.
|
|
85
83
|
|
|
86
84
|
Args:
|
|
87
|
-
|
|
88
|
-
- "
|
|
85
|
+
rename_warning: How to handle renaming of entities that are not compliant with the Information Model.
|
|
86
|
+
- "raise": Raises a warning and renames the entity.
|
|
89
87
|
- "skip": Renames the entity without raising a warning.
|
|
90
88
|
"""
|
|
91
89
|
|
|
92
|
-
def __init__(self,
|
|
93
|
-
self._renaming =
|
|
90
|
+
def __init__(self, rename_warning: Literal["raise", "skip"] = "skip") -> None:
|
|
91
|
+
self._renaming = rename_warning
|
|
94
92
|
|
|
95
93
|
@property
|
|
96
94
|
def description(self) -> str:
|
|
@@ -107,9 +105,9 @@ class ToInformationCompliantEntities(
|
|
|
107
105
|
new_by_old_class_suffix: dict[str, str] = {}
|
|
108
106
|
for cls in copy.classes:
|
|
109
107
|
cls_entity = cast(ClassEntity, cls.class_) # Safe due to the dump above
|
|
110
|
-
if not PATTERNS.
|
|
108
|
+
if not PATTERNS.view_id_compliance.match(cls_entity.suffix):
|
|
111
109
|
new_suffix = self._fix_cls_suffix(cls_entity.suffix)
|
|
112
|
-
if self._renaming == "
|
|
110
|
+
if self._renaming == "raise":
|
|
113
111
|
warnings.warn(
|
|
114
112
|
NeatValueWarning(f"Invalid class name {cls_entity.suffix!r}.Renaming to {new_suffix}"),
|
|
115
113
|
stacklevel=2,
|
|
@@ -123,7 +121,7 @@ class ToInformationCompliantEntities(
|
|
|
123
121
|
cls_.implements[i].suffix = new_by_old_class_suffix[parent.suffix] # type: ignore[union-attr]
|
|
124
122
|
|
|
125
123
|
for prop in copy.properties:
|
|
126
|
-
if not PATTERNS.
|
|
124
|
+
if not PATTERNS.dms_property_id_compliance.match(prop.property_):
|
|
127
125
|
new_property = self._fix_property(prop.property_)
|
|
128
126
|
if self._renaming == "warning":
|
|
129
127
|
warnings.warn(
|
|
@@ -174,6 +172,65 @@ class ToInformationCompliantEntities(
|
|
|
174
172
|
return property_
|
|
175
173
|
|
|
176
174
|
|
|
175
|
+
class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]): # type: ignore[misc]
|
|
176
|
+
"""This transformer standardizes the space and version of the DMSRules.
|
|
177
|
+
|
|
178
|
+
typically used to ensure all the views are moved to the same version as the data model.
|
|
179
|
+
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def description(self) -> str:
|
|
184
|
+
return "Ensures uniform version and space of the views belonging to the data model."
|
|
185
|
+
|
|
186
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
187
|
+
copy = rules.model_copy(deep=True)
|
|
188
|
+
|
|
189
|
+
space = copy.metadata.space
|
|
190
|
+
version = copy.metadata.version
|
|
191
|
+
|
|
192
|
+
copy.views = self._standardize_views(copy.views, space, version)
|
|
193
|
+
copy.properties = self._standardize_properties(copy.properties, space, version)
|
|
194
|
+
return copy
|
|
195
|
+
|
|
196
|
+
def _standardize_views(self, views: SheetList[DMSView], space: str, version: str) -> SheetList[DMSView]:
|
|
197
|
+
for view in views:
|
|
198
|
+
if view.view.space not in COGNITE_SPACES:
|
|
199
|
+
view.view.version = version
|
|
200
|
+
view.view.prefix = space
|
|
201
|
+
|
|
202
|
+
if view.implements:
|
|
203
|
+
for i, parent in enumerate(view.implements):
|
|
204
|
+
if parent.space not in COGNITE_SPACES:
|
|
205
|
+
view.implements[i].version = version
|
|
206
|
+
view.implements[i].prefix = space
|
|
207
|
+
return views
|
|
208
|
+
|
|
209
|
+
def _standardize_properties(
|
|
210
|
+
self, properties: SheetList[DMSProperty], space: str, version: str
|
|
211
|
+
) -> SheetList[DMSProperty]:
|
|
212
|
+
for property_ in properties:
|
|
213
|
+
if property_.view.space not in COGNITE_SPACES:
|
|
214
|
+
property_.view.version = version
|
|
215
|
+
property_.view.prefix = space
|
|
216
|
+
|
|
217
|
+
if isinstance(property_.value_type, ViewEntity) and property_.value_type.space not in COGNITE_SPACES:
|
|
218
|
+
property_.value_type.version = version
|
|
219
|
+
property_.value_type.prefix = space
|
|
220
|
+
|
|
221
|
+
# for edge connection
|
|
222
|
+
if (
|
|
223
|
+
property_.connection
|
|
224
|
+
and isinstance(property_.connection, EdgeEntity)
|
|
225
|
+
and property_.connection.properties
|
|
226
|
+
):
|
|
227
|
+
if property_.connection.properties.space not in COGNITE_SPACES:
|
|
228
|
+
property_.connection.properties.version = version
|
|
229
|
+
property_.connection.properties.prefix = space
|
|
230
|
+
|
|
231
|
+
return properties
|
|
232
|
+
|
|
233
|
+
|
|
177
234
|
class ToCompliantEntities(VerifiedRulesTransformer[InformationRules, InformationRules]): # type: ignore[misc]
|
|
178
235
|
"""Converts input rules to rules with compliant entity IDs that match regex patters used
|
|
179
236
|
by DMS schema components."""
|
|
@@ -575,9 +632,23 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
575
632
|
|
|
576
633
|
# ... however, we do not want to keep the reference containers and properties
|
|
577
634
|
# these we are getting for free through the implements.
|
|
635
|
+
enterprise_containers.sort(key=lambda container: (container.container.space, container.container.external_id))
|
|
578
636
|
enterprise_model.containers = enterprise_containers
|
|
637
|
+
enterprise_properties.sort(
|
|
638
|
+
key=lambda prop: (prop.view.space, prop.view.external_id, prop.view.version, prop.view_property)
|
|
639
|
+
)
|
|
579
640
|
enterprise_model.properties = enterprise_properties
|
|
580
641
|
|
|
642
|
+
# Sorting all your views first.
|
|
643
|
+
enterprise_model.views.sort(
|
|
644
|
+
key=lambda view: (
|
|
645
|
+
# Sorting your views first
|
|
646
|
+
int(view.view.space != self.new_model_id.space),
|
|
647
|
+
view.view.space,
|
|
648
|
+
view.view.external_id,
|
|
649
|
+
view.view.version,
|
|
650
|
+
)
|
|
651
|
+
)
|
|
581
652
|
return enterprise_model
|
|
582
653
|
|
|
583
654
|
@staticmethod
|
|
@@ -635,7 +706,7 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
635
706
|
|
|
636
707
|
container = DMSContainer(container=container_entity)
|
|
637
708
|
|
|
638
|
-
property_id = f"{
|
|
709
|
+
property_id = f"{to_camel_case(view_entity.suffix)}{self.dummy_property}"
|
|
639
710
|
property_ = DMSProperty(
|
|
640
711
|
view=view_entity,
|
|
641
712
|
view_property=property_id,
|
|
@@ -745,13 +816,16 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
745
816
|
|
|
746
817
|
@staticmethod
|
|
747
818
|
def _expand_properties(rules: DMSRules) -> DMSRules:
|
|
748
|
-
probe =
|
|
749
|
-
ancestor_properties_by_view = probe.
|
|
750
|
-
|
|
819
|
+
probe = RulesAnalysis(dms=rules)
|
|
820
|
+
ancestor_properties_by_view = probe.properties_by_view(
|
|
821
|
+
include_ancestors=True,
|
|
822
|
+
include_different_space=True,
|
|
751
823
|
)
|
|
752
824
|
property_ids_by_view = {
|
|
753
825
|
view: {prop.view_property for prop in properties}
|
|
754
|
-
for view, properties in probe.
|
|
826
|
+
for view, properties in probe.properties_by_view(
|
|
827
|
+
include_ancestors=False, include_different_space=True
|
|
828
|
+
).items()
|
|
755
829
|
}
|
|
756
830
|
for view, property_ids in property_ids_by_view.items():
|
|
757
831
|
ancestor_properties = ancestor_properties_by_view.get(view, [])
|
|
@@ -838,7 +912,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
838
912
|
if view.view in read_view_by_new_view:
|
|
839
913
|
read_view = read_view_by_new_view[view.view]
|
|
840
914
|
container_entity = ContainerEntity(space=self.new_model_id.space, externalId=view.view.external_id)
|
|
841
|
-
prefix =
|
|
915
|
+
prefix = to_camel_case(view.view.suffix)
|
|
842
916
|
if self.properties == "repeat" and self.dummy_property:
|
|
843
917
|
property_ = DMSProperty(
|
|
844
918
|
view=view.view,
|
|
@@ -923,7 +997,7 @@ class ToDataProductModel(ToSolutionModel):
|
|
|
923
997
|
self.include = include
|
|
924
998
|
|
|
925
999
|
def transform(self, rules: DMSRules) -> DMSRules:
|
|
926
|
-
# Overwrite
|
|
1000
|
+
# Overwrite transform to avoid the warning.
|
|
927
1001
|
return self._to_solution(rules)
|
|
928
1002
|
|
|
929
1003
|
|
|
@@ -997,7 +1071,7 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
997
1071
|
}
|
|
998
1072
|
new_model = rules.model_copy(deep=True)
|
|
999
1073
|
|
|
1000
|
-
properties_by_view =
|
|
1074
|
+
properties_by_view = RulesAnalysis(dms=new_model).properties_by_view(include_ancestors=True)
|
|
1001
1075
|
|
|
1002
1076
|
new_model.views = SheetList[DMSView]([view for view in new_model.views if view.view not in exclude_views])
|
|
1003
1077
|
new_properties = SheetList[DMSProperty]()
|
|
@@ -1141,6 +1215,7 @@ class ClassicPrepareCore(VerifiedRulesTransformer[InformationRules, InformationR
|
|
|
1141
1215
|
class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
1142
1216
|
description="A source system that provides data to the data model.",
|
|
1143
1217
|
neatId=namespace["ClassicSourceSystem"],
|
|
1218
|
+
instance_source=self.instance_namespace["ClassicSourceSystem"],
|
|
1144
1219
|
)
|
|
1145
1220
|
output.classes.append(source_system_class)
|
|
1146
1221
|
for prop in output.properties:
|
|
@@ -1169,15 +1244,7 @@ class ClassicPrepareCore(VerifiedRulesTransformer[InformationRules, InformationR
|
|
|
1169
1244
|
value_type=String(),
|
|
1170
1245
|
class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
1171
1246
|
max_count=1,
|
|
1172
|
-
instance_source=
|
|
1173
|
-
traversal=SingleProperty(
|
|
1174
|
-
class_=RDFPathEntity(
|
|
1175
|
-
prefix=instance_prefix,
|
|
1176
|
-
suffix="ClassicSourceSystem",
|
|
1177
|
-
),
|
|
1178
|
-
property=RDFPathEntity(prefix=instance_prefix, suffix="name"),
|
|
1179
|
-
),
|
|
1180
|
-
),
|
|
1247
|
+
instance_source=[self.instance_namespace["name"]],
|
|
1181
1248
|
)
|
|
1182
1249
|
)
|
|
1183
1250
|
return output
|
|
@@ -1659,7 +1726,6 @@ class _DMSRulesConverter:
|
|
|
1659
1726
|
classes.append(info_class)
|
|
1660
1727
|
|
|
1661
1728
|
prefixes = get_default_prefixes_and_namespaces()
|
|
1662
|
-
instance_prefix: str | None = None
|
|
1663
1729
|
if self.instance_namespace:
|
|
1664
1730
|
instance_prefix = next((k for k, v in prefixes.items() if v == self.instance_namespace), None)
|
|
1665
1731
|
if instance_prefix is None:
|
|
@@ -1682,15 +1748,6 @@ class _DMSRulesConverter:
|
|
|
1682
1748
|
else:
|
|
1683
1749
|
raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
|
|
1684
1750
|
|
|
1685
|
-
transformation: RDFPath | None = None
|
|
1686
|
-
if instance_prefix is not None:
|
|
1687
|
-
transformation = RDFPath(
|
|
1688
|
-
traversal=SingleProperty(
|
|
1689
|
-
class_=RDFPathEntity(prefix=instance_prefix, suffix=property_.view.external_id),
|
|
1690
|
-
property=RDFPathEntity(prefix=instance_prefix, suffix=property_.view_property),
|
|
1691
|
-
)
|
|
1692
|
-
)
|
|
1693
|
-
|
|
1694
1751
|
info_property = InformationProperty(
|
|
1695
1752
|
# Removing version
|
|
1696
1753
|
class_=ClassEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
|
|
@@ -1699,7 +1756,6 @@ class _DMSRulesConverter:
|
|
|
1699
1756
|
description=property_.description,
|
|
1700
1757
|
min_count=(0 if property_.nullable or property_.nullable is None else 1),
|
|
1701
1758
|
max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
|
|
1702
|
-
instance_source=transformation,
|
|
1703
1759
|
)
|
|
1704
1760
|
|
|
1705
1761
|
# Linking
|
|
@@ -1732,3 +1788,146 @@ class _DMSRulesConverter:
|
|
|
1732
1788
|
created=metadata.created,
|
|
1733
1789
|
updated=metadata.updated,
|
|
1734
1790
|
)
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
class SubsetDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
1794
|
+
"""Subsets DMSRules to only include the specified views."""
|
|
1795
|
+
|
|
1796
|
+
def __init__(self, views: set[ViewEntity]):
|
|
1797
|
+
self._views = views
|
|
1798
|
+
|
|
1799
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
1800
|
+
analysis = RulesAnalysis(dms=rules)
|
|
1801
|
+
|
|
1802
|
+
views_by_view = analysis.view_by_view_entity
|
|
1803
|
+
implements_by_view = analysis.implements_by_view()
|
|
1804
|
+
|
|
1805
|
+
available = analysis.defined_views(include_ancestors=True)
|
|
1806
|
+
subset = available.intersection(self._views)
|
|
1807
|
+
|
|
1808
|
+
ancestors: set[ViewEntity] = set()
|
|
1809
|
+
for view in subset:
|
|
1810
|
+
ancestors = ancestors.union({ancestor for ancestor in get_inheritance_path(view, implements_by_view)})
|
|
1811
|
+
subset = subset.union(ancestors)
|
|
1812
|
+
|
|
1813
|
+
if not subset:
|
|
1814
|
+
raise NeatValueError("None of the requested views are defined in the rules!")
|
|
1815
|
+
|
|
1816
|
+
if nonexisting := self._views - subset:
|
|
1817
|
+
raise NeatValueError(
|
|
1818
|
+
"Following requested views do not exist"
|
|
1819
|
+
f" in the rules: [{','.join([view.external_id for view in nonexisting])}]. Aborting."
|
|
1820
|
+
)
|
|
1821
|
+
|
|
1822
|
+
subsetted_rules: dict[str, Any] = {
|
|
1823
|
+
"metadata": rules.metadata.model_copy(),
|
|
1824
|
+
"views": SheetList[DMSView](),
|
|
1825
|
+
"properties": SheetList[DMSProperty](),
|
|
1826
|
+
"containers": SheetList[DMSContainer](),
|
|
1827
|
+
"enum": rules.enum,
|
|
1828
|
+
"nodes": rules.nodes,
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
# add views
|
|
1832
|
+
for view in subset:
|
|
1833
|
+
subsetted_rules["views"].append(views_by_view[view])
|
|
1834
|
+
|
|
1835
|
+
used_containers = set()
|
|
1836
|
+
|
|
1837
|
+
# add properties
|
|
1838
|
+
for view, properties in analysis.properties_by_view(include_ancestors=False).items():
|
|
1839
|
+
if view not in subset:
|
|
1840
|
+
continue
|
|
1841
|
+
|
|
1842
|
+
for property_ in properties:
|
|
1843
|
+
if (
|
|
1844
|
+
isinstance(property_.value_type, DataType)
|
|
1845
|
+
or isinstance(property_.value_type, DMSUnknownEntity)
|
|
1846
|
+
or (isinstance(property_.value_type, ViewEntity) and property_.value_type in subset)
|
|
1847
|
+
):
|
|
1848
|
+
subsetted_rules["properties"].append(property_)
|
|
1849
|
+
|
|
1850
|
+
if property_.container:
|
|
1851
|
+
used_containers.add(property_.container)
|
|
1852
|
+
|
|
1853
|
+
# add containers
|
|
1854
|
+
if rules.containers:
|
|
1855
|
+
for container in rules.containers:
|
|
1856
|
+
if container.container in used_containers:
|
|
1857
|
+
subsetted_rules["containers"].append(container)
|
|
1858
|
+
|
|
1859
|
+
try:
|
|
1860
|
+
return DMSRules.model_validate(subsetted_rules)
|
|
1861
|
+
except ValidationError as e:
|
|
1862
|
+
raise NeatValueError(f"Cannot subset rules: {e}") from e
|
|
1863
|
+
|
|
1864
|
+
|
|
1865
|
+
class SubsetInformationRules(VerifiedRulesTransformer[InformationRules, InformationRules]):
|
|
1866
|
+
"""Subsets InformationRules to only include the specified classes."""
|
|
1867
|
+
|
|
1868
|
+
def __init__(self, classes: set[ClassEntity]):
|
|
1869
|
+
self._classes = classes
|
|
1870
|
+
|
|
1871
|
+
def transform(self, rules: InformationRules) -> InformationRules:
|
|
1872
|
+
analysis = RulesAnalysis(information=rules)
|
|
1873
|
+
|
|
1874
|
+
class_by_class_entity = analysis.class_by_class_entity
|
|
1875
|
+
parent_entity_by_class_entity = analysis.parents_by_class()
|
|
1876
|
+
|
|
1877
|
+
available = analysis.defined_classes(include_ancestors=True)
|
|
1878
|
+
subset = available.intersection(self._classes)
|
|
1879
|
+
|
|
1880
|
+
# need to add all the parent classes of the desired classes to the possible classes
|
|
1881
|
+
ancestors: set[ClassEntity] = set()
|
|
1882
|
+
for class_ in subset:
|
|
1883
|
+
ancestors = ancestors.union(
|
|
1884
|
+
{ancestor for ancestor in get_inheritance_path(class_, parent_entity_by_class_entity)}
|
|
1885
|
+
)
|
|
1886
|
+
subset = subset.union(ancestors)
|
|
1887
|
+
|
|
1888
|
+
if not subset:
|
|
1889
|
+
raise NeatValueError("None of the requested classes are defined in the rules!")
|
|
1890
|
+
|
|
1891
|
+
if nonexisting := self._classes - subset:
|
|
1892
|
+
raise NeatValueError(
|
|
1893
|
+
"Following requested classes do not exist"
|
|
1894
|
+
f" in the rules: [{','.join([class_.suffix for class_ in nonexisting])}]"
|
|
1895
|
+
". Aborting."
|
|
1896
|
+
)
|
|
1897
|
+
|
|
1898
|
+
subsetted_rules: dict[str, Any] = {
|
|
1899
|
+
"metadata": rules.metadata.model_copy(),
|
|
1900
|
+
"prefixes": (rules.prefixes or {}).copy(),
|
|
1901
|
+
"classes": SheetList[InformationClass](),
|
|
1902
|
+
"properties": SheetList[InformationProperty](),
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
for class_ in subset:
|
|
1906
|
+
subsetted_rules["classes"].append(class_by_class_entity[class_])
|
|
1907
|
+
|
|
1908
|
+
for class_, properties in analysis.properties_by_class(include_ancestors=False).items():
|
|
1909
|
+
if class_ not in subset:
|
|
1910
|
+
continue
|
|
1911
|
+
for property_ in properties:
|
|
1912
|
+
# datatype property can be added directly
|
|
1913
|
+
if (
|
|
1914
|
+
isinstance(property_.value_type, DataType)
|
|
1915
|
+
or (isinstance(property_.value_type, ClassEntity) and property_.value_type in subset)
|
|
1916
|
+
or isinstance(property_.value_type, UnknownEntity)
|
|
1917
|
+
):
|
|
1918
|
+
subsetted_rules["properties"].append(property_)
|
|
1919
|
+
# object property can be added if the value type is in the subset
|
|
1920
|
+
elif isinstance(property_.value_type, MultiValueTypeInfo):
|
|
1921
|
+
allowed = [t for t in property_.value_type.types if t in subset or isinstance(t, DataType)]
|
|
1922
|
+
if allowed:
|
|
1923
|
+
subsetted_rules["properties"].append(
|
|
1924
|
+
property_.model_copy(
|
|
1925
|
+
deep=True,
|
|
1926
|
+
update={"value_type": MultiValueTypeInfo(types=allowed)},
|
|
1927
|
+
)
|
|
1928
|
+
)
|
|
1929
|
+
|
|
1930
|
+
try:
|
|
1931
|
+
return InformationRules.model_validate(subsetted_rules)
|
|
1932
|
+
except ValidationError as e:
|
|
1933
|
+
raise NeatValueError(f"Cannot subset rules: {e}") from e
|
|
@@ -35,27 +35,22 @@ class VerificationTransformer(RulesTransformer[T_ReadInputRules, T_VerifiedRules
|
|
|
35
35
|
in_ = rules.rules
|
|
36
36
|
if in_ is None:
|
|
37
37
|
raise NeatValueError("Cannot verify rules. The reading of the rules failed.")
|
|
38
|
-
error_args = rules.read_context
|
|
39
38
|
verified_rules: T_VerifiedRules | None = None
|
|
40
39
|
# We need to catch issues as we use the error args to provide extra context for the errors/warnings
|
|
41
|
-
# For example, which row in the spreadsheet the error occurred
|
|
42
|
-
with catch_issues(
|
|
40
|
+
# For example, which row in the spreadsheet the error occurred.
|
|
41
|
+
with catch_issues(rules.read_context) as issues:
|
|
43
42
|
rules_cls = self._get_rules_cls(rules)
|
|
44
43
|
dumped = in_.dump()
|
|
45
44
|
verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
|
|
46
45
|
if self.validate:
|
|
47
46
|
validation_cls = self._get_validation_cls(verified_rules) # type: ignore[arg-type]
|
|
48
47
|
if issubclass(validation_cls, DMSValidation):
|
|
49
|
-
validation_issues = DMSValidation(verified_rules, self._client).validate() # type: ignore[arg-type]
|
|
48
|
+
validation_issues = DMSValidation(verified_rules, self._client, rules.read_context).validate() # type: ignore[arg-type]
|
|
50
49
|
elif issubclass(validation_cls, InformationValidation):
|
|
51
|
-
validation_issues = InformationValidation(verified_rules).validate() # type: ignore[arg-type]
|
|
50
|
+
validation_issues = InformationValidation(verified_rules, rules.read_context).validate() # type: ignore[arg-type]
|
|
52
51
|
else:
|
|
53
52
|
raise NeatValueError("Unsupported rule type")
|
|
54
|
-
|
|
55
|
-
# Need to trigger and raise such that the catch_issues can add the extra context
|
|
56
|
-
validation_issues.trigger_warnings()
|
|
57
|
-
if validation_issues.has_errors:
|
|
58
|
-
raise MultiValueError(validation_issues.errors)
|
|
53
|
+
issues.extend(validation_issues)
|
|
59
54
|
|
|
60
55
|
# Raise issues which is expected to be handled outside of this method
|
|
61
56
|
issues.trigger_warnings()
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -16,7 +16,7 @@ from cognite.neat._rules.transformers import (
|
|
|
16
16
|
InformationToDMS,
|
|
17
17
|
MergeDMSRules,
|
|
18
18
|
MergeInformationRules,
|
|
19
|
-
|
|
19
|
+
ToDMSCompliantEntities,
|
|
20
20
|
VerifyInformationRules,
|
|
21
21
|
)
|
|
22
22
|
from cognite.neat._store._rules_store import RulesEntity
|
|
@@ -33,6 +33,7 @@ from ._read import ReadAPI
|
|
|
33
33
|
from ._set import SetAPI
|
|
34
34
|
from ._show import ShowAPI
|
|
35
35
|
from ._state import SessionState
|
|
36
|
+
from ._subset import SubsetAPI
|
|
36
37
|
from ._to import ToAPI
|
|
37
38
|
from .engine import load_neat_engine
|
|
38
39
|
from .exceptions import session_class_wrapper
|
|
@@ -101,6 +102,7 @@ class NeatSession:
|
|
|
101
102
|
self.inspect = InspectAPI(self._state)
|
|
102
103
|
self.mapping = MappingAPI(self._state)
|
|
103
104
|
self.drop = DropAPI(self._state)
|
|
105
|
+
self.subset = SubsetAPI(self._state)
|
|
104
106
|
self.create = CreateAPI(self._state)
|
|
105
107
|
self.opt = OptAPI()
|
|
106
108
|
self.opt._display()
|
|
@@ -236,9 +238,7 @@ class NeatSession:
|
|
|
236
238
|
|
|
237
239
|
def action() -> tuple[InformationRules, DMSRules | None]:
|
|
238
240
|
unverified_information = importer.to_rules()
|
|
239
|
-
unverified_information =
|
|
240
|
-
unverified_information
|
|
241
|
-
)
|
|
241
|
+
unverified_information = ToDMSCompliantEntities(rename_warning="raise").transform(unverified_information)
|
|
242
242
|
|
|
243
243
|
extra_info = VerifyInformationRules().transform(unverified_information)
|
|
244
244
|
if not last_entity:
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from collections.abc import Callable
|
|
2
3
|
from typing import Any
|
|
3
4
|
|
|
@@ -14,6 +15,7 @@ from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
|
|
|
14
15
|
from cognite.neat._issues import IssueList
|
|
15
16
|
from cognite.neat._issues.errors import NeatValueError
|
|
16
17
|
from cognite.neat._rules.transformers import PrefixEntities, StandardizeNaming
|
|
18
|
+
from cognite.neat._rules.transformers._converters import StandardizeSpaceAndVersion
|
|
17
19
|
from cognite.neat._utils.text import humanize_collection
|
|
18
20
|
|
|
19
21
|
from ._state import SessionState
|
|
@@ -270,5 +272,15 @@ class DataModelPrepareAPI:
|
|
|
270
272
|
For classes/views/containers, the naming will be standardized to PascalCase.
|
|
271
273
|
For properties, the naming will be standardized to camelCase.
|
|
272
274
|
"""
|
|
275
|
+
warnings.filterwarnings("default")
|
|
273
276
|
AlphaFlags.standardize_naming.warn()
|
|
274
277
|
return self._state.rule_transform(StandardizeNaming())
|
|
278
|
+
|
|
279
|
+
def standardize_space_and_version(self) -> IssueList:
|
|
280
|
+
"""Standardize space and version in the data model.
|
|
281
|
+
|
|
282
|
+
This method will standardize the space and version in the data model to the Cognite standard.
|
|
283
|
+
"""
|
|
284
|
+
warnings.filterwarnings("default")
|
|
285
|
+
AlphaFlags.standardize_space_and_version.warn()
|
|
286
|
+
return self._state.rule_transform(StandardizeSpaceAndVersion())
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -50,6 +50,7 @@ class ReadAPI:
|
|
|
50
50
|
self.csv = CSVReadAPI(state, verbose)
|
|
51
51
|
self.yaml = YamlReadAPI(state, verbose)
|
|
52
52
|
self.xml = XMLReadAPI(state, verbose)
|
|
53
|
+
self.examples = Examples(state)
|
|
53
54
|
|
|
54
55
|
def session(self, io: Any) -> None:
|
|
55
56
|
"""Reads a Neat Session from a zip file.
|
|
@@ -256,9 +257,6 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
256
257
|
prepare_issues = self._state.rule_store.transform(
|
|
257
258
|
ClassicPrepareCore(namespace, reference_timeseries, reference_files)
|
|
258
259
|
)
|
|
259
|
-
# Update the instance store with the latest rules
|
|
260
|
-
information_rules = self._state.rule_store.last_verified_information_rules
|
|
261
|
-
self._state.instances.store.rules[self._state.instances.store.default_named_graph] = information_rules
|
|
262
260
|
|
|
263
261
|
all_issues = IssueList(extract_issues + prepare_issues)
|
|
264
262
|
# Update the provenance with all issue.
|
|
@@ -286,7 +284,6 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
286
284
|
|
|
287
285
|
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
288
286
|
super().__init__(state, verbose)
|
|
289
|
-
self.examples = ExcelExampleAPI(state, verbose)
|
|
290
287
|
|
|
291
288
|
def __call__(self, io: Any, enable_manual_edit: bool = False) -> IssueList:
|
|
292
289
|
"""Reads a Neat Excel Rules sheet to the graph store. The rules sheet may stem from an Information architect,
|
|
@@ -310,16 +307,6 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
310
307
|
return self._state.rule_import(importers.ExcelImporter(path), enable_manual_edit)
|
|
311
308
|
|
|
312
309
|
|
|
313
|
-
@session_class_wrapper
|
|
314
|
-
class ExcelExampleAPI(BaseReadAPI):
|
|
315
|
-
"""Used as example for reading some data model into the NeatSession."""
|
|
316
|
-
|
|
317
|
-
def pump_example(self) -> IssueList:
|
|
318
|
-
"""Reads the Hello World pump example into the NeatSession."""
|
|
319
|
-
importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)
|
|
320
|
-
return self._state.rule_import(importer)
|
|
321
|
-
|
|
322
|
-
|
|
323
310
|
@session_class_wrapper
|
|
324
311
|
class YamlReadAPI(BaseReadAPI):
|
|
325
312
|
def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
|
|
@@ -519,7 +506,6 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
519
506
|
|
|
520
507
|
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
521
508
|
super().__init__(state, verbose)
|
|
522
|
-
self.examples = RDFExamples(state)
|
|
523
509
|
|
|
524
510
|
def ontology(self, io: Any) -> IssueList:
|
|
525
511
|
"""Reads an OWL ontology source into NeatSession.
|
|
@@ -582,13 +568,31 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
582
568
|
|
|
583
569
|
|
|
584
570
|
@session_class_wrapper
|
|
585
|
-
class
|
|
586
|
-
"""Used as example for reading
|
|
571
|
+
class Examples:
|
|
572
|
+
"""Used as example for reading various sources into NeatSession."""
|
|
587
573
|
|
|
588
574
|
def __init__(self, state: SessionState) -> None:
|
|
589
575
|
self._state = state
|
|
590
576
|
|
|
577
|
+
@property
|
|
578
|
+
def _get_client(self) -> NeatClient:
|
|
579
|
+
if self._state.client is None:
|
|
580
|
+
raise NeatValueError("No client provided. Please provide a client to read a data model.")
|
|
581
|
+
return self._state.client
|
|
582
|
+
|
|
591
583
|
def nordic44(self) -> IssueList:
|
|
592
584
|
"""Reads the Nordic 44 knowledge graph into the NeatSession graph store."""
|
|
593
585
|
self._state.instances.store.write(extractors.RdfFileExtractor(instances_examples.nordic44_knowledge_graph))
|
|
594
586
|
return IssueList()
|
|
587
|
+
|
|
588
|
+
def pump_example(self) -> IssueList:
|
|
589
|
+
"""Reads the Hello World pump example into the NeatSession."""
|
|
590
|
+
importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)
|
|
591
|
+
return self._state.rule_import(importer)
|
|
592
|
+
|
|
593
|
+
def core_data_model(self) -> IssueList:
|
|
594
|
+
"""Reads the core data model example into the NeatSession."""
|
|
595
|
+
|
|
596
|
+
cdm_v1 = DataModelId.load(("cdf_cdm", "CogniteCore", "v1"))
|
|
597
|
+
importer: importers.DMSImporter = importers.DMSImporter.from_data_model_id(self._get_client, cdm_v1)
|
|
598
|
+
return self._state.rule_import(importer)
|