cognite-neat 0.105.2__py3-none-any.whl → 0.107.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_config.py +6 -260
- cognite/neat/_graph/extractors/__init__.py +5 -1
- cognite/neat/_graph/extractors/_base.py +32 -0
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +42 -16
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +78 -8
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +2 -0
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +10 -3
- cognite/neat/_graph/extractors/_dms.py +48 -14
- cognite/neat/_graph/extractors/_dms_graph.py +149 -0
- cognite/neat/_graph/extractors/_rdf_file.py +32 -5
- cognite/neat/_graph/loaders/_rdf2dms.py +119 -20
- cognite/neat/_graph/queries/_construct.py +1 -1
- cognite/neat/_graph/transformers/__init__.py +5 -0
- cognite/neat/_graph/transformers/_base.py +13 -9
- cognite/neat/_graph/transformers/_classic_cdf.py +141 -44
- cognite/neat/_graph/transformers/_rdfpath.py +4 -4
- cognite/neat/_graph/transformers/_value_type.py +54 -44
- cognite/neat/_issues/warnings/_external.py +1 -1
- cognite/neat/_rules/analysis/_base.py +1 -1
- cognite/neat/_rules/analysis/_information.py +14 -13
- cognite/neat/_rules/catalog/__init__.py +1 -0
- cognite/neat/_rules/catalog/classic_model.xlsx +0 -0
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/importers/_dms2rules.py +7 -5
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +5 -3
- cognite/neat/_rules/models/_base_rules.py +0 -12
- cognite/neat/_rules/models/_types.py +5 -0
- cognite/neat/_rules/models/dms/_rules.py +50 -2
- cognite/neat/_rules/models/information/_rules.py +48 -5
- cognite/neat/_rules/models/information/_rules_input.py +1 -1
- cognite/neat/_rules/models/mapping/_classic2core.py +4 -5
- cognite/neat/_rules/models/mapping/_classic2core.yaml +70 -58
- cognite/neat/_rules/transformers/__init__.py +4 -0
- cognite/neat/_rules/transformers/_converters.py +209 -62
- cognite/neat/_rules/transformers/_mapping.py +3 -2
- cognite/neat/_session/_base.py +8 -13
- cognite/neat/_session/_inspect.py +6 -2
- cognite/neat/_session/_mapping.py +22 -13
- cognite/neat/_session/_prepare.py +9 -57
- cognite/neat/_session/_read.py +96 -29
- cognite/neat/_session/_set.py +9 -0
- cognite/neat/_session/_state.py +10 -1
- cognite/neat/_session/_to.py +51 -15
- cognite/neat/_session/exceptions.py +7 -3
- cognite/neat/_store/_graph_store.py +85 -39
- cognite/neat/_store/_rules_store.py +22 -0
- cognite/neat/_utils/auth.py +2 -0
- cognite/neat/_utils/collection_.py +32 -11
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.105.2.dist-info → cognite_neat-0.107.0.dist-info}/METADATA +2 -8
- {cognite_neat-0.105.2.dist-info → cognite_neat-0.107.0.dist-info}/RECORD +54 -52
- {cognite_neat-0.105.2.dist-info → cognite_neat-0.107.0.dist-info}/WHEEL +1 -1
- {cognite_neat-0.105.2.dist-info → cognite_neat-0.107.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.105.2.dist-info → cognite_neat-0.107.0.dist-info}/entry_points.txt +0 -0
|
@@ -9,12 +9,14 @@ from typing import ClassVar, Literal, TypeVar, cast, overload
|
|
|
9
9
|
|
|
10
10
|
from cognite.client.data_classes import data_modeling as dms
|
|
11
11
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
|
|
12
|
+
from rdflib import Namespace
|
|
12
13
|
|
|
13
14
|
from cognite.neat._client import NeatClient
|
|
14
15
|
from cognite.neat._client.data_classes.data_modeling import ContainerApplyDict, ViewApplyDict
|
|
15
16
|
from cognite.neat._constants import (
|
|
16
17
|
COGNITE_MODELS,
|
|
17
18
|
DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
|
|
19
|
+
get_default_prefixes_and_namespaces,
|
|
18
20
|
)
|
|
19
21
|
from cognite.neat._issues.errors import NeatValueError
|
|
20
22
|
from cognite.neat._issues.warnings import NeatValueWarning
|
|
@@ -37,6 +39,8 @@ from cognite.neat._rules.models import (
|
|
|
37
39
|
SheetList,
|
|
38
40
|
data_types,
|
|
39
41
|
)
|
|
42
|
+
from cognite.neat._rules.models._rdfpath import Entity as RDFPathEntity
|
|
43
|
+
from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
|
|
40
44
|
from cognite.neat._rules.models.data_types import AnyURI, DataType, String
|
|
41
45
|
from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSValidation, DMSView
|
|
42
46
|
from cognite.neat._rules.models.dms._rules import DMSContainer
|
|
@@ -248,19 +252,21 @@ class PrefixEntities(RulesTransformer[ReadRules[T_InputRules], ReadRules[T_Input
|
|
|
248
252
|
class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
249
253
|
"""Converts InformationRules to DMSRules."""
|
|
250
254
|
|
|
251
|
-
def __init__(self, ignore_undefined_value_types: bool = False
|
|
255
|
+
def __init__(self, ignore_undefined_value_types: bool = False):
|
|
252
256
|
self.ignore_undefined_value_types = ignore_undefined_value_types
|
|
253
|
-
self.mode = mode
|
|
254
257
|
|
|
255
258
|
def transform(self, rules: InformationRules) -> DMSRules:
|
|
256
|
-
return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types
|
|
259
|
+
return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types)
|
|
257
260
|
|
|
258
261
|
|
|
259
262
|
class DMSToInformation(ConversionTransformer[DMSRules, InformationRules]):
|
|
260
263
|
"""Converts DMSRules to InformationRules."""
|
|
261
264
|
|
|
265
|
+
def __init__(self, instance_namespace: Namespace | None = None):
|
|
266
|
+
self.instance_namespace = instance_namespace
|
|
267
|
+
|
|
262
268
|
def transform(self, rules: DMSRules) -> InformationRules:
|
|
263
|
-
return _DMSRulesConverter(rules).as_information_rules()
|
|
269
|
+
return _DMSRulesConverter(rules, self.instance_namespace).as_information_rules()
|
|
264
270
|
|
|
265
271
|
|
|
266
272
|
class ConvertToRules(ConversionTransformer[VerifiedRules, VerifiedRules]):
|
|
@@ -808,16 +814,107 @@ class AddClassImplements(RulesTransformer[InformationRules, InformationRules]):
|
|
|
808
814
|
return f"Added implements property to classes with suffix {self.suffix}"
|
|
809
815
|
|
|
810
816
|
|
|
817
|
+
class ClassicPrepareCore(RulesTransformer[InformationRules, InformationRules]):
|
|
818
|
+
"""Update the classic data model with the following:
|
|
819
|
+
|
|
820
|
+
This is a special purpose transformer that is only intended to be used with when reading
|
|
821
|
+
from classic cdf using the neat.read.cdf.classic.graph(...).
|
|
822
|
+
|
|
823
|
+
- ClassicTimeseries.isString from boolean to string
|
|
824
|
+
- Add class ClassicSourceSystem, and update all source properties from string to ClassicSourceSystem.
|
|
825
|
+
- Rename externalId properties to classicExternalId
|
|
826
|
+
- Renames the Relationship.sourceExternaId and Relationship.targetExternalId to startNode and endNode
|
|
827
|
+
"""
|
|
828
|
+
|
|
829
|
+
def __init__(self, instance_namespace: Namespace) -> None:
|
|
830
|
+
self.instance_namespace = instance_namespace
|
|
831
|
+
|
|
832
|
+
@property
|
|
833
|
+
def description(self) -> str:
|
|
834
|
+
return "Update the classic data model to the data types in Cognite Core."
|
|
835
|
+
|
|
836
|
+
def transform(self, rules: InformationRules) -> InformationRules:
|
|
837
|
+
output = rules.model_copy(deep=True)
|
|
838
|
+
for prop in output.properties:
|
|
839
|
+
if prop.class_.suffix == "Timeseries" and prop.property_ == "isString":
|
|
840
|
+
prop.value_type = String()
|
|
841
|
+
prefix = output.metadata.prefix
|
|
842
|
+
namespace = output.metadata.namespace
|
|
843
|
+
source_system_class = InformationClass(
|
|
844
|
+
class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
845
|
+
description="A source system that provides data to the data model.",
|
|
846
|
+
neatId=namespace["ClassicSourceSystem"],
|
|
847
|
+
)
|
|
848
|
+
output.classes.append(source_system_class)
|
|
849
|
+
for prop in output.properties:
|
|
850
|
+
if prop.property_ == "source" and prop.class_.suffix != "ClassicSourceSystem":
|
|
851
|
+
prop.value_type = ClassEntity(prefix=prefix, suffix="ClassicSourceSystem")
|
|
852
|
+
elif prop.property_ == "externalId":
|
|
853
|
+
prop.property_ = "classicExternalId"
|
|
854
|
+
elif prop.property_ == "sourceExternalId" and prop.class_.suffix == "ClassicRelationship":
|
|
855
|
+
prop.property_ = "startNode"
|
|
856
|
+
elif prop.property_ == "targetExternalId" and prop.class_.suffix == "ClassicRelationship":
|
|
857
|
+
prop.property_ = "endNode"
|
|
858
|
+
instance_prefix = next(
|
|
859
|
+
(prefix for prefix, namespace in output.prefixes.items() if namespace == self.instance_namespace), None
|
|
860
|
+
)
|
|
861
|
+
if instance_prefix is None:
|
|
862
|
+
raise NeatValueError("Instance namespace not found in the prefixes.")
|
|
863
|
+
|
|
864
|
+
output.properties.append(
|
|
865
|
+
InformationProperty(
|
|
866
|
+
neatId=namespace["ClassicSourceSystem/name"],
|
|
867
|
+
property_="name",
|
|
868
|
+
value_type=String(),
|
|
869
|
+
class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
870
|
+
max_count=1,
|
|
871
|
+
instance_source=RDFPath(
|
|
872
|
+
traversal=SingleProperty(
|
|
873
|
+
class_=RDFPathEntity(
|
|
874
|
+
prefix=instance_prefix,
|
|
875
|
+
suffix="ClassicSourceSystem",
|
|
876
|
+
),
|
|
877
|
+
property=RDFPathEntity(prefix=instance_prefix, suffix="name"),
|
|
878
|
+
),
|
|
879
|
+
),
|
|
880
|
+
)
|
|
881
|
+
)
|
|
882
|
+
return output
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
class ChangeViewPrefix(RulesTransformer[DMSRules, DMSRules]):
|
|
886
|
+
def __init__(self, old: str, new: str) -> None:
|
|
887
|
+
self.old = old
|
|
888
|
+
self.new = new
|
|
889
|
+
|
|
890
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
891
|
+
output = rules.model_copy(deep=True)
|
|
892
|
+
new_by_old: dict[ViewEntity, ViewEntity] = {}
|
|
893
|
+
for view in output.views:
|
|
894
|
+
if view.view.external_id.startswith(self.old):
|
|
895
|
+
new_external_id = f"{self.new}{view.view.external_id.removeprefix(self.old)}"
|
|
896
|
+
new_view_entity = view.view.copy(update={"suffix": new_external_id})
|
|
897
|
+
new_by_old[view.view] = new_view_entity
|
|
898
|
+
view.view = new_view_entity
|
|
899
|
+
for view in output.views:
|
|
900
|
+
if view.implements:
|
|
901
|
+
view.implements = [new_by_old.get(implemented, implemented) for implemented in view.implements]
|
|
902
|
+
for prop in output.properties:
|
|
903
|
+
if prop.view in new_by_old:
|
|
904
|
+
prop.view = new_by_old[prop.view]
|
|
905
|
+
if prop.value_type in new_by_old and isinstance(prop.value_type, ViewEntity):
|
|
906
|
+
prop.value_type = new_by_old[prop.value_type]
|
|
907
|
+
return output
|
|
908
|
+
|
|
909
|
+
|
|
811
910
|
class _InformationRulesConverter:
|
|
812
|
-
|
|
911
|
+
_start_or_end_node: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
|
|
813
912
|
|
|
814
913
|
def __init__(self, information: InformationRules):
|
|
815
914
|
self.rules = information
|
|
816
915
|
self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
|
|
817
916
|
|
|
818
|
-
def as_dms_rules(
|
|
819
|
-
self, ignore_undefined_value_types: bool = False, mode: Literal["edge_properties"] | None = None
|
|
820
|
-
) -> "DMSRules":
|
|
917
|
+
def as_dms_rules(self, ignore_undefined_value_types: bool = False) -> "DMSRules":
|
|
821
918
|
from cognite.neat._rules.models.dms._rules import (
|
|
822
919
|
DMSContainer,
|
|
823
920
|
DMSProperty,
|
|
@@ -829,35 +926,44 @@ class _InformationRulesConverter:
|
|
|
829
926
|
default_version = info_metadata.version
|
|
830
927
|
default_space = self._to_space(info_metadata.prefix)
|
|
831
928
|
dms_metadata = self._convert_metadata_to_dms(info_metadata)
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
929
|
+
|
|
930
|
+
properties_by_class: dict[ClassEntity, set[str]] = defaultdict(set)
|
|
931
|
+
for prop in self.rules.properties:
|
|
932
|
+
properties_by_class[prop.class_].add(prop.property_)
|
|
933
|
+
|
|
934
|
+
# Edge Classes is defined by having both startNode and endNode properties
|
|
935
|
+
edge_classes = {
|
|
936
|
+
cls_
|
|
937
|
+
for cls_, class_properties in properties_by_class.items()
|
|
938
|
+
if ({"startNode", "start_node"} & class_properties) and ({"endNode", "end_node"} & class_properties)
|
|
939
|
+
}
|
|
940
|
+
edge_value_types_by_class_property_pair = {
|
|
941
|
+
(prop.class_, prop.property_): prop.value_type
|
|
942
|
+
for prop in self.rules.properties
|
|
943
|
+
if prop.value_type in edge_classes and isinstance(prop.value_type, ClassEntity)
|
|
944
|
+
}
|
|
945
|
+
end_node_by_edge = {
|
|
946
|
+
prop.class_: prop.value_type
|
|
947
|
+
for prop in self.rules.properties
|
|
948
|
+
if prop.class_ in edge_classes
|
|
949
|
+
and (prop.property_ == "endNode" or prop.property_ == "end_node")
|
|
950
|
+
and isinstance(prop.value_type, ClassEntity)
|
|
951
|
+
}
|
|
851
952
|
|
|
852
953
|
properties_by_class: dict[ClassEntity, list[DMSProperty]] = defaultdict(list)
|
|
853
954
|
referenced_containers: dict[ContainerEntity, Counter[ClassEntity]] = defaultdict(Counter)
|
|
854
955
|
for prop in self.rules.properties:
|
|
855
956
|
if ignore_undefined_value_types and isinstance(prop.value_type, UnknownEntity):
|
|
856
957
|
continue
|
|
857
|
-
if prop.class_ in edge_classes and prop.property_ in self.
|
|
958
|
+
if prop.class_ in edge_classes and prop.property_ in self._start_or_end_node:
|
|
858
959
|
continue
|
|
859
960
|
dms_property = self._as_dms_property(
|
|
860
|
-
prop,
|
|
961
|
+
prop,
|
|
962
|
+
default_space,
|
|
963
|
+
default_version,
|
|
964
|
+
edge_classes,
|
|
965
|
+
edge_value_types_by_class_property_pair,
|
|
966
|
+
end_node_by_edge,
|
|
861
967
|
)
|
|
862
968
|
properties_by_class[prop.class_].append(dms_property)
|
|
863
969
|
if dms_property.container:
|
|
@@ -870,12 +976,10 @@ class _InformationRulesConverter:
|
|
|
870
976
|
name=cls_.name,
|
|
871
977
|
view=cls_.class_.as_view_entity(default_space, default_version),
|
|
872
978
|
description=cls_.description,
|
|
873
|
-
implements=self._get_view_implements(cls_, info_metadata
|
|
979
|
+
implements=self._get_view_implements(cls_, info_metadata),
|
|
874
980
|
)
|
|
875
981
|
|
|
876
982
|
dms_view.logical = cls_.neatId
|
|
877
|
-
cls_.physical = dms_view.neatId
|
|
878
|
-
|
|
879
983
|
views.append(dms_view)
|
|
880
984
|
|
|
881
985
|
class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
|
|
@@ -891,21 +995,34 @@ class _InformationRulesConverter:
|
|
|
891
995
|
)
|
|
892
996
|
most_used_class_entity = class_entities.most_common(1)[0][0]
|
|
893
997
|
class_ = class_by_entity[most_used_class_entity]
|
|
998
|
+
|
|
999
|
+
if len(set(class_entities) - set(edge_classes)) == 0:
|
|
1000
|
+
used_for: Literal["node", "edge", "all"] = "edge"
|
|
1001
|
+
elif len(set(class_entities) - set(edge_classes)) == len(class_entities):
|
|
1002
|
+
used_for = "node"
|
|
1003
|
+
else:
|
|
1004
|
+
used_for = "all"
|
|
1005
|
+
|
|
894
1006
|
container = DMSContainer(
|
|
895
1007
|
container=container_entity,
|
|
896
1008
|
name=class_.name,
|
|
897
1009
|
description=class_.description,
|
|
898
1010
|
constraint=constrains or None,
|
|
1011
|
+
used_for=used_for,
|
|
899
1012
|
)
|
|
900
1013
|
containers.append(container)
|
|
901
1014
|
|
|
902
|
-
|
|
1015
|
+
dms_rules = DMSRules(
|
|
903
1016
|
metadata=dms_metadata,
|
|
904
1017
|
properties=SheetList[DMSProperty]([prop for prop_set in properties_by_class.values() for prop in prop_set]),
|
|
905
1018
|
views=SheetList[DMSView](views),
|
|
906
1019
|
containers=SheetList[DMSContainer](containers),
|
|
907
1020
|
)
|
|
908
1021
|
|
|
1022
|
+
self.rules.sync_with_dms_rules(dms_rules)
|
|
1023
|
+
|
|
1024
|
+
return dms_rules
|
|
1025
|
+
|
|
909
1026
|
@staticmethod
|
|
910
1027
|
def _create_container_constraint(
|
|
911
1028
|
class_entities: Counter[ClassEntity],
|
|
@@ -939,8 +1056,6 @@ class _InformationRulesConverter:
|
|
|
939
1056
|
)
|
|
940
1057
|
|
|
941
1058
|
dms_metadata.logical = metadata.identifier
|
|
942
|
-
metadata.physical = dms_metadata.identifier
|
|
943
|
-
|
|
944
1059
|
return dms_metadata
|
|
945
1060
|
|
|
946
1061
|
def _as_dms_property(
|
|
@@ -949,7 +1064,7 @@ class _InformationRulesConverter:
|
|
|
949
1064
|
default_space: str,
|
|
950
1065
|
default_version: str,
|
|
951
1066
|
edge_classes: set[ClassEntity],
|
|
952
|
-
|
|
1067
|
+
edge_value_types_by_class_property_pair: dict[tuple[ClassEntity, str], ClassEntity],
|
|
953
1068
|
end_node_by_edge: dict[ClassEntity, ClassEntity],
|
|
954
1069
|
) -> "DMSProperty":
|
|
955
1070
|
from cognite.neat._rules.models.dms._rules import DMSProperty
|
|
@@ -963,7 +1078,9 @@ class _InformationRulesConverter:
|
|
|
963
1078
|
end_node_by_edge,
|
|
964
1079
|
)
|
|
965
1080
|
|
|
966
|
-
connection = self._get_connection(
|
|
1081
|
+
connection = self._get_connection(
|
|
1082
|
+
info_property, value_type, edge_value_types_by_class_property_pair, default_space, default_version
|
|
1083
|
+
)
|
|
967
1084
|
|
|
968
1085
|
container: ContainerEntity | None = None
|
|
969
1086
|
container_property: str | None = None
|
|
@@ -992,7 +1109,6 @@ class _InformationRulesConverter:
|
|
|
992
1109
|
|
|
993
1110
|
# linking
|
|
994
1111
|
dms_property.logical = info_property.neatId
|
|
995
|
-
info_property.physical = dms_property.neatId
|
|
996
1112
|
|
|
997
1113
|
return dms_property
|
|
998
1114
|
|
|
@@ -1000,13 +1116,16 @@ class _InformationRulesConverter:
|
|
|
1000
1116
|
def _get_connection(
|
|
1001
1117
|
prop: InformationProperty,
|
|
1002
1118
|
value_type: DataType | ViewEntity | DMSUnknownEntity,
|
|
1003
|
-
|
|
1119
|
+
edge_value_types_by_class_property_pair: dict[tuple[ClassEntity, str], ClassEntity],
|
|
1004
1120
|
default_space: str,
|
|
1005
1121
|
default_version: str,
|
|
1006
1122
|
) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
|
|
1007
|
-
if
|
|
1008
|
-
|
|
1009
|
-
|
|
1123
|
+
if (
|
|
1124
|
+
isinstance(value_type, ViewEntity)
|
|
1125
|
+
and (prop.class_, prop.property_) in edge_value_types_by_class_property_pair
|
|
1126
|
+
):
|
|
1127
|
+
edge_value_type = edge_value_types_by_class_property_pair[(prop.class_, prop.property_)]
|
|
1128
|
+
return EdgeEntity(properties=edge_value_type.as_view_entity(default_space, default_version))
|
|
1010
1129
|
if isinstance(value_type, ViewEntity) and prop.is_list:
|
|
1011
1130
|
return EdgeEntity()
|
|
1012
1131
|
elif isinstance(value_type, ViewEntity):
|
|
@@ -1035,6 +1154,7 @@ class _InformationRulesConverter:
|
|
|
1035
1154
|
elif isinstance(prop.value_type, ClassEntity) and (prop.value_type in edge_classes):
|
|
1036
1155
|
if prop.value_type in end_node_by_edge:
|
|
1037
1156
|
return end_node_by_edge[prop.value_type].as_view_entity(default_space, default_version)
|
|
1157
|
+
# This occurs if the end node is not pointing to a class
|
|
1038
1158
|
warnings.warn(
|
|
1039
1159
|
NeatValueWarning(
|
|
1040
1160
|
f"Edge class {prop.value_type} does not have 'endNode' property, defaulting to DMSUnknownEntity"
|
|
@@ -1091,13 +1211,9 @@ class _InformationRulesConverter:
|
|
|
1091
1211
|
self.property_count_by_container[container_entity] += 1
|
|
1092
1212
|
return container_entity, prop.property_
|
|
1093
1213
|
|
|
1094
|
-
def _get_view_implements(
|
|
1095
|
-
self, cls_: InformationClass, metadata: InformationMetadata, mode: Literal["edge_properties"] | None
|
|
1096
|
-
) -> list[ViewEntity]:
|
|
1214
|
+
def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
|
|
1097
1215
|
implements = []
|
|
1098
1216
|
for parent in cls_.implements or []:
|
|
1099
|
-
if mode == "edge_properties" and parent.suffix == "Edge":
|
|
1100
|
-
continue
|
|
1101
1217
|
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
1102
1218
|
implements.append(view_entity)
|
|
1103
1219
|
return implements
|
|
@@ -1136,8 +1252,9 @@ class _InformationRulesConverter:
|
|
|
1136
1252
|
|
|
1137
1253
|
|
|
1138
1254
|
class _DMSRulesConverter:
|
|
1139
|
-
def __init__(self, dms: DMSRules):
|
|
1255
|
+
def __init__(self, dms: DMSRules, instance_namespace: Namespace | None = None) -> None:
|
|
1140
1256
|
self.dms = dms
|
|
1257
|
+
self.instance_namespace = instance_namespace
|
|
1141
1258
|
|
|
1142
1259
|
def as_information_rules(
|
|
1143
1260
|
self,
|
|
@@ -1152,8 +1269,9 @@ class _DMSRulesConverter:
|
|
|
1152
1269
|
|
|
1153
1270
|
metadata = self._convert_metadata_to_info(dms)
|
|
1154
1271
|
|
|
1155
|
-
classes = [
|
|
1156
|
-
|
|
1272
|
+
classes: list[InformationClass] = []
|
|
1273
|
+
for view in self.dms.views:
|
|
1274
|
+
info_class = InformationClass(
|
|
1157
1275
|
# we do not want a version in class as we use URI for the class
|
|
1158
1276
|
class_=ClassEntity(prefix=view.view.prefix, suffix=view.view.suffix),
|
|
1159
1277
|
description=view.description,
|
|
@@ -1163,8 +1281,19 @@ class _DMSRulesConverter:
|
|
|
1163
1281
|
for implemented_view in view.implements or []
|
|
1164
1282
|
],
|
|
1165
1283
|
)
|
|
1166
|
-
|
|
1167
|
-
|
|
1284
|
+
|
|
1285
|
+
# Linking
|
|
1286
|
+
info_class.physical = view.neatId
|
|
1287
|
+
classes.append(info_class)
|
|
1288
|
+
|
|
1289
|
+
prefixes = get_default_prefixes_and_namespaces()
|
|
1290
|
+
instance_prefix: str | None = None
|
|
1291
|
+
if self.instance_namespace:
|
|
1292
|
+
instance_prefix = next((k for k, v in prefixes.items() if v == self.instance_namespace), None)
|
|
1293
|
+
if instance_prefix is None:
|
|
1294
|
+
# We need to add a new prefix
|
|
1295
|
+
instance_prefix = f"prefix_{len(prefixes) + 1}"
|
|
1296
|
+
prefixes[instance_prefix] = self.instance_namespace
|
|
1168
1297
|
|
|
1169
1298
|
properties: list[InformationProperty] = []
|
|
1170
1299
|
value_type: DataType | ClassEntity | str
|
|
@@ -1181,24 +1310,42 @@ class _DMSRulesConverter:
|
|
|
1181
1310
|
else:
|
|
1182
1311
|
raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
|
|
1183
1312
|
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
min_count=(0 if property_.nullable or property_.nullable is None else 1),
|
|
1192
|
-
max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
|
|
1313
|
+
transformation: RDFPath | None = None
|
|
1314
|
+
if instance_prefix is not None:
|
|
1315
|
+
transformation = RDFPath(
|
|
1316
|
+
traversal=SingleProperty(
|
|
1317
|
+
class_=RDFPathEntity(prefix=instance_prefix, suffix=property_.view.external_id),
|
|
1318
|
+
property=RDFPathEntity(prefix=instance_prefix, suffix=property_.view_property),
|
|
1319
|
+
)
|
|
1193
1320
|
)
|
|
1321
|
+
|
|
1322
|
+
info_property = InformationProperty(
|
|
1323
|
+
# Removing version
|
|
1324
|
+
class_=ClassEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
|
|
1325
|
+
property_=property_.view_property,
|
|
1326
|
+
value_type=value_type,
|
|
1327
|
+
description=property_.description,
|
|
1328
|
+
min_count=(0 if property_.nullable or property_.nullable is None else 1),
|
|
1329
|
+
max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
|
|
1330
|
+
instance_source=transformation,
|
|
1194
1331
|
)
|
|
1195
1332
|
|
|
1196
|
-
|
|
1333
|
+
# Linking
|
|
1334
|
+
info_property.physical = property_.neatId
|
|
1335
|
+
|
|
1336
|
+
properties.append(info_property)
|
|
1337
|
+
|
|
1338
|
+
info_rules = InformationRules(
|
|
1197
1339
|
metadata=metadata,
|
|
1198
1340
|
properties=SheetList[InformationProperty](properties),
|
|
1199
1341
|
classes=SheetList[InformationClass](classes),
|
|
1342
|
+
prefixes=prefixes,
|
|
1200
1343
|
)
|
|
1201
1344
|
|
|
1345
|
+
self.dms.sync_with_info_rules(info_rules)
|
|
1346
|
+
|
|
1347
|
+
return info_rules
|
|
1348
|
+
|
|
1202
1349
|
@classmethod
|
|
1203
1350
|
def _convert_metadata_to_info(cls, metadata: DMSMetadata) -> "InformationMetadata":
|
|
1204
1351
|
from cognite.neat._rules.models.information._rules import InformationMetadata
|
|
@@ -172,8 +172,9 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
|
172
172
|
# All connections must be included in the rules. This is to update the
|
|
173
173
|
# ValueTypes of the implemented views.
|
|
174
174
|
new_rules.properties.append(mapping_prop)
|
|
175
|
-
elif
|
|
176
|
-
# All properties
|
|
175
|
+
elif "guid" in mapping_prop.view_property.casefold():
|
|
176
|
+
# All guid properties are included. Theses are necessary to get an appropriate
|
|
177
|
+
# filter on the resulting view.
|
|
177
178
|
new_rules.properties.append(mapping_prop)
|
|
178
179
|
else:
|
|
179
180
|
# Skipping mapped properties that are not in the input rules.
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -74,16 +74,15 @@ class NeatSession:
|
|
|
74
74
|
verbose: bool = True,
|
|
75
75
|
load_engine: Literal["newest", "cache", "skip"] = "cache",
|
|
76
76
|
) -> None:
|
|
77
|
-
self._client = NeatClient(client) if client else None
|
|
78
77
|
self._verbose = verbose
|
|
79
|
-
self._state = SessionState(store_type=storage)
|
|
80
|
-
self.read = ReadAPI(self._state,
|
|
81
|
-
self.to = ToAPI(self._state,
|
|
82
|
-
self.prepare = PrepareAPI(self.
|
|
78
|
+
self._state = SessionState(store_type=storage, client=NeatClient(client) if client else None)
|
|
79
|
+
self.read = ReadAPI(self._state, verbose)
|
|
80
|
+
self.to = ToAPI(self._state, verbose)
|
|
81
|
+
self.prepare = PrepareAPI(self._state, verbose)
|
|
83
82
|
self.show = ShowAPI(self._state)
|
|
84
83
|
self.set = SetAPI(self._state, verbose)
|
|
85
84
|
self.inspect = InspectAPI(self._state)
|
|
86
|
-
self.mapping = MappingAPI(self._state
|
|
85
|
+
self.mapping = MappingAPI(self._state)
|
|
87
86
|
self.drop = DropAPI(self._state)
|
|
88
87
|
self.opt = OptAPI()
|
|
89
88
|
self.opt._display()
|
|
@@ -119,7 +118,7 @@ class NeatSession:
|
|
|
119
118
|
neat.verify()
|
|
120
119
|
```
|
|
121
120
|
"""
|
|
122
|
-
transformer = VerifyAnyRules(validate=True, client=self.
|
|
121
|
+
transformer = VerifyAnyRules(validate=True, client=self._state.client) # type: ignore[var-annotated]
|
|
123
122
|
issues = self._state.rule_transform(transformer)
|
|
124
123
|
if not issues.has_errors:
|
|
125
124
|
rules = self._state.rule_store.last_verified_rule
|
|
@@ -130,15 +129,11 @@ class NeatSession:
|
|
|
130
129
|
print("You can inspect the issues with the .inspect.issues(...) method.")
|
|
131
130
|
return issues
|
|
132
131
|
|
|
133
|
-
def convert(
|
|
134
|
-
self, target: Literal["dms", "information"], mode: Literal["edge_properties"] | None = None
|
|
135
|
-
) -> IssueList:
|
|
132
|
+
def convert(self, target: Literal["dms", "information"]) -> IssueList:
|
|
136
133
|
"""Converts the last verified data model to the target type.
|
|
137
134
|
|
|
138
135
|
Args:
|
|
139
136
|
target: The target type to convert the data model to.
|
|
140
|
-
mode: If the target is "dms", the mode to use for the conversion. None is used for default conversion.
|
|
141
|
-
"edge_properties" treas classes that implements Edge as edge properties.
|
|
142
137
|
|
|
143
138
|
Example:
|
|
144
139
|
Convert to DMS rules
|
|
@@ -154,7 +149,7 @@ class NeatSession:
|
|
|
154
149
|
"""
|
|
155
150
|
converter: ConversionTransformer
|
|
156
151
|
if target == "dms":
|
|
157
|
-
converter = InformationToDMS(
|
|
152
|
+
converter = InformationToDMS()
|
|
158
153
|
elif target == "information":
|
|
159
154
|
converter = ConvertToRules(InformationRules)
|
|
160
155
|
else:
|
|
@@ -91,9 +91,13 @@ class InspectIssues:
|
|
|
91
91
|
return_dataframe: bool = (False if IN_NOTEBOOK else True), # type: ignore[assignment]
|
|
92
92
|
) -> pd.DataFrame | None:
|
|
93
93
|
"""Returns the issues of the current data model."""
|
|
94
|
-
|
|
95
|
-
|
|
94
|
+
if self._state.rule_store.provenance:
|
|
95
|
+
issues = self._state.rule_store.last_issues
|
|
96
|
+
elif self._state.instances.store.provenance:
|
|
97
|
+
issues = self._state.instances.store.provenance[-1].target_entity.issues
|
|
98
|
+
else:
|
|
96
99
|
self._print("No issues found.")
|
|
100
|
+
return pd.DataFrame() if return_dataframe else None
|
|
97
101
|
|
|
98
102
|
if issues and search is not None:
|
|
99
103
|
unique_types = {type(issue).__name__ for issue in issues}
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
from cognite.neat._client import NeatClient
|
|
2
1
|
from cognite.neat._issues import IssueList
|
|
3
2
|
from cognite.neat._rules.models import DMSRules
|
|
4
3
|
from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
|
|
5
|
-
from cognite.neat._rules.transformers import
|
|
4
|
+
from cognite.neat._rules.transformers import (
|
|
5
|
+
AsParentPropertyId,
|
|
6
|
+
ChangeViewPrefix,
|
|
7
|
+
IncludeReferenced,
|
|
8
|
+
RuleMapper,
|
|
9
|
+
RulesTransformer,
|
|
10
|
+
)
|
|
6
11
|
|
|
7
12
|
from ._state import SessionState
|
|
8
13
|
from .exceptions import NeatSessionError, session_class_wrapper
|
|
@@ -10,17 +15,16 @@ from .exceptions import NeatSessionError, session_class_wrapper
|
|
|
10
15
|
|
|
11
16
|
@session_class_wrapper
|
|
12
17
|
class MappingAPI:
|
|
13
|
-
def __init__(self, state: SessionState
|
|
14
|
-
self.data_model = DataModelMappingAPI(state
|
|
18
|
+
def __init__(self, state: SessionState):
|
|
19
|
+
self.data_model = DataModelMappingAPI(state)
|
|
15
20
|
|
|
16
21
|
|
|
17
22
|
@session_class_wrapper
|
|
18
23
|
class DataModelMappingAPI:
|
|
19
|
-
def __init__(self, state: SessionState
|
|
24
|
+
def __init__(self, state: SessionState):
|
|
20
25
|
self._state = state
|
|
21
|
-
self._client = client
|
|
22
26
|
|
|
23
|
-
def classic_to_core(self, company_prefix: str, use_parent_property_name: bool = True) -> IssueList:
|
|
27
|
+
def classic_to_core(self, company_prefix: str | None = None, use_parent_property_name: bool = True) -> IssueList:
|
|
24
28
|
"""Map classic types to core types.
|
|
25
29
|
|
|
26
30
|
Note this automatically creates an extended CogniteCore model.
|
|
@@ -44,13 +48,18 @@ class DataModelMappingAPI:
|
|
|
44
48
|
# Todo better hint of what you should do.
|
|
45
49
|
raise NeatSessionError(f"Expected DMSRules, got {type(rules)}")
|
|
46
50
|
|
|
47
|
-
if self.
|
|
51
|
+
if self._state.client is None:
|
|
48
52
|
raise NeatSessionError("Client is required to map classic to core")
|
|
49
53
|
|
|
50
|
-
transformers = [
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
+
transformers: list[RulesTransformer] = []
|
|
55
|
+
if company_prefix:
|
|
56
|
+
transformers.append(ChangeViewPrefix("Classic", company_prefix))
|
|
57
|
+
transformers.extend(
|
|
58
|
+
[
|
|
59
|
+
RuleMapper(load_classic_to_core_mapping(company_prefix, rules.metadata.space, rules.metadata.version)),
|
|
60
|
+
IncludeReferenced(self._state.client),
|
|
61
|
+
]
|
|
62
|
+
)
|
|
54
63
|
if use_parent_property_name:
|
|
55
|
-
transformers.append(AsParentPropertyId(self.
|
|
64
|
+
transformers.append(AsParentPropertyId(self._state.client))
|
|
56
65
|
return self._state.rule_transform(*transformers)
|