cognite-neat 0.106.0__py3-none-any.whl → 0.108.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_constants.py +35 -1
- cognite/neat/_graph/_shared.py +4 -0
- cognite/neat/_graph/extractors/__init__.py +5 -1
- cognite/neat/_graph/extractors/_base.py +32 -0
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +128 -14
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +156 -12
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +50 -12
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +26 -1
- cognite/neat/_graph/extractors/_dms.py +196 -47
- cognite/neat/_graph/extractors/_dms_graph.py +199 -0
- cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
- cognite/neat/_graph/extractors/_rdf_file.py +33 -5
- cognite/neat/_graph/loaders/__init__.py +1 -3
- cognite/neat/_graph/loaders/_rdf2dms.py +123 -19
- cognite/neat/_graph/queries/_base.py +140 -84
- cognite/neat/_graph/queries/_construct.py +2 -2
- cognite/neat/_graph/transformers/__init__.py +8 -1
- cognite/neat/_graph/transformers/_base.py +9 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +90 -3
- cognite/neat/_graph/transformers/_rdfpath.py +3 -3
- cognite/neat/_graph/transformers/_value_type.py +106 -45
- cognite/neat/_issues/errors/_resources.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +0 -2
- cognite/neat/_issues/warnings/_models.py +1 -1
- cognite/neat/_issues/warnings/_properties.py +0 -8
- cognite/neat/_rules/analysis/_base.py +1 -1
- cognite/neat/_rules/analysis/_information.py +14 -13
- cognite/neat/_rules/catalog/__init__.py +1 -0
- cognite/neat/_rules/catalog/classic_model.xlsx +0 -0
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2instance_template.py +3 -3
- cognite/neat/_rules/importers/__init__.py +3 -1
- cognite/neat/_rules/importers/_dms2rules.py +7 -5
- cognite/neat/_rules/importers/_dtdl2rules/spec.py +1 -2
- cognite/neat/_rules/importers/_rdf/__init__.py +2 -2
- cognite/neat/_rules/importers/_rdf/_base.py +2 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +242 -19
- cognite/neat/_rules/models/_base_rules.py +13 -15
- cognite/neat/_rules/models/_types.py +5 -0
- cognite/neat/_rules/models/dms/_rules.py +51 -10
- cognite/neat/_rules/models/dms/_rules_input.py +4 -0
- cognite/neat/_rules/models/information/_rules.py +48 -5
- cognite/neat/_rules/models/information/_rules_input.py +6 -1
- cognite/neat/_rules/models/mapping/_classic2core.py +4 -5
- cognite/neat/_rules/transformers/__init__.py +10 -0
- cognite/neat/_rules/transformers/_converters.py +300 -62
- cognite/neat/_session/_base.py +57 -10
- cognite/neat/_session/_drop.py +5 -1
- cognite/neat/_session/_inspect.py +3 -2
- cognite/neat/_session/_mapping.py +17 -6
- cognite/neat/_session/_prepare.py +0 -47
- cognite/neat/_session/_read.py +115 -10
- cognite/neat/_session/_set.py +27 -0
- cognite/neat/_session/_show.py +4 -4
- cognite/neat/_session/_state.py +12 -1
- cognite/neat/_session/_to.py +43 -2
- cognite/neat/_session/_wizard.py +1 -1
- cognite/neat/_session/exceptions.py +8 -3
- cognite/neat/_store/_graph_store.py +331 -136
- cognite/neat/_store/_rules_store.py +130 -1
- cognite/neat/_utils/auth.py +3 -1
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/METADATA +2 -2
- {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/RECORD +67 -65
- {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/WHEEL +1 -1
- {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/entry_points.txt +0 -0
|
@@ -9,12 +9,15 @@ from typing import ClassVar, Literal, TypeVar, cast, overload
|
|
|
9
9
|
|
|
10
10
|
from cognite.client.data_classes import data_modeling as dms
|
|
11
11
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
|
|
12
|
+
from rdflib import Namespace
|
|
12
13
|
|
|
13
14
|
from cognite.neat._client import NeatClient
|
|
14
15
|
from cognite.neat._client.data_classes.data_modeling import ContainerApplyDict, ViewApplyDict
|
|
15
16
|
from cognite.neat._constants import (
|
|
16
17
|
COGNITE_MODELS,
|
|
17
18
|
DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
|
|
19
|
+
DMS_RESERVED_PROPERTIES,
|
|
20
|
+
get_default_prefixes_and_namespaces,
|
|
18
21
|
)
|
|
19
22
|
from cognite.neat._issues.errors import NeatValueError
|
|
20
23
|
from cognite.neat._issues.warnings import NeatValueWarning
|
|
@@ -37,9 +40,11 @@ from cognite.neat._rules.models import (
|
|
|
37
40
|
SheetList,
|
|
38
41
|
data_types,
|
|
39
42
|
)
|
|
40
|
-
from cognite.neat._rules.models.
|
|
43
|
+
from cognite.neat._rules.models._rdfpath import Entity as RDFPathEntity
|
|
44
|
+
from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
|
|
45
|
+
from cognite.neat._rules.models.data_types import AnyURI, DataType, Enum, File, String, Timeseries
|
|
41
46
|
from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSValidation, DMSView
|
|
42
|
-
from cognite.neat._rules.models.dms._rules import DMSContainer
|
|
47
|
+
from cognite.neat._rules.models.dms._rules import DMSContainer, DMSEnum, DMSNode
|
|
43
48
|
from cognite.neat._rules.models.entities import (
|
|
44
49
|
ClassEntity,
|
|
45
50
|
ContainerEntity,
|
|
@@ -248,19 +253,26 @@ class PrefixEntities(RulesTransformer[ReadRules[T_InputRules], ReadRules[T_Input
|
|
|
248
253
|
class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
249
254
|
"""Converts InformationRules to DMSRules."""
|
|
250
255
|
|
|
251
|
-
def __init__(
|
|
256
|
+
def __init__(
|
|
257
|
+
self, ignore_undefined_value_types: bool = False, reserved_properties: Literal["error", "skip"] = "error"
|
|
258
|
+
):
|
|
252
259
|
self.ignore_undefined_value_types = ignore_undefined_value_types
|
|
253
|
-
self.
|
|
260
|
+
self.reserved_properties = reserved_properties
|
|
254
261
|
|
|
255
262
|
def transform(self, rules: InformationRules) -> DMSRules:
|
|
256
|
-
return _InformationRulesConverter(rules).as_dms_rules(
|
|
263
|
+
return _InformationRulesConverter(rules).as_dms_rules(
|
|
264
|
+
self.ignore_undefined_value_types, self.reserved_properties
|
|
265
|
+
)
|
|
257
266
|
|
|
258
267
|
|
|
259
268
|
class DMSToInformation(ConversionTransformer[DMSRules, InformationRules]):
|
|
260
269
|
"""Converts DMSRules to InformationRules."""
|
|
261
270
|
|
|
271
|
+
def __init__(self, instance_namespace: Namespace | None = None):
|
|
272
|
+
self.instance_namespace = instance_namespace
|
|
273
|
+
|
|
262
274
|
def transform(self, rules: DMSRules) -> InformationRules:
|
|
263
|
-
return _DMSRulesConverter(rules).as_information_rules()
|
|
275
|
+
return _DMSRulesConverter(rules, self.instance_namespace).as_information_rules()
|
|
264
276
|
|
|
265
277
|
|
|
266
278
|
class ConvertToRules(ConversionTransformer[VerifiedRules, VerifiedRules]):
|
|
@@ -808,15 +820,184 @@ class AddClassImplements(RulesTransformer[InformationRules, InformationRules]):
|
|
|
808
820
|
return f"Added implements property to classes with suffix {self.suffix}"
|
|
809
821
|
|
|
810
822
|
|
|
823
|
+
class ClassicPrepareCore(RulesTransformer[InformationRules, InformationRules]):
|
|
824
|
+
"""Update the classic data model with the following:
|
|
825
|
+
|
|
826
|
+
This is a special purpose transformer that is only intended to be used with when reading
|
|
827
|
+
from classic cdf using the neat.read.cdf.classic.graph(...).
|
|
828
|
+
|
|
829
|
+
- ClassicTimeseries.isString from boolean to string
|
|
830
|
+
- Add class ClassicSourceSystem, and update all source properties from string to ClassicSourceSystem.
|
|
831
|
+
- Rename externalId properties to classicExternalId
|
|
832
|
+
- Renames the Relationship.sourceExternalId and Relationship.targetExternalId to startNode and endNode
|
|
833
|
+
- If reference_timeseries is True, the classicExternalId property of the TimeSeries class will change type
|
|
834
|
+
from string to timeseries.
|
|
835
|
+
- If reference_files is True, the classicExternalId property of the File class will change type from string to file.
|
|
836
|
+
"""
|
|
837
|
+
|
|
838
|
+
def __init__(
|
|
839
|
+
self,
|
|
840
|
+
instance_namespace: Namespace,
|
|
841
|
+
reference_timeseries: bool = False,
|
|
842
|
+
reference_files: bool = False,
|
|
843
|
+
) -> None:
|
|
844
|
+
self.instance_namespace = instance_namespace
|
|
845
|
+
self.reference_timeseries = reference_timeseries
|
|
846
|
+
self.reference_files = reference_files
|
|
847
|
+
|
|
848
|
+
@property
|
|
849
|
+
def description(self) -> str:
|
|
850
|
+
return "Update the classic data model to the data types in Cognite Core."
|
|
851
|
+
|
|
852
|
+
def transform(self, rules: InformationRules) -> InformationRules:
|
|
853
|
+
output = rules.model_copy(deep=True)
|
|
854
|
+
for prop in output.properties:
|
|
855
|
+
if prop.class_.suffix == "Timeseries" and prop.property_ == "isString":
|
|
856
|
+
prop.value_type = String()
|
|
857
|
+
prefix = output.metadata.prefix
|
|
858
|
+
namespace = output.metadata.namespace
|
|
859
|
+
source_system_class = InformationClass(
|
|
860
|
+
class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
861
|
+
description="A source system that provides data to the data model.",
|
|
862
|
+
neatId=namespace["ClassicSourceSystem"],
|
|
863
|
+
)
|
|
864
|
+
output.classes.append(source_system_class)
|
|
865
|
+
for prop in output.properties:
|
|
866
|
+
if prop.property_ == "source" and prop.class_.suffix != "ClassicSourceSystem":
|
|
867
|
+
prop.value_type = ClassEntity(prefix=prefix, suffix="ClassicSourceSystem")
|
|
868
|
+
elif prop.property_ == "externalId":
|
|
869
|
+
prop.property_ = "classicExternalId"
|
|
870
|
+
if self.reference_timeseries and prop.class_.suffix == "ClassicTimeSeries":
|
|
871
|
+
prop.value_type = Timeseries()
|
|
872
|
+
elif self.reference_files and prop.class_.suffix == "ClassicFile":
|
|
873
|
+
prop.value_type = File()
|
|
874
|
+
elif prop.property_ == "sourceExternalId" and prop.class_.suffix == "ClassicRelationship":
|
|
875
|
+
prop.property_ = "startNode"
|
|
876
|
+
elif prop.property_ == "targetExternalId" and prop.class_.suffix == "ClassicRelationship":
|
|
877
|
+
prop.property_ = "endNode"
|
|
878
|
+
instance_prefix = next(
|
|
879
|
+
(prefix for prefix, namespace in output.prefixes.items() if namespace == self.instance_namespace), None
|
|
880
|
+
)
|
|
881
|
+
if instance_prefix is None:
|
|
882
|
+
raise NeatValueError("Instance namespace not found in the prefixes.")
|
|
883
|
+
|
|
884
|
+
output.properties.append(
|
|
885
|
+
InformationProperty(
|
|
886
|
+
neatId=namespace["ClassicSourceSystem/name"],
|
|
887
|
+
property_="name",
|
|
888
|
+
value_type=String(),
|
|
889
|
+
class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
890
|
+
max_count=1,
|
|
891
|
+
instance_source=RDFPath(
|
|
892
|
+
traversal=SingleProperty(
|
|
893
|
+
class_=RDFPathEntity(
|
|
894
|
+
prefix=instance_prefix,
|
|
895
|
+
suffix="ClassicSourceSystem",
|
|
896
|
+
),
|
|
897
|
+
property=RDFPathEntity(prefix=instance_prefix, suffix="name"),
|
|
898
|
+
),
|
|
899
|
+
),
|
|
900
|
+
)
|
|
901
|
+
)
|
|
902
|
+
return output
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
class ChangeViewPrefix(RulesTransformer[DMSRules, DMSRules]):
|
|
906
|
+
def __init__(self, old: str, new: str) -> None:
|
|
907
|
+
self.old = old
|
|
908
|
+
self.new = new
|
|
909
|
+
|
|
910
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
911
|
+
output = rules.model_copy(deep=True)
|
|
912
|
+
new_by_old: dict[ViewEntity, ViewEntity] = {}
|
|
913
|
+
for view in output.views:
|
|
914
|
+
if view.view.external_id.startswith(self.old):
|
|
915
|
+
new_external_id = f"{self.new}{view.view.external_id.removeprefix(self.old)}"
|
|
916
|
+
new_view_entity = view.view.copy(update={"suffix": new_external_id})
|
|
917
|
+
new_by_old[view.view] = new_view_entity
|
|
918
|
+
view.view = new_view_entity
|
|
919
|
+
for view in output.views:
|
|
920
|
+
if view.implements:
|
|
921
|
+
view.implements = [new_by_old.get(implemented, implemented) for implemented in view.implements]
|
|
922
|
+
for prop in output.properties:
|
|
923
|
+
if prop.view in new_by_old:
|
|
924
|
+
prop.view = new_by_old[prop.view]
|
|
925
|
+
if prop.value_type in new_by_old and isinstance(prop.value_type, ViewEntity):
|
|
926
|
+
prop.value_type = new_by_old[prop.value_type]
|
|
927
|
+
return output
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
class MergeDMSRules(RulesTransformer[DMSRules, DMSRules]):
|
|
931
|
+
def __init__(self, extra: DMSRules) -> None:
|
|
932
|
+
self.extra = extra
|
|
933
|
+
|
|
934
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
935
|
+
output = rules.model_copy(deep=True)
|
|
936
|
+
existing_views = {view.view for view in output.views}
|
|
937
|
+
for view in self.extra.views:
|
|
938
|
+
if view.view not in existing_views:
|
|
939
|
+
output.views.append(view)
|
|
940
|
+
existing_properties = {(prop.view, prop.view_property) for prop in output.properties}
|
|
941
|
+
existing_containers = {container.container for container in output.containers or []}
|
|
942
|
+
existing_enum_collections = {collection.collection for collection in output.enum or []}
|
|
943
|
+
new_containers_by_entity = {container.container: container for container in self.extra.containers or []}
|
|
944
|
+
new_enum_collections_by_entity = {collection.collection: collection for collection in self.extra.enum or []}
|
|
945
|
+
for prop in self.extra.properties:
|
|
946
|
+
if (prop.view, prop.view_property) in existing_properties:
|
|
947
|
+
continue
|
|
948
|
+
output.properties.append(prop)
|
|
949
|
+
if prop.container and prop.container not in existing_containers:
|
|
950
|
+
if output.containers is None:
|
|
951
|
+
output.containers = SheetList[DMSContainer]()
|
|
952
|
+
output.containers.append(new_containers_by_entity[prop.container])
|
|
953
|
+
if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections:
|
|
954
|
+
if output.enum is None:
|
|
955
|
+
output.enum = SheetList[DMSEnum]()
|
|
956
|
+
output.enum.append(new_enum_collections_by_entity[prop.value_type.collection])
|
|
957
|
+
|
|
958
|
+
existing_nodes = {node.node for node in output.nodes or []}
|
|
959
|
+
for node in self.extra.nodes or []:
|
|
960
|
+
if node.node not in existing_nodes:
|
|
961
|
+
if output.nodes is None:
|
|
962
|
+
output.nodes = SheetList[DMSNode]()
|
|
963
|
+
output.nodes.append(node)
|
|
964
|
+
|
|
965
|
+
return output
|
|
966
|
+
|
|
967
|
+
@property
|
|
968
|
+
def description(self) -> str:
|
|
969
|
+
return f"Merged with {self.extra.metadata.as_data_model_id()}"
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
class MergeInformationRules(RulesTransformer[InformationRules, InformationRules]):
|
|
973
|
+
def __init__(self, extra: InformationRules) -> None:
|
|
974
|
+
self.extra = extra
|
|
975
|
+
|
|
976
|
+
def transform(self, rules: InformationRules) -> InformationRules:
|
|
977
|
+
output = rules.model_copy(deep=True)
|
|
978
|
+
existing_classes = {cls.class_ for cls in output.classes}
|
|
979
|
+
for cls in self.extra.classes:
|
|
980
|
+
if cls.class_ not in existing_classes:
|
|
981
|
+
output.classes.append(cls)
|
|
982
|
+
existing_properties = {(prop.class_, prop.property_) for prop in output.properties}
|
|
983
|
+
for prop in self.extra.properties:
|
|
984
|
+
if (prop.class_, prop.property_) not in existing_properties:
|
|
985
|
+
output.properties.append(prop)
|
|
986
|
+
for prefix, namespace in self.extra.prefixes.items():
|
|
987
|
+
if prefix not in output.prefixes:
|
|
988
|
+
output.prefixes[prefix] = namespace
|
|
989
|
+
return output
|
|
990
|
+
|
|
991
|
+
|
|
811
992
|
class _InformationRulesConverter:
|
|
812
|
-
|
|
993
|
+
_start_or_end_node: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
|
|
813
994
|
|
|
814
995
|
def __init__(self, information: InformationRules):
|
|
815
996
|
self.rules = information
|
|
816
997
|
self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
|
|
817
998
|
|
|
818
999
|
def as_dms_rules(
|
|
819
|
-
self, ignore_undefined_value_types: bool = False,
|
|
1000
|
+
self, ignore_undefined_value_types: bool = False, reserved_properties: Literal["error", "skip"] = "error"
|
|
820
1001
|
) -> "DMSRules":
|
|
821
1002
|
from cognite.neat._rules.models.dms._rules import (
|
|
822
1003
|
DMSContainer,
|
|
@@ -829,35 +1010,51 @@ class _InformationRulesConverter:
|
|
|
829
1010
|
default_version = info_metadata.version
|
|
830
1011
|
default_space = self._to_space(info_metadata.prefix)
|
|
831
1012
|
dms_metadata = self._convert_metadata_to_dms(info_metadata)
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
1013
|
+
|
|
1014
|
+
properties_by_class: dict[ClassEntity, set[str]] = defaultdict(set)
|
|
1015
|
+
for prop in self.rules.properties:
|
|
1016
|
+
properties_by_class[prop.class_].add(prop.property_)
|
|
1017
|
+
|
|
1018
|
+
# Edge Classes is defined by having both startNode and endNode properties
|
|
1019
|
+
edge_classes = {
|
|
1020
|
+
cls_
|
|
1021
|
+
for cls_, class_properties in properties_by_class.items()
|
|
1022
|
+
if ({"startNode", "start_node"} & class_properties) and ({"endNode", "end_node"} & class_properties)
|
|
1023
|
+
}
|
|
1024
|
+
edge_value_types_by_class_property_pair = {
|
|
1025
|
+
(prop.class_, prop.property_): prop.value_type
|
|
1026
|
+
for prop in self.rules.properties
|
|
1027
|
+
if prop.value_type in edge_classes and isinstance(prop.value_type, ClassEntity)
|
|
1028
|
+
}
|
|
1029
|
+
end_node_by_edge = {
|
|
1030
|
+
prop.class_: prop.value_type
|
|
1031
|
+
for prop in self.rules.properties
|
|
1032
|
+
if prop.class_ in edge_classes
|
|
1033
|
+
and (prop.property_ == "endNode" or prop.property_ == "end_node")
|
|
1034
|
+
and isinstance(prop.value_type, ClassEntity)
|
|
1035
|
+
}
|
|
851
1036
|
|
|
852
1037
|
properties_by_class: dict[ClassEntity, list[DMSProperty]] = defaultdict(list)
|
|
853
1038
|
referenced_containers: dict[ContainerEntity, Counter[ClassEntity]] = defaultdict(Counter)
|
|
854
1039
|
for prop in self.rules.properties:
|
|
855
1040
|
if ignore_undefined_value_types and isinstance(prop.value_type, UnknownEntity):
|
|
856
1041
|
continue
|
|
857
|
-
if prop.class_ in edge_classes and prop.property_ in self.
|
|
1042
|
+
if prop.class_ in edge_classes and prop.property_ in self._start_or_end_node:
|
|
1043
|
+
continue
|
|
1044
|
+
if prop.property_ in DMS_RESERVED_PROPERTIES:
|
|
1045
|
+
msg = f"Property {prop.property_} is a reserved property in DMS."
|
|
1046
|
+
if reserved_properties == "error":
|
|
1047
|
+
raise NeatValueError(msg)
|
|
1048
|
+
warnings.warn(NeatValueWarning(f"{msg} Skipping..."), stacklevel=2)
|
|
858
1049
|
continue
|
|
1050
|
+
|
|
859
1051
|
dms_property = self._as_dms_property(
|
|
860
|
-
prop,
|
|
1052
|
+
prop,
|
|
1053
|
+
default_space,
|
|
1054
|
+
default_version,
|
|
1055
|
+
edge_classes,
|
|
1056
|
+
edge_value_types_by_class_property_pair,
|
|
1057
|
+
end_node_by_edge,
|
|
861
1058
|
)
|
|
862
1059
|
properties_by_class[prop.class_].append(dms_property)
|
|
863
1060
|
if dms_property.container:
|
|
@@ -870,12 +1067,10 @@ class _InformationRulesConverter:
|
|
|
870
1067
|
name=cls_.name,
|
|
871
1068
|
view=cls_.class_.as_view_entity(default_space, default_version),
|
|
872
1069
|
description=cls_.description,
|
|
873
|
-
implements=self._get_view_implements(cls_, info_metadata
|
|
1070
|
+
implements=self._get_view_implements(cls_, info_metadata),
|
|
874
1071
|
)
|
|
875
1072
|
|
|
876
1073
|
dms_view.logical = cls_.neatId
|
|
877
|
-
cls_.physical = dms_view.neatId
|
|
878
|
-
|
|
879
1074
|
views.append(dms_view)
|
|
880
1075
|
|
|
881
1076
|
class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
|
|
@@ -891,21 +1086,34 @@ class _InformationRulesConverter:
|
|
|
891
1086
|
)
|
|
892
1087
|
most_used_class_entity = class_entities.most_common(1)[0][0]
|
|
893
1088
|
class_ = class_by_entity[most_used_class_entity]
|
|
1089
|
+
|
|
1090
|
+
if len(set(class_entities) - set(edge_classes)) == 0:
|
|
1091
|
+
used_for: Literal["node", "edge", "all"] = "edge"
|
|
1092
|
+
elif len(set(class_entities) - set(edge_classes)) == len(class_entities):
|
|
1093
|
+
used_for = "node"
|
|
1094
|
+
else:
|
|
1095
|
+
used_for = "all"
|
|
1096
|
+
|
|
894
1097
|
container = DMSContainer(
|
|
895
1098
|
container=container_entity,
|
|
896
1099
|
name=class_.name,
|
|
897
1100
|
description=class_.description,
|
|
898
1101
|
constraint=constrains or None,
|
|
1102
|
+
used_for=used_for,
|
|
899
1103
|
)
|
|
900
1104
|
containers.append(container)
|
|
901
1105
|
|
|
902
|
-
|
|
1106
|
+
dms_rules = DMSRules(
|
|
903
1107
|
metadata=dms_metadata,
|
|
904
1108
|
properties=SheetList[DMSProperty]([prop for prop_set in properties_by_class.values() for prop in prop_set]),
|
|
905
1109
|
views=SheetList[DMSView](views),
|
|
906
1110
|
containers=SheetList[DMSContainer](containers),
|
|
907
1111
|
)
|
|
908
1112
|
|
|
1113
|
+
self.rules.sync_with_dms_rules(dms_rules)
|
|
1114
|
+
|
|
1115
|
+
return dms_rules
|
|
1116
|
+
|
|
909
1117
|
@staticmethod
|
|
910
1118
|
def _create_container_constraint(
|
|
911
1119
|
class_entities: Counter[ClassEntity],
|
|
@@ -939,8 +1147,6 @@ class _InformationRulesConverter:
|
|
|
939
1147
|
)
|
|
940
1148
|
|
|
941
1149
|
dms_metadata.logical = metadata.identifier
|
|
942
|
-
metadata.physical = dms_metadata.identifier
|
|
943
|
-
|
|
944
1150
|
return dms_metadata
|
|
945
1151
|
|
|
946
1152
|
def _as_dms_property(
|
|
@@ -949,7 +1155,7 @@ class _InformationRulesConverter:
|
|
|
949
1155
|
default_space: str,
|
|
950
1156
|
default_version: str,
|
|
951
1157
|
edge_classes: set[ClassEntity],
|
|
952
|
-
|
|
1158
|
+
edge_value_types_by_class_property_pair: dict[tuple[ClassEntity, str], ClassEntity],
|
|
953
1159
|
end_node_by_edge: dict[ClassEntity, ClassEntity],
|
|
954
1160
|
) -> "DMSProperty":
|
|
955
1161
|
from cognite.neat._rules.models.dms._rules import DMSProperty
|
|
@@ -963,7 +1169,9 @@ class _InformationRulesConverter:
|
|
|
963
1169
|
end_node_by_edge,
|
|
964
1170
|
)
|
|
965
1171
|
|
|
966
|
-
connection = self._get_connection(
|
|
1172
|
+
connection = self._get_connection(
|
|
1173
|
+
info_property, value_type, edge_value_types_by_class_property_pair, default_space, default_version
|
|
1174
|
+
)
|
|
967
1175
|
|
|
968
1176
|
container: ContainerEntity | None = None
|
|
969
1177
|
container_property: str | None = None
|
|
@@ -992,7 +1200,6 @@ class _InformationRulesConverter:
|
|
|
992
1200
|
|
|
993
1201
|
# linking
|
|
994
1202
|
dms_property.logical = info_property.neatId
|
|
995
|
-
info_property.physical = dms_property.neatId
|
|
996
1203
|
|
|
997
1204
|
return dms_property
|
|
998
1205
|
|
|
@@ -1000,13 +1207,16 @@ class _InformationRulesConverter:
|
|
|
1000
1207
|
def _get_connection(
|
|
1001
1208
|
prop: InformationProperty,
|
|
1002
1209
|
value_type: DataType | ViewEntity | DMSUnknownEntity,
|
|
1003
|
-
|
|
1210
|
+
edge_value_types_by_class_property_pair: dict[tuple[ClassEntity, str], ClassEntity],
|
|
1004
1211
|
default_space: str,
|
|
1005
1212
|
default_version: str,
|
|
1006
1213
|
) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
|
|
1007
|
-
if
|
|
1008
|
-
|
|
1009
|
-
|
|
1214
|
+
if (
|
|
1215
|
+
isinstance(value_type, ViewEntity)
|
|
1216
|
+
and (prop.class_, prop.property_) in edge_value_types_by_class_property_pair
|
|
1217
|
+
):
|
|
1218
|
+
edge_value_type = edge_value_types_by_class_property_pair[(prop.class_, prop.property_)]
|
|
1219
|
+
return EdgeEntity(properties=edge_value_type.as_view_entity(default_space, default_version))
|
|
1010
1220
|
if isinstance(value_type, ViewEntity) and prop.is_list:
|
|
1011
1221
|
return EdgeEntity()
|
|
1012
1222
|
elif isinstance(value_type, ViewEntity):
|
|
@@ -1035,6 +1245,7 @@ class _InformationRulesConverter:
|
|
|
1035
1245
|
elif isinstance(prop.value_type, ClassEntity) and (prop.value_type in edge_classes):
|
|
1036
1246
|
if prop.value_type in end_node_by_edge:
|
|
1037
1247
|
return end_node_by_edge[prop.value_type].as_view_entity(default_space, default_version)
|
|
1248
|
+
# This occurs if the end node is not pointing to a class
|
|
1038
1249
|
warnings.warn(
|
|
1039
1250
|
NeatValueWarning(
|
|
1040
1251
|
f"Edge class {prop.value_type} does not have 'endNode' property, defaulting to DMSUnknownEntity"
|
|
@@ -1091,13 +1302,9 @@ class _InformationRulesConverter:
|
|
|
1091
1302
|
self.property_count_by_container[container_entity] += 1
|
|
1092
1303
|
return container_entity, prop.property_
|
|
1093
1304
|
|
|
1094
|
-
def _get_view_implements(
|
|
1095
|
-
self, cls_: InformationClass, metadata: InformationMetadata, mode: Literal["edge_properties"] | None
|
|
1096
|
-
) -> list[ViewEntity]:
|
|
1305
|
+
def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
|
|
1097
1306
|
implements = []
|
|
1098
1307
|
for parent in cls_.implements or []:
|
|
1099
|
-
if mode == "edge_properties" and parent.suffix == "Edge":
|
|
1100
|
-
continue
|
|
1101
1308
|
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
1102
1309
|
implements.append(view_entity)
|
|
1103
1310
|
return implements
|
|
@@ -1136,8 +1343,9 @@ class _InformationRulesConverter:
|
|
|
1136
1343
|
|
|
1137
1344
|
|
|
1138
1345
|
class _DMSRulesConverter:
|
|
1139
|
-
def __init__(self, dms: DMSRules):
|
|
1346
|
+
def __init__(self, dms: DMSRules, instance_namespace: Namespace | None = None) -> None:
|
|
1140
1347
|
self.dms = dms
|
|
1348
|
+
self.instance_namespace = instance_namespace
|
|
1141
1349
|
|
|
1142
1350
|
def as_information_rules(
|
|
1143
1351
|
self,
|
|
@@ -1152,8 +1360,9 @@ class _DMSRulesConverter:
|
|
|
1152
1360
|
|
|
1153
1361
|
metadata = self._convert_metadata_to_info(dms)
|
|
1154
1362
|
|
|
1155
|
-
classes = [
|
|
1156
|
-
|
|
1363
|
+
classes: list[InformationClass] = []
|
|
1364
|
+
for view in self.dms.views:
|
|
1365
|
+
info_class = InformationClass(
|
|
1157
1366
|
# we do not want a version in class as we use URI for the class
|
|
1158
1367
|
class_=ClassEntity(prefix=view.view.prefix, suffix=view.view.suffix),
|
|
1159
1368
|
description=view.description,
|
|
@@ -1163,8 +1372,19 @@ class _DMSRulesConverter:
|
|
|
1163
1372
|
for implemented_view in view.implements or []
|
|
1164
1373
|
],
|
|
1165
1374
|
)
|
|
1166
|
-
|
|
1167
|
-
|
|
1375
|
+
|
|
1376
|
+
# Linking
|
|
1377
|
+
info_class.physical = view.neatId
|
|
1378
|
+
classes.append(info_class)
|
|
1379
|
+
|
|
1380
|
+
prefixes = get_default_prefixes_and_namespaces()
|
|
1381
|
+
instance_prefix: str | None = None
|
|
1382
|
+
if self.instance_namespace:
|
|
1383
|
+
instance_prefix = next((k for k, v in prefixes.items() if v == self.instance_namespace), None)
|
|
1384
|
+
if instance_prefix is None:
|
|
1385
|
+
# We need to add a new prefix
|
|
1386
|
+
instance_prefix = f"prefix_{len(prefixes) + 1}"
|
|
1387
|
+
prefixes[instance_prefix] = self.instance_namespace
|
|
1168
1388
|
|
|
1169
1389
|
properties: list[InformationProperty] = []
|
|
1170
1390
|
value_type: DataType | ClassEntity | str
|
|
@@ -1181,24 +1401,42 @@ class _DMSRulesConverter:
|
|
|
1181
1401
|
else:
|
|
1182
1402
|
raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
|
|
1183
1403
|
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
min_count=(0 if property_.nullable or property_.nullable is None else 1),
|
|
1192
|
-
max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
|
|
1404
|
+
transformation: RDFPath | None = None
|
|
1405
|
+
if instance_prefix is not None:
|
|
1406
|
+
transformation = RDFPath(
|
|
1407
|
+
traversal=SingleProperty(
|
|
1408
|
+
class_=RDFPathEntity(prefix=instance_prefix, suffix=property_.view.external_id),
|
|
1409
|
+
property=RDFPathEntity(prefix=instance_prefix, suffix=property_.view_property),
|
|
1410
|
+
)
|
|
1193
1411
|
)
|
|
1412
|
+
|
|
1413
|
+
info_property = InformationProperty(
|
|
1414
|
+
# Removing version
|
|
1415
|
+
class_=ClassEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
|
|
1416
|
+
property_=property_.view_property,
|
|
1417
|
+
value_type=value_type,
|
|
1418
|
+
description=property_.description,
|
|
1419
|
+
min_count=(0 if property_.nullable or property_.nullable is None else 1),
|
|
1420
|
+
max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
|
|
1421
|
+
instance_source=transformation,
|
|
1194
1422
|
)
|
|
1195
1423
|
|
|
1196
|
-
|
|
1424
|
+
# Linking
|
|
1425
|
+
info_property.physical = property_.neatId
|
|
1426
|
+
|
|
1427
|
+
properties.append(info_property)
|
|
1428
|
+
|
|
1429
|
+
info_rules = InformationRules(
|
|
1197
1430
|
metadata=metadata,
|
|
1198
1431
|
properties=SheetList[InformationProperty](properties),
|
|
1199
1432
|
classes=SheetList[InformationClass](classes),
|
|
1433
|
+
prefixes=prefixes,
|
|
1200
1434
|
)
|
|
1201
1435
|
|
|
1436
|
+
self.dms.sync_with_info_rules(info_rules)
|
|
1437
|
+
|
|
1438
|
+
return info_rules
|
|
1439
|
+
|
|
1202
1440
|
@classmethod
|
|
1203
1441
|
def _convert_metadata_to_info(cls, metadata: DMSMetadata) -> "InformationMetadata":
|
|
1204
1442
|
from cognite.neat._rules.models.information._rules import InformationMetadata
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -7,11 +7,21 @@ from cognite.neat import _version
|
|
|
7
7
|
from cognite.neat._client import NeatClient
|
|
8
8
|
from cognite.neat._issues import IssueList
|
|
9
9
|
from cognite.neat._issues.errors import RegexViolationError
|
|
10
|
+
from cognite.neat._issues.errors._general import NeatImportError
|
|
10
11
|
from cognite.neat._rules import importers
|
|
11
12
|
from cognite.neat._rules.models._base_input import InputRules
|
|
12
13
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
13
|
-
from cognite.neat._rules.transformers import
|
|
14
|
-
|
|
14
|
+
from cognite.neat._rules.transformers import (
|
|
15
|
+
ConversionTransformer,
|
|
16
|
+
ConvertToRules,
|
|
17
|
+
InformationToDMS,
|
|
18
|
+
MergeDMSRules,
|
|
19
|
+
MergeInformationRules,
|
|
20
|
+
VerifyAnyRules,
|
|
21
|
+
VerifyInformationRules,
|
|
22
|
+
)
|
|
23
|
+
from cognite.neat._store._rules_store import ModelEntity
|
|
24
|
+
from cognite.neat._utils.auxiliary import local_import
|
|
15
25
|
|
|
16
26
|
from ._collector import _COLLECTOR, Collector
|
|
17
27
|
from ._drop import DropAPI
|
|
@@ -70,12 +80,15 @@ class NeatSession:
|
|
|
70
80
|
def __init__(
|
|
71
81
|
self,
|
|
72
82
|
client: CogniteClient | None = None,
|
|
73
|
-
storage: Literal["memory", "oxigraph"] =
|
|
83
|
+
storage: Literal["memory", "oxigraph"] | None = None,
|
|
74
84
|
verbose: bool = True,
|
|
75
85
|
load_engine: Literal["newest", "cache", "skip"] = "cache",
|
|
76
86
|
) -> None:
|
|
77
87
|
self._verbose = verbose
|
|
78
|
-
self._state = SessionState(
|
|
88
|
+
self._state = SessionState(
|
|
89
|
+
store_type=storage or self._select_most_performant_store(),
|
|
90
|
+
client=NeatClient(client) if client else None,
|
|
91
|
+
)
|
|
79
92
|
self.read = ReadAPI(self._state, verbose)
|
|
80
93
|
self.to = ToAPI(self._state, verbose)
|
|
81
94
|
self.prepare = PrepareAPI(self._state, verbose)
|
|
@@ -89,6 +102,16 @@ class NeatSession:
|
|
|
89
102
|
if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
|
|
90
103
|
print(f"Neat Engine {engine_version} loaded.")
|
|
91
104
|
|
|
105
|
+
def _select_most_performant_store(self) -> Literal["memory", "oxigraph"]:
|
|
106
|
+
"""Select the most performant store based on the current environment."""
|
|
107
|
+
|
|
108
|
+
try:
|
|
109
|
+
local_import("pyoxigraph", "oxi")
|
|
110
|
+
local_import("oxrdflib", "oxi")
|
|
111
|
+
return "oxigraph"
|
|
112
|
+
except NeatImportError:
|
|
113
|
+
return "memory"
|
|
114
|
+
|
|
92
115
|
@property
|
|
93
116
|
def version(self) -> str:
|
|
94
117
|
"""Get the current version of neat.
|
|
@@ -129,15 +152,11 @@ class NeatSession:
|
|
|
129
152
|
print("You can inspect the issues with the .inspect.issues(...) method.")
|
|
130
153
|
return issues
|
|
131
154
|
|
|
132
|
-
def convert(
|
|
133
|
-
self, target: Literal["dms", "information"], mode: Literal["edge_properties"] | None = None
|
|
134
|
-
) -> IssueList:
|
|
155
|
+
def convert(self, target: Literal["dms", "information"]) -> IssueList:
|
|
135
156
|
"""Converts the last verified data model to the target type.
|
|
136
157
|
|
|
137
158
|
Args:
|
|
138
159
|
target: The target type to convert the data model to.
|
|
139
|
-
mode: If the target is "dms", the mode to use for the conversion. None is used for default conversion.
|
|
140
|
-
"edge_properties" treas classes that implements Edge as edge properties.
|
|
141
160
|
|
|
142
161
|
Example:
|
|
143
162
|
Convert to DMS rules
|
|
@@ -153,7 +172,7 @@ class NeatSession:
|
|
|
153
172
|
"""
|
|
154
173
|
converter: ConversionTransformer
|
|
155
174
|
if target == "dms":
|
|
156
|
-
converter = InformationToDMS(
|
|
175
|
+
converter = InformationToDMS()
|
|
157
176
|
elif target == "information":
|
|
158
177
|
converter = ConvertToRules(InformationRules)
|
|
159
178
|
else:
|
|
@@ -203,6 +222,34 @@ class NeatSession:
|
|
|
203
222
|
)
|
|
204
223
|
return self._state.rule_import(importer)
|
|
205
224
|
|
|
225
|
+
def _infer_subclasses(self) -> IssueList:
|
|
226
|
+
"""Infer the subclass of instances."""
|
|
227
|
+
last_information = self._state.rule_store.last_verified_information_rules
|
|
228
|
+
issue_list = IssueList()
|
|
229
|
+
importer = importers.SubclassInferenceImporter(
|
|
230
|
+
issue_list=issue_list,
|
|
231
|
+
graph=self._state.instances.store.graph(),
|
|
232
|
+
rules=last_information,
|
|
233
|
+
max_number_of_instance=-1,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
unverified_information = importer.to_rules()
|
|
237
|
+
verified_information = VerifyInformationRules().transform(unverified_information)
|
|
238
|
+
|
|
239
|
+
# Hack into the last information rules to merge the rules with the last verified information rules.
|
|
240
|
+
# This is to be able to populate the instances store with the inferred subclasses.
|
|
241
|
+
provenance = self._state.rule_store.provenance
|
|
242
|
+
for change in reversed(provenance):
|
|
243
|
+
target_entity = change.target_entity
|
|
244
|
+
if isinstance(target_entity, ModelEntity) and isinstance(target_entity.result, InformationRules):
|
|
245
|
+
last_information_rules = change.target_entity.result
|
|
246
|
+
new_information_rules = MergeInformationRules(verified_information).transform(last_information_rules)
|
|
247
|
+
object.__setattr__(change.target_entity, "result", new_information_rules)
|
|
248
|
+
break
|
|
249
|
+
|
|
250
|
+
dms_rules = InformationToDMS(reserved_properties="skip").transform(verified_information)
|
|
251
|
+
return self._state.rule_transform(MergeDMSRules(dms_rules))
|
|
252
|
+
|
|
206
253
|
def _repr_html_(self) -> str:
|
|
207
254
|
state = self._state
|
|
208
255
|
if (
|