cognite-neat 0.106.0__py3-none-any.whl → 0.108.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (67) hide show
  1. cognite/neat/_constants.py +35 -1
  2. cognite/neat/_graph/_shared.py +4 -0
  3. cognite/neat/_graph/extractors/__init__.py +5 -1
  4. cognite/neat/_graph/extractors/_base.py +32 -0
  5. cognite/neat/_graph/extractors/_classic_cdf/_base.py +128 -14
  6. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +156 -12
  7. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +50 -12
  8. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +26 -1
  9. cognite/neat/_graph/extractors/_dms.py +196 -47
  10. cognite/neat/_graph/extractors/_dms_graph.py +199 -0
  11. cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
  12. cognite/neat/_graph/extractors/_rdf_file.py +33 -5
  13. cognite/neat/_graph/loaders/__init__.py +1 -3
  14. cognite/neat/_graph/loaders/_rdf2dms.py +123 -19
  15. cognite/neat/_graph/queries/_base.py +140 -84
  16. cognite/neat/_graph/queries/_construct.py +2 -2
  17. cognite/neat/_graph/transformers/__init__.py +8 -1
  18. cognite/neat/_graph/transformers/_base.py +9 -1
  19. cognite/neat/_graph/transformers/_classic_cdf.py +90 -3
  20. cognite/neat/_graph/transformers/_rdfpath.py +3 -3
  21. cognite/neat/_graph/transformers/_value_type.py +106 -45
  22. cognite/neat/_issues/errors/_resources.py +1 -1
  23. cognite/neat/_issues/warnings/__init__.py +0 -2
  24. cognite/neat/_issues/warnings/_models.py +1 -1
  25. cognite/neat/_issues/warnings/_properties.py +0 -8
  26. cognite/neat/_rules/analysis/_base.py +1 -1
  27. cognite/neat/_rules/analysis/_information.py +14 -13
  28. cognite/neat/_rules/catalog/__init__.py +1 -0
  29. cognite/neat/_rules/catalog/classic_model.xlsx +0 -0
  30. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  31. cognite/neat/_rules/exporters/_rules2instance_template.py +3 -3
  32. cognite/neat/_rules/importers/__init__.py +3 -1
  33. cognite/neat/_rules/importers/_dms2rules.py +7 -5
  34. cognite/neat/_rules/importers/_dtdl2rules/spec.py +1 -2
  35. cognite/neat/_rules/importers/_rdf/__init__.py +2 -2
  36. cognite/neat/_rules/importers/_rdf/_base.py +2 -2
  37. cognite/neat/_rules/importers/_rdf/_inference2rules.py +242 -19
  38. cognite/neat/_rules/models/_base_rules.py +13 -15
  39. cognite/neat/_rules/models/_types.py +5 -0
  40. cognite/neat/_rules/models/dms/_rules.py +51 -10
  41. cognite/neat/_rules/models/dms/_rules_input.py +4 -0
  42. cognite/neat/_rules/models/information/_rules.py +48 -5
  43. cognite/neat/_rules/models/information/_rules_input.py +6 -1
  44. cognite/neat/_rules/models/mapping/_classic2core.py +4 -5
  45. cognite/neat/_rules/transformers/__init__.py +10 -0
  46. cognite/neat/_rules/transformers/_converters.py +300 -62
  47. cognite/neat/_session/_base.py +57 -10
  48. cognite/neat/_session/_drop.py +5 -1
  49. cognite/neat/_session/_inspect.py +3 -2
  50. cognite/neat/_session/_mapping.py +17 -6
  51. cognite/neat/_session/_prepare.py +0 -47
  52. cognite/neat/_session/_read.py +115 -10
  53. cognite/neat/_session/_set.py +27 -0
  54. cognite/neat/_session/_show.py +4 -4
  55. cognite/neat/_session/_state.py +12 -1
  56. cognite/neat/_session/_to.py +43 -2
  57. cognite/neat/_session/_wizard.py +1 -1
  58. cognite/neat/_session/exceptions.py +8 -3
  59. cognite/neat/_store/_graph_store.py +331 -136
  60. cognite/neat/_store/_rules_store.py +130 -1
  61. cognite/neat/_utils/auth.py +3 -1
  62. cognite/neat/_version.py +1 -1
  63. {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/METADATA +2 -2
  64. {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/RECORD +67 -65
  65. {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/WHEEL +1 -1
  66. {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/LICENSE +0 -0
  67. {cognite_neat-0.106.0.dist-info → cognite_neat-0.108.0.dist-info}/entry_points.txt +0 -0
@@ -9,12 +9,15 @@ from typing import ClassVar, Literal, TypeVar, cast, overload
9
9
 
10
10
  from cognite.client.data_classes import data_modeling as dms
11
11
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
12
+ from rdflib import Namespace
12
13
 
13
14
  from cognite.neat._client import NeatClient
14
15
  from cognite.neat._client.data_classes.data_modeling import ContainerApplyDict, ViewApplyDict
15
16
  from cognite.neat._constants import (
16
17
  COGNITE_MODELS,
17
18
  DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
19
+ DMS_RESERVED_PROPERTIES,
20
+ get_default_prefixes_and_namespaces,
18
21
  )
19
22
  from cognite.neat._issues.errors import NeatValueError
20
23
  from cognite.neat._issues.warnings import NeatValueWarning
@@ -37,9 +40,11 @@ from cognite.neat._rules.models import (
37
40
  SheetList,
38
41
  data_types,
39
42
  )
40
- from cognite.neat._rules.models.data_types import AnyURI, DataType, String
43
+ from cognite.neat._rules.models._rdfpath import Entity as RDFPathEntity
44
+ from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
45
+ from cognite.neat._rules.models.data_types import AnyURI, DataType, Enum, File, String, Timeseries
41
46
  from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSValidation, DMSView
42
- from cognite.neat._rules.models.dms._rules import DMSContainer
47
+ from cognite.neat._rules.models.dms._rules import DMSContainer, DMSEnum, DMSNode
43
48
  from cognite.neat._rules.models.entities import (
44
49
  ClassEntity,
45
50
  ContainerEntity,
@@ -248,19 +253,26 @@ class PrefixEntities(RulesTransformer[ReadRules[T_InputRules], ReadRules[T_Input
248
253
  class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
249
254
  """Converts InformationRules to DMSRules."""
250
255
 
251
- def __init__(self, ignore_undefined_value_types: bool = False, mode: Literal["edge_properties"] | None = None):
256
+ def __init__(
257
+ self, ignore_undefined_value_types: bool = False, reserved_properties: Literal["error", "skip"] = "error"
258
+ ):
252
259
  self.ignore_undefined_value_types = ignore_undefined_value_types
253
- self.mode = mode
260
+ self.reserved_properties = reserved_properties
254
261
 
255
262
  def transform(self, rules: InformationRules) -> DMSRules:
256
- return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types, self.mode)
263
+ return _InformationRulesConverter(rules).as_dms_rules(
264
+ self.ignore_undefined_value_types, self.reserved_properties
265
+ )
257
266
 
258
267
 
259
268
  class DMSToInformation(ConversionTransformer[DMSRules, InformationRules]):
260
269
  """Converts DMSRules to InformationRules."""
261
270
 
271
+ def __init__(self, instance_namespace: Namespace | None = None):
272
+ self.instance_namespace = instance_namespace
273
+
262
274
  def transform(self, rules: DMSRules) -> InformationRules:
263
- return _DMSRulesConverter(rules).as_information_rules()
275
+ return _DMSRulesConverter(rules, self.instance_namespace).as_information_rules()
264
276
 
265
277
 
266
278
  class ConvertToRules(ConversionTransformer[VerifiedRules, VerifiedRules]):
@@ -808,15 +820,184 @@ class AddClassImplements(RulesTransformer[InformationRules, InformationRules]):
808
820
  return f"Added implements property to classes with suffix {self.suffix}"
809
821
 
810
822
 
823
+ class ClassicPrepareCore(RulesTransformer[InformationRules, InformationRules]):
824
+ """Update the classic data model with the following:
825
+
826
+ This is a special purpose transformer that is only intended to be used with when reading
827
+ from classic cdf using the neat.read.cdf.classic.graph(...).
828
+
829
+ - ClassicTimeseries.isString from boolean to string
830
+ - Add class ClassicSourceSystem, and update all source properties from string to ClassicSourceSystem.
831
+ - Rename externalId properties to classicExternalId
832
+ - Renames the Relationship.sourceExternalId and Relationship.targetExternalId to startNode and endNode
833
+ - If reference_timeseries is True, the classicExternalId property of the TimeSeries class will change type
834
+ from string to timeseries.
835
+ - If reference_files is True, the classicExternalId property of the File class will change type from string to file.
836
+ """
837
+
838
+ def __init__(
839
+ self,
840
+ instance_namespace: Namespace,
841
+ reference_timeseries: bool = False,
842
+ reference_files: bool = False,
843
+ ) -> None:
844
+ self.instance_namespace = instance_namespace
845
+ self.reference_timeseries = reference_timeseries
846
+ self.reference_files = reference_files
847
+
848
+ @property
849
+ def description(self) -> str:
850
+ return "Update the classic data model to the data types in Cognite Core."
851
+
852
+ def transform(self, rules: InformationRules) -> InformationRules:
853
+ output = rules.model_copy(deep=True)
854
+ for prop in output.properties:
855
+ if prop.class_.suffix == "Timeseries" and prop.property_ == "isString":
856
+ prop.value_type = String()
857
+ prefix = output.metadata.prefix
858
+ namespace = output.metadata.namespace
859
+ source_system_class = InformationClass(
860
+ class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
861
+ description="A source system that provides data to the data model.",
862
+ neatId=namespace["ClassicSourceSystem"],
863
+ )
864
+ output.classes.append(source_system_class)
865
+ for prop in output.properties:
866
+ if prop.property_ == "source" and prop.class_.suffix != "ClassicSourceSystem":
867
+ prop.value_type = ClassEntity(prefix=prefix, suffix="ClassicSourceSystem")
868
+ elif prop.property_ == "externalId":
869
+ prop.property_ = "classicExternalId"
870
+ if self.reference_timeseries and prop.class_.suffix == "ClassicTimeSeries":
871
+ prop.value_type = Timeseries()
872
+ elif self.reference_files and prop.class_.suffix == "ClassicFile":
873
+ prop.value_type = File()
874
+ elif prop.property_ == "sourceExternalId" and prop.class_.suffix == "ClassicRelationship":
875
+ prop.property_ = "startNode"
876
+ elif prop.property_ == "targetExternalId" and prop.class_.suffix == "ClassicRelationship":
877
+ prop.property_ = "endNode"
878
+ instance_prefix = next(
879
+ (prefix for prefix, namespace in output.prefixes.items() if namespace == self.instance_namespace), None
880
+ )
881
+ if instance_prefix is None:
882
+ raise NeatValueError("Instance namespace not found in the prefixes.")
883
+
884
+ output.properties.append(
885
+ InformationProperty(
886
+ neatId=namespace["ClassicSourceSystem/name"],
887
+ property_="name",
888
+ value_type=String(),
889
+ class_=ClassEntity(prefix=prefix, suffix="ClassicSourceSystem"),
890
+ max_count=1,
891
+ instance_source=RDFPath(
892
+ traversal=SingleProperty(
893
+ class_=RDFPathEntity(
894
+ prefix=instance_prefix,
895
+ suffix="ClassicSourceSystem",
896
+ ),
897
+ property=RDFPathEntity(prefix=instance_prefix, suffix="name"),
898
+ ),
899
+ ),
900
+ )
901
+ )
902
+ return output
903
+
904
+
905
+ class ChangeViewPrefix(RulesTransformer[DMSRules, DMSRules]):
906
+ def __init__(self, old: str, new: str) -> None:
907
+ self.old = old
908
+ self.new = new
909
+
910
+ def transform(self, rules: DMSRules) -> DMSRules:
911
+ output = rules.model_copy(deep=True)
912
+ new_by_old: dict[ViewEntity, ViewEntity] = {}
913
+ for view in output.views:
914
+ if view.view.external_id.startswith(self.old):
915
+ new_external_id = f"{self.new}{view.view.external_id.removeprefix(self.old)}"
916
+ new_view_entity = view.view.copy(update={"suffix": new_external_id})
917
+ new_by_old[view.view] = new_view_entity
918
+ view.view = new_view_entity
919
+ for view in output.views:
920
+ if view.implements:
921
+ view.implements = [new_by_old.get(implemented, implemented) for implemented in view.implements]
922
+ for prop in output.properties:
923
+ if prop.view in new_by_old:
924
+ prop.view = new_by_old[prop.view]
925
+ if prop.value_type in new_by_old and isinstance(prop.value_type, ViewEntity):
926
+ prop.value_type = new_by_old[prop.value_type]
927
+ return output
928
+
929
+
930
+ class MergeDMSRules(RulesTransformer[DMSRules, DMSRules]):
931
+ def __init__(self, extra: DMSRules) -> None:
932
+ self.extra = extra
933
+
934
+ def transform(self, rules: DMSRules) -> DMSRules:
935
+ output = rules.model_copy(deep=True)
936
+ existing_views = {view.view for view in output.views}
937
+ for view in self.extra.views:
938
+ if view.view not in existing_views:
939
+ output.views.append(view)
940
+ existing_properties = {(prop.view, prop.view_property) for prop in output.properties}
941
+ existing_containers = {container.container for container in output.containers or []}
942
+ existing_enum_collections = {collection.collection for collection in output.enum or []}
943
+ new_containers_by_entity = {container.container: container for container in self.extra.containers or []}
944
+ new_enum_collections_by_entity = {collection.collection: collection for collection in self.extra.enum or []}
945
+ for prop in self.extra.properties:
946
+ if (prop.view, prop.view_property) in existing_properties:
947
+ continue
948
+ output.properties.append(prop)
949
+ if prop.container and prop.container not in existing_containers:
950
+ if output.containers is None:
951
+ output.containers = SheetList[DMSContainer]()
952
+ output.containers.append(new_containers_by_entity[prop.container])
953
+ if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections:
954
+ if output.enum is None:
955
+ output.enum = SheetList[DMSEnum]()
956
+ output.enum.append(new_enum_collections_by_entity[prop.value_type.collection])
957
+
958
+ existing_nodes = {node.node for node in output.nodes or []}
959
+ for node in self.extra.nodes or []:
960
+ if node.node not in existing_nodes:
961
+ if output.nodes is None:
962
+ output.nodes = SheetList[DMSNode]()
963
+ output.nodes.append(node)
964
+
965
+ return output
966
+
967
+ @property
968
+ def description(self) -> str:
969
+ return f"Merged with {self.extra.metadata.as_data_model_id()}"
970
+
971
+
972
+ class MergeInformationRules(RulesTransformer[InformationRules, InformationRules]):
973
+ def __init__(self, extra: InformationRules) -> None:
974
+ self.extra = extra
975
+
976
+ def transform(self, rules: InformationRules) -> InformationRules:
977
+ output = rules.model_copy(deep=True)
978
+ existing_classes = {cls.class_ for cls in output.classes}
979
+ for cls in self.extra.classes:
980
+ if cls.class_ not in existing_classes:
981
+ output.classes.append(cls)
982
+ existing_properties = {(prop.class_, prop.property_) for prop in output.properties}
983
+ for prop in self.extra.properties:
984
+ if (prop.class_, prop.property_) not in existing_properties:
985
+ output.properties.append(prop)
986
+ for prefix, namespace in self.extra.prefixes.items():
987
+ if prefix not in output.prefixes:
988
+ output.prefixes[prefix] = namespace
989
+ return output
990
+
991
+
811
992
  class _InformationRulesConverter:
812
- _edge_properties: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
993
+ _start_or_end_node: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
813
994
 
814
995
  def __init__(self, information: InformationRules):
815
996
  self.rules = information
816
997
  self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
817
998
 
818
999
  def as_dms_rules(
819
- self, ignore_undefined_value_types: bool = False, mode: Literal["edge_properties"] | None = None
1000
+ self, ignore_undefined_value_types: bool = False, reserved_properties: Literal["error", "skip"] = "error"
820
1001
  ) -> "DMSRules":
821
1002
  from cognite.neat._rules.models.dms._rules import (
822
1003
  DMSContainer,
@@ -829,35 +1010,51 @@ class _InformationRulesConverter:
829
1010
  default_version = info_metadata.version
830
1011
  default_space = self._to_space(info_metadata.prefix)
831
1012
  dms_metadata = self._convert_metadata_to_dms(info_metadata)
832
- edge_classes: set[ClassEntity] = set()
833
- property_to_edge: dict[tuple[ClassEntity, str], ClassEntity] = {}
834
- end_node_by_edge: dict[ClassEntity, ClassEntity] = {}
835
- if mode == "edge_properties":
836
- edge_classes = {
837
- cls_.class_ for cls_ in self.rules.classes if cls_.implements and cls_.implements[0].suffix == "Edge"
838
- }
839
- property_to_edge = {
840
- (prop.class_, prop.property_): prop.value_type
841
- for prop in self.rules.properties
842
- if prop.value_type in edge_classes and isinstance(prop.value_type, ClassEntity)
843
- }
844
- end_node_by_edge = {
845
- prop.class_: prop.value_type
846
- for prop in self.rules.properties
847
- if prop.class_ in edge_classes
848
- and (prop.property_ == "endNode" or prop.property_ == "end_node")
849
- and isinstance(prop.value_type, ClassEntity)
850
- }
1013
+
1014
+ properties_by_class: dict[ClassEntity, set[str]] = defaultdict(set)
1015
+ for prop in self.rules.properties:
1016
+ properties_by_class[prop.class_].add(prop.property_)
1017
+
1018
+ # Edge Classes is defined by having both startNode and endNode properties
1019
+ edge_classes = {
1020
+ cls_
1021
+ for cls_, class_properties in properties_by_class.items()
1022
+ if ({"startNode", "start_node"} & class_properties) and ({"endNode", "end_node"} & class_properties)
1023
+ }
1024
+ edge_value_types_by_class_property_pair = {
1025
+ (prop.class_, prop.property_): prop.value_type
1026
+ for prop in self.rules.properties
1027
+ if prop.value_type in edge_classes and isinstance(prop.value_type, ClassEntity)
1028
+ }
1029
+ end_node_by_edge = {
1030
+ prop.class_: prop.value_type
1031
+ for prop in self.rules.properties
1032
+ if prop.class_ in edge_classes
1033
+ and (prop.property_ == "endNode" or prop.property_ == "end_node")
1034
+ and isinstance(prop.value_type, ClassEntity)
1035
+ }
851
1036
 
852
1037
  properties_by_class: dict[ClassEntity, list[DMSProperty]] = defaultdict(list)
853
1038
  referenced_containers: dict[ContainerEntity, Counter[ClassEntity]] = defaultdict(Counter)
854
1039
  for prop in self.rules.properties:
855
1040
  if ignore_undefined_value_types and isinstance(prop.value_type, UnknownEntity):
856
1041
  continue
857
- if prop.class_ in edge_classes and prop.property_ in self._edge_properties:
1042
+ if prop.class_ in edge_classes and prop.property_ in self._start_or_end_node:
1043
+ continue
1044
+ if prop.property_ in DMS_RESERVED_PROPERTIES:
1045
+ msg = f"Property {prop.property_} is a reserved property in DMS."
1046
+ if reserved_properties == "error":
1047
+ raise NeatValueError(msg)
1048
+ warnings.warn(NeatValueWarning(f"{msg} Skipping..."), stacklevel=2)
858
1049
  continue
1050
+
859
1051
  dms_property = self._as_dms_property(
860
- prop, default_space, default_version, edge_classes, property_to_edge, end_node_by_edge
1052
+ prop,
1053
+ default_space,
1054
+ default_version,
1055
+ edge_classes,
1056
+ edge_value_types_by_class_property_pair,
1057
+ end_node_by_edge,
861
1058
  )
862
1059
  properties_by_class[prop.class_].append(dms_property)
863
1060
  if dms_property.container:
@@ -870,12 +1067,10 @@ class _InformationRulesConverter:
870
1067
  name=cls_.name,
871
1068
  view=cls_.class_.as_view_entity(default_space, default_version),
872
1069
  description=cls_.description,
873
- implements=self._get_view_implements(cls_, info_metadata, mode),
1070
+ implements=self._get_view_implements(cls_, info_metadata),
874
1071
  )
875
1072
 
876
1073
  dms_view.logical = cls_.neatId
877
- cls_.physical = dms_view.neatId
878
-
879
1074
  views.append(dms_view)
880
1075
 
881
1076
  class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
@@ -891,21 +1086,34 @@ class _InformationRulesConverter:
891
1086
  )
892
1087
  most_used_class_entity = class_entities.most_common(1)[0][0]
893
1088
  class_ = class_by_entity[most_used_class_entity]
1089
+
1090
+ if len(set(class_entities) - set(edge_classes)) == 0:
1091
+ used_for: Literal["node", "edge", "all"] = "edge"
1092
+ elif len(set(class_entities) - set(edge_classes)) == len(class_entities):
1093
+ used_for = "node"
1094
+ else:
1095
+ used_for = "all"
1096
+
894
1097
  container = DMSContainer(
895
1098
  container=container_entity,
896
1099
  name=class_.name,
897
1100
  description=class_.description,
898
1101
  constraint=constrains or None,
1102
+ used_for=used_for,
899
1103
  )
900
1104
  containers.append(container)
901
1105
 
902
- return DMSRules(
1106
+ dms_rules = DMSRules(
903
1107
  metadata=dms_metadata,
904
1108
  properties=SheetList[DMSProperty]([prop for prop_set in properties_by_class.values() for prop in prop_set]),
905
1109
  views=SheetList[DMSView](views),
906
1110
  containers=SheetList[DMSContainer](containers),
907
1111
  )
908
1112
 
1113
+ self.rules.sync_with_dms_rules(dms_rules)
1114
+
1115
+ return dms_rules
1116
+
909
1117
  @staticmethod
910
1118
  def _create_container_constraint(
911
1119
  class_entities: Counter[ClassEntity],
@@ -939,8 +1147,6 @@ class _InformationRulesConverter:
939
1147
  )
940
1148
 
941
1149
  dms_metadata.logical = metadata.identifier
942
- metadata.physical = dms_metadata.identifier
943
-
944
1150
  return dms_metadata
945
1151
 
946
1152
  def _as_dms_property(
@@ -949,7 +1155,7 @@ class _InformationRulesConverter:
949
1155
  default_space: str,
950
1156
  default_version: str,
951
1157
  edge_classes: set[ClassEntity],
952
- property_to_edge: dict[tuple[ClassEntity, str], ClassEntity],
1158
+ edge_value_types_by_class_property_pair: dict[tuple[ClassEntity, str], ClassEntity],
953
1159
  end_node_by_edge: dict[ClassEntity, ClassEntity],
954
1160
  ) -> "DMSProperty":
955
1161
  from cognite.neat._rules.models.dms._rules import DMSProperty
@@ -963,7 +1169,9 @@ class _InformationRulesConverter:
963
1169
  end_node_by_edge,
964
1170
  )
965
1171
 
966
- connection = self._get_connection(info_property, value_type, property_to_edge, default_space, default_version)
1172
+ connection = self._get_connection(
1173
+ info_property, value_type, edge_value_types_by_class_property_pair, default_space, default_version
1174
+ )
967
1175
 
968
1176
  container: ContainerEntity | None = None
969
1177
  container_property: str | None = None
@@ -992,7 +1200,6 @@ class _InformationRulesConverter:
992
1200
 
993
1201
  # linking
994
1202
  dms_property.logical = info_property.neatId
995
- info_property.physical = dms_property.neatId
996
1203
 
997
1204
  return dms_property
998
1205
 
@@ -1000,13 +1207,16 @@ class _InformationRulesConverter:
1000
1207
  def _get_connection(
1001
1208
  prop: InformationProperty,
1002
1209
  value_type: DataType | ViewEntity | DMSUnknownEntity,
1003
- property_to_edge: dict[tuple[ClassEntity, str], ClassEntity],
1210
+ edge_value_types_by_class_property_pair: dict[tuple[ClassEntity, str], ClassEntity],
1004
1211
  default_space: str,
1005
1212
  default_version: str,
1006
1213
  ) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
1007
- if isinstance(value_type, ViewEntity) and (prop.class_, prop.property_) in property_to_edge:
1008
- edge_properties = property_to_edge[(prop.class_, prop.property_)]
1009
- return EdgeEntity(properties=edge_properties.as_view_entity(default_space, default_version))
1214
+ if (
1215
+ isinstance(value_type, ViewEntity)
1216
+ and (prop.class_, prop.property_) in edge_value_types_by_class_property_pair
1217
+ ):
1218
+ edge_value_type = edge_value_types_by_class_property_pair[(prop.class_, prop.property_)]
1219
+ return EdgeEntity(properties=edge_value_type.as_view_entity(default_space, default_version))
1010
1220
  if isinstance(value_type, ViewEntity) and prop.is_list:
1011
1221
  return EdgeEntity()
1012
1222
  elif isinstance(value_type, ViewEntity):
@@ -1035,6 +1245,7 @@ class _InformationRulesConverter:
1035
1245
  elif isinstance(prop.value_type, ClassEntity) and (prop.value_type in edge_classes):
1036
1246
  if prop.value_type in end_node_by_edge:
1037
1247
  return end_node_by_edge[prop.value_type].as_view_entity(default_space, default_version)
1248
+ # This occurs if the end node is not pointing to a class
1038
1249
  warnings.warn(
1039
1250
  NeatValueWarning(
1040
1251
  f"Edge class {prop.value_type} does not have 'endNode' property, defaulting to DMSUnknownEntity"
@@ -1091,13 +1302,9 @@ class _InformationRulesConverter:
1091
1302
  self.property_count_by_container[container_entity] += 1
1092
1303
  return container_entity, prop.property_
1093
1304
 
1094
- def _get_view_implements(
1095
- self, cls_: InformationClass, metadata: InformationMetadata, mode: Literal["edge_properties"] | None
1096
- ) -> list[ViewEntity]:
1305
+ def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
1097
1306
  implements = []
1098
1307
  for parent in cls_.implements or []:
1099
- if mode == "edge_properties" and parent.suffix == "Edge":
1100
- continue
1101
1308
  view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
1102
1309
  implements.append(view_entity)
1103
1310
  return implements
@@ -1136,8 +1343,9 @@ class _InformationRulesConverter:
1136
1343
 
1137
1344
 
1138
1345
  class _DMSRulesConverter:
1139
- def __init__(self, dms: DMSRules):
1346
+ def __init__(self, dms: DMSRules, instance_namespace: Namespace | None = None) -> None:
1140
1347
  self.dms = dms
1348
+ self.instance_namespace = instance_namespace
1141
1349
 
1142
1350
  def as_information_rules(
1143
1351
  self,
@@ -1152,8 +1360,9 @@ class _DMSRulesConverter:
1152
1360
 
1153
1361
  metadata = self._convert_metadata_to_info(dms)
1154
1362
 
1155
- classes = [
1156
- InformationClass(
1363
+ classes: list[InformationClass] = []
1364
+ for view in self.dms.views:
1365
+ info_class = InformationClass(
1157
1366
  # we do not want a version in class as we use URI for the class
1158
1367
  class_=ClassEntity(prefix=view.view.prefix, suffix=view.view.suffix),
1159
1368
  description=view.description,
@@ -1163,8 +1372,19 @@ class _DMSRulesConverter:
1163
1372
  for implemented_view in view.implements or []
1164
1373
  ],
1165
1374
  )
1166
- for view in self.dms.views
1167
- ]
1375
+
1376
+ # Linking
1377
+ info_class.physical = view.neatId
1378
+ classes.append(info_class)
1379
+
1380
+ prefixes = get_default_prefixes_and_namespaces()
1381
+ instance_prefix: str | None = None
1382
+ if self.instance_namespace:
1383
+ instance_prefix = next((k for k, v in prefixes.items() if v == self.instance_namespace), None)
1384
+ if instance_prefix is None:
1385
+ # We need to add a new prefix
1386
+ instance_prefix = f"prefix_{len(prefixes) + 1}"
1387
+ prefixes[instance_prefix] = self.instance_namespace
1168
1388
 
1169
1389
  properties: list[InformationProperty] = []
1170
1390
  value_type: DataType | ClassEntity | str
@@ -1181,24 +1401,42 @@ class _DMSRulesConverter:
1181
1401
  else:
1182
1402
  raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
1183
1403
 
1184
- properties.append(
1185
- InformationProperty(
1186
- # Removing version
1187
- class_=ClassEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
1188
- property_=property_.view_property,
1189
- value_type=value_type,
1190
- description=property_.description,
1191
- min_count=(0 if property_.nullable or property_.nullable is None else 1),
1192
- max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
1404
+ transformation: RDFPath | None = None
1405
+ if instance_prefix is not None:
1406
+ transformation = RDFPath(
1407
+ traversal=SingleProperty(
1408
+ class_=RDFPathEntity(prefix=instance_prefix, suffix=property_.view.external_id),
1409
+ property=RDFPathEntity(prefix=instance_prefix, suffix=property_.view_property),
1410
+ )
1193
1411
  )
1412
+
1413
+ info_property = InformationProperty(
1414
+ # Removing version
1415
+ class_=ClassEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
1416
+ property_=property_.view_property,
1417
+ value_type=value_type,
1418
+ description=property_.description,
1419
+ min_count=(0 if property_.nullable or property_.nullable is None else 1),
1420
+ max_count=(float("inf") if property_.is_list or property_.nullable is None else 1),
1421
+ instance_source=transformation,
1194
1422
  )
1195
1423
 
1196
- return InformationRules(
1424
+ # Linking
1425
+ info_property.physical = property_.neatId
1426
+
1427
+ properties.append(info_property)
1428
+
1429
+ info_rules = InformationRules(
1197
1430
  metadata=metadata,
1198
1431
  properties=SheetList[InformationProperty](properties),
1199
1432
  classes=SheetList[InformationClass](classes),
1433
+ prefixes=prefixes,
1200
1434
  )
1201
1435
 
1436
+ self.dms.sync_with_info_rules(info_rules)
1437
+
1438
+ return info_rules
1439
+
1202
1440
  @classmethod
1203
1441
  def _convert_metadata_to_info(cls, metadata: DMSMetadata) -> "InformationMetadata":
1204
1442
  from cognite.neat._rules.models.information._rules import InformationMetadata
@@ -7,11 +7,21 @@ from cognite.neat import _version
7
7
  from cognite.neat._client import NeatClient
8
8
  from cognite.neat._issues import IssueList
9
9
  from cognite.neat._issues.errors import RegexViolationError
10
+ from cognite.neat._issues.errors._general import NeatImportError
10
11
  from cognite.neat._rules import importers
11
12
  from cognite.neat._rules.models._base_input import InputRules
12
13
  from cognite.neat._rules.models.information._rules import InformationRules
13
- from cognite.neat._rules.transformers import ConvertToRules, InformationToDMS, VerifyAnyRules
14
- from cognite.neat._rules.transformers._converters import ConversionTransformer
14
+ from cognite.neat._rules.transformers import (
15
+ ConversionTransformer,
16
+ ConvertToRules,
17
+ InformationToDMS,
18
+ MergeDMSRules,
19
+ MergeInformationRules,
20
+ VerifyAnyRules,
21
+ VerifyInformationRules,
22
+ )
23
+ from cognite.neat._store._rules_store import ModelEntity
24
+ from cognite.neat._utils.auxiliary import local_import
15
25
 
16
26
  from ._collector import _COLLECTOR, Collector
17
27
  from ._drop import DropAPI
@@ -70,12 +80,15 @@ class NeatSession:
70
80
  def __init__(
71
81
  self,
72
82
  client: CogniteClient | None = None,
73
- storage: Literal["memory", "oxigraph"] = "memory",
83
+ storage: Literal["memory", "oxigraph"] | None = None,
74
84
  verbose: bool = True,
75
85
  load_engine: Literal["newest", "cache", "skip"] = "cache",
76
86
  ) -> None:
77
87
  self._verbose = verbose
78
- self._state = SessionState(store_type=storage, client=NeatClient(client) if client else None)
88
+ self._state = SessionState(
89
+ store_type=storage or self._select_most_performant_store(),
90
+ client=NeatClient(client) if client else None,
91
+ )
79
92
  self.read = ReadAPI(self._state, verbose)
80
93
  self.to = ToAPI(self._state, verbose)
81
94
  self.prepare = PrepareAPI(self._state, verbose)
@@ -89,6 +102,16 @@ class NeatSession:
89
102
  if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
90
103
  print(f"Neat Engine {engine_version} loaded.")
91
104
 
105
+ def _select_most_performant_store(self) -> Literal["memory", "oxigraph"]:
106
+ """Select the most performant store based on the current environment."""
107
+
108
+ try:
109
+ local_import("pyoxigraph", "oxi")
110
+ local_import("oxrdflib", "oxi")
111
+ return "oxigraph"
112
+ except NeatImportError:
113
+ return "memory"
114
+
92
115
  @property
93
116
  def version(self) -> str:
94
117
  """Get the current version of neat.
@@ -129,15 +152,11 @@ class NeatSession:
129
152
  print("You can inspect the issues with the .inspect.issues(...) method.")
130
153
  return issues
131
154
 
132
- def convert(
133
- self, target: Literal["dms", "information"], mode: Literal["edge_properties"] | None = None
134
- ) -> IssueList:
155
+ def convert(self, target: Literal["dms", "information"]) -> IssueList:
135
156
  """Converts the last verified data model to the target type.
136
157
 
137
158
  Args:
138
159
  target: The target type to convert the data model to.
139
- mode: If the target is "dms", the mode to use for the conversion. None is used for default conversion.
140
- "edge_properties" treas classes that implements Edge as edge properties.
141
160
 
142
161
  Example:
143
162
  Convert to DMS rules
@@ -153,7 +172,7 @@ class NeatSession:
153
172
  """
154
173
  converter: ConversionTransformer
155
174
  if target == "dms":
156
- converter = InformationToDMS(mode=mode)
175
+ converter = InformationToDMS()
157
176
  elif target == "information":
158
177
  converter = ConvertToRules(InformationRules)
159
178
  else:
@@ -203,6 +222,34 @@ class NeatSession:
203
222
  )
204
223
  return self._state.rule_import(importer)
205
224
 
225
+ def _infer_subclasses(self) -> IssueList:
226
+ """Infer the subclass of instances."""
227
+ last_information = self._state.rule_store.last_verified_information_rules
228
+ issue_list = IssueList()
229
+ importer = importers.SubclassInferenceImporter(
230
+ issue_list=issue_list,
231
+ graph=self._state.instances.store.graph(),
232
+ rules=last_information,
233
+ max_number_of_instance=-1,
234
+ )
235
+
236
+ unverified_information = importer.to_rules()
237
+ verified_information = VerifyInformationRules().transform(unverified_information)
238
+
239
+ # Hack into the last information rules to merge the rules with the last verified information rules.
240
+ # This is to be able to populate the instances store with the inferred subclasses.
241
+ provenance = self._state.rule_store.provenance
242
+ for change in reversed(provenance):
243
+ target_entity = change.target_entity
244
+ if isinstance(target_entity, ModelEntity) and isinstance(target_entity.result, InformationRules):
245
+ last_information_rules = change.target_entity.result
246
+ new_information_rules = MergeInformationRules(verified_information).transform(last_information_rules)
247
+ object.__setattr__(change.target_entity, "result", new_information_rules)
248
+ break
249
+
250
+ dms_rules = InformationToDMS(reserved_properties="skip").transform(verified_information)
251
+ return self._state.rule_transform(MergeDMSRules(dms_rules))
252
+
206
253
  def _repr_html_(self) -> str:
207
254
  state = self._state
208
255
  if (