cognite-neat 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (84) hide show
  1. cognite/neat/_client/_api/data_modeling_loaders.py +390 -116
  2. cognite/neat/_client/_api/schema.py +63 -2
  3. cognite/neat/_client/data_classes/data_modeling.py +4 -0
  4. cognite/neat/_client/data_classes/schema.py +2 -348
  5. cognite/neat/_constants.py +27 -4
  6. cognite/neat/_graph/extractors/_base.py +7 -0
  7. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +28 -18
  8. cognite/neat/_graph/loaders/_rdf2dms.py +52 -13
  9. cognite/neat/_graph/transformers/__init__.py +3 -3
  10. cognite/neat/_graph/transformers/_classic_cdf.py +135 -56
  11. cognite/neat/_issues/_base.py +26 -17
  12. cognite/neat/_issues/errors/__init__.py +4 -2
  13. cognite/neat/_issues/errors/_external.py +7 -0
  14. cognite/neat/_issues/errors/_properties.py +2 -7
  15. cognite/neat/_issues/errors/_resources.py +1 -1
  16. cognite/neat/_issues/warnings/__init__.py +6 -2
  17. cognite/neat/_issues/warnings/_external.py +9 -1
  18. cognite/neat/_issues/warnings/_resources.py +41 -2
  19. cognite/neat/_issues/warnings/user_modeling.py +4 -4
  20. cognite/neat/_rules/_constants.py +2 -6
  21. cognite/neat/_rules/analysis/_base.py +15 -5
  22. cognite/neat/_rules/analysis/_dms.py +20 -0
  23. cognite/neat/_rules/analysis/_information.py +22 -0
  24. cognite/neat/_rules/exporters/_base.py +3 -5
  25. cognite/neat/_rules/exporters/_rules2dms.py +190 -200
  26. cognite/neat/_rules/importers/__init__.py +1 -3
  27. cognite/neat/_rules/importers/_base.py +1 -1
  28. cognite/neat/_rules/importers/_dms2rules.py +3 -25
  29. cognite/neat/_rules/importers/_rdf/__init__.py +5 -0
  30. cognite/neat/_rules/importers/_rdf/_base.py +34 -11
  31. cognite/neat/_rules/importers/_rdf/_imf2rules.py +91 -0
  32. cognite/neat/_rules/importers/_rdf/_inference2rules.py +40 -7
  33. cognite/neat/_rules/importers/_rdf/_owl2rules.py +80 -0
  34. cognite/neat/_rules/importers/_rdf/_shared.py +138 -441
  35. cognite/neat/_rules/models/_base_rules.py +19 -0
  36. cognite/neat/_rules/models/_types.py +5 -0
  37. cognite/neat/_rules/models/dms/__init__.py +2 -0
  38. cognite/neat/_rules/models/dms/_exporter.py +247 -123
  39. cognite/neat/_rules/models/dms/_rules.py +7 -49
  40. cognite/neat/_rules/models/dms/_rules_input.py +8 -3
  41. cognite/neat/_rules/models/dms/_validation.py +421 -123
  42. cognite/neat/_rules/models/entities/_multi_value.py +3 -0
  43. cognite/neat/_rules/models/information/__init__.py +2 -0
  44. cognite/neat/_rules/models/information/_rules.py +17 -61
  45. cognite/neat/_rules/models/information/_rules_input.py +11 -2
  46. cognite/neat/_rules/models/information/_validation.py +107 -11
  47. cognite/neat/_rules/models/mapping/_classic2core.py +1 -1
  48. cognite/neat/_rules/models/mapping/_classic2core.yaml +8 -4
  49. cognite/neat/_rules/transformers/__init__.py +2 -1
  50. cognite/neat/_rules/transformers/_converters.py +163 -61
  51. cognite/neat/_rules/transformers/_mapping.py +132 -2
  52. cognite/neat/_rules/transformers/_pipelines.py +1 -1
  53. cognite/neat/_rules/transformers/_verification.py +29 -4
  54. cognite/neat/_session/_base.py +46 -60
  55. cognite/neat/_session/_mapping.py +105 -5
  56. cognite/neat/_session/_prepare.py +49 -14
  57. cognite/neat/_session/_read.py +50 -4
  58. cognite/neat/_session/_set.py +1 -0
  59. cognite/neat/_session/_to.py +38 -12
  60. cognite/neat/_session/_wizard.py +5 -0
  61. cognite/neat/_session/engine/_interface.py +3 -2
  62. cognite/neat/_session/exceptions.py +4 -0
  63. cognite/neat/_store/_base.py +79 -19
  64. cognite/neat/_utils/collection_.py +22 -0
  65. cognite/neat/_utils/rdf_.py +30 -4
  66. cognite/neat/_version.py +2 -2
  67. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +3 -91
  68. cognite/neat/_workflows/steps/lib/current/rules_importer.py +2 -16
  69. cognite/neat/_workflows/steps/lib/current/rules_validator.py +3 -5
  70. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/METADATA +1 -1
  71. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/RECORD +74 -82
  72. cognite/neat/_rules/importers/_rdf/_imf2rules/__init__.py +0 -3
  73. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +0 -86
  74. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +0 -29
  75. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +0 -130
  76. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2rules.py +0 -154
  77. cognite/neat/_rules/importers/_rdf/_owl2rules/__init__.py +0 -3
  78. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +0 -58
  79. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +0 -65
  80. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +0 -59
  81. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2rules.py +0 -39
  82. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/LICENSE +0 -0
  83. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/WHEEL +0 -0
  84. {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/entry_points.txt +0 -0
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
4
4
  from collections import Counter, defaultdict
5
5
  from collections.abc import Collection, Mapping
6
6
  from datetime import date, datetime
7
- from typing import Literal, TypeVar, cast, overload
7
+ from typing import ClassVar, Literal, TypeVar, cast, overload
8
8
 
9
9
  from cognite.client.data_classes import data_modeling as dms
10
10
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
@@ -15,6 +15,7 @@ from cognite.neat._constants import (
15
15
  )
16
16
  from cognite.neat._issues._base import IssueList
17
17
  from cognite.neat._issues.errors import NeatValueError
18
+ from cognite.neat._issues.warnings import NeatValueWarning
18
19
  from cognite.neat._issues.warnings._models import (
19
20
  EnterpriseModelNotBuildOnTopOfCDMWarning,
20
21
  SolutionModelBuildOnTopOfCDMWarning,
@@ -34,7 +35,7 @@ from cognite.neat._rules.models import (
34
35
  SheetList,
35
36
  data_types,
36
37
  )
37
- from cognite.neat._rules.models.data_types import DataType, String
38
+ from cognite.neat._rules.models.data_types import AnyURI, DataType, String
38
39
  from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSView
39
40
  from cognite.neat._rules.models.dms._rules import DMSContainer
40
41
  from cognite.neat._rules.models.entities import (
@@ -249,11 +250,12 @@ class PrefixEntities(RulesTransformer[InputRules, InputRules]): # type: ignore[
249
250
  class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
250
251
  """Converts InformationRules to DMSRules."""
251
252
 
252
- def __init__(self, ignore_undefined_value_types: bool = False):
253
+ def __init__(self, ignore_undefined_value_types: bool = False, mode: Literal["edge_properties"] | None = None):
253
254
  self.ignore_undefined_value_types = ignore_undefined_value_types
255
+ self.mode = mode
254
256
 
255
257
  def _transform(self, rules: InformationRules) -> DMSRules:
256
- return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types)
258
+ return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types, self.mode)
257
259
 
258
260
 
259
261
  class DMSToInformation(ConversionTransformer[DMSRules, InformationRules]):
@@ -637,11 +639,15 @@ class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
637
639
 
638
640
 
639
641
  class _InformationRulesConverter:
642
+ _edge_properties: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
643
+
640
644
  def __init__(self, information: InformationRules):
641
645
  self.rules = information
642
646
  self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
643
647
 
644
- def as_dms_rules(self, ignore_undefined_value_types: bool = False) -> "DMSRules":
648
+ def as_dms_rules(
649
+ self, ignore_undefined_value_types: bool = False, mode: Literal["edge_properties"] | None = None
650
+ ) -> "DMSRules":
645
651
  from cognite.neat._rules.models.dms._rules import (
646
652
  DMSContainer,
647
653
  DMSProperty,
@@ -652,27 +658,55 @@ class _InformationRulesConverter:
652
658
  info_metadata = self.rules.metadata
653
659
  default_version = info_metadata.version
654
660
  default_space = self._to_space(info_metadata.prefix)
655
- metadata = self._convert_metadata_to_dms(info_metadata)
661
+ dms_metadata = self._convert_metadata_to_dms(info_metadata)
662
+ edge_classes: set[ClassEntity] = set()
663
+ property_to_edge: dict[tuple[ClassEntity, str], ClassEntity] = {}
664
+ end_node_by_edge: dict[ClassEntity, ClassEntity] = {}
665
+ if mode == "edge_properties":
666
+ edge_classes = {
667
+ cls_.class_ for cls_ in self.rules.classes if cls_.implements and cls_.implements[0].suffix == "Edge"
668
+ }
669
+ property_to_edge = {
670
+ (prop.class_, prop.property_): prop.value_type
671
+ for prop in self.rules.properties
672
+ if prop.value_type in edge_classes and isinstance(prop.value_type, ClassEntity)
673
+ }
674
+ end_node_by_edge = {
675
+ prop.class_: prop.value_type
676
+ for prop in self.rules.properties
677
+ if prop.class_ in edge_classes
678
+ and (prop.property_ == "endNode" or prop.property_ == "end_node")
679
+ and isinstance(prop.value_type, ClassEntity)
680
+ }
656
681
 
657
682
  properties_by_class: dict[ClassEntity, list[DMSProperty]] = defaultdict(list)
658
683
  referenced_containers: dict[ContainerEntity, Counter[ClassEntity]] = defaultdict(Counter)
659
684
  for prop in self.rules.properties:
660
685
  if ignore_undefined_value_types and isinstance(prop.value_type, UnknownEntity):
661
686
  continue
662
- dms_property = self._as_dms_property(prop, default_space, default_version)
687
+ if prop.class_ in edge_classes and prop.property_ in self._edge_properties:
688
+ continue
689
+ dms_property = self._as_dms_property(
690
+ prop, default_space, default_version, edge_classes, property_to_edge, end_node_by_edge
691
+ )
663
692
  properties_by_class[prop.class_].append(dms_property)
664
693
  if dms_property.container:
665
694
  referenced_containers[dms_property.container][prop.class_] += 1
666
695
 
667
- views: list[DMSView] = [
668
- DMSView(
696
+ views: list[DMSView] = []
697
+
698
+ for cls_ in self.rules.classes:
699
+ dms_view = DMSView(
669
700
  name=cls_.name,
670
701
  view=cls_.class_.as_view_entity(default_space, default_version),
671
702
  description=cls_.description,
672
- implements=self._get_view_implements(cls_, info_metadata),
703
+ implements=self._get_view_implements(cls_, info_metadata, mode),
673
704
  )
674
- for cls_ in self.rules.classes
675
- ]
705
+
706
+ dms_view.logical = cls_.neatId
707
+ cls_.physical = dms_view.neatId
708
+
709
+ views.append(dms_view)
676
710
 
677
711
  class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
678
712
 
@@ -696,7 +730,7 @@ class _InformationRulesConverter:
696
730
  containers.append(container)
697
731
 
698
732
  return DMSRules(
699
- metadata=metadata,
733
+ metadata=dms_metadata,
700
734
  properties=SheetList[DMSProperty]([prop for prop_set in properties_by_class.values() for prop in prop_set]),
701
735
  views=SheetList[DMSView](views),
702
736
  containers=SheetList[DMSContainer](containers),
@@ -724,7 +758,7 @@ class _InformationRulesConverter:
724
758
  DMSMetadata,
725
759
  )
726
760
 
727
- return DMSMetadata(
761
+ dms_metadata = DMSMetadata(
728
762
  space=metadata.space,
729
763
  version=metadata.version,
730
764
  external_id=metadata.external_id,
@@ -734,74 +768,138 @@ class _InformationRulesConverter:
734
768
  updated=metadata.updated,
735
769
  )
736
770
 
737
- def _as_dms_property(self, prop: InformationProperty, default_space: str, default_version: str) -> "DMSProperty":
738
- """This creates the first"""
771
+ dms_metadata.logical = metadata.identifier
772
+ metadata.physical = dms_metadata.identifier
773
+
774
+ return dms_metadata
739
775
 
776
+ def _as_dms_property(
777
+ self,
778
+ info_property: InformationProperty,
779
+ default_space: str,
780
+ default_version: str,
781
+ edge_classes: set[ClassEntity],
782
+ property_to_edge: dict[tuple[ClassEntity, str], ClassEntity],
783
+ end_node_by_edge: dict[ClassEntity, ClassEntity],
784
+ ) -> "DMSProperty":
740
785
  from cognite.neat._rules.models.dms._rules import DMSProperty
741
786
 
742
787
  # returns property type, which can be ObjectProperty or DatatypeProperty
743
- value_type: DataType | ViewEntity | DMSUnknownEntity
788
+ value_type = self._get_value_type(
789
+ info_property,
790
+ default_space,
791
+ default_version,
792
+ edge_classes,
793
+ end_node_by_edge,
794
+ )
795
+
796
+ connection = self._get_connection(info_property, value_type, property_to_edge, default_space, default_version)
797
+
798
+ container: ContainerEntity | None = None
799
+ container_property: str | None = None
800
+ is_list: bool | None = info_property.is_list
801
+ nullable: bool | None = not info_property.is_mandatory
802
+ if isinstance(connection, EdgeEntity):
803
+ nullable = None
804
+ elif connection == "direct":
805
+ nullable = True
806
+ container, container_property = self._get_container(info_property, default_space)
807
+ else:
808
+ container, container_property = self._get_container(info_property, default_space)
809
+
810
+ dms_property = DMSProperty(
811
+ name=info_property.name,
812
+ value_type=value_type,
813
+ nullable=nullable,
814
+ is_list=is_list,
815
+ connection=connection,
816
+ default=info_property.default,
817
+ container=container,
818
+ container_property=container_property,
819
+ view=info_property.class_.as_view_entity(default_space, default_version),
820
+ view_property=info_property.property_,
821
+ )
822
+
823
+ # linking
824
+ dms_property.logical = info_property.neatId
825
+ info_property.physical = dms_property.neatId
826
+
827
+ return dms_property
828
+
829
+ @staticmethod
830
+ def _get_connection(
831
+ prop: InformationProperty,
832
+ value_type: DataType | ViewEntity | DMSUnknownEntity,
833
+ property_to_edge: dict[tuple[ClassEntity, str], ClassEntity],
834
+ default_space: str,
835
+ default_version: str,
836
+ ) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
837
+ if isinstance(value_type, ViewEntity) and (prop.class_, prop.property_) in property_to_edge:
838
+ edge_properties = property_to_edge[(prop.class_, prop.property_)]
839
+ return EdgeEntity(properties=edge_properties.as_view_entity(default_space, default_version))
840
+ if isinstance(value_type, ViewEntity) and prop.is_list:
841
+ return EdgeEntity()
842
+ elif isinstance(value_type, ViewEntity):
843
+ return "direct"
844
+ # defaulting to direct connection
845
+ elif isinstance(value_type, DMSUnknownEntity):
846
+ return "direct"
847
+ return None
848
+
849
+ def _get_value_type(
850
+ self,
851
+ prop: InformationProperty,
852
+ default_space: str,
853
+ default_version: str,
854
+ edge_classes: set[ClassEntity],
855
+ end_node_by_edge: dict[ClassEntity, ClassEntity],
856
+ ) -> DataType | ViewEntity | DMSUnknownEntity:
744
857
  if isinstance(prop.value_type, DataType):
745
- value_type = prop.value_type
858
+ return prop.value_type
746
859
 
747
860
  # UnknownEntity should resolve to DMSUnknownEntity
748
861
  # meaning end node type is unknown
749
862
  elif isinstance(prop.value_type, UnknownEntity):
750
- value_type = DMSUnknownEntity()
751
-
863
+ return DMSUnknownEntity()
864
+
865
+ elif isinstance(prop.value_type, ClassEntity) and (prop.value_type in edge_classes):
866
+ if prop.value_type in end_node_by_edge:
867
+ return end_node_by_edge[prop.value_type].as_view_entity(default_space, default_version)
868
+ warnings.warn(
869
+ NeatValueWarning(
870
+ f"Edge class {prop.value_type} does not have 'endNode' property, defaulting to DMSUnknownEntity"
871
+ ),
872
+ stacklevel=2,
873
+ )
874
+ return DMSUnknownEntity()
752
875
  elif isinstance(prop.value_type, ClassEntity):
753
- value_type = prop.value_type.as_view_entity(default_space, default_version)
876
+ return prop.value_type.as_view_entity(default_space, default_version)
754
877
 
755
878
  elif isinstance(prop.value_type, MultiValueTypeInfo):
756
879
  # Multi Object type should resolve to DMSUnknownEntity
757
880
  # meaning end node type is unknown
758
881
  if prop.value_type.is_multi_object_type():
759
- value_type = DMSUnknownEntity()
882
+ non_unknown = [type_ for type_ in prop.value_type.types if isinstance(type_, UnknownEntity)]
883
+ if list(non_unknown) == 1:
884
+ #
885
+ return non_unknown[0].as_view_entity(default_space, default_version)
886
+ return DMSUnknownEntity()
760
887
 
761
888
  # Multi Data type should resolve to a single data type, or it should
762
889
  elif prop.value_type.is_multi_data_type():
763
- value_type = self.convert_multi_data_type(prop.value_type)
890
+ return self.convert_multi_data_type(prop.value_type)
764
891
 
765
892
  # Mixed types default to string
766
893
  else:
767
- value_type = String()
768
-
769
- else:
770
- raise ValueError(f"Unsupported value type: {prop.value_type.type_}")
894
+ non_any_uri = [type_ for type_ in prop.value_type.types if type_ != AnyURI()]
895
+ if list(non_any_uri) == 1:
896
+ if isinstance(non_any_uri[0], ClassEntity):
897
+ return non_any_uri[0].as_view_entity(default_space, default_version)
898
+ else:
899
+ return non_any_uri[0]
900
+ return String()
771
901
 
772
- connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None = None
773
- if isinstance(value_type, ViewEntity):
774
- # Default connection type.
775
- connection = EdgeEntity() if prop.is_list else "direct"
776
-
777
- # defaulting to direct connection
778
- elif isinstance(value_type, DMSUnknownEntity):
779
- connection = "direct"
780
-
781
- container: ContainerEntity | None = None
782
- container_property: str | None = None
783
- is_list: bool | None = prop.is_list
784
- nullable: bool | None = not prop.is_mandatory
785
- if isinstance(connection, EdgeEntity):
786
- nullable = None
787
- elif connection == "direct":
788
- nullable = True
789
- container, container_property = self._get_container(prop, default_space)
790
- else:
791
- container, container_property = self._get_container(prop, default_space)
792
-
793
- return DMSProperty(
794
- name=prop.name,
795
- value_type=value_type,
796
- nullable=nullable,
797
- is_list=is_list,
798
- connection=connection,
799
- default=prop.default,
800
- container=container,
801
- container_property=container_property,
802
- view=prop.class_.as_view_entity(default_space, default_version),
803
- view_property=prop.property_,
804
- )
902
+ raise ValueError(f"Unsupported value type: {prop.value_type.type_}")
805
903
 
806
904
  @classmethod
807
905
  def _to_space(cls, prefix: str) -> str:
@@ -823,9 +921,13 @@ class _InformationRulesConverter:
823
921
  self.property_count_by_container[container_entity] += 1
824
922
  return container_entity, prop.property_
825
923
 
826
- def _get_view_implements(self, cls_: InformationClass, metadata: InformationMetadata) -> list[ViewEntity]:
924
+ def _get_view_implements(
925
+ self, cls_: InformationClass, metadata: InformationMetadata, mode: Literal["edge_properties"] | None
926
+ ) -> list[ViewEntity]:
827
927
  implements = []
828
928
  for parent in cls_.implements or []:
929
+ if mode == "edge_properties" and parent.suffix == "Edge":
930
+ continue
829
931
  view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
830
932
  implements.append(view_entity)
831
933
  return implements
@@ -4,13 +4,16 @@ from collections import defaultdict
4
4
  from functools import cached_property
5
5
  from typing import Any, ClassVar, Literal
6
6
 
7
- from cognite.neat._issues.errors import NeatValueError
7
+ from cognite.client import data_modeling as dm
8
+
9
+ from cognite.neat._client import NeatClient
10
+ from cognite.neat._issues.errors import CDFMissingClientError, NeatValueError, ResourceNotFoundError
8
11
  from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
9
12
  from cognite.neat._rules._shared import JustRules, OutRules
10
13
  from cognite.neat._rules.models import DMSRules, SheetList
11
14
  from cognite.neat._rules.models.data_types import Enum
12
15
  from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
13
- from cognite.neat._rules.models.entities import ViewEntity
16
+ from cognite.neat._rules.models.entities import ContainerEntity, ViewEntity
14
17
 
15
18
  from ._base import RulesTransformer
16
19
 
@@ -208,3 +211,130 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
208
211
  # These are used for warnings so we use the alias to make it more readable for the user
209
212
  conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
210
213
  return to_overwrite, conflicts
214
+
215
+
216
+ class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
217
+ """Looks up all view properties that map to the same container property,
218
+ and changes the child view property id to match the parent property id.
219
+ """
220
+
221
+ def __init__(self, client: NeatClient | None = None) -> None:
222
+ self._client = client
223
+
224
+ def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
225
+ input_rules = self._to_rules(rules)
226
+ new_rules = input_rules.model_copy(deep=True)
227
+ new_rules.metadata.version += "_as_parent_name"
228
+
229
+ path_by_view = self._inheritance_path_by_view(new_rules)
230
+ view_by_container_property = self._view_by_container_properties(new_rules)
231
+
232
+ parent_view_property_by_container_property = self._get_parent_view_property_by_container_property(
233
+ path_by_view, view_by_container_property
234
+ )
235
+
236
+ for prop in new_rules.properties:
237
+ if prop.container and prop.container_property:
238
+ if parent_name := parent_view_property_by_container_property.get(
239
+ (prop.container, prop.container_property)
240
+ ):
241
+ prop.view_property = parent_name
242
+
243
+ return JustRules(new_rules)
244
+
245
+ # Todo: Move into Probe class. Note this means that the Probe class must take a NeatClient as an argument.
246
+ def _inheritance_path_by_view(self, rules: DMSRules) -> dict[ViewEntity, list[ViewEntity]]:
247
+ parents_by_view: dict[ViewEntity, list[ViewEntity]] = {view.view: view.implements or [] for view in rules.views}
248
+
249
+ path_by_view: dict[ViewEntity, list[ViewEntity]] = {}
250
+ for view in rules.views:
251
+ path_by_view[view.view] = self._get_inheritance_path(
252
+ view.view, parents_by_view, rules.metadata.as_data_model_id()
253
+ )
254
+ return path_by_view
255
+
256
+ def _get_inheritance_path(
257
+ self, view: ViewEntity, parents_by_view: dict[ViewEntity, list[ViewEntity]], data_model_id: dm.DataModelId
258
+ ) -> list[ViewEntity]:
259
+ if parents_by_view.get(view) == []:
260
+ # We found the root.
261
+ return [view]
262
+ if view not in parents_by_view and self._client is not None:
263
+ # Lookup the parent
264
+ view_id = view.as_id()
265
+ read_views = self._client.loaders.views.retrieve([view_id])
266
+ if not read_views:
267
+ # Warning? Should be caught by validation
268
+ raise ResourceNotFoundError(view_id, "view", data_model_id, "data model")
269
+ parent_view_latest = max(read_views, key=lambda view: view.created_time)
270
+ parents_by_view[ViewEntity.from_id(parent_view_latest.as_id())] = [
271
+ ViewEntity.from_id(grand_parent) for grand_parent in parent_view_latest.implements or []
272
+ ]
273
+ elif view not in parents_by_view:
274
+ raise CDFMissingClientError(
275
+ f"The data model {data_model_id} is referencing a view that is not in the data model."
276
+ f"Please provide a client to lookup the view."
277
+ )
278
+
279
+ inheritance_path = [view]
280
+ seen = {view}
281
+ if view in parents_by_view:
282
+ for parent in parents_by_view[view]:
283
+ parent_path = self._get_inheritance_path(parent, parents_by_view, data_model_id)
284
+ inheritance_path.extend([p for p in parent_path if p not in seen])
285
+ seen.update(parent_path)
286
+ return inheritance_path
287
+
288
+ def _view_by_container_properties(
289
+ self, rules: DMSRules
290
+ ) -> dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]:
291
+ view_properties_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]] = (
292
+ defaultdict(list)
293
+ )
294
+ view_with_properties: set[ViewEntity] = set()
295
+ for prop in rules.properties:
296
+ if not prop.container or not prop.container_property:
297
+ continue
298
+ view_properties_by_container_properties[(prop.container, prop.container_property)].append(
299
+ (prop.view, prop.view_property)
300
+ )
301
+ view_with_properties.add(prop.view)
302
+
303
+ # We need to look up all parent properties.
304
+ to_lookup = {view.view.as_id() for view in rules.views if view.view not in view_with_properties}
305
+ if to_lookup and self._client is None:
306
+ raise CDFMissingClientError(
307
+ f"Views {to_lookup} are not in the data model. Please provide a client to lookup the views."
308
+ )
309
+ elif to_lookup and self._client:
310
+ read_views = self._client.loaders.views.retrieve(list(to_lookup), include_ancestor=True)
311
+ write_views = [self._client.loaders.views.as_write(read_view) for read_view in read_views]
312
+ # We use the write/request format of the views as the read/response format contains all properties
313
+ # including ancestor properties. The goal is to find the property name used in the parent
314
+ # and thus we cannot have that repeated in the child views.
315
+ for write_view in write_views:
316
+ view_id = write_view.as_id()
317
+ view_entity = ViewEntity.from_id(view_id)
318
+
319
+ for property_id, property_ in (write_view.properties or {}).items():
320
+ if not isinstance(property_, dm.MappedPropertyApply):
321
+ continue
322
+ container_entity = ContainerEntity.from_id(property_.container)
323
+ view_properties_by_container_properties[
324
+ (container_entity, property_.container_property_identifier)
325
+ ].append((view_entity, property_id))
326
+
327
+ return view_properties_by_container_properties
328
+
329
+ @staticmethod
330
+ def _get_parent_view_property_by_container_property(
331
+ path_by_view, view_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]
332
+ ) -> dict[tuple[ContainerEntity, str], str]:
333
+ parent_name_by_container_property: dict[tuple[ContainerEntity, str], str] = {}
334
+ for (container, container_property), view_properties in view_by_container_properties.items():
335
+ if len(view_properties) == 1:
336
+ continue
337
+ # Shortest path is the parent
338
+ _, prop_name = min(view_properties, key=lambda prop: len(path_by_view[prop[0]]))
339
+ parent_name_by_container_property[(container, container_property)] = prop_name
340
+ return parent_name_by_container_property
@@ -23,7 +23,7 @@ class ImporterPipeline(RulesPipeline[InputRules, VerifiedRules]):
23
23
 
24
24
  @classmethod
25
25
  def _create_pipeline(cls, importer: BaseImporter[InputRules], role: RoleTypes | None = None) -> "ImporterPipeline":
26
- items: list[RulesTransformer] = [VerifyAnyRules(errors="continue")]
26
+ items: list[RulesTransformer] = [VerifyAnyRules(errors="continue", validate=True)]
27
27
  if role is not None:
28
28
  out_cls = VERIFIED_RULES_BY_ROLE[role]
29
29
  items.append(ConvertToRules(out_cls))
@@ -1,7 +1,7 @@
1
1
  from abc import ABC
2
2
  from typing import Any, Literal
3
3
 
4
- from cognite.neat._issues import IssueList, NeatError, NeatWarning, catch_issues
4
+ from cognite.neat._issues import IssueList, MultiValueError, NeatError, NeatWarning, catch_issues
5
5
  from cognite.neat._issues.errors import NeatTypeError
6
6
  from cognite.neat._rules._shared import (
7
7
  InputRules,
@@ -18,6 +18,8 @@ from cognite.neat._rules.models import (
18
18
  InformationInputRules,
19
19
  InformationRules,
20
20
  )
21
+ from cognite.neat._rules.models.dms import DMSValidation
22
+ from cognite.neat._rules.models.information import InformationValidation
21
23
 
22
24
  from ._base import RulesTransformer
23
25
 
@@ -26,10 +28,11 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
26
28
  """Base class for all verification transformers."""
27
29
 
28
30
  _rules_cls: type[T_VerifiedRules]
31
+ _validation_cls: type
29
32
 
30
- def __init__(self, errors: Literal["raise", "continue"], post_validate: bool = True) -> None:
33
+ def __init__(self, errors: Literal["raise", "continue"], validate: bool = True) -> None:
31
34
  self.errors = errors
32
- self.post_validate = post_validate
35
+ self.validate = validate
33
36
 
34
37
  def transform(self, rules: T_InputRules | OutRules[T_InputRules]) -> MaybeRules[T_VerifiedRules]:
35
38
  issues = IssueList()
@@ -41,8 +44,17 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
41
44
  with catch_issues(issues, NeatError, NeatWarning, error_args) as future:
42
45
  rules_cls = self._get_rules_cls(in_)
43
46
  dumped = in_.dump()
44
- dumped["post_validate"] = self.post_validate
45
47
  verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
48
+ if self.validate:
49
+ validation_cls = self._get_validation_cls(verified_rules) # type: ignore[arg-type]
50
+ validation_issues = validation_cls(verified_rules).validate()
51
+ # We need to trigger warnings are raise exceptions such that they are caught by the context manager
52
+ # and processed with the read context
53
+ if validation_issues.warnings:
54
+ validation_issues.trigger_warnings()
55
+ if validation_issues.has_errors:
56
+ verified_rules = None
57
+ raise MultiValueError(validation_issues.errors)
46
58
 
47
59
  if (future.result == "failure" or issues.has_errors or verified_rules is None) and self.errors == "raise":
48
60
  raise issues.as_errors()
@@ -54,17 +66,22 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
54
66
  def _get_rules_cls(self, in_: T_InputRules) -> type[T_VerifiedRules]:
55
67
  return self._rules_cls
56
68
 
69
+ def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
70
+ return self._validation_cls
71
+
57
72
 
58
73
  class VerifyDMSRules(VerificationTransformer[DMSInputRules, DMSRules]):
59
74
  """Class to verify DMS rules."""
60
75
 
61
76
  _rules_cls = DMSRules
77
+ _validation_cls = DMSValidation
62
78
 
63
79
 
64
80
  class VerifyInformationRules(VerificationTransformer[InformationInputRules, InformationRules]):
65
81
  """Class to verify Information rules."""
66
82
 
67
83
  _rules_cls = InformationRules
84
+ _validation_cls = InformationValidation
68
85
 
69
86
 
70
87
  class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
@@ -77,3 +94,11 @@ class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
77
94
  return DMSRules
78
95
  else:
79
96
  raise NeatTypeError(f"Unsupported rules type: {type(in_)}")
97
+
98
+ def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
99
+ if isinstance(rules, InformationRules):
100
+ return InformationValidation
101
+ elif isinstance(rules, DMSRules):
102
+ return DMSValidation
103
+ else:
104
+ raise NeatTypeError(f"Unsupported rules type: {type(rules)}")