cognite-neat 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +390 -116
- cognite/neat/_client/_api/schema.py +63 -2
- cognite/neat/_client/data_classes/data_modeling.py +4 -0
- cognite/neat/_client/data_classes/schema.py +2 -348
- cognite/neat/_constants.py +27 -4
- cognite/neat/_graph/extractors/_base.py +7 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +28 -18
- cognite/neat/_graph/loaders/_rdf2dms.py +52 -13
- cognite/neat/_graph/transformers/__init__.py +3 -3
- cognite/neat/_graph/transformers/_classic_cdf.py +135 -56
- cognite/neat/_issues/_base.py +26 -17
- cognite/neat/_issues/errors/__init__.py +4 -2
- cognite/neat/_issues/errors/_external.py +7 -0
- cognite/neat/_issues/errors/_properties.py +2 -7
- cognite/neat/_issues/errors/_resources.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -2
- cognite/neat/_issues/warnings/_external.py +9 -1
- cognite/neat/_issues/warnings/_resources.py +41 -2
- cognite/neat/_issues/warnings/user_modeling.py +4 -4
- cognite/neat/_rules/_constants.py +2 -6
- cognite/neat/_rules/analysis/_base.py +15 -5
- cognite/neat/_rules/analysis/_dms.py +20 -0
- cognite/neat/_rules/analysis/_information.py +22 -0
- cognite/neat/_rules/exporters/_base.py +3 -5
- cognite/neat/_rules/exporters/_rules2dms.py +190 -200
- cognite/neat/_rules/importers/__init__.py +1 -3
- cognite/neat/_rules/importers/_base.py +1 -1
- cognite/neat/_rules/importers/_dms2rules.py +3 -25
- cognite/neat/_rules/importers/_rdf/__init__.py +5 -0
- cognite/neat/_rules/importers/_rdf/_base.py +34 -11
- cognite/neat/_rules/importers/_rdf/_imf2rules.py +91 -0
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +40 -7
- cognite/neat/_rules/importers/_rdf/_owl2rules.py +80 -0
- cognite/neat/_rules/importers/_rdf/_shared.py +138 -441
- cognite/neat/_rules/models/_base_rules.py +19 -0
- cognite/neat/_rules/models/_types.py +5 -0
- cognite/neat/_rules/models/dms/__init__.py +2 -0
- cognite/neat/_rules/models/dms/_exporter.py +247 -123
- cognite/neat/_rules/models/dms/_rules.py +7 -49
- cognite/neat/_rules/models/dms/_rules_input.py +8 -3
- cognite/neat/_rules/models/dms/_validation.py +421 -123
- cognite/neat/_rules/models/entities/_multi_value.py +3 -0
- cognite/neat/_rules/models/information/__init__.py +2 -0
- cognite/neat/_rules/models/information/_rules.py +17 -61
- cognite/neat/_rules/models/information/_rules_input.py +11 -2
- cognite/neat/_rules/models/information/_validation.py +107 -11
- cognite/neat/_rules/models/mapping/_classic2core.py +1 -1
- cognite/neat/_rules/models/mapping/_classic2core.yaml +8 -4
- cognite/neat/_rules/transformers/__init__.py +2 -1
- cognite/neat/_rules/transformers/_converters.py +163 -61
- cognite/neat/_rules/transformers/_mapping.py +132 -2
- cognite/neat/_rules/transformers/_pipelines.py +1 -1
- cognite/neat/_rules/transformers/_verification.py +29 -4
- cognite/neat/_session/_base.py +46 -60
- cognite/neat/_session/_mapping.py +105 -5
- cognite/neat/_session/_prepare.py +49 -14
- cognite/neat/_session/_read.py +50 -4
- cognite/neat/_session/_set.py +1 -0
- cognite/neat/_session/_to.py +38 -12
- cognite/neat/_session/_wizard.py +5 -0
- cognite/neat/_session/engine/_interface.py +3 -2
- cognite/neat/_session/exceptions.py +4 -0
- cognite/neat/_store/_base.py +79 -19
- cognite/neat/_utils/collection_.py +22 -0
- cognite/neat/_utils/rdf_.py +30 -4
- cognite/neat/_version.py +2 -2
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +3 -91
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +2 -16
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +3 -5
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/RECORD +74 -82
- cognite/neat/_rules/importers/_rdf/_imf2rules/__init__.py +0 -3
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +0 -86
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +0 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +0 -130
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2rules.py +0 -154
- cognite/neat/_rules/importers/_rdf/_owl2rules/__init__.py +0 -3
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +0 -58
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +0 -65
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +0 -59
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2rules.py +0 -39
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/entry_points.txt +0 -0
|
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
|
|
|
4
4
|
from collections import Counter, defaultdict
|
|
5
5
|
from collections.abc import Collection, Mapping
|
|
6
6
|
from datetime import date, datetime
|
|
7
|
-
from typing import Literal, TypeVar, cast, overload
|
|
7
|
+
from typing import ClassVar, Literal, TypeVar, cast, overload
|
|
8
8
|
|
|
9
9
|
from cognite.client.data_classes import data_modeling as dms
|
|
10
10
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
|
|
@@ -15,6 +15,7 @@ from cognite.neat._constants import (
|
|
|
15
15
|
)
|
|
16
16
|
from cognite.neat._issues._base import IssueList
|
|
17
17
|
from cognite.neat._issues.errors import NeatValueError
|
|
18
|
+
from cognite.neat._issues.warnings import NeatValueWarning
|
|
18
19
|
from cognite.neat._issues.warnings._models import (
|
|
19
20
|
EnterpriseModelNotBuildOnTopOfCDMWarning,
|
|
20
21
|
SolutionModelBuildOnTopOfCDMWarning,
|
|
@@ -34,7 +35,7 @@ from cognite.neat._rules.models import (
|
|
|
34
35
|
SheetList,
|
|
35
36
|
data_types,
|
|
36
37
|
)
|
|
37
|
-
from cognite.neat._rules.models.data_types import DataType, String
|
|
38
|
+
from cognite.neat._rules.models.data_types import AnyURI, DataType, String
|
|
38
39
|
from cognite.neat._rules.models.dms import DMSMetadata, DMSProperty, DMSView
|
|
39
40
|
from cognite.neat._rules.models.dms._rules import DMSContainer
|
|
40
41
|
from cognite.neat._rules.models.entities import (
|
|
@@ -249,11 +250,12 @@ class PrefixEntities(RulesTransformer[InputRules, InputRules]): # type: ignore[
|
|
|
249
250
|
class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
250
251
|
"""Converts InformationRules to DMSRules."""
|
|
251
252
|
|
|
252
|
-
def __init__(self, ignore_undefined_value_types: bool = False):
|
|
253
|
+
def __init__(self, ignore_undefined_value_types: bool = False, mode: Literal["edge_properties"] | None = None):
|
|
253
254
|
self.ignore_undefined_value_types = ignore_undefined_value_types
|
|
255
|
+
self.mode = mode
|
|
254
256
|
|
|
255
257
|
def _transform(self, rules: InformationRules) -> DMSRules:
|
|
256
|
-
return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types)
|
|
258
|
+
return _InformationRulesConverter(rules).as_dms_rules(self.ignore_undefined_value_types, self.mode)
|
|
257
259
|
|
|
258
260
|
|
|
259
261
|
class DMSToInformation(ConversionTransformer[DMSRules, InformationRules]):
|
|
@@ -637,11 +639,15 @@ class ReduceCogniteModel(RulesTransformer[DMSRules, DMSRules]):
|
|
|
637
639
|
|
|
638
640
|
|
|
639
641
|
class _InformationRulesConverter:
|
|
642
|
+
_edge_properties: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
|
|
643
|
+
|
|
640
644
|
def __init__(self, information: InformationRules):
|
|
641
645
|
self.rules = information
|
|
642
646
|
self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
|
|
643
647
|
|
|
644
|
-
def as_dms_rules(
|
|
648
|
+
def as_dms_rules(
|
|
649
|
+
self, ignore_undefined_value_types: bool = False, mode: Literal["edge_properties"] | None = None
|
|
650
|
+
) -> "DMSRules":
|
|
645
651
|
from cognite.neat._rules.models.dms._rules import (
|
|
646
652
|
DMSContainer,
|
|
647
653
|
DMSProperty,
|
|
@@ -652,27 +658,55 @@ class _InformationRulesConverter:
|
|
|
652
658
|
info_metadata = self.rules.metadata
|
|
653
659
|
default_version = info_metadata.version
|
|
654
660
|
default_space = self._to_space(info_metadata.prefix)
|
|
655
|
-
|
|
661
|
+
dms_metadata = self._convert_metadata_to_dms(info_metadata)
|
|
662
|
+
edge_classes: set[ClassEntity] = set()
|
|
663
|
+
property_to_edge: dict[tuple[ClassEntity, str], ClassEntity] = {}
|
|
664
|
+
end_node_by_edge: dict[ClassEntity, ClassEntity] = {}
|
|
665
|
+
if mode == "edge_properties":
|
|
666
|
+
edge_classes = {
|
|
667
|
+
cls_.class_ for cls_ in self.rules.classes if cls_.implements and cls_.implements[0].suffix == "Edge"
|
|
668
|
+
}
|
|
669
|
+
property_to_edge = {
|
|
670
|
+
(prop.class_, prop.property_): prop.value_type
|
|
671
|
+
for prop in self.rules.properties
|
|
672
|
+
if prop.value_type in edge_classes and isinstance(prop.value_type, ClassEntity)
|
|
673
|
+
}
|
|
674
|
+
end_node_by_edge = {
|
|
675
|
+
prop.class_: prop.value_type
|
|
676
|
+
for prop in self.rules.properties
|
|
677
|
+
if prop.class_ in edge_classes
|
|
678
|
+
and (prop.property_ == "endNode" or prop.property_ == "end_node")
|
|
679
|
+
and isinstance(prop.value_type, ClassEntity)
|
|
680
|
+
}
|
|
656
681
|
|
|
657
682
|
properties_by_class: dict[ClassEntity, list[DMSProperty]] = defaultdict(list)
|
|
658
683
|
referenced_containers: dict[ContainerEntity, Counter[ClassEntity]] = defaultdict(Counter)
|
|
659
684
|
for prop in self.rules.properties:
|
|
660
685
|
if ignore_undefined_value_types and isinstance(prop.value_type, UnknownEntity):
|
|
661
686
|
continue
|
|
662
|
-
|
|
687
|
+
if prop.class_ in edge_classes and prop.property_ in self._edge_properties:
|
|
688
|
+
continue
|
|
689
|
+
dms_property = self._as_dms_property(
|
|
690
|
+
prop, default_space, default_version, edge_classes, property_to_edge, end_node_by_edge
|
|
691
|
+
)
|
|
663
692
|
properties_by_class[prop.class_].append(dms_property)
|
|
664
693
|
if dms_property.container:
|
|
665
694
|
referenced_containers[dms_property.container][prop.class_] += 1
|
|
666
695
|
|
|
667
|
-
views: list[DMSView] = [
|
|
668
|
-
|
|
696
|
+
views: list[DMSView] = []
|
|
697
|
+
|
|
698
|
+
for cls_ in self.rules.classes:
|
|
699
|
+
dms_view = DMSView(
|
|
669
700
|
name=cls_.name,
|
|
670
701
|
view=cls_.class_.as_view_entity(default_space, default_version),
|
|
671
702
|
description=cls_.description,
|
|
672
|
-
implements=self._get_view_implements(cls_, info_metadata),
|
|
703
|
+
implements=self._get_view_implements(cls_, info_metadata, mode),
|
|
673
704
|
)
|
|
674
|
-
|
|
675
|
-
|
|
705
|
+
|
|
706
|
+
dms_view.logical = cls_.neatId
|
|
707
|
+
cls_.physical = dms_view.neatId
|
|
708
|
+
|
|
709
|
+
views.append(dms_view)
|
|
676
710
|
|
|
677
711
|
class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
|
|
678
712
|
|
|
@@ -696,7 +730,7 @@ class _InformationRulesConverter:
|
|
|
696
730
|
containers.append(container)
|
|
697
731
|
|
|
698
732
|
return DMSRules(
|
|
699
|
-
metadata=
|
|
733
|
+
metadata=dms_metadata,
|
|
700
734
|
properties=SheetList[DMSProperty]([prop for prop_set in properties_by_class.values() for prop in prop_set]),
|
|
701
735
|
views=SheetList[DMSView](views),
|
|
702
736
|
containers=SheetList[DMSContainer](containers),
|
|
@@ -724,7 +758,7 @@ class _InformationRulesConverter:
|
|
|
724
758
|
DMSMetadata,
|
|
725
759
|
)
|
|
726
760
|
|
|
727
|
-
|
|
761
|
+
dms_metadata = DMSMetadata(
|
|
728
762
|
space=metadata.space,
|
|
729
763
|
version=metadata.version,
|
|
730
764
|
external_id=metadata.external_id,
|
|
@@ -734,74 +768,138 @@ class _InformationRulesConverter:
|
|
|
734
768
|
updated=metadata.updated,
|
|
735
769
|
)
|
|
736
770
|
|
|
737
|
-
|
|
738
|
-
|
|
771
|
+
dms_metadata.logical = metadata.identifier
|
|
772
|
+
metadata.physical = dms_metadata.identifier
|
|
773
|
+
|
|
774
|
+
return dms_metadata
|
|
739
775
|
|
|
776
|
+
def _as_dms_property(
|
|
777
|
+
self,
|
|
778
|
+
info_property: InformationProperty,
|
|
779
|
+
default_space: str,
|
|
780
|
+
default_version: str,
|
|
781
|
+
edge_classes: set[ClassEntity],
|
|
782
|
+
property_to_edge: dict[tuple[ClassEntity, str], ClassEntity],
|
|
783
|
+
end_node_by_edge: dict[ClassEntity, ClassEntity],
|
|
784
|
+
) -> "DMSProperty":
|
|
740
785
|
from cognite.neat._rules.models.dms._rules import DMSProperty
|
|
741
786
|
|
|
742
787
|
# returns property type, which can be ObjectProperty or DatatypeProperty
|
|
743
|
-
value_type
|
|
788
|
+
value_type = self._get_value_type(
|
|
789
|
+
info_property,
|
|
790
|
+
default_space,
|
|
791
|
+
default_version,
|
|
792
|
+
edge_classes,
|
|
793
|
+
end_node_by_edge,
|
|
794
|
+
)
|
|
795
|
+
|
|
796
|
+
connection = self._get_connection(info_property, value_type, property_to_edge, default_space, default_version)
|
|
797
|
+
|
|
798
|
+
container: ContainerEntity | None = None
|
|
799
|
+
container_property: str | None = None
|
|
800
|
+
is_list: bool | None = info_property.is_list
|
|
801
|
+
nullable: bool | None = not info_property.is_mandatory
|
|
802
|
+
if isinstance(connection, EdgeEntity):
|
|
803
|
+
nullable = None
|
|
804
|
+
elif connection == "direct":
|
|
805
|
+
nullable = True
|
|
806
|
+
container, container_property = self._get_container(info_property, default_space)
|
|
807
|
+
else:
|
|
808
|
+
container, container_property = self._get_container(info_property, default_space)
|
|
809
|
+
|
|
810
|
+
dms_property = DMSProperty(
|
|
811
|
+
name=info_property.name,
|
|
812
|
+
value_type=value_type,
|
|
813
|
+
nullable=nullable,
|
|
814
|
+
is_list=is_list,
|
|
815
|
+
connection=connection,
|
|
816
|
+
default=info_property.default,
|
|
817
|
+
container=container,
|
|
818
|
+
container_property=container_property,
|
|
819
|
+
view=info_property.class_.as_view_entity(default_space, default_version),
|
|
820
|
+
view_property=info_property.property_,
|
|
821
|
+
)
|
|
822
|
+
|
|
823
|
+
# linking
|
|
824
|
+
dms_property.logical = info_property.neatId
|
|
825
|
+
info_property.physical = dms_property.neatId
|
|
826
|
+
|
|
827
|
+
return dms_property
|
|
828
|
+
|
|
829
|
+
@staticmethod
|
|
830
|
+
def _get_connection(
|
|
831
|
+
prop: InformationProperty,
|
|
832
|
+
value_type: DataType | ViewEntity | DMSUnknownEntity,
|
|
833
|
+
property_to_edge: dict[tuple[ClassEntity, str], ClassEntity],
|
|
834
|
+
default_space: str,
|
|
835
|
+
default_version: str,
|
|
836
|
+
) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
|
|
837
|
+
if isinstance(value_type, ViewEntity) and (prop.class_, prop.property_) in property_to_edge:
|
|
838
|
+
edge_properties = property_to_edge[(prop.class_, prop.property_)]
|
|
839
|
+
return EdgeEntity(properties=edge_properties.as_view_entity(default_space, default_version))
|
|
840
|
+
if isinstance(value_type, ViewEntity) and prop.is_list:
|
|
841
|
+
return EdgeEntity()
|
|
842
|
+
elif isinstance(value_type, ViewEntity):
|
|
843
|
+
return "direct"
|
|
844
|
+
# defaulting to direct connection
|
|
845
|
+
elif isinstance(value_type, DMSUnknownEntity):
|
|
846
|
+
return "direct"
|
|
847
|
+
return None
|
|
848
|
+
|
|
849
|
+
def _get_value_type(
|
|
850
|
+
self,
|
|
851
|
+
prop: InformationProperty,
|
|
852
|
+
default_space: str,
|
|
853
|
+
default_version: str,
|
|
854
|
+
edge_classes: set[ClassEntity],
|
|
855
|
+
end_node_by_edge: dict[ClassEntity, ClassEntity],
|
|
856
|
+
) -> DataType | ViewEntity | DMSUnknownEntity:
|
|
744
857
|
if isinstance(prop.value_type, DataType):
|
|
745
|
-
|
|
858
|
+
return prop.value_type
|
|
746
859
|
|
|
747
860
|
# UnknownEntity should resolve to DMSUnknownEntity
|
|
748
861
|
# meaning end node type is unknown
|
|
749
862
|
elif isinstance(prop.value_type, UnknownEntity):
|
|
750
|
-
|
|
751
|
-
|
|
863
|
+
return DMSUnknownEntity()
|
|
864
|
+
|
|
865
|
+
elif isinstance(prop.value_type, ClassEntity) and (prop.value_type in edge_classes):
|
|
866
|
+
if prop.value_type in end_node_by_edge:
|
|
867
|
+
return end_node_by_edge[prop.value_type].as_view_entity(default_space, default_version)
|
|
868
|
+
warnings.warn(
|
|
869
|
+
NeatValueWarning(
|
|
870
|
+
f"Edge class {prop.value_type} does not have 'endNode' property, defaulting to DMSUnknownEntity"
|
|
871
|
+
),
|
|
872
|
+
stacklevel=2,
|
|
873
|
+
)
|
|
874
|
+
return DMSUnknownEntity()
|
|
752
875
|
elif isinstance(prop.value_type, ClassEntity):
|
|
753
|
-
|
|
876
|
+
return prop.value_type.as_view_entity(default_space, default_version)
|
|
754
877
|
|
|
755
878
|
elif isinstance(prop.value_type, MultiValueTypeInfo):
|
|
756
879
|
# Multi Object type should resolve to DMSUnknownEntity
|
|
757
880
|
# meaning end node type is unknown
|
|
758
881
|
if prop.value_type.is_multi_object_type():
|
|
759
|
-
|
|
882
|
+
non_unknown = [type_ for type_ in prop.value_type.types if isinstance(type_, UnknownEntity)]
|
|
883
|
+
if list(non_unknown) == 1:
|
|
884
|
+
#
|
|
885
|
+
return non_unknown[0].as_view_entity(default_space, default_version)
|
|
886
|
+
return DMSUnknownEntity()
|
|
760
887
|
|
|
761
888
|
# Multi Data type should resolve to a single data type, or it should
|
|
762
889
|
elif prop.value_type.is_multi_data_type():
|
|
763
|
-
|
|
890
|
+
return self.convert_multi_data_type(prop.value_type)
|
|
764
891
|
|
|
765
892
|
# Mixed types default to string
|
|
766
893
|
else:
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
894
|
+
non_any_uri = [type_ for type_ in prop.value_type.types if type_ != AnyURI()]
|
|
895
|
+
if list(non_any_uri) == 1:
|
|
896
|
+
if isinstance(non_any_uri[0], ClassEntity):
|
|
897
|
+
return non_any_uri[0].as_view_entity(default_space, default_version)
|
|
898
|
+
else:
|
|
899
|
+
return non_any_uri[0]
|
|
900
|
+
return String()
|
|
771
901
|
|
|
772
|
-
|
|
773
|
-
if isinstance(value_type, ViewEntity):
|
|
774
|
-
# Default connection type.
|
|
775
|
-
connection = EdgeEntity() if prop.is_list else "direct"
|
|
776
|
-
|
|
777
|
-
# defaulting to direct connection
|
|
778
|
-
elif isinstance(value_type, DMSUnknownEntity):
|
|
779
|
-
connection = "direct"
|
|
780
|
-
|
|
781
|
-
container: ContainerEntity | None = None
|
|
782
|
-
container_property: str | None = None
|
|
783
|
-
is_list: bool | None = prop.is_list
|
|
784
|
-
nullable: bool | None = not prop.is_mandatory
|
|
785
|
-
if isinstance(connection, EdgeEntity):
|
|
786
|
-
nullable = None
|
|
787
|
-
elif connection == "direct":
|
|
788
|
-
nullable = True
|
|
789
|
-
container, container_property = self._get_container(prop, default_space)
|
|
790
|
-
else:
|
|
791
|
-
container, container_property = self._get_container(prop, default_space)
|
|
792
|
-
|
|
793
|
-
return DMSProperty(
|
|
794
|
-
name=prop.name,
|
|
795
|
-
value_type=value_type,
|
|
796
|
-
nullable=nullable,
|
|
797
|
-
is_list=is_list,
|
|
798
|
-
connection=connection,
|
|
799
|
-
default=prop.default,
|
|
800
|
-
container=container,
|
|
801
|
-
container_property=container_property,
|
|
802
|
-
view=prop.class_.as_view_entity(default_space, default_version),
|
|
803
|
-
view_property=prop.property_,
|
|
804
|
-
)
|
|
902
|
+
raise ValueError(f"Unsupported value type: {prop.value_type.type_}")
|
|
805
903
|
|
|
806
904
|
@classmethod
|
|
807
905
|
def _to_space(cls, prefix: str) -> str:
|
|
@@ -823,9 +921,13 @@ class _InformationRulesConverter:
|
|
|
823
921
|
self.property_count_by_container[container_entity] += 1
|
|
824
922
|
return container_entity, prop.property_
|
|
825
923
|
|
|
826
|
-
def _get_view_implements(
|
|
924
|
+
def _get_view_implements(
|
|
925
|
+
self, cls_: InformationClass, metadata: InformationMetadata, mode: Literal["edge_properties"] | None
|
|
926
|
+
) -> list[ViewEntity]:
|
|
827
927
|
implements = []
|
|
828
928
|
for parent in cls_.implements or []:
|
|
929
|
+
if mode == "edge_properties" and parent.suffix == "Edge":
|
|
930
|
+
continue
|
|
829
931
|
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
830
932
|
implements.append(view_entity)
|
|
831
933
|
return implements
|
|
@@ -4,13 +4,16 @@ from collections import defaultdict
|
|
|
4
4
|
from functools import cached_property
|
|
5
5
|
from typing import Any, ClassVar, Literal
|
|
6
6
|
|
|
7
|
-
from cognite.
|
|
7
|
+
from cognite.client import data_modeling as dm
|
|
8
|
+
|
|
9
|
+
from cognite.neat._client import NeatClient
|
|
10
|
+
from cognite.neat._issues.errors import CDFMissingClientError, NeatValueError, ResourceNotFoundError
|
|
8
11
|
from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
|
|
9
12
|
from cognite.neat._rules._shared import JustRules, OutRules
|
|
10
13
|
from cognite.neat._rules.models import DMSRules, SheetList
|
|
11
14
|
from cognite.neat._rules.models.data_types import Enum
|
|
12
15
|
from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
|
|
13
|
-
from cognite.neat._rules.models.entities import ViewEntity
|
|
16
|
+
from cognite.neat._rules.models.entities import ContainerEntity, ViewEntity
|
|
14
17
|
|
|
15
18
|
from ._base import RulesTransformer
|
|
16
19
|
|
|
@@ -208,3 +211,130 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
|
208
211
|
# These are used for warnings so we use the alias to make it more readable for the user
|
|
209
212
|
conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
|
|
210
213
|
return to_overwrite, conflicts
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
|
|
217
|
+
"""Looks up all view properties that map to the same container property,
|
|
218
|
+
and changes the child view property id to match the parent property id.
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
def __init__(self, client: NeatClient | None = None) -> None:
|
|
222
|
+
self._client = client
|
|
223
|
+
|
|
224
|
+
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
225
|
+
input_rules = self._to_rules(rules)
|
|
226
|
+
new_rules = input_rules.model_copy(deep=True)
|
|
227
|
+
new_rules.metadata.version += "_as_parent_name"
|
|
228
|
+
|
|
229
|
+
path_by_view = self._inheritance_path_by_view(new_rules)
|
|
230
|
+
view_by_container_property = self._view_by_container_properties(new_rules)
|
|
231
|
+
|
|
232
|
+
parent_view_property_by_container_property = self._get_parent_view_property_by_container_property(
|
|
233
|
+
path_by_view, view_by_container_property
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
for prop in new_rules.properties:
|
|
237
|
+
if prop.container and prop.container_property:
|
|
238
|
+
if parent_name := parent_view_property_by_container_property.get(
|
|
239
|
+
(prop.container, prop.container_property)
|
|
240
|
+
):
|
|
241
|
+
prop.view_property = parent_name
|
|
242
|
+
|
|
243
|
+
return JustRules(new_rules)
|
|
244
|
+
|
|
245
|
+
# Todo: Move into Probe class. Note this means that the Probe class must take a NeatClient as an argument.
|
|
246
|
+
def _inheritance_path_by_view(self, rules: DMSRules) -> dict[ViewEntity, list[ViewEntity]]:
|
|
247
|
+
parents_by_view: dict[ViewEntity, list[ViewEntity]] = {view.view: view.implements or [] for view in rules.views}
|
|
248
|
+
|
|
249
|
+
path_by_view: dict[ViewEntity, list[ViewEntity]] = {}
|
|
250
|
+
for view in rules.views:
|
|
251
|
+
path_by_view[view.view] = self._get_inheritance_path(
|
|
252
|
+
view.view, parents_by_view, rules.metadata.as_data_model_id()
|
|
253
|
+
)
|
|
254
|
+
return path_by_view
|
|
255
|
+
|
|
256
|
+
def _get_inheritance_path(
|
|
257
|
+
self, view: ViewEntity, parents_by_view: dict[ViewEntity, list[ViewEntity]], data_model_id: dm.DataModelId
|
|
258
|
+
) -> list[ViewEntity]:
|
|
259
|
+
if parents_by_view.get(view) == []:
|
|
260
|
+
# We found the root.
|
|
261
|
+
return [view]
|
|
262
|
+
if view not in parents_by_view and self._client is not None:
|
|
263
|
+
# Lookup the parent
|
|
264
|
+
view_id = view.as_id()
|
|
265
|
+
read_views = self._client.loaders.views.retrieve([view_id])
|
|
266
|
+
if not read_views:
|
|
267
|
+
# Warning? Should be caught by validation
|
|
268
|
+
raise ResourceNotFoundError(view_id, "view", data_model_id, "data model")
|
|
269
|
+
parent_view_latest = max(read_views, key=lambda view: view.created_time)
|
|
270
|
+
parents_by_view[ViewEntity.from_id(parent_view_latest.as_id())] = [
|
|
271
|
+
ViewEntity.from_id(grand_parent) for grand_parent in parent_view_latest.implements or []
|
|
272
|
+
]
|
|
273
|
+
elif view not in parents_by_view:
|
|
274
|
+
raise CDFMissingClientError(
|
|
275
|
+
f"The data model {data_model_id} is referencing a view that is not in the data model."
|
|
276
|
+
f"Please provide a client to lookup the view."
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
inheritance_path = [view]
|
|
280
|
+
seen = {view}
|
|
281
|
+
if view in parents_by_view:
|
|
282
|
+
for parent in parents_by_view[view]:
|
|
283
|
+
parent_path = self._get_inheritance_path(parent, parents_by_view, data_model_id)
|
|
284
|
+
inheritance_path.extend([p for p in parent_path if p not in seen])
|
|
285
|
+
seen.update(parent_path)
|
|
286
|
+
return inheritance_path
|
|
287
|
+
|
|
288
|
+
def _view_by_container_properties(
|
|
289
|
+
self, rules: DMSRules
|
|
290
|
+
) -> dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]:
|
|
291
|
+
view_properties_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]] = (
|
|
292
|
+
defaultdict(list)
|
|
293
|
+
)
|
|
294
|
+
view_with_properties: set[ViewEntity] = set()
|
|
295
|
+
for prop in rules.properties:
|
|
296
|
+
if not prop.container or not prop.container_property:
|
|
297
|
+
continue
|
|
298
|
+
view_properties_by_container_properties[(prop.container, prop.container_property)].append(
|
|
299
|
+
(prop.view, prop.view_property)
|
|
300
|
+
)
|
|
301
|
+
view_with_properties.add(prop.view)
|
|
302
|
+
|
|
303
|
+
# We need to look up all parent properties.
|
|
304
|
+
to_lookup = {view.view.as_id() for view in rules.views if view.view not in view_with_properties}
|
|
305
|
+
if to_lookup and self._client is None:
|
|
306
|
+
raise CDFMissingClientError(
|
|
307
|
+
f"Views {to_lookup} are not in the data model. Please provide a client to lookup the views."
|
|
308
|
+
)
|
|
309
|
+
elif to_lookup and self._client:
|
|
310
|
+
read_views = self._client.loaders.views.retrieve(list(to_lookup), include_ancestor=True)
|
|
311
|
+
write_views = [self._client.loaders.views.as_write(read_view) for read_view in read_views]
|
|
312
|
+
# We use the write/request format of the views as the read/response format contains all properties
|
|
313
|
+
# including ancestor properties. The goal is to find the property name used in the parent
|
|
314
|
+
# and thus we cannot have that repeated in the child views.
|
|
315
|
+
for write_view in write_views:
|
|
316
|
+
view_id = write_view.as_id()
|
|
317
|
+
view_entity = ViewEntity.from_id(view_id)
|
|
318
|
+
|
|
319
|
+
for property_id, property_ in (write_view.properties or {}).items():
|
|
320
|
+
if not isinstance(property_, dm.MappedPropertyApply):
|
|
321
|
+
continue
|
|
322
|
+
container_entity = ContainerEntity.from_id(property_.container)
|
|
323
|
+
view_properties_by_container_properties[
|
|
324
|
+
(container_entity, property_.container_property_identifier)
|
|
325
|
+
].append((view_entity, property_id))
|
|
326
|
+
|
|
327
|
+
return view_properties_by_container_properties
|
|
328
|
+
|
|
329
|
+
@staticmethod
|
|
330
|
+
def _get_parent_view_property_by_container_property(
|
|
331
|
+
path_by_view, view_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]
|
|
332
|
+
) -> dict[tuple[ContainerEntity, str], str]:
|
|
333
|
+
parent_name_by_container_property: dict[tuple[ContainerEntity, str], str] = {}
|
|
334
|
+
for (container, container_property), view_properties in view_by_container_properties.items():
|
|
335
|
+
if len(view_properties) == 1:
|
|
336
|
+
continue
|
|
337
|
+
# Shortest path is the parent
|
|
338
|
+
_, prop_name = min(view_properties, key=lambda prop: len(path_by_view[prop[0]]))
|
|
339
|
+
parent_name_by_container_property[(container, container_property)] = prop_name
|
|
340
|
+
return parent_name_by_container_property
|
|
@@ -23,7 +23,7 @@ class ImporterPipeline(RulesPipeline[InputRules, VerifiedRules]):
|
|
|
23
23
|
|
|
24
24
|
@classmethod
|
|
25
25
|
def _create_pipeline(cls, importer: BaseImporter[InputRules], role: RoleTypes | None = None) -> "ImporterPipeline":
|
|
26
|
-
items: list[RulesTransformer] = [VerifyAnyRules(errors="continue")]
|
|
26
|
+
items: list[RulesTransformer] = [VerifyAnyRules(errors="continue", validate=True)]
|
|
27
27
|
if role is not None:
|
|
28
28
|
out_cls = VERIFIED_RULES_BY_ROLE[role]
|
|
29
29
|
items.append(ConvertToRules(out_cls))
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from abc import ABC
|
|
2
2
|
from typing import Any, Literal
|
|
3
3
|
|
|
4
|
-
from cognite.neat._issues import IssueList, NeatError, NeatWarning, catch_issues
|
|
4
|
+
from cognite.neat._issues import IssueList, MultiValueError, NeatError, NeatWarning, catch_issues
|
|
5
5
|
from cognite.neat._issues.errors import NeatTypeError
|
|
6
6
|
from cognite.neat._rules._shared import (
|
|
7
7
|
InputRules,
|
|
@@ -18,6 +18,8 @@ from cognite.neat._rules.models import (
|
|
|
18
18
|
InformationInputRules,
|
|
19
19
|
InformationRules,
|
|
20
20
|
)
|
|
21
|
+
from cognite.neat._rules.models.dms import DMSValidation
|
|
22
|
+
from cognite.neat._rules.models.information import InformationValidation
|
|
21
23
|
|
|
22
24
|
from ._base import RulesTransformer
|
|
23
25
|
|
|
@@ -26,10 +28,11 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
|
|
|
26
28
|
"""Base class for all verification transformers."""
|
|
27
29
|
|
|
28
30
|
_rules_cls: type[T_VerifiedRules]
|
|
31
|
+
_validation_cls: type
|
|
29
32
|
|
|
30
|
-
def __init__(self, errors: Literal["raise", "continue"],
|
|
33
|
+
def __init__(self, errors: Literal["raise", "continue"], validate: bool = True) -> None:
|
|
31
34
|
self.errors = errors
|
|
32
|
-
self.
|
|
35
|
+
self.validate = validate
|
|
33
36
|
|
|
34
37
|
def transform(self, rules: T_InputRules | OutRules[T_InputRules]) -> MaybeRules[T_VerifiedRules]:
|
|
35
38
|
issues = IssueList()
|
|
@@ -41,8 +44,17 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
|
|
|
41
44
|
with catch_issues(issues, NeatError, NeatWarning, error_args) as future:
|
|
42
45
|
rules_cls = self._get_rules_cls(in_)
|
|
43
46
|
dumped = in_.dump()
|
|
44
|
-
dumped["post_validate"] = self.post_validate
|
|
45
47
|
verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
|
|
48
|
+
if self.validate:
|
|
49
|
+
validation_cls = self._get_validation_cls(verified_rules) # type: ignore[arg-type]
|
|
50
|
+
validation_issues = validation_cls(verified_rules).validate()
|
|
51
|
+
# We need to trigger warnings are raise exceptions such that they are caught by the context manager
|
|
52
|
+
# and processed with the read context
|
|
53
|
+
if validation_issues.warnings:
|
|
54
|
+
validation_issues.trigger_warnings()
|
|
55
|
+
if validation_issues.has_errors:
|
|
56
|
+
verified_rules = None
|
|
57
|
+
raise MultiValueError(validation_issues.errors)
|
|
46
58
|
|
|
47
59
|
if (future.result == "failure" or issues.has_errors or verified_rules is None) and self.errors == "raise":
|
|
48
60
|
raise issues.as_errors()
|
|
@@ -54,17 +66,22 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
|
|
|
54
66
|
def _get_rules_cls(self, in_: T_InputRules) -> type[T_VerifiedRules]:
|
|
55
67
|
return self._rules_cls
|
|
56
68
|
|
|
69
|
+
def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
|
|
70
|
+
return self._validation_cls
|
|
71
|
+
|
|
57
72
|
|
|
58
73
|
class VerifyDMSRules(VerificationTransformer[DMSInputRules, DMSRules]):
|
|
59
74
|
"""Class to verify DMS rules."""
|
|
60
75
|
|
|
61
76
|
_rules_cls = DMSRules
|
|
77
|
+
_validation_cls = DMSValidation
|
|
62
78
|
|
|
63
79
|
|
|
64
80
|
class VerifyInformationRules(VerificationTransformer[InformationInputRules, InformationRules]):
|
|
65
81
|
"""Class to verify Information rules."""
|
|
66
82
|
|
|
67
83
|
_rules_cls = InformationRules
|
|
84
|
+
_validation_cls = InformationValidation
|
|
68
85
|
|
|
69
86
|
|
|
70
87
|
class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
|
|
@@ -77,3 +94,11 @@ class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
|
|
|
77
94
|
return DMSRules
|
|
78
95
|
else:
|
|
79
96
|
raise NeatTypeError(f"Unsupported rules type: {type(in_)}")
|
|
97
|
+
|
|
98
|
+
def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
|
|
99
|
+
if isinstance(rules, InformationRules):
|
|
100
|
+
return InformationValidation
|
|
101
|
+
elif isinstance(rules, DMSRules):
|
|
102
|
+
return DMSValidation
|
|
103
|
+
else:
|
|
104
|
+
raise NeatTypeError(f"Unsupported rules type: {type(rules)}")
|