cognite-neat 0.85.12__py3-none-any.whl → 0.87.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/routers/core.py +4 -4
- cognite/neat/constants.py +11 -9
- cognite/neat/graph/extractors/_mock_graph_generator.py +8 -9
- cognite/neat/graph/loaders/__init__.py +5 -2
- cognite/neat/graph/loaders/_base.py +13 -5
- cognite/neat/graph/loaders/_rdf2asset.py +94 -20
- cognite/neat/graph/loaders/_rdf2dms.py +3 -16
- cognite/neat/graph/queries/_base.py +58 -5
- cognite/neat/graph/queries/_construct.py +17 -15
- cognite/neat/graph/queries/_shared.py +20 -6
- cognite/neat/graph/stores/_base.py +19 -10
- cognite/neat/graph/transformers/_rdfpath.py +7 -0
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
- cognite/neat/legacy/graph/stores/_base.py +24 -8
- cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
- cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +48 -15
- cognite/neat/legacy/rules/importers/_graph2rules.py +34 -7
- cognite/neat/legacy/rules/models/raw_rules.py +18 -6
- cognite/neat/legacy/rules/models/rules.py +32 -12
- cognite/neat/rules/_shared.py +6 -1
- cognite/neat/rules/analysis/__init__.py +4 -4
- cognite/neat/rules/analysis/_asset.py +128 -0
- cognite/neat/rules/analysis/_base.py +385 -6
- cognite/neat/rules/analysis/_information.py +155 -0
- cognite/neat/rules/exporters/_base.py +4 -4
- cognite/neat/rules/exporters/_rules2dms.py +1 -1
- cognite/neat/rules/exporters/_rules2ontology.py +5 -5
- cognite/neat/rules/importers/_base.py +4 -4
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
- cognite/neat/rules/importers/_inference2rules.py +2 -2
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +1 -1
- cognite/neat/rules/importers/_spreadsheet2rules.py +5 -5
- cognite/neat/rules/models/__init__.py +3 -3
- cognite/neat/rules/models/_base.py +10 -10
- cognite/neat/rules/models/asset/_rules.py +9 -10
- cognite/neat/rules/models/dms/_converter.py +4 -5
- cognite/neat/rules/models/dms/_rules.py +6 -3
- cognite/neat/rules/models/domain.py +5 -2
- cognite/neat/rules/models/entities.py +2 -9
- cognite/neat/rules/models/information/_converter.py +3 -3
- cognite/neat/rules/models/information/_rules.py +13 -11
- cognite/neat/rules/models/information/_rules_input.py +1 -2
- cognite/neat/rules/models/information/_validation.py +1 -1
- cognite/neat/utils/utils.py +54 -18
- cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +129 -27
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/RECORD +56 -54
- cognite/neat/rules/analysis/_information_rules.py +0 -469
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.85.12.dist-info → cognite_neat-0.87.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import warnings
|
|
3
|
+
from typing import Any, cast
|
|
4
|
+
|
|
5
|
+
from pydantic import ValidationError
|
|
6
|
+
|
|
7
|
+
from cognite.neat.rules.models import SchemaCompleteness
|
|
8
|
+
from cognite.neat.rules.models._rdfpath import Hop, RDFPath, SingleProperty
|
|
9
|
+
from cognite.neat.rules.models.entities import ClassEntity, ReferenceEntity
|
|
10
|
+
from cognite.neat.rules.models.information import (
|
|
11
|
+
InformationClass,
|
|
12
|
+
InformationProperty,
|
|
13
|
+
InformationRules,
|
|
14
|
+
)
|
|
15
|
+
from cognite.neat.utils.utils import get_inheritance_path
|
|
16
|
+
|
|
17
|
+
from ._base import BaseAnalysis
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, InformationProperty, ClassEntity, str]):
|
|
21
|
+
"""Assumes analysis over only the complete schema"""
|
|
22
|
+
|
|
23
|
+
def _get_object(self, property_: InformationProperty) -> ClassEntity | None:
|
|
24
|
+
return property_.value_type if isinstance(property_.value_type, ClassEntity) else None
|
|
25
|
+
|
|
26
|
+
def _get_max_occurrence(self, property_: InformationProperty) -> int | float | None:
|
|
27
|
+
return property_.max_count
|
|
28
|
+
|
|
29
|
+
def _get_reference(self, class_or_property: InformationClass | InformationProperty) -> ReferenceEntity | None:
|
|
30
|
+
return class_or_property.reference if isinstance(class_or_property.reference, ReferenceEntity) else None
|
|
31
|
+
|
|
32
|
+
def _get_cls_entity(self, class_: InformationClass | InformationProperty) -> ClassEntity:
|
|
33
|
+
return class_.class_
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def _set_cls_entity(cls, property_: InformationProperty, class_: ClassEntity) -> None:
|
|
37
|
+
property_.class_ = class_
|
|
38
|
+
|
|
39
|
+
def _get_prop_entity(self, property_: InformationProperty) -> str:
|
|
40
|
+
return property_.property_
|
|
41
|
+
|
|
42
|
+
def _get_cls_parents(self, class_: InformationClass) -> list[ClassEntity] | None:
|
|
43
|
+
return list(class_.parent or []) or None
|
|
44
|
+
|
|
45
|
+
def _get_reference_rules(self) -> InformationRules | None:
|
|
46
|
+
return self.rules.reference
|
|
47
|
+
|
|
48
|
+
def _get_properties(self) -> list[InformationProperty]:
|
|
49
|
+
return list(self.rules.properties)
|
|
50
|
+
|
|
51
|
+
def _get_classes(self) -> list[InformationClass]:
|
|
52
|
+
return list(self.rules.classes)
|
|
53
|
+
|
|
54
|
+
def has_hop_transformations(self):
|
|
55
|
+
return any(
|
|
56
|
+
prop_.transformation and isinstance(prop_.transformation.traversal, Hop) for prop_ in self.rules.properties
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def define_property_renaming_config(self, class_: ClassEntity) -> dict[str, str]:
|
|
60
|
+
property_renaming_configuration = {}
|
|
61
|
+
|
|
62
|
+
if definitions := self.class_property_pairs(only_rdfpath=True, consider_inheritance=True).get(class_, None):
|
|
63
|
+
for property_id, definition in definitions.items():
|
|
64
|
+
if isinstance(
|
|
65
|
+
cast(RDFPath, definition.transformation).traversal,
|
|
66
|
+
SingleProperty,
|
|
67
|
+
):
|
|
68
|
+
graph_property = cast(
|
|
69
|
+
SingleProperty,
|
|
70
|
+
cast(RDFPath, definition.transformation).traversal,
|
|
71
|
+
).property.suffix
|
|
72
|
+
|
|
73
|
+
else:
|
|
74
|
+
graph_property = property_id
|
|
75
|
+
|
|
76
|
+
property_renaming_configuration[graph_property] = property_id
|
|
77
|
+
|
|
78
|
+
return property_renaming_configuration
|
|
79
|
+
|
|
80
|
+
def subset_rules(self, desired_classes: set[ClassEntity]) -> InformationRules:
|
|
81
|
+
"""
|
|
82
|
+
Subset rules to only include desired classes and their properties.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
desired_classes: Desired classes to include in the reduced data model
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
Instance of InformationRules
|
|
89
|
+
|
|
90
|
+
!!! note "Inheritance"
|
|
91
|
+
If desired classes contain a class that is a subclass of another class(es), the parent class(es)
|
|
92
|
+
will be included in the reduced data model as well even though the parent class(es) are
|
|
93
|
+
not in the desired classes set. This is to ensure that the reduced data model is
|
|
94
|
+
consistent and complete.
|
|
95
|
+
|
|
96
|
+
!!! note "Partial Reduction"
|
|
97
|
+
This method does not perform checks if classes that are value types of desired classes
|
|
98
|
+
properties are part of desired classes. If a class is not part of desired classes, but it
|
|
99
|
+
is a value type of a property of a class that is part of desired classes, derived reduced
|
|
100
|
+
rules will be marked as partial.
|
|
101
|
+
|
|
102
|
+
!!! note "Validation"
|
|
103
|
+
This method will attempt to validate the reduced rules with custom validations.
|
|
104
|
+
If it fails, it will return a partial rules with a warning message, validated
|
|
105
|
+
only with base Pydantic validators.
|
|
106
|
+
"""
|
|
107
|
+
if self.rules.metadata.schema_ is not SchemaCompleteness.complete:
|
|
108
|
+
raise ValueError("Rules are not complete cannot perform reduction!")
|
|
109
|
+
class_as_dict = self.as_class_dict()
|
|
110
|
+
class_parents_pairs = self.class_parent_pairs()
|
|
111
|
+
defined_classes = self.defined_classes(consider_inheritance=True)
|
|
112
|
+
|
|
113
|
+
possible_classes = defined_classes.intersection(desired_classes)
|
|
114
|
+
impossible_classes = desired_classes - possible_classes
|
|
115
|
+
|
|
116
|
+
# need to add all the parent classes of the desired classes to the possible classes
|
|
117
|
+
parents: set[ClassEntity] = set()
|
|
118
|
+
for class_ in possible_classes:
|
|
119
|
+
parents = parents.union({parent for parent in get_inheritance_path(class_, class_parents_pairs)})
|
|
120
|
+
possible_classes = possible_classes.union(parents)
|
|
121
|
+
|
|
122
|
+
if not possible_classes:
|
|
123
|
+
logging.error("None of the desired classes are defined in the data model!")
|
|
124
|
+
raise ValueError("None of the desired classes are defined in the data model!")
|
|
125
|
+
|
|
126
|
+
if impossible_classes:
|
|
127
|
+
logging.warning(f"Could not find the following classes defined in the data model: {impossible_classes}")
|
|
128
|
+
warnings.warn(
|
|
129
|
+
f"Could not find the following classes defined in the data model: {impossible_classes}",
|
|
130
|
+
stacklevel=2,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
reduced_data_model: dict[str, Any] = {
|
|
134
|
+
"metadata": self.rules.metadata.model_copy(),
|
|
135
|
+
"prefixes": (self.rules.prefixes or {}).copy(),
|
|
136
|
+
"classes": [],
|
|
137
|
+
"properties": [],
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
logging.info(f"Reducing data model to only include the following classes: {possible_classes}")
|
|
141
|
+
for class_ in possible_classes:
|
|
142
|
+
reduced_data_model["classes"].append(class_as_dict[str(class_.suffix)])
|
|
143
|
+
|
|
144
|
+
class_property_pairs = self.classes_with_properties(consider_inheritance=False)
|
|
145
|
+
|
|
146
|
+
for class_, properties in class_property_pairs.items():
|
|
147
|
+
if class_ in possible_classes:
|
|
148
|
+
reduced_data_model["properties"].extend(properties)
|
|
149
|
+
|
|
150
|
+
try:
|
|
151
|
+
return type(self.rules)(**reduced_data_model)
|
|
152
|
+
except ValidationError as e:
|
|
153
|
+
warnings.warn(f"Reduced data model is not complete: {e}", stacklevel=2)
|
|
154
|
+
reduced_data_model["metadata"].schema_ = SchemaCompleteness.partial
|
|
155
|
+
return type(self.rules).model_construct(**reduced_data_model)
|
|
@@ -28,10 +28,10 @@ class BaseExporter(ABC, Generic[T_Export]):
|
|
|
28
28
|
def _convert_to_output_role(self, rules: Rules, output_role: RoleTypes | None = None) -> Rules:
|
|
29
29
|
if rules.metadata.role is output_role or output_role is None:
|
|
30
30
|
return rules
|
|
31
|
-
elif output_role is RoleTypes.
|
|
32
|
-
return rules.
|
|
33
|
-
elif output_role is RoleTypes.
|
|
34
|
-
return rules.
|
|
31
|
+
elif output_role is RoleTypes.dms and isinstance(rules, InformationRules):
|
|
32
|
+
return rules.as_dms_rules()
|
|
33
|
+
elif output_role is RoleTypes.information and isinstance(rules, DMSRules):
|
|
34
|
+
return rules.as_information_rules()
|
|
35
35
|
else:
|
|
36
36
|
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
37
37
|
|
|
@@ -118,7 +118,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
118
118
|
if isinstance(rules, DMSRules):
|
|
119
119
|
dms_rules = rules
|
|
120
120
|
elif isinstance(rules, InformationRules):
|
|
121
|
-
dms_rules = rules.
|
|
121
|
+
dms_rules = rules.as_dms_rules()
|
|
122
122
|
else:
|
|
123
123
|
raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
|
|
124
124
|
return dms_rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
|
|
@@ -10,7 +10,7 @@ from rdflib.collection import Collection as GraphCollection
|
|
|
10
10
|
|
|
11
11
|
from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
|
|
12
12
|
from cognite.neat.rules import exceptions
|
|
13
|
-
from cognite.neat.rules.analysis import
|
|
13
|
+
from cognite.neat.rules.analysis import InformationAnalysis
|
|
14
14
|
from cognite.neat.rules.models import DMSRules
|
|
15
15
|
from cognite.neat.rules.models.data_types import DataType
|
|
16
16
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
|
|
@@ -95,7 +95,7 @@ class Ontology(OntologyModel):
|
|
|
95
95
|
if isinstance(input_rules, InformationRules):
|
|
96
96
|
rules = input_rules
|
|
97
97
|
elif isinstance(input_rules, DMSRules):
|
|
98
|
-
rules = input_rules.
|
|
98
|
+
rules = input_rules.as_information_rules()
|
|
99
99
|
else:
|
|
100
100
|
raise ValueError(f"{type(input_rules).__name__} cannot be exported to Ontology")
|
|
101
101
|
|
|
@@ -109,11 +109,11 @@ class Ontology(OntologyModel):
|
|
|
109
109
|
if rules.metadata.namespace is None:
|
|
110
110
|
raise exceptions.MissingDataModelPrefixOrNamespace()
|
|
111
111
|
|
|
112
|
-
class_dict =
|
|
112
|
+
class_dict = InformationAnalysis(rules).as_class_dict()
|
|
113
113
|
return cls(
|
|
114
114
|
properties=[
|
|
115
115
|
OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
|
|
116
|
-
for definition in
|
|
116
|
+
for definition in InformationAnalysis(rules).as_property_dict().values()
|
|
117
117
|
],
|
|
118
118
|
classes=[
|
|
119
119
|
OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
|
|
@@ -125,7 +125,7 @@ class Ontology(OntologyModel):
|
|
|
125
125
|
list(properties.values()),
|
|
126
126
|
rules.metadata.namespace,
|
|
127
127
|
)
|
|
128
|
-
for class_, properties in
|
|
128
|
+
for class_, properties in InformationAnalysis(rules).class_property_pairs().items()
|
|
129
129
|
]
|
|
130
130
|
+ [
|
|
131
131
|
SHACLNodeShape.from_rules(
|
|
@@ -55,10 +55,10 @@ class BaseImporter(ABC):
|
|
|
55
55
|
|
|
56
56
|
if rules.metadata.role is role or role is None:
|
|
57
57
|
output = rules
|
|
58
|
-
elif isinstance(rules, DMSRules) or isinstance(rules, AssetRules) and role is RoleTypes.
|
|
59
|
-
output = rules.
|
|
60
|
-
elif isinstance(rules, InformationRules) or isinstance(rules, AssetRules) and role is RoleTypes.
|
|
61
|
-
output = rules.
|
|
58
|
+
elif isinstance(rules, DMSRules) or isinstance(rules, AssetRules) and role is RoleTypes.information:
|
|
59
|
+
output = rules.as_information_rules()
|
|
60
|
+
elif isinstance(rules, InformationRules) or isinstance(rules, AssetRules) and role is RoleTypes.dms:
|
|
61
|
+
output = rules.as_dms_rules()
|
|
62
62
|
else:
|
|
63
63
|
raise NotImplementedError(f"Role {role} is not supported for {type(rules).__name__} rules")
|
|
64
64
|
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
|
-
from typing import cast
|
|
4
3
|
|
|
5
4
|
import cognite.neat.rules.issues.importing
|
|
6
5
|
from cognite.neat.rules import issues
|
|
@@ -22,7 +21,7 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
22
21
|
)
|
|
23
22
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
24
23
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
25
|
-
from cognite.neat.rules.models.entities import ClassEntity
|
|
24
|
+
from cognite.neat.rules.models.entities import ClassEntity
|
|
26
25
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
27
26
|
|
|
28
27
|
|
|
@@ -89,12 +88,7 @@ class _DTDLConverter:
|
|
|
89
88
|
name=item.display_name,
|
|
90
89
|
description=item.description,
|
|
91
90
|
comment=item.comment,
|
|
92
|
-
parent=[
|
|
93
|
-
cast(ParentClassEntity, parent_entity)
|
|
94
|
-
for parent in item.extends or []
|
|
95
|
-
if isinstance(parent_entity := ParentClassEntity.load(parent.as_class_id()), ParentClassEntity)
|
|
96
|
-
]
|
|
97
|
-
or None,
|
|
91
|
+
parent=[parent.as_class_id() for parent in item.extends or []] or None,
|
|
98
92
|
)
|
|
99
93
|
self.classes.append(class_)
|
|
100
94
|
for sub_item_or_id in item.contents or []:
|
|
@@ -7,7 +7,7 @@ from rdflib import Graph, Namespace, URIRef
|
|
|
7
7
|
from rdflib import Literal as RdfLiteral
|
|
8
8
|
|
|
9
9
|
import cognite.neat.rules.issues as issues
|
|
10
|
-
from cognite.neat.constants import DEFAULT_NAMESPACE,
|
|
10
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
11
11
|
from cognite.neat.graph.stores import NeatGraphStore
|
|
12
12
|
from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
|
|
13
13
|
from cognite.neat.rules.issues import IssueList
|
|
@@ -204,7 +204,7 @@ class InferenceImporter(BaseImporter):
|
|
|
204
204
|
"""
|
|
205
205
|
classes: dict[str, dict] = {}
|
|
206
206
|
properties: dict[str, dict] = {}
|
|
207
|
-
prefixes: dict[str, Namespace] =
|
|
207
|
+
prefixes: dict[str, Namespace] = get_default_prefixes()
|
|
208
208
|
|
|
209
209
|
query = INSTANCE_PROPERTIES_JSON_DEFINITION if self.check_for_json_string else INSTANCE_PROPERTIES_DEFINITION
|
|
210
210
|
# Adds default namespace to prefixes
|
|
@@ -51,7 +51,7 @@ def parse_owl_metadata(graph: Graph) -> dict:
|
|
|
51
51
|
|
|
52
52
|
raw_metadata = convert_rdflib_content(
|
|
53
53
|
{
|
|
54
|
-
"role": RoleTypes.
|
|
54
|
+
"role": RoleTypes.information,
|
|
55
55
|
"schema": SchemaCompleteness.partial,
|
|
56
56
|
"prefix": results[1].pop(),
|
|
57
57
|
"namespace": Namespace(results[0].pop()),
|
|
@@ -36,9 +36,9 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
|
|
|
36
36
|
"Properties",
|
|
37
37
|
{
|
|
38
38
|
RoleTypes.domain_expert: "Property",
|
|
39
|
-
RoleTypes.
|
|
40
|
-
RoleTypes.
|
|
41
|
-
RoleTypes.
|
|
39
|
+
RoleTypes.information: "Property",
|
|
40
|
+
RoleTypes.asset: "Property",
|
|
41
|
+
RoleTypes.dms: "View Property",
|
|
42
42
|
},
|
|
43
43
|
),
|
|
44
44
|
("Classes", "Classes", "Class"),
|
|
@@ -334,9 +334,9 @@ class GoogleSheetImporter(BaseImporter):
|
|
|
334
334
|
|
|
335
335
|
if role == RoleTypes.domain_expert:
|
|
336
336
|
output = rules_model.model_validate(sheets)
|
|
337
|
-
elif role == RoleTypes.
|
|
337
|
+
elif role == RoleTypes.information:
|
|
338
338
|
output = rules_model.model_validate(sheets)
|
|
339
|
-
elif role == RoleTypes.
|
|
339
|
+
elif role == RoleTypes.dms:
|
|
340
340
|
output = rules_model.model_validate(sheets)
|
|
341
341
|
else:
|
|
342
342
|
raise ValueError(f"Role {role} is not valid.")
|
|
@@ -8,9 +8,9 @@ from .dms._schema import DMSSchema
|
|
|
8
8
|
|
|
9
9
|
RULES_PER_ROLE: dict[RoleTypes, type[DomainRules] | type[InformationRules] | type[AssetRules] | type[DMSRules]] = {
|
|
10
10
|
RoleTypes.domain_expert: DomainRules,
|
|
11
|
-
RoleTypes.
|
|
12
|
-
RoleTypes.
|
|
13
|
-
RoleTypes.
|
|
11
|
+
RoleTypes.information: InformationRules,
|
|
12
|
+
RoleTypes.asset: AssetRules,
|
|
13
|
+
RoleTypes.dms: DMSRules,
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
|
|
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|
|
7
7
|
import math
|
|
8
8
|
import sys
|
|
9
9
|
import types
|
|
10
|
-
from abc import abstractmethod
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
11
|
from collections.abc import Callable, Iterator
|
|
12
12
|
from functools import wraps
|
|
13
13
|
from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
|
|
@@ -18,7 +18,6 @@ from pydantic import (
|
|
|
18
18
|
BeforeValidator,
|
|
19
19
|
ConfigDict,
|
|
20
20
|
Field,
|
|
21
|
-
HttpUrl,
|
|
22
21
|
PlainSerializer,
|
|
23
22
|
constr,
|
|
24
23
|
field_validator,
|
|
@@ -150,9 +149,9 @@ class DataModelType(StrEnum):
|
|
|
150
149
|
|
|
151
150
|
class RoleTypes(StrEnum):
|
|
152
151
|
domain_expert = "domain expert"
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
152
|
+
information = "information architect"
|
|
153
|
+
asset = "asset architect"
|
|
154
|
+
dms = "DMS Architect"
|
|
156
155
|
|
|
157
156
|
|
|
158
157
|
class MatchType(StrEnum):
|
|
@@ -219,10 +218,6 @@ class RuleModel(BaseModel):
|
|
|
219
218
|
return headers_by_sheet
|
|
220
219
|
|
|
221
220
|
|
|
222
|
-
class URL(BaseModel):
|
|
223
|
-
url: HttpUrl
|
|
224
|
-
|
|
225
|
-
|
|
226
221
|
class BaseMetadata(RuleModel):
|
|
227
222
|
"""
|
|
228
223
|
Metadata model for data model
|
|
@@ -252,8 +247,13 @@ class BaseMetadata(RuleModel):
|
|
|
252
247
|
"""Returns a unique identifier for the metadata."""
|
|
253
248
|
raise NotImplementedError()
|
|
254
249
|
|
|
250
|
+
@abstractmethod
|
|
251
|
+
def get_prefix(self) -> str:
|
|
252
|
+
"""Returns the prefix for the metadata."""
|
|
253
|
+
raise NotImplementedError()
|
|
254
|
+
|
|
255
255
|
|
|
256
|
-
class BaseRules(RuleModel):
|
|
256
|
+
class BaseRules(RuleModel, ABC):
|
|
257
257
|
"""
|
|
258
258
|
Rules is a core concept in `neat`. This represents fusion of data model
|
|
259
259
|
definitions and (optionally) the transformation rules used to transform the data/graph
|
|
@@ -5,7 +5,7 @@ from pydantic import Field, field_validator, model_validator
|
|
|
5
5
|
from pydantic.main import IncEx
|
|
6
6
|
from rdflib import Namespace
|
|
7
7
|
|
|
8
|
-
from cognite.neat.constants import
|
|
8
|
+
from cognite.neat.constants import get_default_prefixes
|
|
9
9
|
from cognite.neat.issues import MultiValueError
|
|
10
10
|
from cognite.neat.rules import issues
|
|
11
11
|
from cognite.neat.rules.models._base import BaseRules, RoleTypes, SheetList
|
|
@@ -14,7 +14,6 @@ from cognite.neat.rules.models.entities import (
|
|
|
14
14
|
CdfResourceEntityList,
|
|
15
15
|
ClassEntity,
|
|
16
16
|
MultiValueTypeInfo,
|
|
17
|
-
ParentClassEntity,
|
|
18
17
|
Undefined,
|
|
19
18
|
)
|
|
20
19
|
from cognite.neat.rules.models.information import (
|
|
@@ -35,7 +34,7 @@ else:
|
|
|
35
34
|
|
|
36
35
|
|
|
37
36
|
class AssetMetadata(InformationMetadata):
|
|
38
|
-
role: ClassVar[RoleTypes] = RoleTypes.
|
|
37
|
+
role: ClassVar[RoleTypes] = RoleTypes.asset
|
|
39
38
|
|
|
40
39
|
|
|
41
40
|
class AssetClass(InformationClass): ...
|
|
@@ -61,14 +60,14 @@ class AssetProperty(InformationProperty):
|
|
|
61
60
|
implementation: Details on how given class-property is implemented in the classic CDF
|
|
62
61
|
"""
|
|
63
62
|
|
|
64
|
-
implementation: CdfResourceEntityList
|
|
63
|
+
implementation: CdfResourceEntityList = Field(alias="Implementation")
|
|
65
64
|
|
|
66
65
|
|
|
67
66
|
class AssetRules(BaseRules):
|
|
68
67
|
metadata: AssetMetadata = Field(alias="Metadata")
|
|
69
68
|
properties: SheetList[AssetProperty] = Field(alias="Properties")
|
|
70
69
|
classes: SheetList[AssetClass] = Field(alias="Classes")
|
|
71
|
-
prefixes: dict[str, Namespace] = Field(default_factory=
|
|
70
|
+
prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes)
|
|
72
71
|
last: "AssetRules | None" = Field(None, alias="Last")
|
|
73
72
|
reference: "AssetRules | None" = Field(None, alias="Reference")
|
|
74
73
|
|
|
@@ -94,7 +93,7 @@ class AssetRules(BaseRules):
|
|
|
94
93
|
# update parent classes
|
|
95
94
|
for class_ in self.classes:
|
|
96
95
|
if class_.parent:
|
|
97
|
-
for parent in
|
|
96
|
+
for parent in class_.parent:
|
|
98
97
|
if not isinstance(parent.prefix, str):
|
|
99
98
|
parent.prefix = self.metadata.prefix
|
|
100
99
|
if class_.class_.prefix is Undefined:
|
|
@@ -148,12 +147,12 @@ class AssetRules(BaseRules):
|
|
|
148
147
|
def as_domain_rules(self) -> DomainRules:
|
|
149
148
|
from ._converter import _AssetRulesConverter
|
|
150
149
|
|
|
151
|
-
return _AssetRulesConverter(self.
|
|
150
|
+
return _AssetRulesConverter(self.as_information_rules()).as_domain_rules()
|
|
152
151
|
|
|
153
|
-
def
|
|
152
|
+
def as_dms_rules(self) -> "DMSRules":
|
|
154
153
|
from ._converter import _AssetRulesConverter
|
|
155
154
|
|
|
156
|
-
return _AssetRulesConverter(self.
|
|
155
|
+
return _AssetRulesConverter(self.as_information_rules()).as_dms_rules()
|
|
157
156
|
|
|
158
|
-
def
|
|
157
|
+
def as_information_rules(self) -> InformationRules:
|
|
159
158
|
return InformationRules.model_validate(self.model_dump())
|
|
@@ -11,7 +11,6 @@ from cognite.neat.rules.models.entities import (
|
|
|
11
11
|
ClassEntity,
|
|
12
12
|
ContainerEntity,
|
|
13
13
|
DMSUnknownEntity,
|
|
14
|
-
ParentClassEntity,
|
|
15
14
|
ReferenceEntity,
|
|
16
15
|
UnknownEntity,
|
|
17
16
|
ViewEntity,
|
|
@@ -32,7 +31,7 @@ class _DMSRulesConverter:
|
|
|
32
31
|
def as_domain_rules(self) -> "DomainRules":
|
|
33
32
|
raise NotImplementedError("DomainRules not implemented yet")
|
|
34
33
|
|
|
35
|
-
def
|
|
34
|
+
def as_information_rules(
|
|
36
35
|
self,
|
|
37
36
|
) -> "InformationRules":
|
|
38
37
|
from cognite.neat.rules.models.information._rules import (
|
|
@@ -52,7 +51,7 @@ class _DMSRulesConverter:
|
|
|
52
51
|
description=view.description,
|
|
53
52
|
parent=[
|
|
54
53
|
# we do not want a version in class as we use URI for the class
|
|
55
|
-
|
|
54
|
+
implemented_view.as_class(skip_version=True)
|
|
56
55
|
# We only want parents in the same namespace, parent in a different namespace is a reference
|
|
57
56
|
for implemented_view in view.implements or []
|
|
58
57
|
if implemented_view.prefix == view.class_.prefix
|
|
@@ -94,8 +93,8 @@ class _DMSRulesConverter:
|
|
|
94
93
|
metadata=metadata,
|
|
95
94
|
properties=SheetList[InformationProperty](data=properties),
|
|
96
95
|
classes=SheetList[InformationClass](data=classes),
|
|
97
|
-
last=self.dms.last.
|
|
98
|
-
reference=self.dms.reference.
|
|
96
|
+
last=self.dms.last.as_information_rules() if self.dms.last else None,
|
|
97
|
+
reference=self.dms.reference.as_information_rules() if self.dms.reference else None,
|
|
99
98
|
)
|
|
100
99
|
|
|
101
100
|
@classmethod
|
|
@@ -58,7 +58,7 @@ _DEFAULT_VERSION = "1"
|
|
|
58
58
|
|
|
59
59
|
|
|
60
60
|
class DMSMetadata(BaseMetadata):
|
|
61
|
-
role: ClassVar[RoleTypes] = RoleTypes.
|
|
61
|
+
role: ClassVar[RoleTypes] = RoleTypes.dms
|
|
62
62
|
data_model_type: DataModelType = Field(DataModelType.enterprise, alias="dataModelType")
|
|
63
63
|
schema_: SchemaCompleteness = Field(alias="schema")
|
|
64
64
|
extension: ExtensionCategory = ExtensionCategory.addition
|
|
@@ -165,6 +165,9 @@ class DMSMetadata(BaseMetadata):
|
|
|
165
165
|
updated=datetime.now(),
|
|
166
166
|
)
|
|
167
167
|
|
|
168
|
+
def get_prefix(self) -> str:
|
|
169
|
+
return self.space
|
|
170
|
+
|
|
168
171
|
|
|
169
172
|
class DMSProperty(SheetEntity):
|
|
170
173
|
view: ViewEntity = Field(alias="View")
|
|
@@ -382,10 +385,10 @@ class DMSRules(BaseRules):
|
|
|
382
385
|
|
|
383
386
|
return _DMSExporter(self, include_pipeline, instance_space).to_schema()
|
|
384
387
|
|
|
385
|
-
def
|
|
388
|
+
def as_information_rules(self) -> "InformationRules":
|
|
386
389
|
from ._converter import _DMSRulesConverter
|
|
387
390
|
|
|
388
|
-
return _DMSRulesConverter(self).
|
|
391
|
+
return _DMSRulesConverter(self).as_information_rules()
|
|
389
392
|
|
|
390
393
|
def as_domain_expert_rules(self) -> DomainRules:
|
|
391
394
|
from ._converter import _DMSRulesConverter
|
|
@@ -5,7 +5,7 @@ from pydantic import Field, field_serializer, field_validator, model_serializer
|
|
|
5
5
|
from pydantic_core.core_schema import SerializationInfo
|
|
6
6
|
|
|
7
7
|
from cognite.neat.rules.models.data_types import DataType
|
|
8
|
-
from cognite.neat.rules.models.entities import ClassEntity,
|
|
8
|
+
from cognite.neat.rules.models.entities import ClassEntity, ClassEntityList
|
|
9
9
|
|
|
10
10
|
from ._base import (
|
|
11
11
|
BaseMetadata,
|
|
@@ -24,6 +24,9 @@ class DomainMetadata(BaseMetadata):
|
|
|
24
24
|
def as_identifier(self) -> str:
|
|
25
25
|
return "DomainRules"
|
|
26
26
|
|
|
27
|
+
def get_prefix(self) -> str:
|
|
28
|
+
return "domain"
|
|
29
|
+
|
|
27
30
|
|
|
28
31
|
class DomainProperty(SheetEntity):
|
|
29
32
|
class_: ClassEntity = Field(alias="Class")
|
|
@@ -51,7 +54,7 @@ class DomainClass(SheetEntity):
|
|
|
51
54
|
class_: ClassEntity = Field(alias="Class")
|
|
52
55
|
name: str | None = Field(alias="Name", default=None)
|
|
53
56
|
description: str | None = Field(None, alias="Description")
|
|
54
|
-
parent:
|
|
57
|
+
parent: ClassEntityList | None = Field(alias="Parent Class")
|
|
55
58
|
|
|
56
59
|
|
|
57
60
|
class DomainRules(BaseRules):
|
|
@@ -252,13 +252,6 @@ class ClassEntity(Entity):
|
|
|
252
252
|
return ContainerEntity(space=space, externalId=str(self.suffix))
|
|
253
253
|
|
|
254
254
|
|
|
255
|
-
class ParentClassEntity(ClassEntity):
|
|
256
|
-
type_: ClassVar[EntityTypes] = EntityTypes.parent_class
|
|
257
|
-
|
|
258
|
-
def as_class_entity(self) -> ClassEntity:
|
|
259
|
-
return ClassEntity(prefix=self.prefix, suffix=self.suffix, version=self.version)
|
|
260
|
-
|
|
261
|
-
|
|
262
255
|
class UnknownEntity(ClassEntity):
|
|
263
256
|
type_: ClassVar[EntityTypes] = EntityTypes.undefined
|
|
264
257
|
prefix: _UndefinedType = Undefined
|
|
@@ -555,8 +548,8 @@ def _generate_cdf_resource_list(v: Any) -> list[AssetEntity | RelationshipEntity
|
|
|
555
548
|
return results # type: ignore
|
|
556
549
|
|
|
557
550
|
|
|
558
|
-
|
|
559
|
-
list[
|
|
551
|
+
ClassEntityList = Annotated[
|
|
552
|
+
list[ClassEntity],
|
|
560
553
|
BeforeValidator(_split_str),
|
|
561
554
|
PlainSerializer(
|
|
562
555
|
_join_str,
|
|
@@ -79,7 +79,7 @@ class _InformationRulesConverter:
|
|
|
79
79
|
prefixes=self.rules.prefixes,
|
|
80
80
|
)
|
|
81
81
|
|
|
82
|
-
def
|
|
82
|
+
def as_dms_rules(self) -> "DMSRules":
|
|
83
83
|
from cognite.neat.rules.models.dms._rules import (
|
|
84
84
|
DMSContainer,
|
|
85
85
|
DMSProperty,
|
|
@@ -112,8 +112,8 @@ class _InformationRulesConverter:
|
|
|
112
112
|
for cls_ in self.rules.classes
|
|
113
113
|
]
|
|
114
114
|
|
|
115
|
-
last_dms_rules = self.rules.last.
|
|
116
|
-
ref_dms_rules = self.rules.reference.
|
|
115
|
+
last_dms_rules = self.rules.last.as_dms_rules() if self.rules.last else None
|
|
116
|
+
ref_dms_rules = self.rules.reference.as_dms_rules() if self.rules.reference else None
|
|
117
117
|
|
|
118
118
|
class_by_entity = {cls_.class_: cls_ for cls_ in self.rules.classes}
|
|
119
119
|
if self.rules.last:
|