cognite-neat 0.86.0__py3-none-any.whl → 0.87.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/configuration.py +1 -10
- cognite/neat/app/api/routers/data_exploration.py +1 -1
- cognite/neat/config.py +84 -17
- cognite/neat/constants.py +11 -9
- cognite/neat/graph/extractors/_classic_cdf/_assets.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_events.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_files.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_labels.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_relationships.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_sequences.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +1 -1
- cognite/neat/graph/extractors/_dexpi.py +1 -1
- cognite/neat/graph/extractors/_mock_graph_generator.py +8 -9
- cognite/neat/graph/loaders/__init__.py +5 -2
- cognite/neat/graph/loaders/_base.py +13 -5
- cognite/neat/graph/loaders/_rdf2asset.py +185 -55
- cognite/neat/graph/loaders/_rdf2dms.py +7 -7
- cognite/neat/graph/queries/_base.py +20 -11
- cognite/neat/graph/queries/_construct.py +5 -5
- cognite/neat/graph/queries/_shared.py +21 -7
- cognite/neat/graph/stores/_base.py +16 -4
- cognite/neat/graph/transformers/__init__.py +3 -0
- cognite/neat/graph/transformers/_rdfpath.py +42 -0
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
- cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -1
- cognite/neat/legacy/graph/loaders/_asset_loader.py +2 -2
- cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +5 -2
- cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +4 -1
- cognite/neat/legacy/graph/loaders/rdf_to_dms.py +3 -1
- cognite/neat/legacy/graph/stores/_base.py +24 -8
- cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
- cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +49 -16
- cognite/neat/legacy/graph/transformations/transformer.py +1 -1
- cognite/neat/legacy/rules/exporters/_rules2dms.py +8 -3
- cognite/neat/legacy/rules/exporters/_rules2graphql.py +1 -1
- cognite/neat/legacy/rules/exporters/_rules2ontology.py +2 -1
- cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +3 -4
- cognite/neat/legacy/rules/importers/_dms2rules.py +4 -1
- cognite/neat/legacy/rules/importers/_graph2rules.py +3 -3
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +2 -1
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/legacy/rules/models/raw_rules.py +19 -7
- cognite/neat/legacy/rules/models/rules.py +32 -12
- cognite/neat/rules/_shared.py +6 -1
- cognite/neat/rules/analysis/__init__.py +4 -4
- cognite/neat/rules/analysis/_asset.py +143 -0
- cognite/neat/rules/analysis/_base.py +385 -6
- cognite/neat/rules/analysis/_information.py +183 -0
- cognite/neat/rules/exporters/_rules2dms.py +1 -1
- cognite/neat/rules/exporters/_rules2ontology.py +6 -5
- cognite/neat/rules/importers/_dms2rules.py +3 -1
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
- cognite/neat/rules/importers/_inference2rules.py +3 -7
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -1
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/rules/issues/spreadsheet.py +35 -0
- cognite/neat/rules/models/_base.py +7 -7
- cognite/neat/rules/models/_rdfpath.py +17 -21
- cognite/neat/rules/models/asset/_rules.py +4 -5
- cognite/neat/rules/models/asset/_validation.py +38 -1
- cognite/neat/rules/models/dms/_converter.py +1 -2
- cognite/neat/rules/models/dms/_exporter.py +7 -3
- cognite/neat/rules/models/dms/_rules.py +3 -0
- cognite/neat/rules/models/dms/_schema.py +5 -4
- cognite/neat/rules/models/domain.py +5 -2
- cognite/neat/rules/models/entities.py +28 -17
- cognite/neat/rules/models/information/_rules.py +10 -8
- cognite/neat/rules/models/information/_rules_input.py +1 -2
- cognite/neat/rules/models/information/_validation.py +2 -2
- cognite/neat/utils/__init__.py +0 -3
- cognite/neat/utils/auth.py +47 -28
- cognite/neat/utils/auxiliary.py +141 -1
- cognite/neat/utils/cdf/__init__.py +0 -0
- cognite/neat/utils/{cdf_classes.py → cdf/data_classes.py} +122 -2
- cognite/neat/utils/{cdf_loaders → cdf/loaders}/_data_modeling.py +37 -0
- cognite/neat/utils/{cdf_loaders → cdf/loaders}/_ingestion.py +2 -1
- cognite/neat/utils/collection_.py +18 -0
- cognite/neat/utils/rdf_.py +165 -0
- cognite/neat/utils/text.py +4 -0
- cognite/neat/utils/time_.py +17 -0
- cognite/neat/utils/upload.py +13 -1
- cognite/neat/workflows/_exceptions.py +5 -5
- cognite/neat/workflows/base.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
- cognite/neat/workflows/steps/lib/current/rules_validator.py +2 -2
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +130 -28
- cognite/neat/workflows/steps/lib/legacy/graph_loader.py +1 -1
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
- cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +1 -1
- cognite/neat/workflows/steps/lib/legacy/rules_importer.py +1 -1
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/METADATA +2 -2
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/RECORD +103 -102
- cognite/neat/rules/analysis/_information_rules.py +0 -476
- cognite/neat/utils/cdf.py +0 -59
- cognite/neat/utils/cdf_loaders/data_classes.py +0 -121
- cognite/neat/utils/exceptions.py +0 -41
- cognite/neat/utils/utils.py +0 -429
- /cognite/neat/utils/{cdf_loaders → cdf/loaders}/__init__.py +0 -0
- /cognite/neat/utils/{cdf_loaders → cdf/loaders}/_base.py +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import warnings
|
|
3
|
+
from typing import Any, cast
|
|
4
|
+
|
|
5
|
+
from pydantic import ValidationError
|
|
6
|
+
from rdflib import URIRef
|
|
7
|
+
|
|
8
|
+
from cognite.neat.rules.models import SchemaCompleteness
|
|
9
|
+
from cognite.neat.rules.models._rdfpath import (
|
|
10
|
+
Hop,
|
|
11
|
+
RDFPath,
|
|
12
|
+
SelfReferenceProperty,
|
|
13
|
+
SingleProperty,
|
|
14
|
+
)
|
|
15
|
+
from cognite.neat.rules.models.entities import ClassEntity, ReferenceEntity
|
|
16
|
+
from cognite.neat.rules.models.information import (
|
|
17
|
+
InformationClass,
|
|
18
|
+
InformationProperty,
|
|
19
|
+
InformationRules,
|
|
20
|
+
)
|
|
21
|
+
from cognite.neat.utils.rdf_ import get_inheritance_path
|
|
22
|
+
|
|
23
|
+
from ._base import BaseAnalysis
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, InformationProperty, ClassEntity, str]):
|
|
27
|
+
"""Assumes analysis over only the complete schema"""
|
|
28
|
+
|
|
29
|
+
def _get_object(self, property_: InformationProperty) -> ClassEntity | None:
|
|
30
|
+
return property_.value_type if isinstance(property_.value_type, ClassEntity) else None
|
|
31
|
+
|
|
32
|
+
def _get_max_occurrence(self, property_: InformationProperty) -> int | float | None:
|
|
33
|
+
return property_.max_count
|
|
34
|
+
|
|
35
|
+
def _get_reference(self, class_or_property: InformationClass | InformationProperty) -> ReferenceEntity | None:
|
|
36
|
+
return class_or_property.reference if isinstance(class_or_property.reference, ReferenceEntity) else None
|
|
37
|
+
|
|
38
|
+
def _get_cls_entity(self, class_: InformationClass | InformationProperty) -> ClassEntity:
|
|
39
|
+
return class_.class_
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def _set_cls_entity(cls, property_: InformationProperty, class_: ClassEntity) -> None:
|
|
43
|
+
property_.class_ = class_
|
|
44
|
+
|
|
45
|
+
def _get_prop_entity(self, property_: InformationProperty) -> str:
|
|
46
|
+
return property_.property_
|
|
47
|
+
|
|
48
|
+
def _get_cls_parents(self, class_: InformationClass) -> list[ClassEntity] | None:
|
|
49
|
+
return list(class_.parent or []) or None
|
|
50
|
+
|
|
51
|
+
def _get_reference_rules(self) -> InformationRules | None:
|
|
52
|
+
return self.rules.reference
|
|
53
|
+
|
|
54
|
+
def _get_properties(self) -> list[InformationProperty]:
|
|
55
|
+
return list(self.rules.properties)
|
|
56
|
+
|
|
57
|
+
def _get_classes(self) -> list[InformationClass]:
|
|
58
|
+
return list(self.rules.classes)
|
|
59
|
+
|
|
60
|
+
def has_hop_transformations(self):
|
|
61
|
+
return any(
|
|
62
|
+
prop_.transformation and isinstance(prop_.transformation.traversal, Hop) for prop_ in self.rules.properties
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def has_self_reference_property_transformations(self):
|
|
66
|
+
return any(
|
|
67
|
+
prop_.transformation and isinstance(prop_.transformation.traversal, SelfReferenceProperty)
|
|
68
|
+
for prop_ in self.rules.properties
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
def all_reference_transformations(self):
|
|
72
|
+
return [
|
|
73
|
+
prop_
|
|
74
|
+
for prop_ in self.rules.properties
|
|
75
|
+
if prop_.transformation and isinstance(prop_.transformation.traversal, SelfReferenceProperty)
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
def define_property_renaming_config(self, class_: ClassEntity) -> dict[str | URIRef, str]:
|
|
79
|
+
property_renaming_configuration: dict[str | URIRef, str] = {}
|
|
80
|
+
|
|
81
|
+
if definitions := self.class_property_pairs(only_rdfpath=True, consider_inheritance=True).get(class_, None):
|
|
82
|
+
for property_id, definition in definitions.items():
|
|
83
|
+
transformation = cast(RDFPath, definition.transformation)
|
|
84
|
+
|
|
85
|
+
# use case we have a single property rdf path, and defined prefix
|
|
86
|
+
# in either metadata or prefixes of rules
|
|
87
|
+
if isinstance(
|
|
88
|
+
transformation.traversal,
|
|
89
|
+
SingleProperty,
|
|
90
|
+
) and (
|
|
91
|
+
transformation.traversal.property.prefix in self.rules.prefixes
|
|
92
|
+
or transformation.traversal.property.prefix == self.rules.metadata.prefix
|
|
93
|
+
):
|
|
94
|
+
namespace = (
|
|
95
|
+
self.rules.metadata.namespace
|
|
96
|
+
if transformation.traversal.property.prefix == self.rules.metadata.prefix
|
|
97
|
+
else self.rules.prefixes[transformation.traversal.property.prefix]
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
property_renaming_configuration[namespace[transformation.traversal.property.suffix]] = property_id
|
|
101
|
+
|
|
102
|
+
# otherwise we default to the property id
|
|
103
|
+
else:
|
|
104
|
+
property_renaming_configuration[property_id] = property_id
|
|
105
|
+
|
|
106
|
+
return property_renaming_configuration
|
|
107
|
+
|
|
108
|
+
def subset_rules(self, desired_classes: set[ClassEntity]) -> InformationRules:
|
|
109
|
+
"""
|
|
110
|
+
Subset rules to only include desired classes and their properties.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
desired_classes: Desired classes to include in the reduced data model
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
Instance of InformationRules
|
|
117
|
+
|
|
118
|
+
!!! note "Inheritance"
|
|
119
|
+
If desired classes contain a class that is a subclass of another class(es), the parent class(es)
|
|
120
|
+
will be included in the reduced data model as well even though the parent class(es) are
|
|
121
|
+
not in the desired classes set. This is to ensure that the reduced data model is
|
|
122
|
+
consistent and complete.
|
|
123
|
+
|
|
124
|
+
!!! note "Partial Reduction"
|
|
125
|
+
This method does not perform checks if classes that are value types of desired classes
|
|
126
|
+
properties are part of desired classes. If a class is not part of desired classes, but it
|
|
127
|
+
is a value type of a property of a class that is part of desired classes, derived reduced
|
|
128
|
+
rules will be marked as partial.
|
|
129
|
+
|
|
130
|
+
!!! note "Validation"
|
|
131
|
+
This method will attempt to validate the reduced rules with custom validations.
|
|
132
|
+
If it fails, it will return a partial rules with a warning message, validated
|
|
133
|
+
only with base Pydantic validators.
|
|
134
|
+
"""
|
|
135
|
+
if self.rules.metadata.schema_ is not SchemaCompleteness.complete:
|
|
136
|
+
raise ValueError("Rules are not complete cannot perform reduction!")
|
|
137
|
+
class_as_dict = self.as_class_dict()
|
|
138
|
+
class_parents_pairs = self.class_parent_pairs()
|
|
139
|
+
defined_classes = self.defined_classes(consider_inheritance=True)
|
|
140
|
+
|
|
141
|
+
possible_classes = defined_classes.intersection(desired_classes)
|
|
142
|
+
impossible_classes = desired_classes - possible_classes
|
|
143
|
+
|
|
144
|
+
# need to add all the parent classes of the desired classes to the possible classes
|
|
145
|
+
parents: set[ClassEntity] = set()
|
|
146
|
+
for class_ in possible_classes:
|
|
147
|
+
parents = parents.union({parent for parent in get_inheritance_path(class_, class_parents_pairs)})
|
|
148
|
+
possible_classes = possible_classes.union(parents)
|
|
149
|
+
|
|
150
|
+
if not possible_classes:
|
|
151
|
+
logging.error("None of the desired classes are defined in the data model!")
|
|
152
|
+
raise ValueError("None of the desired classes are defined in the data model!")
|
|
153
|
+
|
|
154
|
+
if impossible_classes:
|
|
155
|
+
logging.warning(f"Could not find the following classes defined in the data model: {impossible_classes}")
|
|
156
|
+
warnings.warn(
|
|
157
|
+
f"Could not find the following classes defined in the data model: {impossible_classes}",
|
|
158
|
+
stacklevel=2,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
reduced_data_model: dict[str, Any] = {
|
|
162
|
+
"metadata": self.rules.metadata.model_copy(),
|
|
163
|
+
"prefixes": (self.rules.prefixes or {}).copy(),
|
|
164
|
+
"classes": [],
|
|
165
|
+
"properties": [],
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
logging.info(f"Reducing data model to only include the following classes: {possible_classes}")
|
|
169
|
+
for class_ in possible_classes:
|
|
170
|
+
reduced_data_model["classes"].append(class_as_dict[str(class_.suffix)])
|
|
171
|
+
|
|
172
|
+
class_property_pairs = self.classes_with_properties(consider_inheritance=False)
|
|
173
|
+
|
|
174
|
+
for class_, properties in class_property_pairs.items():
|
|
175
|
+
if class_ in possible_classes:
|
|
176
|
+
reduced_data_model["properties"].extend(properties)
|
|
177
|
+
|
|
178
|
+
try:
|
|
179
|
+
return type(self.rules)(**reduced_data_model)
|
|
180
|
+
except ValidationError as e:
|
|
181
|
+
warnings.warn(f"Reduced data model is not complete: {e}", stacklevel=2)
|
|
182
|
+
reduced_data_model["metadata"].schema_ = SchemaCompleteness.partial
|
|
183
|
+
return type(self.rules).model_construct(**reduced_data_model)
|
|
@@ -20,7 +20,7 @@ from cognite.neat.rules._shared import Rules
|
|
|
20
20
|
from cognite.neat.rules.issues import IssueList
|
|
21
21
|
from cognite.neat.rules.models import InformationRules
|
|
22
22
|
from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
|
|
23
|
-
from cognite.neat.utils.
|
|
23
|
+
from cognite.neat.utils.cdf.loaders import (
|
|
24
24
|
ContainerLoader,
|
|
25
25
|
DataModelingLoader,
|
|
26
26
|
DataModelLoader,
|
|
@@ -10,7 +10,7 @@ from rdflib.collection import Collection as GraphCollection
|
|
|
10
10
|
|
|
11
11
|
from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
|
|
12
12
|
from cognite.neat.rules import exceptions
|
|
13
|
-
from cognite.neat.rules.analysis import
|
|
13
|
+
from cognite.neat.rules.analysis import InformationAnalysis
|
|
14
14
|
from cognite.neat.rules.models import DMSRules
|
|
15
15
|
from cognite.neat.rules.models.data_types import DataType
|
|
16
16
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
|
|
@@ -20,7 +20,8 @@ from cognite.neat.rules.models.information import (
|
|
|
20
20
|
InformationProperty,
|
|
21
21
|
InformationRules,
|
|
22
22
|
)
|
|
23
|
-
from cognite.neat.utils.
|
|
23
|
+
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
24
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
24
25
|
|
|
25
26
|
from ._base import BaseExporter
|
|
26
27
|
from ._validation import are_properties_redefined
|
|
@@ -109,11 +110,11 @@ class Ontology(OntologyModel):
|
|
|
109
110
|
if rules.metadata.namespace is None:
|
|
110
111
|
raise exceptions.MissingDataModelPrefixOrNamespace()
|
|
111
112
|
|
|
112
|
-
class_dict =
|
|
113
|
+
class_dict = InformationAnalysis(rules).as_class_dict()
|
|
113
114
|
return cls(
|
|
114
115
|
properties=[
|
|
115
116
|
OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
|
|
116
|
-
for definition in
|
|
117
|
+
for definition in InformationAnalysis(rules).as_property_dict().values()
|
|
117
118
|
],
|
|
118
119
|
classes=[
|
|
119
120
|
OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
|
|
@@ -125,7 +126,7 @@ class Ontology(OntologyModel):
|
|
|
125
126
|
list(properties.values()),
|
|
126
127
|
rules.metadata.namespace,
|
|
127
128
|
)
|
|
128
|
-
for class_, properties in
|
|
129
|
+
for class_, properties in InformationAnalysis(rules).class_property_pairs().items()
|
|
129
130
|
]
|
|
130
131
|
+ [
|
|
131
132
|
SHACLNodeShape.from_rules(
|
|
@@ -8,6 +8,7 @@ from cognite.client import CogniteClient
|
|
|
8
8
|
from cognite.client import data_modeling as dm
|
|
9
9
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
|
|
10
10
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
|
|
11
|
+
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
|
|
11
12
|
from cognite.client.data_classes.data_modeling.views import (
|
|
12
13
|
MultiEdgeConnectionApply,
|
|
13
14
|
MultiReverseDirectRelationApply,
|
|
@@ -403,7 +404,8 @@ class DMSImporter(BaseImporter):
|
|
|
403
404
|
|
|
404
405
|
def _get_is_list(self, prop: ViewPropertyApply) -> bool | None:
|
|
405
406
|
if isinstance(prop, dm.MappedPropertyApply):
|
|
406
|
-
|
|
407
|
+
prop_type = self._container_prop_unsafe(prop).type
|
|
408
|
+
return isinstance(prop_type, ListablePropertyType) and prop_type.is_list
|
|
407
409
|
elif isinstance(prop, MultiEdgeConnectionApply | MultiReverseDirectRelationApply):
|
|
408
410
|
return True
|
|
409
411
|
elif isinstance(prop, SingleEdgeConnectionApply | SingleReverseDirectRelationApply):
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
|
-
from typing import cast
|
|
4
3
|
|
|
5
4
|
import cognite.neat.rules.issues.importing
|
|
6
5
|
from cognite.neat.rules import issues
|
|
@@ -22,7 +21,7 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
22
21
|
)
|
|
23
22
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
24
23
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
25
|
-
from cognite.neat.rules.models.entities import ClassEntity
|
|
24
|
+
from cognite.neat.rules.models.entities import ClassEntity
|
|
26
25
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
27
26
|
|
|
28
27
|
|
|
@@ -89,12 +88,7 @@ class _DTDLConverter:
|
|
|
89
88
|
name=item.display_name,
|
|
90
89
|
description=item.description,
|
|
91
90
|
comment=item.comment,
|
|
92
|
-
parent=[
|
|
93
|
-
cast(ParentClassEntity, parent_entity)
|
|
94
|
-
for parent in item.extends or []
|
|
95
|
-
if isinstance(parent_entity := ParentClassEntity.load(parent.as_class_id()), ParentClassEntity)
|
|
96
|
-
]
|
|
97
|
-
or None,
|
|
91
|
+
parent=[parent.as_class_id() for parent in item.extends or []] or None,
|
|
98
92
|
)
|
|
99
93
|
self.classes.append(class_)
|
|
100
94
|
for sub_item_or_id in item.contents or []:
|
|
@@ -7,7 +7,7 @@ from rdflib import Graph, Namespace, URIRef
|
|
|
7
7
|
from rdflib import Literal as RdfLiteral
|
|
8
8
|
|
|
9
9
|
import cognite.neat.rules.issues as issues
|
|
10
|
-
from cognite.neat.constants import DEFAULT_NAMESPACE,
|
|
10
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
11
11
|
from cognite.neat.graph.stores import NeatGraphStore
|
|
12
12
|
from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
|
|
13
13
|
from cognite.neat.rules.issues import IssueList
|
|
@@ -17,11 +17,7 @@ from cognite.neat.rules.models.information import (
|
|
|
17
17
|
InformationMetadata,
|
|
18
18
|
InformationRulesInput,
|
|
19
19
|
)
|
|
20
|
-
from cognite.neat.utils.
|
|
21
|
-
get_namespace,
|
|
22
|
-
remove_namespace_from_uri,
|
|
23
|
-
uri_to_short_form,
|
|
24
|
-
)
|
|
20
|
+
from cognite.neat.utils.rdf_ import get_namespace, remove_namespace_from_uri, uri_to_short_form
|
|
25
21
|
|
|
26
22
|
ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
|
|
27
23
|
WHERE { ?s a ?class . }
|
|
@@ -204,7 +200,7 @@ class InferenceImporter(BaseImporter):
|
|
|
204
200
|
"""
|
|
205
201
|
classes: dict[str, dict] = {}
|
|
206
202
|
properties: dict[str, dict] = {}
|
|
207
|
-
prefixes: dict[str, Namespace] =
|
|
203
|
+
prefixes: dict[str, Namespace] = get_default_prefixes()
|
|
208
204
|
|
|
209
205
|
query = INSTANCE_PROPERTIES_JSON_DEFINITION if self.check_for_json_string else INSTANCE_PROPERTIES_DEFINITION
|
|
210
206
|
# Adds default namespace to prefixes
|
|
@@ -5,7 +5,7 @@ import pandas as pd
|
|
|
5
5
|
from rdflib import OWL, Graph
|
|
6
6
|
|
|
7
7
|
from cognite.neat.rules.models._base import MatchType
|
|
8
|
-
from cognite.neat.utils.
|
|
8
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def parse_owl_classes(graph: Graph, language: str = "en") -> list[dict]:
|
|
@@ -9,7 +9,8 @@ from cognite.neat.rules.models._types._base import (
|
|
|
9
9
|
PREFIX_COMPLIANCE_REGEX,
|
|
10
10
|
VERSION_COMPLIANCE_REGEX,
|
|
11
11
|
)
|
|
12
|
-
from cognite.neat.utils.
|
|
12
|
+
from cognite.neat.utils.collection_ import remove_none_elements_from_set
|
|
13
|
+
from cognite.neat.utils.rdf_ import convert_rdflib_content
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
def parse_owl_metadata(graph: Graph) -> dict:
|
|
@@ -5,7 +5,7 @@ import pandas as pd
|
|
|
5
5
|
from rdflib import Graph
|
|
6
6
|
|
|
7
7
|
from cognite.neat.rules.models._base import MatchType
|
|
8
|
-
from cognite.neat.utils.
|
|
8
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
9
9
|
|
|
10
10
|
from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
|
|
11
11
|
|
|
@@ -295,6 +295,41 @@ class ClassNoPropertiesNoParentError(NeatValidationError):
|
|
|
295
295
|
return f"Class {self.classes[0]} have no direct or inherited properties. This may be a mistake."
|
|
296
296
|
|
|
297
297
|
|
|
298
|
+
@dataclass(frozen=True)
|
|
299
|
+
class AssetRulesHaveCircularDependencyError(NeatValidationError):
|
|
300
|
+
description = "Asset rules have circular dependencies."
|
|
301
|
+
fix = "Linking between classes via property that maps to parent_external_id must yield hierarchy structure."
|
|
302
|
+
|
|
303
|
+
classes: list[str]
|
|
304
|
+
|
|
305
|
+
def dump(self) -> dict[str, list[tuple[str, str]]]:
|
|
306
|
+
output = super().dump()
|
|
307
|
+
output["classes"] = self.classes
|
|
308
|
+
return output
|
|
309
|
+
|
|
310
|
+
def message(self) -> str:
|
|
311
|
+
return f"Asset rules have circular dependencies between classes {', '.join(self.classes)}."
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@dataclass(frozen=True)
|
|
315
|
+
class AssetParentPropertyPointsToDataValueTypeError(NeatValidationError):
|
|
316
|
+
description = "Parent property points to a data value type instead of a class."
|
|
317
|
+
fix = "Make sure that the parent property points to a class."
|
|
318
|
+
|
|
319
|
+
class_property_with_data_value_type: list[tuple[str, str]]
|
|
320
|
+
|
|
321
|
+
def dump(self) -> dict[str, list[tuple[str, str]]]:
|
|
322
|
+
output = super().dump()
|
|
323
|
+
output["class_property"] = self.class_property_with_data_value_type
|
|
324
|
+
return output
|
|
325
|
+
|
|
326
|
+
def message(self) -> str:
|
|
327
|
+
text = [
|
|
328
|
+
f"class {class_} property {property_}" for class_, property_ in self.class_property_with_data_value_type
|
|
329
|
+
]
|
|
330
|
+
return f"Following {', and'.join(text)} point to data value type instead to classes. This is a mistake."
|
|
331
|
+
|
|
332
|
+
|
|
298
333
|
@dataclass(frozen=True)
|
|
299
334
|
class ParentClassesNotDefinedError(NeatValidationError):
|
|
300
335
|
description = "Parent classes are not defined."
|
|
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|
|
7
7
|
import math
|
|
8
8
|
import sys
|
|
9
9
|
import types
|
|
10
|
-
from abc import abstractmethod
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
11
|
from collections.abc import Callable, Iterator
|
|
12
12
|
from functools import wraps
|
|
13
13
|
from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
|
|
@@ -18,7 +18,6 @@ from pydantic import (
|
|
|
18
18
|
BeforeValidator,
|
|
19
19
|
ConfigDict,
|
|
20
20
|
Field,
|
|
21
|
-
HttpUrl,
|
|
22
21
|
PlainSerializer,
|
|
23
22
|
constr,
|
|
24
23
|
field_validator,
|
|
@@ -219,10 +218,6 @@ class RuleModel(BaseModel):
|
|
|
219
218
|
return headers_by_sheet
|
|
220
219
|
|
|
221
220
|
|
|
222
|
-
class URL(BaseModel):
|
|
223
|
-
url: HttpUrl
|
|
224
|
-
|
|
225
|
-
|
|
226
221
|
class BaseMetadata(RuleModel):
|
|
227
222
|
"""
|
|
228
223
|
Metadata model for data model
|
|
@@ -252,8 +247,13 @@ class BaseMetadata(RuleModel):
|
|
|
252
247
|
"""Returns a unique identifier for the metadata."""
|
|
253
248
|
raise NotImplementedError()
|
|
254
249
|
|
|
250
|
+
@abstractmethod
|
|
251
|
+
def get_prefix(self) -> str:
|
|
252
|
+
"""Returns the prefix for the metadata."""
|
|
253
|
+
raise NotImplementedError()
|
|
254
|
+
|
|
255
255
|
|
|
256
|
-
class BaseRules(RuleModel):
|
|
256
|
+
class BaseRules(RuleModel, ABC):
|
|
257
257
|
"""
|
|
258
258
|
Rules is a core concept in `neat`. This represents fusion of data model
|
|
259
259
|
definitions and (optionally) the transformation rules used to transform the data/graph
|
|
@@ -232,21 +232,12 @@ class SingleProperty(Traversal):
|
|
|
232
232
|
return f"{self.class_}({self.property})"
|
|
233
233
|
|
|
234
234
|
|
|
235
|
-
class
|
|
235
|
+
class SelfReferenceProperty(Traversal):
|
|
236
236
|
@classmethod
|
|
237
237
|
def from_string(cls, class_: str) -> Self:
|
|
238
238
|
return cls(class_=Entity.from_string(class_))
|
|
239
239
|
|
|
240
240
|
|
|
241
|
-
class AllProperties(Traversal):
|
|
242
|
-
@classmethod
|
|
243
|
-
def from_string(cls, class_: str) -> Self:
|
|
244
|
-
return cls(class_=Entity.from_string(class_))
|
|
245
|
-
|
|
246
|
-
def __str__(self) -> str:
|
|
247
|
-
return f"{self.class_}(*)"
|
|
248
|
-
|
|
249
|
-
|
|
250
241
|
class Origin(BaseModel):
|
|
251
242
|
class_: Entity
|
|
252
243
|
|
|
@@ -290,7 +281,7 @@ class Query(BaseModel):
|
|
|
290
281
|
|
|
291
282
|
|
|
292
283
|
class RDFPath(Rule):
|
|
293
|
-
traversal: SingleProperty |
|
|
284
|
+
traversal: SingleProperty | SelfReferenceProperty | Hop
|
|
294
285
|
|
|
295
286
|
def __str__(self) -> str:
|
|
296
287
|
return f"{self.traversal}"
|
|
@@ -311,14 +302,13 @@ class SPARQLQuery(RDFPath):
|
|
|
311
302
|
traversal: Query
|
|
312
303
|
|
|
313
304
|
|
|
314
|
-
def parse_traversal(raw: str) ->
|
|
305
|
+
def parse_traversal(raw: str) -> SelfReferenceProperty | SingleProperty | Hop:
|
|
315
306
|
if result := CLASS_ID_REGEX_COMPILED.match(raw):
|
|
316
|
-
return
|
|
317
|
-
elif result := ALL_PROPERTIES_REGEX_COMPILED.match(raw):
|
|
318
|
-
return AllProperties.from_string(class_=result.group(EntityTypes.class_))
|
|
307
|
+
return SelfReferenceProperty.from_string(class_=result.group(EntityTypes.class_))
|
|
319
308
|
elif result := SINGLE_PROPERTY_REGEX_COMPILED.match(raw):
|
|
320
309
|
return SingleProperty.from_string(
|
|
321
|
-
class_=result.group(EntityTypes.class_),
|
|
310
|
+
class_=result.group(EntityTypes.class_),
|
|
311
|
+
property_=result.group(EntityTypes.property_),
|
|
322
312
|
)
|
|
323
313
|
elif result := HOP_REGEX_COMPILED.match(raw):
|
|
324
314
|
return Hop.from_string(class_=result.group("origin"), traversal=result.group(_traversal))
|
|
@@ -329,7 +319,9 @@ def parse_traversal(raw: str) -> AllReferences | AllProperties | SingleProperty
|
|
|
329
319
|
def parse_table_lookup(raw: str) -> TableLookup:
|
|
330
320
|
if result := TABLE_REGEX_COMPILED.match(raw):
|
|
331
321
|
return TableLookup(
|
|
332
|
-
name=result.group(Lookup.table),
|
|
322
|
+
name=result.group(Lookup.table),
|
|
323
|
+
key=result.group(Lookup.key),
|
|
324
|
+
value=result.group(Lookup.value),
|
|
333
325
|
)
|
|
334
326
|
raise exceptions.NotValidTableLookUp(raw).to_pydantic_custom_error()
|
|
335
327
|
|
|
@@ -344,7 +336,10 @@ def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPa
|
|
|
344
336
|
if Counter(rule_raw).get("|") != 1:
|
|
345
337
|
raise exceptions.NotValidRAWLookUp(rule_raw).to_pydantic_custom_error()
|
|
346
338
|
traversal, table_lookup = rule_raw.split("|")
|
|
347
|
-
return RawLookup(
|
|
339
|
+
return RawLookup(
|
|
340
|
+
traversal=parse_traversal(traversal),
|
|
341
|
+
table=parse_table_lookup(table_lookup),
|
|
342
|
+
)
|
|
348
343
|
case TransformationRuleType.sparql:
|
|
349
344
|
return SPARQLQuery(traversal=Query(query=rule_raw))
|
|
350
345
|
case None:
|
|
@@ -352,9 +347,10 @@ def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPa
|
|
|
352
347
|
|
|
353
348
|
|
|
354
349
|
def is_valid_rule(rule_type: TransformationRuleType, rule_raw: str) -> bool:
|
|
355
|
-
is_valid_rule = {
|
|
356
|
-
|
|
357
|
-
|
|
350
|
+
is_valid_rule = {
|
|
351
|
+
TransformationRuleType.rdfpath: is_rdfpath,
|
|
352
|
+
TransformationRuleType.rawlookup: is_rawlookup,
|
|
353
|
+
}[rule_type]
|
|
358
354
|
return is_valid_rule(rule_raw)
|
|
359
355
|
|
|
360
356
|
|
|
@@ -5,7 +5,7 @@ from pydantic import Field, field_validator, model_validator
|
|
|
5
5
|
from pydantic.main import IncEx
|
|
6
6
|
from rdflib import Namespace
|
|
7
7
|
|
|
8
|
-
from cognite.neat.constants import
|
|
8
|
+
from cognite.neat.constants import get_default_prefixes
|
|
9
9
|
from cognite.neat.issues import MultiValueError
|
|
10
10
|
from cognite.neat.rules import issues
|
|
11
11
|
from cognite.neat.rules.models._base import BaseRules, RoleTypes, SheetList
|
|
@@ -14,7 +14,6 @@ from cognite.neat.rules.models.entities import (
|
|
|
14
14
|
CdfResourceEntityList,
|
|
15
15
|
ClassEntity,
|
|
16
16
|
MultiValueTypeInfo,
|
|
17
|
-
ParentClassEntity,
|
|
18
17
|
Undefined,
|
|
19
18
|
)
|
|
20
19
|
from cognite.neat.rules.models.information import (
|
|
@@ -61,14 +60,14 @@ class AssetProperty(InformationProperty):
|
|
|
61
60
|
implementation: Details on how given class-property is implemented in the classic CDF
|
|
62
61
|
"""
|
|
63
62
|
|
|
64
|
-
implementation: CdfResourceEntityList
|
|
63
|
+
implementation: CdfResourceEntityList = Field(alias="Implementation")
|
|
65
64
|
|
|
66
65
|
|
|
67
66
|
class AssetRules(BaseRules):
|
|
68
67
|
metadata: AssetMetadata = Field(alias="Metadata")
|
|
69
68
|
properties: SheetList[AssetProperty] = Field(alias="Properties")
|
|
70
69
|
classes: SheetList[AssetClass] = Field(alias="Classes")
|
|
71
|
-
prefixes: dict[str, Namespace] = Field(default_factory=
|
|
70
|
+
prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes)
|
|
72
71
|
last: "AssetRules | None" = Field(None, alias="Last")
|
|
73
72
|
reference: "AssetRules | None" = Field(None, alias="Reference")
|
|
74
73
|
|
|
@@ -94,7 +93,7 @@ class AssetRules(BaseRules):
|
|
|
94
93
|
# update parent classes
|
|
95
94
|
for class_ in self.classes:
|
|
96
95
|
if class_.parent:
|
|
97
|
-
for parent in
|
|
96
|
+
for parent in class_.parent:
|
|
98
97
|
if not isinstance(parent.prefix, str):
|
|
99
98
|
parent.prefix = self.metadata.prefix
|
|
100
99
|
if class_.class_.prefix is Undefined:
|
|
@@ -1,4 +1,41 @@
|
|
|
1
|
+
from graphlib import CycleError
|
|
2
|
+
from typing import cast
|
|
3
|
+
|
|
4
|
+
from cognite.neat.rules import issues
|
|
5
|
+
from cognite.neat.rules.issues.base import IssueList
|
|
6
|
+
from cognite.neat.rules.models._base import SheetList
|
|
7
|
+
from cognite.neat.rules.models.asset._rules import AssetProperty, AssetRules
|
|
8
|
+
from cognite.neat.rules.models.entities import AssetEntity, AssetFields, ClassEntity
|
|
1
9
|
from cognite.neat.rules.models.information._validation import InformationPostValidation
|
|
2
10
|
|
|
3
11
|
|
|
4
|
-
class AssetPostValidation(InformationPostValidation):
|
|
12
|
+
class AssetPostValidation(InformationPostValidation):
|
|
13
|
+
def validate(self) -> IssueList:
|
|
14
|
+
self.issue_list = super().validate()
|
|
15
|
+
self._parent_property_point_to_class()
|
|
16
|
+
self._circular_dependency()
|
|
17
|
+
return self.issue_list
|
|
18
|
+
|
|
19
|
+
def _parent_property_point_to_class(self) -> None:
|
|
20
|
+
class_property_with_data_value_type = []
|
|
21
|
+
for property_ in cast(SheetList[AssetProperty], self.properties):
|
|
22
|
+
for implementation in property_.implementation:
|
|
23
|
+
if (
|
|
24
|
+
isinstance(implementation, AssetEntity)
|
|
25
|
+
and implementation.property_ == AssetFields.parentExternalId
|
|
26
|
+
and not isinstance(property_.value_type, ClassEntity)
|
|
27
|
+
):
|
|
28
|
+
class_property_with_data_value_type.append((property_.class_.suffix, property_.property_))
|
|
29
|
+
|
|
30
|
+
if class_property_with_data_value_type:
|
|
31
|
+
self.issue_list.append(
|
|
32
|
+
issues.spreadsheet.AssetParentPropertyPointsToDataValueTypeError(class_property_with_data_value_type)
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
def _circular_dependency(self) -> None:
|
|
36
|
+
from cognite.neat.rules.analysis import AssetAnalysis
|
|
37
|
+
|
|
38
|
+
try:
|
|
39
|
+
_ = AssetAnalysis(cast(AssetRules, self.rules)).class_topological_sort()
|
|
40
|
+
except CycleError as error:
|
|
41
|
+
self.issue_list.append(issues.spreadsheet.AssetRulesHaveCircularDependencyError(error.args[1]))
|
|
@@ -11,7 +11,6 @@ from cognite.neat.rules.models.entities import (
|
|
|
11
11
|
ClassEntity,
|
|
12
12
|
ContainerEntity,
|
|
13
13
|
DMSUnknownEntity,
|
|
14
|
-
ParentClassEntity,
|
|
15
14
|
ReferenceEntity,
|
|
16
15
|
UnknownEntity,
|
|
17
16
|
ViewEntity,
|
|
@@ -52,7 +51,7 @@ class _DMSRulesConverter:
|
|
|
52
51
|
description=view.description,
|
|
53
52
|
parent=[
|
|
54
53
|
# we do not want a version in class as we use URI for the class
|
|
55
|
-
|
|
54
|
+
implemented_view.as_class(skip_version=True)
|
|
56
55
|
# We only want parents in the same namespace, parent in a different namespace is a reference
|
|
57
56
|
for implemented_view in view.implements or []
|
|
58
57
|
if implemented_view.prefix == view.class_.prefix
|
|
@@ -5,6 +5,7 @@ from typing import Any, cast
|
|
|
5
5
|
|
|
6
6
|
from cognite.client.data_classes import data_modeling as dm
|
|
7
7
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex
|
|
8
|
+
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
|
|
8
9
|
from cognite.client.data_classes.data_modeling.views import (
|
|
9
10
|
SingleEdgeConnectionApply,
|
|
10
11
|
SingleReverseDirectRelationApply,
|
|
@@ -23,7 +24,7 @@ from cognite.neat.rules.models.entities import (
|
|
|
23
24
|
ViewPropertyEntity,
|
|
24
25
|
)
|
|
25
26
|
from cognite.neat.rules.models.wrapped_entities import DMSFilter, HasDataFilter, NodeTypeFilter
|
|
26
|
-
from cognite.neat.utils.
|
|
27
|
+
from cognite.neat.utils.cdf.data_classes import ContainerApplyDict, NodeApplyDict, SpaceApplyDict, ViewApplyDict
|
|
27
28
|
|
|
28
29
|
from ._rules import DMSMetadata, DMSProperty, DMSRules, DMSView
|
|
29
30
|
from ._schema import DMSSchema, PipelineSchema
|
|
@@ -282,8 +283,11 @@ class _DMSExporter:
|
|
|
282
283
|
type_cls = prop.value_type.dms
|
|
283
284
|
else:
|
|
284
285
|
type_cls = dm.DirectRelation
|
|
285
|
-
|
|
286
|
-
|
|
286
|
+
type_: dm.PropertyType
|
|
287
|
+
if issubclass(type_cls, ListablePropertyType):
|
|
288
|
+
type_ = type_cls(is_list=prop.is_list or False)
|
|
289
|
+
else:
|
|
290
|
+
type_ = type_cls()
|
|
287
291
|
container.properties[prop.container_property] = dm.ContainerProperty(
|
|
288
292
|
type=type_,
|
|
289
293
|
nullable=prop.nullable if prop.nullable is not None else True,
|