cognite-neat 0.121.1__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/_data_model/_constants.py +23 -16
- cognite/neat/core/_data_model/_shared.py +33 -17
- cognite/neat/core/_data_model/analysis/__init__.py +2 -2
- cognite/neat/core/_data_model/analysis/_base.py +186 -183
- cognite/neat/core/_data_model/catalog/__init__.py +1 -1
- cognite/neat/core/_data_model/exporters/__init__.py +5 -5
- cognite/neat/core/_data_model/exporters/_base.py +10 -8
- cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
- cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
- cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +4 -4
- cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
- cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
- cognite/neat/core/_data_model/importers/__init__.py +6 -6
- cognite/neat/core/_data_model/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +40 -20
- cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
- cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/_data_model/importers/_rdf/_base.py +9 -9
- cognite/neat/core/_data_model/importers/_rdf/_imf2rules.py +15 -15
- cognite/neat/core/_data_model/importers/_rdf/_inference2rules.py +36 -36
- cognite/neat/core/_data_model/importers/_rdf/_owl2rules.py +12 -12
- cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +72 -12
- cognite/neat/core/_data_model/models/__init__.py +8 -8
- cognite/neat/core/_data_model/models/_base_unverified.py +1 -1
- cognite/neat/core/_data_model/models/_base_verified.py +3 -3
- cognite/neat/core/_data_model/models/_types.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
- cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
- cognite/neat/core/_data_model/models/conceptual/_validation.py +87 -77
- cognite/neat/core/_data_model/models/conceptual/_verified.py +53 -51
- cognite/neat/core/_data_model/models/data_types.py +2 -2
- cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
- cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
- cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
- cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
- cognite/neat/core/_data_model/models/entities/_types.py +9 -3
- cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
- cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +71 -52
- cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
- cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +13 -11
- cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
- cognite/neat/core/_data_model/transformers/__init__.py +27 -23
- cognite/neat/core/_data_model/transformers/_base.py +26 -19
- cognite/neat/core/_data_model/transformers/_converters.py +703 -618
- cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
- cognite/neat/core/_data_model/transformers/_verification.py +63 -54
- cognite/neat/core/_instances/extractors/_base.py +1 -1
- cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +8 -8
- cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
- cognite/neat/core/_instances/extractors/_mock_graph_generator.py +98 -95
- cognite/neat/core/_instances/loaders/_base.py +2 -2
- cognite/neat/core/_instances/loaders/_rdf2dms.py +6 -6
- cognite/neat/core/_instances/transformers/_base.py +7 -4
- cognite/neat/core/_instances/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +2 -2
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/text.py +1 -1
- cognite/neat/session/_base.py +22 -20
- cognite/neat/session/_drop.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +8 -6
- cognite/neat/session/_read.py +2 -2
- cognite/neat/session/_set.py +3 -3
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +13 -13
- cognite/neat/session/_subset.py +12 -9
- cognite/neat/session/_template.py +13 -13
- cognite/neat/session/_to.py +17 -17
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/RECORD +95 -93
- cognite/neat/core/_data_model/exporters/_validation.py +0 -14
- cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
- /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -8,11 +8,14 @@ from rdflib import Namespace, URIRef
|
|
|
8
8
|
from cognite.neat.core._client import NeatClient
|
|
9
9
|
from cognite.neat.core._constants import COGNITE_SPACES, DEFAULT_NAMESPACE
|
|
10
10
|
from cognite.neat.core._data_model.importers import DMSImporter
|
|
11
|
-
from cognite.neat.core._data_model.models import ConceptualDataModel,
|
|
11
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
|
|
12
12
|
from cognite.neat.core._data_model.models.conceptual import ConceptualProperty
|
|
13
13
|
from cognite.neat.core._data_model.models.data_types import Json
|
|
14
14
|
from cognite.neat.core._data_model.models.entities import UnknownEntity
|
|
15
|
-
from cognite.neat.core._data_model.transformers import
|
|
15
|
+
from cognite.neat.core._data_model.transformers import (
|
|
16
|
+
PhysicalToConceptual,
|
|
17
|
+
VerifyPhysicalDataModel,
|
|
18
|
+
)
|
|
16
19
|
from cognite.neat.core._issues import IssueList, NeatIssue, catch_warnings
|
|
17
20
|
from cognite.neat.core._issues.warnings import (
|
|
18
21
|
CDFAuthWarning,
|
|
@@ -47,8 +50,8 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
47
50
|
self._str_to_ideal_type = str_to_ideal_type
|
|
48
51
|
|
|
49
52
|
self._views: list[dm.View] | None = None
|
|
50
|
-
self.
|
|
51
|
-
self.
|
|
53
|
+
self._conceptual_data_model: ConceptualDataModel | None = None
|
|
54
|
+
self._physical_data_model: PhysicalDataModel | None = None
|
|
52
55
|
|
|
53
56
|
@classmethod
|
|
54
57
|
def from_data_model_id(
|
|
@@ -164,32 +167,32 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
164
167
|
self._issues.append(ResourceNotFoundWarning(dm_view, "view", data_model_id, "data model"))
|
|
165
168
|
return views
|
|
166
169
|
|
|
167
|
-
def
|
|
168
|
-
"""Returns the
|
|
169
|
-
if self.
|
|
170
|
-
self.
|
|
171
|
-
return self.
|
|
170
|
+
def get_conceptual_data_model(self) -> ConceptualDataModel:
|
|
171
|
+
"""Returns the conceptual data model that the extractor uses."""
|
|
172
|
+
if self._conceptual_data_model is None:
|
|
173
|
+
self._conceptual_data_model, self._physical_data_model = self._create_data_models()
|
|
174
|
+
return self._conceptual_data_model
|
|
172
175
|
|
|
173
|
-
def
|
|
174
|
-
"""Returns the
|
|
175
|
-
if self.
|
|
176
|
-
self.
|
|
177
|
-
return self.
|
|
176
|
+
def get_physical_data_model(self) -> PhysicalDataModel:
|
|
177
|
+
"""Returns the physical data model that the extractor uses."""
|
|
178
|
+
if self._physical_data_model is None:
|
|
179
|
+
self._conceptual_data_model, self._physical_data_model = self._create_data_models()
|
|
180
|
+
return self._physical_data_model
|
|
178
181
|
|
|
179
182
|
def get_issues(self) -> IssueList:
|
|
180
183
|
"""Returns the issues that occurred during the extraction."""
|
|
181
184
|
return self._issues
|
|
182
185
|
|
|
183
|
-
def
|
|
184
|
-
# The
|
|
186
|
+
def _create_data_models(self) -> tuple[ConceptualDataModel, PhysicalDataModel]:
|
|
187
|
+
# The physical and conceptual data model must be created together to link them property.
|
|
185
188
|
importer = DMSImporter.from_data_model(self._client, self._data_model)
|
|
186
|
-
|
|
187
|
-
if self._unpack_json and (
|
|
188
|
-
# Drop the JSON properties from the
|
|
189
|
+
imported_physical_data_model = importer.to_data_model()
|
|
190
|
+
if self._unpack_json and (unverified_physical_data_model := imported_physical_data_model.unverified_data_model):
|
|
191
|
+
# Drop the JSON properties from the physical data model as these are no longer valid.
|
|
189
192
|
json_name = Json().name # To avoid instantiating Json multiple times.
|
|
190
|
-
|
|
193
|
+
unverified_physical_data_model.properties = [
|
|
191
194
|
prop
|
|
192
|
-
for prop in
|
|
195
|
+
for prop in unverified_physical_data_model.properties
|
|
193
196
|
if not (
|
|
194
197
|
(
|
|
195
198
|
isinstance(prop.value_type, Json)
|
|
@@ -202,23 +205,28 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
202
205
|
|
|
203
206
|
with catch_warnings() as issues:
|
|
204
207
|
# Any errors occur will be raised and caught outside the extractor.
|
|
205
|
-
|
|
206
|
-
|
|
208
|
+
verified_physical_data_model = VerifyPhysicalDataModel(client=self._client).transform(
|
|
209
|
+
imported_physical_data_model
|
|
210
|
+
)
|
|
211
|
+
verified_conceptual_data_model = PhysicalToConceptual(self._namespace).transform(
|
|
212
|
+
verified_physical_data_model
|
|
213
|
+
)
|
|
207
214
|
|
|
208
|
-
# We need to sync the metadata between the two
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
215
|
+
# We need to sync the metadata between the two data model, such that the
|
|
216
|
+
# `.sync_with_conceptual_data_model` method works.
|
|
217
|
+
verified_conceptual_data_model.metadata.physical = verified_physical_data_model.metadata.identifier
|
|
218
|
+
verified_physical_data_model.metadata.conceptual = verified_conceptual_data_model.metadata.identifier
|
|
219
|
+
verified_physical_data_model.sync_with_conceptual_data_model(verified_conceptual_data_model)
|
|
212
220
|
|
|
213
|
-
# Adding startNode and endNode to the
|
|
214
|
-
|
|
221
|
+
# Adding startNode and endNode to the conceptual data model for views that are used for edges.
|
|
222
|
+
concepts_by_prefix = {concept.concept.prefix: concept for concept in verified_conceptual_data_model.concepts}
|
|
215
223
|
for view in self._model_views:
|
|
216
|
-
if view.used_for == "edge" and view.external_id in
|
|
217
|
-
cls_ =
|
|
224
|
+
if view.used_for == "edge" and view.external_id in concepts_by_prefix:
|
|
225
|
+
cls_ = concepts_by_prefix[view.external_id]
|
|
218
226
|
for property_ in ("startNode", "endNode"):
|
|
219
|
-
|
|
227
|
+
verified_conceptual_data_model.properties.append(
|
|
220
228
|
ConceptualProperty(
|
|
221
|
-
|
|
229
|
+
concept=cls_.concept,
|
|
222
230
|
property_=property_,
|
|
223
231
|
value_type=UnknownEntity(),
|
|
224
232
|
min_count=0,
|
|
@@ -227,4 +235,4 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
|
|
|
227
235
|
)
|
|
228
236
|
|
|
229
237
|
self._issues.extend(issues)
|
|
230
|
-
return
|
|
238
|
+
return verified_conceptual_data_model, verified_physical_data_model
|
|
@@ -12,12 +12,12 @@ import pandas as pd
|
|
|
12
12
|
from rdflib import RDF, Literal, Namespace, URIRef
|
|
13
13
|
|
|
14
14
|
from cognite.neat.core._data_model._constants import EntityTypes
|
|
15
|
-
from cognite.neat.core._data_model.analysis import
|
|
16
|
-
from cognite.neat.core._data_model.models import ConceptualDataModel,
|
|
15
|
+
from cognite.neat.core._data_model.analysis import DataModelAnalysis
|
|
16
|
+
from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
|
|
17
17
|
from cognite.neat.core._data_model.models.conceptual import ConceptualProperty
|
|
18
18
|
from cognite.neat.core._data_model.models.data_types import DataType
|
|
19
|
-
from cognite.neat.core._data_model.models.entities import
|
|
20
|
-
from cognite.neat.core._data_model.transformers import
|
|
19
|
+
from cognite.neat.core._data_model.models.entities import ConceptEntity
|
|
20
|
+
from cognite.neat.core._data_model.transformers import SubsetConceptualDataModel
|
|
21
21
|
from cognite.neat.core._shared import Triple
|
|
22
22
|
from cognite.neat.core._utils.rdf_ import remove_namespace_from_uri
|
|
23
23
|
|
|
@@ -38,31 +38,31 @@ class MockGraphGenerator(BaseExtractor):
|
|
|
38
38
|
|
|
39
39
|
def __init__(
|
|
40
40
|
self,
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
data_model: ConceptualDataModel | PhysicalDataModel,
|
|
42
|
+
concept_count: dict[str | ConceptEntity, int] | None = None,
|
|
43
43
|
stop_on_exception: bool = False,
|
|
44
44
|
allow_isolated_classes: bool = True,
|
|
45
45
|
):
|
|
46
|
-
if isinstance(
|
|
46
|
+
if isinstance(data_model, PhysicalDataModel):
|
|
47
47
|
# fixes potential issues with circular dependencies
|
|
48
|
-
from cognite.neat.core._data_model.transformers import
|
|
48
|
+
from cognite.neat.core._data_model.transformers import PhysicalToConceptual
|
|
49
49
|
|
|
50
|
-
self.rules =
|
|
51
|
-
elif isinstance(
|
|
52
|
-
self.rules =
|
|
50
|
+
self.rules = PhysicalToConceptual().transform(data_model)
|
|
51
|
+
elif isinstance(data_model, ConceptualDataModel):
|
|
52
|
+
self.rules = data_model
|
|
53
53
|
else:
|
|
54
54
|
raise ValueError("Rules must be of type InformationRules or DMSRules!")
|
|
55
55
|
|
|
56
|
-
if not
|
|
57
|
-
self.
|
|
58
|
-
|
|
56
|
+
if not concept_count:
|
|
57
|
+
self.concept_count = {
|
|
58
|
+
concept: 1 for concept in DataModelAnalysis(self.rules).defined_concepts(include_ancestors=True)
|
|
59
59
|
}
|
|
60
|
-
elif all(isinstance(key, str) for key in
|
|
61
|
-
self.
|
|
62
|
-
|
|
60
|
+
elif all(isinstance(key, str) for key in concept_count.keys()):
|
|
61
|
+
self.concept_count = {
|
|
62
|
+
ConceptEntity.load(f"{self.rules.metadata.prefix}:{key}"): value for key, value in concept_count.items()
|
|
63
63
|
}
|
|
64
|
-
elif all(isinstance(key,
|
|
65
|
-
self.
|
|
64
|
+
elif all(isinstance(key, ConceptEntity) for key in concept_count.keys()):
|
|
65
|
+
self.concept_count = cast(dict[ConceptEntity, int], concept_count)
|
|
66
66
|
else:
|
|
67
67
|
raise ValueError("Class count keys must be of type str! or ClassEntity! or empty dict!")
|
|
68
68
|
|
|
@@ -78,87 +78,90 @@ class MockGraphGenerator(BaseExtractor):
|
|
|
78
78
|
"""
|
|
79
79
|
return generate_triples(
|
|
80
80
|
self.rules,
|
|
81
|
-
self.
|
|
81
|
+
self.concept_count,
|
|
82
82
|
stop_on_exception=self.stop_on_exception,
|
|
83
|
-
|
|
83
|
+
allow_isolated_concepts=self.allow_isolated_classes,
|
|
84
84
|
)
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
def generate_triples(
|
|
88
|
-
|
|
89
|
-
|
|
88
|
+
data_model: ConceptualDataModel,
|
|
89
|
+
concept_count: dict[ConceptEntity, int],
|
|
90
90
|
stop_on_exception: bool = False,
|
|
91
|
-
|
|
91
|
+
allow_isolated_concepts: bool = True,
|
|
92
92
|
) -> list[Triple]:
|
|
93
|
-
"""Generate mock triples based on data model defined
|
|
93
|
+
"""Generate mock triples based on the conceptual data model defined and desired number
|
|
94
94
|
of class instances
|
|
95
95
|
|
|
96
96
|
Args:
|
|
97
97
|
rules : Rules defining the data model
|
|
98
|
-
|
|
98
|
+
concept_count: Target class count for each class in the ontology
|
|
99
99
|
stop_on_exception: To stop if exception is encountered or not, default is False
|
|
100
|
-
|
|
100
|
+
allow_isolated_concepts: To allow generation of instances for classes that are not
|
|
101
101
|
connected to any other class, default is True
|
|
102
102
|
|
|
103
103
|
Returns:
|
|
104
104
|
List of RDF triples, represented as tuples `(subject, predicate, object)`, that define data model instances
|
|
105
105
|
"""
|
|
106
106
|
|
|
107
|
-
namespace =
|
|
108
|
-
analysis =
|
|
109
|
-
|
|
107
|
+
namespace = data_model.metadata.namespace
|
|
108
|
+
analysis = DataModelAnalysis(data_model)
|
|
109
|
+
defined_concepts = analysis.defined_concepts(include_ancestors=True)
|
|
110
110
|
|
|
111
|
-
if
|
|
112
|
-
msg =
|
|
111
|
+
if non_existing_concepts := set(concept_count.keys()) - defined_concepts:
|
|
112
|
+
msg = (
|
|
113
|
+
f"Concept count contains concepts {non_existing_concepts} for which"
|
|
114
|
+
" properties are not defined in Data Model!"
|
|
115
|
+
)
|
|
113
116
|
if stop_on_exception:
|
|
114
117
|
raise ValueError(msg)
|
|
115
118
|
else:
|
|
116
119
|
msg += " These classes will be ignored."
|
|
117
120
|
warnings.warn(msg, stacklevel=2)
|
|
118
|
-
for
|
|
119
|
-
|
|
121
|
+
for concept in non_existing_concepts:
|
|
122
|
+
concept_count.pop(concept)
|
|
120
123
|
|
|
121
124
|
# Subset data model to only classes that are defined in class count
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
if
|
|
125
|
-
else
|
|
125
|
+
data_model = (
|
|
126
|
+
SubsetConceptualDataModel(concepts=set(concept_count.keys())).transform(data_model)
|
|
127
|
+
if defined_concepts != set(concept_count.keys())
|
|
128
|
+
else data_model
|
|
126
129
|
)
|
|
127
130
|
|
|
128
|
-
|
|
131
|
+
concept_linkage = analysis.concept_linkage().to_pandas()
|
|
129
132
|
|
|
130
133
|
# Remove one of symmetric pairs from class linkage to maintain proper linking
|
|
131
134
|
# among instances of symmetrically linked classes
|
|
132
|
-
if sym_pairs := analysis.
|
|
133
|
-
|
|
135
|
+
if sym_pairs := analysis.symmetrically_connected_concepts():
|
|
136
|
+
concept_linkage = _remove_higher_occurring_sym_pair(concept_linkage, sym_pairs)
|
|
134
137
|
|
|
135
138
|
# Remove any of symmetric pairs containing classes that are not present class count
|
|
136
|
-
|
|
139
|
+
concept_linkage = _remove_non_requested_sym_pairs(concept_linkage, concept_count)
|
|
137
140
|
|
|
138
141
|
# Generate generation order for classes instances
|
|
139
|
-
generation_order = _prettify_generation_order(_get_generation_order(
|
|
142
|
+
generation_order = _prettify_generation_order(_get_generation_order(concept_linkage))
|
|
140
143
|
|
|
141
144
|
# Generated simple view of data model
|
|
142
|
-
|
|
145
|
+
properties_by_concepts = analysis.properties_by_concepts(include_ancestors=True)
|
|
143
146
|
|
|
144
147
|
# pregenerate instance ids for each remaining class
|
|
145
148
|
instance_ids = {
|
|
146
|
-
key: [URIRef(namespace[f"{key.suffix}-{i + 1}"]) for i in range(value)] for key, value in
|
|
149
|
+
key: [URIRef(namespace[f"{key.suffix}-{i + 1}"]) for i in range(value)] for key, value in concept_count.items()
|
|
147
150
|
}
|
|
148
151
|
|
|
149
152
|
# create triple for each class instance defining its type
|
|
150
153
|
triples: list[Triple] = []
|
|
151
|
-
for
|
|
154
|
+
for concept in concept_count:
|
|
152
155
|
triples += [
|
|
153
|
-
(
|
|
154
|
-
for
|
|
156
|
+
(concept_instance_id, RDF.type, URIRef(namespace[str(concept.suffix)]))
|
|
157
|
+
for concept_instance_id in instance_ids[concept]
|
|
155
158
|
]
|
|
156
159
|
|
|
157
160
|
# generate triples for connected classes
|
|
158
|
-
for
|
|
161
|
+
for concept in generation_order:
|
|
159
162
|
triples += _generate_triples_per_class(
|
|
160
|
-
|
|
161
|
-
|
|
163
|
+
concept,
|
|
164
|
+
properties_by_concepts,
|
|
162
165
|
sym_pairs,
|
|
163
166
|
instance_ids,
|
|
164
167
|
namespace,
|
|
@@ -166,11 +169,11 @@ def generate_triples(
|
|
|
166
169
|
)
|
|
167
170
|
|
|
168
171
|
# generate triples for isolated classes
|
|
169
|
-
if
|
|
170
|
-
for
|
|
172
|
+
if allow_isolated_concepts:
|
|
173
|
+
for concept in set(concept_count.keys()) - set(generation_order):
|
|
171
174
|
triples += _generate_triples_per_class(
|
|
172
|
-
|
|
173
|
-
|
|
175
|
+
concept,
|
|
176
|
+
properties_by_concepts,
|
|
174
177
|
sym_pairs,
|
|
175
178
|
instance_ids,
|
|
176
179
|
namespace,
|
|
@@ -181,11 +184,11 @@ def generate_triples(
|
|
|
181
184
|
|
|
182
185
|
|
|
183
186
|
def _get_generation_order(
|
|
184
|
-
|
|
187
|
+
concept_linkage: pd.DataFrame,
|
|
185
188
|
parent_col: str = "source_class",
|
|
186
189
|
child_col: str = "target_class",
|
|
187
190
|
) -> dict:
|
|
188
|
-
parent_child_list: list[list[str]] =
|
|
191
|
+
parent_child_list: list[list[str]] = concept_linkage[[parent_col, child_col]].values.tolist() # type: ignore[assignment]
|
|
189
192
|
# Build a directed graph and a list of all names that have no parent
|
|
190
193
|
graph: dict[str, set] = {name: set() for tup in parent_child_list for name in tup}
|
|
191
194
|
has_parent: dict[str, bool] = {name: False for tup in parent_child_list for name in tup}
|
|
@@ -217,44 +220,44 @@ def _prettify_generation_order(generation_order: dict, depth: dict | None = None
|
|
|
217
220
|
|
|
218
221
|
|
|
219
222
|
def _remove_higher_occurring_sym_pair(
|
|
220
|
-
|
|
223
|
+
concept_linkage: pd.DataFrame, sym_pairs: set[tuple[ConceptEntity, ConceptEntity]]
|
|
221
224
|
) -> pd.DataFrame:
|
|
222
225
|
"""Remove symmetric pair which is higher in occurrence."""
|
|
223
226
|
rows_to_remove = set()
|
|
224
227
|
for source, target in sym_pairs:
|
|
225
|
-
first_sym_property_occurrence =
|
|
226
|
-
(
|
|
228
|
+
first_sym_property_occurrence = concept_linkage[
|
|
229
|
+
(concept_linkage.source_class == source) & (concept_linkage.target_class == target)
|
|
227
230
|
].max_occurrence.values[0]
|
|
228
|
-
second_sym_property_occurrence =
|
|
229
|
-
(
|
|
231
|
+
second_sym_property_occurrence = concept_linkage[
|
|
232
|
+
(concept_linkage.source_class == target) & (concept_linkage.target_class == source)
|
|
230
233
|
].max_occurrence.values[0]
|
|
231
234
|
|
|
232
235
|
if first_sym_property_occurrence is None:
|
|
233
236
|
# this means that source occurrence is unbounded
|
|
234
|
-
index =
|
|
235
|
-
(
|
|
237
|
+
index = concept_linkage[
|
|
238
|
+
(concept_linkage.source_class == source) & (concept_linkage.target_class == target)
|
|
236
239
|
].index.values[0]
|
|
237
240
|
elif second_sym_property_occurrence is None or (
|
|
238
241
|
first_sym_property_occurrence <= second_sym_property_occurrence
|
|
239
242
|
and second_sym_property_occurrence > first_sym_property_occurrence
|
|
240
243
|
):
|
|
241
244
|
# this means that target occurrence is unbounded
|
|
242
|
-
index =
|
|
243
|
-
(
|
|
245
|
+
index = concept_linkage[
|
|
246
|
+
(concept_linkage.source_class == target) & (concept_linkage.target_class == source)
|
|
244
247
|
].index.values[0]
|
|
245
248
|
else:
|
|
246
|
-
index =
|
|
247
|
-
(
|
|
249
|
+
index = concept_linkage[
|
|
250
|
+
(concept_linkage.source_class == source) & (concept_linkage.target_class == target)
|
|
248
251
|
].index.values[0]
|
|
249
252
|
rows_to_remove.add(index)
|
|
250
253
|
|
|
251
|
-
return
|
|
254
|
+
return concept_linkage.drop(list(rows_to_remove))
|
|
252
255
|
|
|
253
256
|
|
|
254
|
-
def _remove_non_requested_sym_pairs(class_linkage: pd.DataFrame,
|
|
257
|
+
def _remove_non_requested_sym_pairs(class_linkage: pd.DataFrame, concept_count: dict) -> pd.DataFrame:
|
|
255
258
|
"""Remove symmetric pairs which classes are not found in class count."""
|
|
256
|
-
rows_to_remove = set(class_linkage[~(class_linkage["source_class"].isin(set(
|
|
257
|
-
rows_to_remove |= set(class_linkage[~(class_linkage["target_class"].isin(set(
|
|
259
|
+
rows_to_remove = set(class_linkage[~(class_linkage["source_class"].isin(set(concept_count.keys())))].index.values)
|
|
260
|
+
rows_to_remove |= set(class_linkage[~(class_linkage["target_class"].isin(set(concept_count.keys())))].index.values)
|
|
258
261
|
|
|
259
262
|
return class_linkage.drop(list(rows_to_remove))
|
|
260
263
|
|
|
@@ -293,41 +296,41 @@ def _generate_mock_data_property_triples(
|
|
|
293
296
|
|
|
294
297
|
|
|
295
298
|
def _generate_mock_object_property_triples(
|
|
296
|
-
|
|
299
|
+
concept: ConceptEntity,
|
|
297
300
|
property_definition: ConceptualProperty,
|
|
298
|
-
|
|
299
|
-
sym_pairs: set[tuple[
|
|
300
|
-
instance_ids: dict[
|
|
301
|
+
concept_property_pairs: dict[ConceptEntity, list[ConceptualProperty]],
|
|
302
|
+
sym_pairs: set[tuple[ConceptEntity, ConceptEntity]],
|
|
303
|
+
instance_ids: dict[ConceptEntity, list[URIRef]],
|
|
301
304
|
namespace: Namespace,
|
|
302
305
|
stop_on_exception: bool,
|
|
303
306
|
) -> list[tuple[URIRef, URIRef, URIRef]]:
|
|
304
307
|
"""Generates triples for object properties."""
|
|
305
308
|
if property_definition.value_type not in instance_ids:
|
|
306
|
-
msg = f"
|
|
309
|
+
msg = f"Concept {property_definition.value_type} not found in concept count! "
|
|
307
310
|
if stop_on_exception:
|
|
308
311
|
raise ValueError(msg)
|
|
309
312
|
else:
|
|
310
313
|
msg += (
|
|
311
314
|
f"Skipping creating triples for property {property_definition.name} "
|
|
312
|
-
f"of
|
|
315
|
+
f"of concept {concept.suffix} which expects values of this type!"
|
|
313
316
|
)
|
|
314
317
|
warnings.warn(msg, stacklevel=2)
|
|
315
318
|
return []
|
|
316
319
|
|
|
317
320
|
# Handling symmetric property
|
|
318
321
|
|
|
319
|
-
if tuple((
|
|
320
|
-
|
|
322
|
+
if tuple((concept, property_definition.value_type)) in sym_pairs:
|
|
323
|
+
symmetric_concept_properties = concept_property_pairs[cast(ConceptEntity, property_definition.value_type)]
|
|
321
324
|
candidates = list(
|
|
322
325
|
filter(
|
|
323
|
-
lambda instance: instance.value_type ==
|
|
324
|
-
|
|
326
|
+
lambda instance: instance.value_type == concept,
|
|
327
|
+
symmetric_concept_properties,
|
|
325
328
|
)
|
|
326
329
|
)
|
|
327
330
|
symmetric_property = candidates[0]
|
|
328
331
|
if len(candidates) > 1:
|
|
329
332
|
warnings.warn(
|
|
330
|
-
f"Multiple symmetric properties found for
|
|
333
|
+
f"Multiple symmetric properties found for concept {property_definition.value_type}! "
|
|
331
334
|
f"Only one will be used for creating symmetric triples.",
|
|
332
335
|
stacklevel=2,
|
|
333
336
|
)
|
|
@@ -336,9 +339,9 @@ def _generate_mock_object_property_triples(
|
|
|
336
339
|
|
|
337
340
|
triples = []
|
|
338
341
|
|
|
339
|
-
for i, source in enumerate(instance_ids[
|
|
340
|
-
target = instance_ids[cast(
|
|
341
|
-
i % len(instance_ids[cast(
|
|
342
|
+
for i, source in enumerate(instance_ids[concept]):
|
|
343
|
+
target = instance_ids[cast(ConceptEntity, property_definition.value_type)][
|
|
344
|
+
i % len(instance_ids[cast(ConceptEntity, property_definition.value_type)])
|
|
342
345
|
]
|
|
343
346
|
triples += [
|
|
344
347
|
(
|
|
@@ -358,26 +361,26 @@ def _generate_mock_object_property_triples(
|
|
|
358
361
|
]
|
|
359
362
|
|
|
360
363
|
if symmetric_property:
|
|
361
|
-
|
|
364
|
+
concept_property_pairs[cast(ConceptEntity, property_definition.value_type)].remove(symmetric_property)
|
|
362
365
|
|
|
363
366
|
return triples
|
|
364
367
|
|
|
365
368
|
|
|
366
369
|
def _generate_triples_per_class(
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
sym_pairs: set[tuple[
|
|
370
|
-
instance_ids: dict[
|
|
370
|
+
concept: ConceptEntity,
|
|
371
|
+
concept_properties_pairs: dict[ConceptEntity, list[ConceptualProperty]],
|
|
372
|
+
sym_pairs: set[tuple[ConceptEntity, ConceptEntity]],
|
|
373
|
+
instance_ids: dict[ConceptEntity, list[URIRef]],
|
|
371
374
|
namespace: Namespace,
|
|
372
375
|
stop_on_exception: bool,
|
|
373
376
|
) -> list[Triple]:
|
|
374
377
|
"""Generate triples for a given class."""
|
|
375
378
|
triples: list[Triple] = []
|
|
376
379
|
|
|
377
|
-
for property_ in
|
|
380
|
+
for property_ in concept_properties_pairs[concept]:
|
|
378
381
|
if property_.type_ == EntityTypes.data_property:
|
|
379
382
|
triples += _generate_mock_data_property_triples(
|
|
380
|
-
instance_ids[
|
|
383
|
+
instance_ids[concept],
|
|
381
384
|
property_.property_,
|
|
382
385
|
namespace,
|
|
383
386
|
cast(DataType, property_.value_type),
|
|
@@ -385,9 +388,9 @@ def _generate_triples_per_class(
|
|
|
385
388
|
|
|
386
389
|
elif property_.type_ == EntityTypes.object_property:
|
|
387
390
|
triples += _generate_mock_object_property_triples(
|
|
388
|
-
|
|
391
|
+
concept,
|
|
389
392
|
property_,
|
|
390
|
-
|
|
393
|
+
concept_properties_pairs,
|
|
391
394
|
sym_pairs,
|
|
392
395
|
instance_ids,
|
|
393
396
|
namespace,
|
|
@@ -8,7 +8,7 @@ from cognite.client.data_classes.capabilities import Capability
|
|
|
8
8
|
|
|
9
9
|
from cognite.neat.core._issues import IssueList, NeatIssue
|
|
10
10
|
from cognite.neat.core._issues.errors import AuthorizationError
|
|
11
|
-
from cognite.neat.core._store import
|
|
11
|
+
from cognite.neat.core._store import NeatInstanceStore
|
|
12
12
|
from cognite.neat.core._utils.auxiliary import class_html_doc
|
|
13
13
|
from cognite.neat.core._utils.upload import UploadResult, UploadResultList
|
|
14
14
|
|
|
@@ -29,7 +29,7 @@ class BaseLoader(ABC, Generic[T_Output]):
|
|
|
29
29
|
_new_line = "\n"
|
|
30
30
|
_encoding = "utf-8"
|
|
31
31
|
|
|
32
|
-
def __init__(self, graph_store:
|
|
32
|
+
def __init__(self, graph_store: NeatInstanceStore):
|
|
33
33
|
self.graph_store = graph_store
|
|
34
34
|
|
|
35
35
|
@abstractmethod
|
|
@@ -25,9 +25,9 @@ from cognite.neat.core._constants import (
|
|
|
25
25
|
DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
|
|
26
26
|
is_readonly_property,
|
|
27
27
|
)
|
|
28
|
-
from cognite.neat.core._data_model.analysis import
|
|
28
|
+
from cognite.neat.core._data_model.analysis import DataModelAnalysis
|
|
29
29
|
from cognite.neat.core._data_model.analysis._base import ViewQuery, ViewQueryDict
|
|
30
|
-
from cognite.neat.core._data_model.models import
|
|
30
|
+
from cognite.neat.core._data_model.models import PhysicalDataModel
|
|
31
31
|
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
32
32
|
ConceptualDataModel,
|
|
33
33
|
)
|
|
@@ -51,7 +51,7 @@ from cognite.neat.core._issues.warnings import (
|
|
|
51
51
|
ResourceNeatWarning,
|
|
52
52
|
)
|
|
53
53
|
from cognite.neat.core._shared import InstanceType
|
|
54
|
-
from cognite.neat.core._store import
|
|
54
|
+
from cognite.neat.core._store import NeatInstanceStore
|
|
55
55
|
from cognite.neat.core._utils.auxiliary import create_sha256_hash
|
|
56
56
|
from cognite.neat.core._utils.collection_ import (
|
|
57
57
|
iterate_progress_bar_if_above_config_threshold,
|
|
@@ -115,9 +115,9 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
|
|
|
115
115
|
|
|
116
116
|
def __init__(
|
|
117
117
|
self,
|
|
118
|
-
dms_rules:
|
|
118
|
+
dms_rules: PhysicalDataModel,
|
|
119
119
|
info_rules: ConceptualDataModel,
|
|
120
|
-
graph_store:
|
|
120
|
+
graph_store: NeatInstanceStore,
|
|
121
121
|
instance_space: str,
|
|
122
122
|
space_property: str | None = None,
|
|
123
123
|
use_source_space: bool = False,
|
|
@@ -209,7 +209,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
|
|
|
209
209
|
yield _END_OF_CLASS
|
|
210
210
|
|
|
211
211
|
def _create_view_iterations(self) -> tuple[list[_ViewIterator], IssueList]:
|
|
212
|
-
view_query_by_id =
|
|
212
|
+
view_query_by_id = DataModelAnalysis(self.info_rules, self.dms_rules).view_query_by_id
|
|
213
213
|
iterations_by_view_id = self._select_views_with_instances(view_query_by_id)
|
|
214
214
|
if self._client:
|
|
215
215
|
issues = IssueList()
|
|
@@ -100,10 +100,13 @@ class BaseTransformerStandardised(ABC):
|
|
|
100
100
|
if self._skip_count_query():
|
|
101
101
|
skipped_count_res = list(graph.query(self._skip_count_query()))
|
|
102
102
|
skipped_count = int(skipped_count_res[0][0]) # type: ignore [index, arg-type]
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
103
|
+
if skipped_count > 0:
|
|
104
|
+
warnings.warn(
|
|
105
|
+
NeatValueWarning(
|
|
106
|
+
f"Skipping {skipped_count} properties in transformation {self.__class__.__name__}"
|
|
107
|
+
),
|
|
108
|
+
stacklevel=2,
|
|
109
|
+
)
|
|
107
110
|
outcome.skipped = skipped_count
|
|
108
111
|
|
|
109
112
|
if iteration_count == 0:
|
|
@@ -10,11 +10,7 @@ from rdflib.query import ResultRow
|
|
|
10
10
|
from cognite.neat.core._constants import NEAT
|
|
11
11
|
from cognite.neat.core._issues.warnings import PropertyDataTypeConversionWarning
|
|
12
12
|
from cognite.neat.core._utils.auxiliary import string_to_ideal_type
|
|
13
|
-
from cognite.neat.core._utils.rdf_ import
|
|
14
|
-
Triple,
|
|
15
|
-
get_namespace,
|
|
16
|
-
remove_namespace_from_uri,
|
|
17
|
-
)
|
|
13
|
+
from cognite.neat.core._utils.rdf_ import Triple, get_namespace, remove_namespace_from_uri, uri_to_cdf_id
|
|
18
14
|
|
|
19
15
|
from ._base import BaseTransformerStandardised, RowTransformationOutput
|
|
20
16
|
|
|
@@ -303,7 +299,7 @@ class ConnectionToLiteral(BaseTransformerStandardised):
|
|
|
303
299
|
row_output = RowTransformationOutput()
|
|
304
300
|
|
|
305
301
|
instance, object_entity = cast(tuple[URIRef, URIRef], query_result_row)
|
|
306
|
-
value =
|
|
302
|
+
value = uri_to_cdf_id(object_entity)
|
|
307
303
|
|
|
308
304
|
row_output.add_triples.add((instance, self.subject_predicate, rdflib.Literal(value)))
|
|
309
305
|
row_output.remove_triples.add((instance, self.subject_predicate, object_entity))
|
|
@@ -119,7 +119,7 @@ class NeatIssue:
|
|
|
119
119
|
|
|
120
120
|
@classmethod
|
|
121
121
|
def _dump_value(cls, value: Any) -> list | int | bool | float | str | dict:
|
|
122
|
-
from cognite.neat.core._data_model.models.entities import
|
|
122
|
+
from cognite.neat.core._data_model.models.entities import ConceptualEntity
|
|
123
123
|
|
|
124
124
|
if isinstance(value, str | int | bool | float):
|
|
125
125
|
return value
|
|
@@ -131,7 +131,7 @@ class NeatIssue:
|
|
|
131
131
|
return [cls._dump_value(item) for item in value]
|
|
132
132
|
elif isinstance(value, ViewId | ContainerId):
|
|
133
133
|
return value.dump(camel_case=True, include_type=True)
|
|
134
|
-
elif isinstance(value,
|
|
134
|
+
elif isinstance(value, ConceptualEntity):
|
|
135
135
|
return value.dump()
|
|
136
136
|
elif isinstance(value, PropertyId):
|
|
137
137
|
return value.dump(camel_case=True)
|
|
@@ -173,7 +173,7 @@ class NeatIssue:
|
|
|
173
173
|
|
|
174
174
|
@classmethod
|
|
175
175
|
def _load_value(cls, type_: Any, value: Any) -> Any:
|
|
176
|
-
from cognite.neat.core._data_model.models.entities import
|
|
176
|
+
from cognite.neat.core._data_model.models.entities import ConceptualEntity
|
|
177
177
|
|
|
178
178
|
if isinstance(type_, UnionType) or get_origin(type_) is UnionType:
|
|
179
179
|
args = get_args(type_)
|
|
@@ -194,7 +194,7 @@ class NeatIssue:
|
|
|
194
194
|
return PropertyId.load(value)
|
|
195
195
|
elif type_ is ContainerId:
|
|
196
196
|
return ContainerId.load(value)
|
|
197
|
-
elif inspect.isclass(type_) and issubclass(type_,
|
|
197
|
+
elif inspect.isclass(type_) and issubclass(type_, ConceptualEntity):
|
|
198
198
|
return type_.load(value)
|
|
199
199
|
elif type_ is NeatError:
|
|
200
200
|
return cls.load(value)
|