cognite-neat 0.99.1__py3-none-any.whl → 0.100.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +403 -182
- cognite/neat/_client/data_classes/data_modeling.py +4 -0
- cognite/neat/_graph/extractors/_base.py +7 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -13
- cognite/neat/_graph/loaders/_rdf2dms.py +50 -11
- cognite/neat/_graph/transformers/__init__.py +3 -3
- cognite/neat/_graph/transformers/_classic_cdf.py +120 -52
- cognite/neat/_issues/warnings/__init__.py +2 -0
- cognite/neat/_issues/warnings/_resources.py +15 -0
- cognite/neat/_rules/analysis/_base.py +15 -5
- cognite/neat/_rules/analysis/_dms.py +20 -0
- cognite/neat/_rules/analysis/_information.py +22 -0
- cognite/neat/_rules/exporters/_base.py +3 -5
- cognite/neat/_rules/exporters/_rules2dms.py +192 -200
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +22 -5
- cognite/neat/_rules/models/_base_rules.py +19 -0
- cognite/neat/_rules/models/_types.py +5 -0
- cognite/neat/_rules/models/dms/_exporter.py +215 -93
- cognite/neat/_rules/models/dms/_rules.py +4 -4
- cognite/neat/_rules/models/dms/_rules_input.py +8 -3
- cognite/neat/_rules/models/dms/_validation.py +42 -11
- cognite/neat/_rules/models/entities/_multi_value.py +3 -0
- cognite/neat/_rules/models/information/_rules.py +17 -2
- cognite/neat/_rules/models/information/_rules_input.py +11 -2
- cognite/neat/_rules/models/information/_validation.py +99 -3
- cognite/neat/_rules/models/mapping/_classic2core.yaml +1 -1
- cognite/neat/_rules/transformers/__init__.py +2 -1
- cognite/neat/_rules/transformers/_converters.py +163 -61
- cognite/neat/_rules/transformers/_mapping.py +132 -2
- cognite/neat/_session/_base.py +42 -31
- cognite/neat/_session/_mapping.py +105 -5
- cognite/neat/_session/_prepare.py +43 -9
- cognite/neat/_session/_read.py +50 -4
- cognite/neat/_session/_set.py +1 -0
- cognite/neat/_session/_to.py +36 -13
- cognite/neat/_session/_wizard.py +5 -0
- cognite/neat/_session/engine/_interface.py +3 -2
- cognite/neat/_store/_base.py +79 -19
- cognite/neat/_utils/collection_.py +22 -0
- cognite/neat/_utils/rdf_.py +24 -0
- cognite/neat/_version.py +2 -2
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +3 -3
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/METADATA +1 -1
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/RECORD +47 -47
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/LICENSE +0 -0
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/WHEEL +0 -0
- {cognite_neat-0.99.1.dist-info → cognite_neat-0.100.1.dist-info}/entry_points.txt +0 -0
|
@@ -4,13 +4,16 @@ from collections import defaultdict
|
|
|
4
4
|
from functools import cached_property
|
|
5
5
|
from typing import Any, ClassVar, Literal
|
|
6
6
|
|
|
7
|
-
from cognite.
|
|
7
|
+
from cognite.client import data_modeling as dm
|
|
8
|
+
|
|
9
|
+
from cognite.neat._client import NeatClient
|
|
10
|
+
from cognite.neat._issues.errors import CDFMissingClientError, NeatValueError, ResourceNotFoundError
|
|
8
11
|
from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
|
|
9
12
|
from cognite.neat._rules._shared import JustRules, OutRules
|
|
10
13
|
from cognite.neat._rules.models import DMSRules, SheetList
|
|
11
14
|
from cognite.neat._rules.models.data_types import Enum
|
|
12
15
|
from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
|
|
13
|
-
from cognite.neat._rules.models.entities import ViewEntity
|
|
16
|
+
from cognite.neat._rules.models.entities import ContainerEntity, ViewEntity
|
|
14
17
|
|
|
15
18
|
from ._base import RulesTransformer
|
|
16
19
|
|
|
@@ -208,3 +211,130 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
|
208
211
|
# These are used for warnings so we use the alias to make it more readable for the user
|
|
209
212
|
conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
|
|
210
213
|
return to_overwrite, conflicts
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
|
|
217
|
+
"""Looks up all view properties that map to the same container property,
|
|
218
|
+
and changes the child view property id to match the parent property id.
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
def __init__(self, client: NeatClient | None = None) -> None:
|
|
222
|
+
self._client = client
|
|
223
|
+
|
|
224
|
+
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
225
|
+
input_rules = self._to_rules(rules)
|
|
226
|
+
new_rules = input_rules.model_copy(deep=True)
|
|
227
|
+
new_rules.metadata.version += "_as_parent_name"
|
|
228
|
+
|
|
229
|
+
path_by_view = self._inheritance_path_by_view(new_rules)
|
|
230
|
+
view_by_container_property = self._view_by_container_properties(new_rules)
|
|
231
|
+
|
|
232
|
+
parent_view_property_by_container_property = self._get_parent_view_property_by_container_property(
|
|
233
|
+
path_by_view, view_by_container_property
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
for prop in new_rules.properties:
|
|
237
|
+
if prop.container and prop.container_property:
|
|
238
|
+
if parent_name := parent_view_property_by_container_property.get(
|
|
239
|
+
(prop.container, prop.container_property)
|
|
240
|
+
):
|
|
241
|
+
prop.view_property = parent_name
|
|
242
|
+
|
|
243
|
+
return JustRules(new_rules)
|
|
244
|
+
|
|
245
|
+
# Todo: Move into Probe class. Note this means that the Probe class must take a NeatClient as an argument.
|
|
246
|
+
def _inheritance_path_by_view(self, rules: DMSRules) -> dict[ViewEntity, list[ViewEntity]]:
|
|
247
|
+
parents_by_view: dict[ViewEntity, list[ViewEntity]] = {view.view: view.implements or [] for view in rules.views}
|
|
248
|
+
|
|
249
|
+
path_by_view: dict[ViewEntity, list[ViewEntity]] = {}
|
|
250
|
+
for view in rules.views:
|
|
251
|
+
path_by_view[view.view] = self._get_inheritance_path(
|
|
252
|
+
view.view, parents_by_view, rules.metadata.as_data_model_id()
|
|
253
|
+
)
|
|
254
|
+
return path_by_view
|
|
255
|
+
|
|
256
|
+
def _get_inheritance_path(
|
|
257
|
+
self, view: ViewEntity, parents_by_view: dict[ViewEntity, list[ViewEntity]], data_model_id: dm.DataModelId
|
|
258
|
+
) -> list[ViewEntity]:
|
|
259
|
+
if parents_by_view.get(view) == []:
|
|
260
|
+
# We found the root.
|
|
261
|
+
return [view]
|
|
262
|
+
if view not in parents_by_view and self._client is not None:
|
|
263
|
+
# Lookup the parent
|
|
264
|
+
view_id = view.as_id()
|
|
265
|
+
read_views = self._client.loaders.views.retrieve([view_id])
|
|
266
|
+
if not read_views:
|
|
267
|
+
# Warning? Should be caught by validation
|
|
268
|
+
raise ResourceNotFoundError(view_id, "view", data_model_id, "data model")
|
|
269
|
+
parent_view_latest = max(read_views, key=lambda view: view.created_time)
|
|
270
|
+
parents_by_view[ViewEntity.from_id(parent_view_latest.as_id())] = [
|
|
271
|
+
ViewEntity.from_id(grand_parent) for grand_parent in parent_view_latest.implements or []
|
|
272
|
+
]
|
|
273
|
+
elif view not in parents_by_view:
|
|
274
|
+
raise CDFMissingClientError(
|
|
275
|
+
f"The data model {data_model_id} is referencing a view that is not in the data model."
|
|
276
|
+
f"Please provide a client to lookup the view."
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
inheritance_path = [view]
|
|
280
|
+
seen = {view}
|
|
281
|
+
if view in parents_by_view:
|
|
282
|
+
for parent in parents_by_view[view]:
|
|
283
|
+
parent_path = self._get_inheritance_path(parent, parents_by_view, data_model_id)
|
|
284
|
+
inheritance_path.extend([p for p in parent_path if p not in seen])
|
|
285
|
+
seen.update(parent_path)
|
|
286
|
+
return inheritance_path
|
|
287
|
+
|
|
288
|
+
def _view_by_container_properties(
|
|
289
|
+
self, rules: DMSRules
|
|
290
|
+
) -> dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]:
|
|
291
|
+
view_properties_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]] = (
|
|
292
|
+
defaultdict(list)
|
|
293
|
+
)
|
|
294
|
+
view_with_properties: set[ViewEntity] = set()
|
|
295
|
+
for prop in rules.properties:
|
|
296
|
+
if not prop.container or not prop.container_property:
|
|
297
|
+
continue
|
|
298
|
+
view_properties_by_container_properties[(prop.container, prop.container_property)].append(
|
|
299
|
+
(prop.view, prop.view_property)
|
|
300
|
+
)
|
|
301
|
+
view_with_properties.add(prop.view)
|
|
302
|
+
|
|
303
|
+
# We need to look up all parent properties.
|
|
304
|
+
to_lookup = {view.view.as_id() for view in rules.views if view.view not in view_with_properties}
|
|
305
|
+
if to_lookup and self._client is None:
|
|
306
|
+
raise CDFMissingClientError(
|
|
307
|
+
f"Views {to_lookup} are not in the data model. Please provide a client to lookup the views."
|
|
308
|
+
)
|
|
309
|
+
elif to_lookup and self._client:
|
|
310
|
+
read_views = self._client.loaders.views.retrieve(list(to_lookup), include_ancestor=True)
|
|
311
|
+
write_views = [self._client.loaders.views.as_write(read_view) for read_view in read_views]
|
|
312
|
+
# We use the write/request format of the views as the read/response format contains all properties
|
|
313
|
+
# including ancestor properties. The goal is to find the property name used in the parent
|
|
314
|
+
# and thus we cannot have that repeated in the child views.
|
|
315
|
+
for write_view in write_views:
|
|
316
|
+
view_id = write_view.as_id()
|
|
317
|
+
view_entity = ViewEntity.from_id(view_id)
|
|
318
|
+
|
|
319
|
+
for property_id, property_ in (write_view.properties or {}).items():
|
|
320
|
+
if not isinstance(property_, dm.MappedPropertyApply):
|
|
321
|
+
continue
|
|
322
|
+
container_entity = ContainerEntity.from_id(property_.container)
|
|
323
|
+
view_properties_by_container_properties[
|
|
324
|
+
(container_entity, property_.container_property_identifier)
|
|
325
|
+
].append((view_entity, property_id))
|
|
326
|
+
|
|
327
|
+
return view_properties_by_container_properties
|
|
328
|
+
|
|
329
|
+
@staticmethod
|
|
330
|
+
def _get_parent_view_property_by_container_property(
|
|
331
|
+
path_by_view, view_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]
|
|
332
|
+
) -> dict[tuple[ContainerEntity, str], str]:
|
|
333
|
+
parent_name_by_container_property: dict[tuple[ContainerEntity, str], str] = {}
|
|
334
|
+
for (container, container_property), view_properties in view_by_container_properties.items():
|
|
335
|
+
if len(view_properties) == 1:
|
|
336
|
+
continue
|
|
337
|
+
# Shortest path is the parent
|
|
338
|
+
_, prop_name = min(view_properties, key=lambda prop: len(path_by_view[prop[0]]))
|
|
339
|
+
parent_name_by_container_property[(container, container_property)] = prop_name
|
|
340
|
+
return parent_name_by_container_property
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -15,7 +15,7 @@ from cognite.neat._rules.models.dms import DMSValidation
|
|
|
15
15
|
from cognite.neat._rules.models.information import InformationValidation
|
|
16
16
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
17
17
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
18
|
-
from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
|
|
18
|
+
from cognite.neat._rules.transformers import ConvertToRules, InformationToDMS, VerifyAnyRules
|
|
19
19
|
from cognite.neat._rules.transformers._converters import ConversionTransformer
|
|
20
20
|
from cognite.neat._store._provenance import (
|
|
21
21
|
INSTANCES_ENTITY,
|
|
@@ -54,7 +54,7 @@ class NeatSession:
|
|
|
54
54
|
self.show = ShowAPI(self._state)
|
|
55
55
|
self.set = SetAPI(self._state, verbose)
|
|
56
56
|
self.inspect = InspectAPI(self._state)
|
|
57
|
-
self.mapping = MappingAPI(self._state)
|
|
57
|
+
self.mapping = MappingAPI(self._state, self._client)
|
|
58
58
|
self.drop = DropAPI(self._state)
|
|
59
59
|
self.opt = OptAPI()
|
|
60
60
|
self.opt._display()
|
|
@@ -70,35 +70,37 @@ class NeatSession:
|
|
|
70
70
|
transformer = VerifyAnyRules("continue", validate=False)
|
|
71
71
|
start = datetime.now(timezone.utc)
|
|
72
72
|
output = transformer.try_transform(last_unverified_rule)
|
|
73
|
-
if isinstance(output.rules, DMSRules):
|
|
74
|
-
issues = DMSValidation(output.rules, self._client).validate()
|
|
75
|
-
elif isinstance(output.rules, InformationRules):
|
|
76
|
-
issues = InformationValidation(output.rules).validate()
|
|
77
|
-
else:
|
|
78
|
-
raise NeatSessionError("Unsupported rule type")
|
|
79
|
-
if issues.has_errors:
|
|
80
|
-
# This is up for discussion, but I think we should not return rules that
|
|
81
|
-
# only pass the verification but not the validation.
|
|
82
|
-
output.rules = None
|
|
83
|
-
output.issues.extend(issues)
|
|
84
|
-
|
|
85
|
-
end = datetime.now(timezone.utc)
|
|
86
73
|
|
|
87
74
|
if output.rules:
|
|
88
|
-
|
|
89
|
-
output.rules,
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
75
|
+
if isinstance(output.rules, DMSRules):
|
|
76
|
+
issues = DMSValidation(output.rules, self._client).validate()
|
|
77
|
+
elif isinstance(output.rules, InformationRules):
|
|
78
|
+
issues = InformationValidation(output.rules).validate()
|
|
79
|
+
else:
|
|
80
|
+
raise NeatSessionError("Unsupported rule type")
|
|
81
|
+
if issues.has_errors:
|
|
82
|
+
# This is up for discussion, but I think we should not return rules that
|
|
83
|
+
# only pass the verification but not the validation.
|
|
84
|
+
output.rules = None
|
|
85
|
+
output.issues.extend(issues)
|
|
86
|
+
|
|
87
|
+
end = datetime.now(timezone.utc)
|
|
88
|
+
|
|
89
|
+
if output.rules:
|
|
90
|
+
change = Change.from_rules_activity(
|
|
91
|
+
output.rules,
|
|
92
|
+
transformer.agent,
|
|
93
|
+
start,
|
|
94
|
+
end,
|
|
95
|
+
f"Verified data model {source_id} as {output.rules.metadata.identifier}",
|
|
96
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
97
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
self._state.data_model.write(output.rules, change)
|
|
101
|
+
|
|
102
|
+
if isinstance(output.rules, InformationRules):
|
|
103
|
+
self._state.instances.store.add_rules(output.rules)
|
|
102
104
|
|
|
103
105
|
output.issues.action = "verify"
|
|
104
106
|
self._state.data_model.issue_lists.append(output.issues)
|
|
@@ -106,7 +108,16 @@ class NeatSession:
|
|
|
106
108
|
print("You can inspect the issues with the .inspect.issues(...) method.")
|
|
107
109
|
return output.issues
|
|
108
110
|
|
|
109
|
-
def convert(
|
|
111
|
+
def convert(
|
|
112
|
+
self, target: Literal["dms", "information"], mode: Literal["edge_properties"] | None = None
|
|
113
|
+
) -> IssueList:
|
|
114
|
+
"""Converts the last verified data model to the target type.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
target: The target type to convert the data model to.
|
|
118
|
+
mode: If the target is "dms", the mode to use for the conversion. None is used for default conversion.
|
|
119
|
+
"edge_properties" treas classes that implements Edge as edge properties.
|
|
120
|
+
"""
|
|
110
121
|
start = datetime.now(timezone.utc)
|
|
111
122
|
issues = IssueList()
|
|
112
123
|
converter: ConversionTransformer | None = None
|
|
@@ -114,7 +125,7 @@ class NeatSession:
|
|
|
114
125
|
with catch_issues(issues):
|
|
115
126
|
if target == "dms":
|
|
116
127
|
source_id, info_rules = self._state.data_model.last_verified_information_rules
|
|
117
|
-
converter =
|
|
128
|
+
converter = InformationToDMS(mode=mode)
|
|
118
129
|
converted_rules = converter.transform(info_rules).rules
|
|
119
130
|
elif target == "information":
|
|
120
131
|
source_id, dms_rules = self._state.data_model.last_verified_dms_rules
|
|
@@ -1,28 +1,50 @@
|
|
|
1
1
|
from datetime import datetime, timezone
|
|
2
2
|
|
|
3
|
+
from cognite.neat._client import NeatClient
|
|
4
|
+
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
5
|
+
from cognite.neat._rules.importers import DMSImporter
|
|
6
|
+
from cognite.neat._rules.models.dms import DMSValidation
|
|
3
7
|
from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
|
|
4
|
-
from cognite.neat._rules.transformers import RuleMapper
|
|
8
|
+
from cognite.neat._rules.transformers import AsParentPropertyId, RuleMapper, VerifyDMSRules
|
|
9
|
+
from cognite.neat._store._provenance import Agent as ProvenanceAgent
|
|
5
10
|
from cognite.neat._store._provenance import Change
|
|
6
11
|
|
|
7
12
|
from ._state import SessionState
|
|
8
|
-
from .exceptions import session_class_wrapper
|
|
13
|
+
from .exceptions import NeatSessionError, session_class_wrapper
|
|
9
14
|
|
|
10
15
|
|
|
11
16
|
@session_class_wrapper
|
|
12
17
|
class MappingAPI:
|
|
13
|
-
def __init__(self, state: SessionState):
|
|
18
|
+
def __init__(self, state: SessionState, client: NeatClient | None = None):
|
|
19
|
+
self.data_model = DataModelMappingAPI(state, client)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@session_class_wrapper
|
|
23
|
+
class DataModelMappingAPI:
|
|
24
|
+
def __init__(self, state: SessionState, client: NeatClient | None = None):
|
|
14
25
|
self._state = state
|
|
26
|
+
self._client = client
|
|
15
27
|
|
|
16
|
-
def classic_to_core(self,
|
|
28
|
+
def classic_to_core(self, company_prefix: str, use_parent_property_name: bool = True) -> None:
|
|
17
29
|
"""Map classic types to core types.
|
|
18
30
|
|
|
19
31
|
Note this automatically creates an extended CogniteCore model.
|
|
20
32
|
|
|
33
|
+
Args:
|
|
34
|
+
company_prefix: Prefix used for all extended CogniteCore types.
|
|
35
|
+
use_parent_property_name: Whether to use the parent property name in the extended CogniteCore model.
|
|
36
|
+
See below for more information.
|
|
37
|
+
|
|
38
|
+
If you extend CogniteAsset, with for example, ClassicAsset. You will map the property `parentId` to `parent`.
|
|
39
|
+
If you set `user_parent_property_name` to True, the `parentId` will be renamed to `parent` after the
|
|
40
|
+
mapping is done. If you set it to False, the property will remain `parentId`.
|
|
21
41
|
"""
|
|
22
42
|
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
23
43
|
|
|
24
44
|
start = datetime.now(timezone.utc)
|
|
25
|
-
transformer = RuleMapper(
|
|
45
|
+
transformer = RuleMapper(
|
|
46
|
+
load_classic_to_core_mapping(company_prefix, rules.metadata.space, rules.metadata.version)
|
|
47
|
+
)
|
|
26
48
|
output = transformer.transform(rules)
|
|
27
49
|
end = datetime.now(timezone.utc)
|
|
28
50
|
|
|
@@ -37,3 +59,81 @@ class MappingAPI:
|
|
|
37
59
|
)
|
|
38
60
|
|
|
39
61
|
self._state.data_model.write(output.rules, change)
|
|
62
|
+
|
|
63
|
+
start = datetime.now(timezone.utc)
|
|
64
|
+
|
|
65
|
+
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
66
|
+
view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
|
|
67
|
+
if not (view_ids or container_ids):
|
|
68
|
+
print(
|
|
69
|
+
f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
|
|
70
|
+
f"that is not already included in the data model."
|
|
71
|
+
)
|
|
72
|
+
return
|
|
73
|
+
if self._client is None:
|
|
74
|
+
raise NeatSessionError(
|
|
75
|
+
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
76
|
+
"NEAT needs a client to lookup the definitions. "
|
|
77
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
78
|
+
)
|
|
79
|
+
schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
|
|
80
|
+
copy_ = rules.model_copy(deep=True)
|
|
81
|
+
copy_.metadata.version = f"{rules.metadata.version}_completed"
|
|
82
|
+
importer = DMSImporter(schema)
|
|
83
|
+
imported = importer.to_rules()
|
|
84
|
+
if imported.rules is None:
|
|
85
|
+
self._state.data_model.issue_lists.append(imported.issues)
|
|
86
|
+
raise NeatSessionError(
|
|
87
|
+
"Could not import the referenced views and containers. "
|
|
88
|
+
"See `neat.inspect.issues()` for more information."
|
|
89
|
+
)
|
|
90
|
+
verified = VerifyDMSRules("continue", validate=False).transform(imported.rules)
|
|
91
|
+
if verified.rules is None:
|
|
92
|
+
self._state.data_model.issue_lists.append(verified.issues)
|
|
93
|
+
raise NeatSessionError(
|
|
94
|
+
"Could not verify the referenced views and containers. "
|
|
95
|
+
"See `neat.inspect.issues()` for more information."
|
|
96
|
+
)
|
|
97
|
+
if copy_.containers is None:
|
|
98
|
+
copy_.containers = verified.rules.containers
|
|
99
|
+
else:
|
|
100
|
+
existing_containers = {c.container for c in copy_.containers}
|
|
101
|
+
copy_.containers.extend(
|
|
102
|
+
[c for c in verified.rules.containers or [] if c.container not in existing_containers]
|
|
103
|
+
)
|
|
104
|
+
existing_views = {v.view for v in copy_.views}
|
|
105
|
+
copy_.views.extend([v for v in verified.rules.views if v.view not in existing_views])
|
|
106
|
+
end = datetime.now(timezone.utc)
|
|
107
|
+
|
|
108
|
+
change = Change.from_rules_activity(
|
|
109
|
+
copy_,
|
|
110
|
+
ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
|
|
111
|
+
start,
|
|
112
|
+
end,
|
|
113
|
+
(f"Included referenced views and containers in the data model {rules.metadata.as_data_model_id()}"),
|
|
114
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
115
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
self._state.data_model.write(copy_, change)
|
|
119
|
+
|
|
120
|
+
if not use_parent_property_name:
|
|
121
|
+
return
|
|
122
|
+
|
|
123
|
+
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
124
|
+
start = datetime.now(timezone.utc)
|
|
125
|
+
transformer = AsParentPropertyId(self._client)
|
|
126
|
+
output = transformer.transform(rules)
|
|
127
|
+
end = datetime.now(timezone.utc)
|
|
128
|
+
|
|
129
|
+
change = Change.from_rules_activity(
|
|
130
|
+
output,
|
|
131
|
+
transformer.agent,
|
|
132
|
+
start,
|
|
133
|
+
end,
|
|
134
|
+
"Renaming property names to parent name",
|
|
135
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
136
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
self._state.data_model.write(output.rules, change)
|
|
@@ -8,12 +8,13 @@ from rdflib import URIRef
|
|
|
8
8
|
|
|
9
9
|
from cognite.neat._client import NeatClient
|
|
10
10
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
11
|
-
from cognite.neat._graph.transformers import
|
|
11
|
+
from cognite.neat._graph.transformers import RelationshipAsEdgeTransformer
|
|
12
12
|
from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
|
|
13
13
|
from cognite.neat._rules._shared import InputRules, ReadRules
|
|
14
14
|
from cognite.neat._rules.importers import DMSImporter
|
|
15
15
|
from cognite.neat._rules.models import DMSRules
|
|
16
16
|
from cognite.neat._rules.models.dms import DMSValidation
|
|
17
|
+
from cognite.neat._rules.models.entities import ClassEntity
|
|
17
18
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
18
19
|
from cognite.neat._rules.transformers import (
|
|
19
20
|
PrefixEntities,
|
|
@@ -113,20 +114,23 @@ class InstancePrepareAPI:
|
|
|
113
114
|
raise NeatSessionError(f"Property {property_} is not defined for type {type_}. Cannot make connection")
|
|
114
115
|
return type_uri[0], property_uri[0]
|
|
115
116
|
|
|
116
|
-
def
|
|
117
|
+
def relationships_as_edges(self, min_relationship_types: int = 1, limit_per_type: int | None = None) -> None:
|
|
117
118
|
"""This assumes that you have read a classic CDF knowledge graph including relationships.
|
|
118
119
|
|
|
119
|
-
This
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
properties are replaced by a schema that contains the properties as attributes.
|
|
120
|
+
This method converts relationships into edges in the graph. This is useful as the
|
|
121
|
+
edges will be picked up as part of the schema connected to Assets, Events, Files, Sequences,
|
|
122
|
+
and TimeSeries in the InferenceImporter.
|
|
123
123
|
|
|
124
124
|
Args:
|
|
125
|
-
|
|
126
|
-
to
|
|
125
|
+
min_relationship_types: The minimum number of relationship types that must exists to convert those
|
|
126
|
+
relationships to edges. For example, if there is only 5 relationships between Assets and TimeSeries,
|
|
127
|
+
and limit is 10, those relationships will not be converted to edges.
|
|
128
|
+
limit_per_type: The number of conversions to perform per relationship type. For example, if there are 10
|
|
129
|
+
relationships between Assets and TimeSeries, and limit_per_type is 1, only 1 of those relationships
|
|
130
|
+
will be converted to an edge. If None, all relationships will be converted.
|
|
127
131
|
|
|
128
132
|
"""
|
|
129
|
-
transformer =
|
|
133
|
+
transformer = RelationshipAsEdgeTransformer(min_relationship_types, limit_per_type)
|
|
130
134
|
self._state.instances.store.transform(transformer)
|
|
131
135
|
|
|
132
136
|
|
|
@@ -463,3 +467,33 @@ class DataModelPrepareAPI:
|
|
|
463
467
|
)
|
|
464
468
|
|
|
465
469
|
self._state.data_model.write(copy_, change)
|
|
470
|
+
|
|
471
|
+
def add_implements_to_classes(self, suffix: Literal["Edge"], implements: str = "Edge") -> None:
|
|
472
|
+
"""All classes with the suffix will have the implements property set to the given value.
|
|
473
|
+
|
|
474
|
+
Args:
|
|
475
|
+
suffix: The suffix of the classes to add the implements property to.
|
|
476
|
+
implements: The value of the implements property to set.
|
|
477
|
+
|
|
478
|
+
"""
|
|
479
|
+
source_id, rules = self._state.data_model.last_verified_information_rules
|
|
480
|
+
start = datetime.now(timezone.utc)
|
|
481
|
+
|
|
482
|
+
output = rules.model_copy(deep=True)
|
|
483
|
+
for class_ in output.classes:
|
|
484
|
+
if class_.class_.suffix.endswith(suffix):
|
|
485
|
+
class_.implements = [ClassEntity(prefix=class_.class_.prefix, suffix=implements)]
|
|
486
|
+
output.metadata.version = f"{rules.metadata.version}.implements_{implements}"
|
|
487
|
+
end = datetime.now(timezone.utc)
|
|
488
|
+
|
|
489
|
+
change = Change.from_rules_activity(
|
|
490
|
+
output,
|
|
491
|
+
ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
|
|
492
|
+
start,
|
|
493
|
+
end,
|
|
494
|
+
(f"Added implements property to classes with suffix {suffix}"),
|
|
495
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
496
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
self._state.data_model.write(output, change)
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -20,7 +20,7 @@ from cognite.neat._store._provenance import Entity as ProvenanceEntity
|
|
|
20
20
|
from cognite.neat._utils.reader import GitHubReader, NeatReader, PathReader
|
|
21
21
|
|
|
22
22
|
from ._state import SessionState
|
|
23
|
-
from ._wizard import NeatObjectType, RDFFileType, object_wizard, rdf_dm_wizard
|
|
23
|
+
from ._wizard import NeatObjectType, RDFFileType, XMLFileType, object_wizard, rdf_dm_wizard, xml_format_wizard
|
|
24
24
|
from .engine import import_engine
|
|
25
25
|
from .exceptions import NeatSessionError, session_class_wrapper
|
|
26
26
|
|
|
@@ -35,6 +35,7 @@ class ReadAPI:
|
|
|
35
35
|
self.excel = ExcelReadAPI(state, client, verbose)
|
|
36
36
|
self.csv = CSVReadAPI(state, client, verbose)
|
|
37
37
|
self.yaml = YamlReadAPI(state, client, verbose)
|
|
38
|
+
self.xml = XMLReadAPI(state, client, verbose)
|
|
38
39
|
|
|
39
40
|
|
|
40
41
|
@session_class_wrapper
|
|
@@ -118,7 +119,7 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
118
119
|
raise ValueError("No client provided. Please provide a client to read a data model.")
|
|
119
120
|
return self._client
|
|
120
121
|
|
|
121
|
-
def graph(self, root_asset_external_id: str) -> None:
|
|
122
|
+
def graph(self, root_asset_external_id: str, limit_per_type: int | None = None) -> None:
|
|
122
123
|
"""Reads the classic knowledge graph from CDF.
|
|
123
124
|
|
|
124
125
|
The Classic Graph consists of the following core resource type.
|
|
@@ -153,9 +154,12 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
153
154
|
|
|
154
155
|
Args:
|
|
155
156
|
root_asset_external_id: The external id of the root asset
|
|
157
|
+
limit_per_type: The maximum number of nodes to extract per core node type. If None, all nodes are extracted.
|
|
156
158
|
|
|
157
159
|
"""
|
|
158
|
-
extractor = extractors.ClassicGraphExtractor(
|
|
160
|
+
extractor = extractors.ClassicGraphExtractor(
|
|
161
|
+
self._get_client, root_asset_external_id=root_asset_external_id, limit_per_type=limit_per_type
|
|
162
|
+
)
|
|
159
163
|
|
|
160
164
|
self._state.instances.store.write(extractor)
|
|
161
165
|
if self._verbose:
|
|
@@ -248,7 +252,7 @@ class CSVReadAPI(BaseReadAPI):
|
|
|
248
252
|
else:
|
|
249
253
|
raise NeatValueError("Only file paths are supported for CSV files")
|
|
250
254
|
engine = import_engine()
|
|
251
|
-
engine.set.
|
|
255
|
+
engine.set.format = "csv"
|
|
252
256
|
engine.set.file = path
|
|
253
257
|
engine.set.type = type
|
|
254
258
|
engine.set.primary_key = primary_key
|
|
@@ -257,6 +261,48 @@ class CSVReadAPI(BaseReadAPI):
|
|
|
257
261
|
self._state.instances.store.write(extractor)
|
|
258
262
|
|
|
259
263
|
|
|
264
|
+
@session_class_wrapper
|
|
265
|
+
class XMLReadAPI(BaseReadAPI):
|
|
266
|
+
def __call__(
|
|
267
|
+
self,
|
|
268
|
+
io: Any,
|
|
269
|
+
format: XMLFileType | None = None,
|
|
270
|
+
) -> None:
|
|
271
|
+
reader = NeatReader.create(io)
|
|
272
|
+
if isinstance(reader, GitHubReader):
|
|
273
|
+
path = Path(tempfile.gettempdir()).resolve() / reader.name
|
|
274
|
+
path.write_text(reader.read_text())
|
|
275
|
+
elif isinstance(reader, PathReader):
|
|
276
|
+
path = reader.path
|
|
277
|
+
else:
|
|
278
|
+
raise NeatValueError("Only file paths are supported for XML files")
|
|
279
|
+
if format is None:
|
|
280
|
+
format = xml_format_wizard()
|
|
281
|
+
|
|
282
|
+
if format.lower() == "dexpi":
|
|
283
|
+
return self.dexpi(path)
|
|
284
|
+
|
|
285
|
+
if format.lower() == "aml":
|
|
286
|
+
return self.aml(path)
|
|
287
|
+
|
|
288
|
+
else:
|
|
289
|
+
raise NeatValueError("Only support XML files of DEXPI format at the moment.")
|
|
290
|
+
|
|
291
|
+
def dexpi(self, path):
|
|
292
|
+
engine = import_engine()
|
|
293
|
+
engine.set.format = "dexpi"
|
|
294
|
+
engine.set.file = path
|
|
295
|
+
extractor = engine.create_extractor()
|
|
296
|
+
self._state.instances.store.write(extractor)
|
|
297
|
+
|
|
298
|
+
def aml(self, path):
|
|
299
|
+
engine = import_engine()
|
|
300
|
+
engine.set.format = "aml"
|
|
301
|
+
engine.set.file = path
|
|
302
|
+
extractor = engine.create_extractor()
|
|
303
|
+
self._state.instances.store.write(extractor)
|
|
304
|
+
|
|
305
|
+
|
|
260
306
|
@session_class_wrapper
|
|
261
307
|
class RDFReadAPI(BaseReadAPI):
|
|
262
308
|
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|