cognite-neat 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +390 -116
- cognite/neat/_client/_api/schema.py +63 -2
- cognite/neat/_client/data_classes/data_modeling.py +4 -0
- cognite/neat/_client/data_classes/schema.py +2 -348
- cognite/neat/_constants.py +27 -4
- cognite/neat/_graph/extractors/_base.py +7 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +28 -18
- cognite/neat/_graph/loaders/_rdf2dms.py +52 -13
- cognite/neat/_graph/transformers/__init__.py +3 -3
- cognite/neat/_graph/transformers/_classic_cdf.py +135 -56
- cognite/neat/_issues/_base.py +26 -17
- cognite/neat/_issues/errors/__init__.py +4 -2
- cognite/neat/_issues/errors/_external.py +7 -0
- cognite/neat/_issues/errors/_properties.py +2 -7
- cognite/neat/_issues/errors/_resources.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -2
- cognite/neat/_issues/warnings/_external.py +9 -1
- cognite/neat/_issues/warnings/_resources.py +41 -2
- cognite/neat/_issues/warnings/user_modeling.py +4 -4
- cognite/neat/_rules/_constants.py +2 -6
- cognite/neat/_rules/analysis/_base.py +15 -5
- cognite/neat/_rules/analysis/_dms.py +20 -0
- cognite/neat/_rules/analysis/_information.py +22 -0
- cognite/neat/_rules/exporters/_base.py +3 -5
- cognite/neat/_rules/exporters/_rules2dms.py +190 -200
- cognite/neat/_rules/importers/__init__.py +1 -3
- cognite/neat/_rules/importers/_base.py +1 -1
- cognite/neat/_rules/importers/_dms2rules.py +3 -25
- cognite/neat/_rules/importers/_rdf/__init__.py +5 -0
- cognite/neat/_rules/importers/_rdf/_base.py +34 -11
- cognite/neat/_rules/importers/_rdf/_imf2rules.py +91 -0
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +40 -7
- cognite/neat/_rules/importers/_rdf/_owl2rules.py +80 -0
- cognite/neat/_rules/importers/_rdf/_shared.py +138 -441
- cognite/neat/_rules/models/_base_rules.py +19 -0
- cognite/neat/_rules/models/_types.py +5 -0
- cognite/neat/_rules/models/dms/__init__.py +2 -0
- cognite/neat/_rules/models/dms/_exporter.py +247 -123
- cognite/neat/_rules/models/dms/_rules.py +7 -49
- cognite/neat/_rules/models/dms/_rules_input.py +8 -3
- cognite/neat/_rules/models/dms/_validation.py +421 -123
- cognite/neat/_rules/models/entities/_multi_value.py +3 -0
- cognite/neat/_rules/models/information/__init__.py +2 -0
- cognite/neat/_rules/models/information/_rules.py +17 -61
- cognite/neat/_rules/models/information/_rules_input.py +11 -2
- cognite/neat/_rules/models/information/_validation.py +107 -11
- cognite/neat/_rules/models/mapping/_classic2core.py +1 -1
- cognite/neat/_rules/models/mapping/_classic2core.yaml +8 -4
- cognite/neat/_rules/transformers/__init__.py +2 -1
- cognite/neat/_rules/transformers/_converters.py +163 -61
- cognite/neat/_rules/transformers/_mapping.py +132 -2
- cognite/neat/_rules/transformers/_pipelines.py +1 -1
- cognite/neat/_rules/transformers/_verification.py +29 -4
- cognite/neat/_session/_base.py +46 -60
- cognite/neat/_session/_mapping.py +105 -5
- cognite/neat/_session/_prepare.py +49 -14
- cognite/neat/_session/_read.py +50 -4
- cognite/neat/_session/_set.py +1 -0
- cognite/neat/_session/_to.py +38 -12
- cognite/neat/_session/_wizard.py +5 -0
- cognite/neat/_session/engine/_interface.py +3 -2
- cognite/neat/_session/exceptions.py +4 -0
- cognite/neat/_store/_base.py +79 -19
- cognite/neat/_utils/collection_.py +22 -0
- cognite/neat/_utils/rdf_.py +30 -4
- cognite/neat/_version.py +2 -2
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +3 -91
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +2 -16
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +3 -5
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/RECORD +74 -82
- cognite/neat/_rules/importers/_rdf/_imf2rules/__init__.py +0 -3
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +0 -86
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +0 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +0 -130
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2rules.py +0 -154
- cognite/neat/_rules/importers/_rdf/_owl2rules/__init__.py +0 -3
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +0 -58
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +0 -65
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +0 -59
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2rules.py +0 -39
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/entry_points.txt +0 -0
cognite/neat/_session/_base.py
CHANGED
|
@@ -10,11 +10,12 @@ from cognite.neat._issues import IssueList, catch_issues
|
|
|
10
10
|
from cognite.neat._issues.errors import RegexViolationError
|
|
11
11
|
from cognite.neat._rules import importers
|
|
12
12
|
from cognite.neat._rules._shared import ReadRules, VerifiedRules
|
|
13
|
-
from cognite.neat._rules.
|
|
14
|
-
from cognite.neat._rules.models import
|
|
13
|
+
from cognite.neat._rules.models import DMSRules
|
|
14
|
+
from cognite.neat._rules.models.dms import DMSValidation
|
|
15
|
+
from cognite.neat._rules.models.information import InformationValidation
|
|
15
16
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
16
17
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
17
|
-
from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
|
|
18
|
+
from cognite.neat._rules.transformers import ConvertToRules, InformationToDMS, VerifyAnyRules
|
|
18
19
|
from cognite.neat._rules.transformers._converters import ConversionTransformer
|
|
19
20
|
from cognite.neat._store._provenance import (
|
|
20
21
|
INSTANCES_ENTITY,
|
|
@@ -53,7 +54,7 @@ class NeatSession:
|
|
|
53
54
|
self.show = ShowAPI(self._state)
|
|
54
55
|
self.set = SetAPI(self._state, verbose)
|
|
55
56
|
self.inspect = InspectAPI(self._state)
|
|
56
|
-
self.mapping = MappingAPI(self._state)
|
|
57
|
+
self.mapping = MappingAPI(self._state, self._client)
|
|
57
58
|
self.drop = DropAPI(self._state)
|
|
58
59
|
self.opt = OptAPI()
|
|
59
60
|
self.opt._display()
|
|
@@ -66,64 +67,40 @@ class NeatSession:
|
|
|
66
67
|
|
|
67
68
|
def verify(self) -> IssueList:
|
|
68
69
|
source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
|
|
69
|
-
|
|
70
|
-
reference_rules: DMSInputRules | None = None
|
|
71
|
-
if isinstance(last_unverified_rule.rules, DMSInputRules):
|
|
72
|
-
dms_rules = last_unverified_rule.rules
|
|
73
|
-
views_ids, containers_ids = dms_rules.imported_views_and_containers_ids()
|
|
74
|
-
if views_ids or containers_ids:
|
|
75
|
-
if self._client is None:
|
|
76
|
-
raise NeatSessionError(
|
|
77
|
-
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
78
|
-
"NEAT needs a client to lookup the definitions. "
|
|
79
|
-
"Please set the client in the session, NeatSession(client=client)."
|
|
80
|
-
)
|
|
81
|
-
schema = self._client.schema.retrieve(list(views_ids), list(containers_ids))
|
|
82
|
-
|
|
83
|
-
importer = DMSImporter(schema)
|
|
84
|
-
reference_rules = importer.to_rules().rules
|
|
85
|
-
|
|
86
|
-
if reference_rules is not None:
|
|
87
|
-
dms_rules.views.extend(reference_rules.views)
|
|
88
|
-
if dms_rules.containers:
|
|
89
|
-
dms_rules.containers.extend(reference_rules.containers or [])
|
|
90
|
-
|
|
91
|
-
transformer = VerifyAnyRules("continue")
|
|
70
|
+
transformer = VerifyAnyRules("continue", validate=False)
|
|
92
71
|
start = datetime.now(timezone.utc)
|
|
93
72
|
output = transformer.try_transform(last_unverified_rule)
|
|
94
|
-
end = datetime.now(timezone.utc)
|
|
95
73
|
|
|
96
74
|
if output.rules:
|
|
97
|
-
|
|
98
|
-
output.rules,
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
self._state.instances.store.add_rules(output.rules)
|
|
75
|
+
if isinstance(output.rules, DMSRules):
|
|
76
|
+
issues = DMSValidation(output.rules, self._client).validate()
|
|
77
|
+
elif isinstance(output.rules, InformationRules):
|
|
78
|
+
issues = InformationValidation(output.rules).validate()
|
|
79
|
+
else:
|
|
80
|
+
raise NeatSessionError("Unsupported rule type")
|
|
81
|
+
if issues.has_errors:
|
|
82
|
+
# This is up for discussion, but I think we should not return rules that
|
|
83
|
+
# only pass the verification but not the validation.
|
|
84
|
+
output.rules = None
|
|
85
|
+
output.issues.extend(issues)
|
|
86
|
+
|
|
87
|
+
end = datetime.now(timezone.utc)
|
|
88
|
+
|
|
89
|
+
if output.rules:
|
|
90
|
+
change = Change.from_rules_activity(
|
|
91
|
+
output.rules,
|
|
92
|
+
transformer.agent,
|
|
93
|
+
start,
|
|
94
|
+
end,
|
|
95
|
+
f"Verified data model {source_id} as {output.rules.metadata.identifier}",
|
|
96
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
97
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
self._state.data_model.write(output.rules, change)
|
|
101
|
+
|
|
102
|
+
if isinstance(output.rules, InformationRules):
|
|
103
|
+
self._state.instances.store.add_rules(output.rules)
|
|
127
104
|
|
|
128
105
|
output.issues.action = "verify"
|
|
129
106
|
self._state.data_model.issue_lists.append(output.issues)
|
|
@@ -131,7 +108,16 @@ class NeatSession:
|
|
|
131
108
|
print("You can inspect the issues with the .inspect.issues(...) method.")
|
|
132
109
|
return output.issues
|
|
133
110
|
|
|
134
|
-
def convert(
|
|
111
|
+
def convert(
|
|
112
|
+
self, target: Literal["dms", "information"], mode: Literal["edge_properties"] | None = None
|
|
113
|
+
) -> IssueList:
|
|
114
|
+
"""Converts the last verified data model to the target type.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
target: The target type to convert the data model to.
|
|
118
|
+
mode: If the target is "dms", the mode to use for the conversion. None is used for default conversion.
|
|
119
|
+
"edge_properties" treas classes that implements Edge as edge properties.
|
|
120
|
+
"""
|
|
135
121
|
start = datetime.now(timezone.utc)
|
|
136
122
|
issues = IssueList()
|
|
137
123
|
converter: ConversionTransformer | None = None
|
|
@@ -139,7 +125,7 @@ class NeatSession:
|
|
|
139
125
|
with catch_issues(issues):
|
|
140
126
|
if target == "dms":
|
|
141
127
|
source_id, info_rules = self._state.data_model.last_verified_information_rules
|
|
142
|
-
converter =
|
|
128
|
+
converter = InformationToDMS(mode=mode)
|
|
143
129
|
converted_rules = converter.transform(info_rules).rules
|
|
144
130
|
elif target == "information":
|
|
145
131
|
source_id, dms_rules = self._state.data_model.last_verified_dms_rules
|
|
@@ -1,28 +1,50 @@
|
|
|
1
1
|
from datetime import datetime, timezone
|
|
2
2
|
|
|
3
|
+
from cognite.neat._client import NeatClient
|
|
4
|
+
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
5
|
+
from cognite.neat._rules.importers import DMSImporter
|
|
6
|
+
from cognite.neat._rules.models.dms import DMSValidation
|
|
3
7
|
from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
|
|
4
|
-
from cognite.neat._rules.transformers import RuleMapper
|
|
8
|
+
from cognite.neat._rules.transformers import AsParentPropertyId, RuleMapper, VerifyDMSRules
|
|
9
|
+
from cognite.neat._store._provenance import Agent as ProvenanceAgent
|
|
5
10
|
from cognite.neat._store._provenance import Change
|
|
6
11
|
|
|
7
12
|
from ._state import SessionState
|
|
8
|
-
from .exceptions import session_class_wrapper
|
|
13
|
+
from .exceptions import NeatSessionError, session_class_wrapper
|
|
9
14
|
|
|
10
15
|
|
|
11
16
|
@session_class_wrapper
|
|
12
17
|
class MappingAPI:
|
|
13
|
-
def __init__(self, state: SessionState):
|
|
18
|
+
def __init__(self, state: SessionState, client: NeatClient | None = None):
|
|
19
|
+
self.data_model = DataModelMappingAPI(state, client)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@session_class_wrapper
|
|
23
|
+
class DataModelMappingAPI:
|
|
24
|
+
def __init__(self, state: SessionState, client: NeatClient | None = None):
|
|
14
25
|
self._state = state
|
|
26
|
+
self._client = client
|
|
15
27
|
|
|
16
|
-
def classic_to_core(self,
|
|
28
|
+
def classic_to_core(self, company_prefix: str, use_parent_property_name: bool = True) -> None:
|
|
17
29
|
"""Map classic types to core types.
|
|
18
30
|
|
|
19
31
|
Note this automatically creates an extended CogniteCore model.
|
|
20
32
|
|
|
33
|
+
Args:
|
|
34
|
+
company_prefix: Prefix used for all extended CogniteCore types.
|
|
35
|
+
use_parent_property_name: Whether to use the parent property name in the extended CogniteCore model.
|
|
36
|
+
See below for more information.
|
|
37
|
+
|
|
38
|
+
If you extend CogniteAsset, with for example, ClassicAsset. You will map the property `parentId` to `parent`.
|
|
39
|
+
If you set `user_parent_property_name` to True, the `parentId` will be renamed to `parent` after the
|
|
40
|
+
mapping is done. If you set it to False, the property will remain `parentId`.
|
|
21
41
|
"""
|
|
22
42
|
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
23
43
|
|
|
24
44
|
start = datetime.now(timezone.utc)
|
|
25
|
-
transformer = RuleMapper(
|
|
45
|
+
transformer = RuleMapper(
|
|
46
|
+
load_classic_to_core_mapping(company_prefix, rules.metadata.space, rules.metadata.version)
|
|
47
|
+
)
|
|
26
48
|
output = transformer.transform(rules)
|
|
27
49
|
end = datetime.now(timezone.utc)
|
|
28
50
|
|
|
@@ -37,3 +59,81 @@ class MappingAPI:
|
|
|
37
59
|
)
|
|
38
60
|
|
|
39
61
|
self._state.data_model.write(output.rules, change)
|
|
62
|
+
|
|
63
|
+
start = datetime.now(timezone.utc)
|
|
64
|
+
|
|
65
|
+
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
66
|
+
view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
|
|
67
|
+
if not (view_ids or container_ids):
|
|
68
|
+
print(
|
|
69
|
+
f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
|
|
70
|
+
f"that is not already included in the data model."
|
|
71
|
+
)
|
|
72
|
+
return
|
|
73
|
+
if self._client is None:
|
|
74
|
+
raise NeatSessionError(
|
|
75
|
+
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
76
|
+
"NEAT needs a client to lookup the definitions. "
|
|
77
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
78
|
+
)
|
|
79
|
+
schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
|
|
80
|
+
copy_ = rules.model_copy(deep=True)
|
|
81
|
+
copy_.metadata.version = f"{rules.metadata.version}_completed"
|
|
82
|
+
importer = DMSImporter(schema)
|
|
83
|
+
imported = importer.to_rules()
|
|
84
|
+
if imported.rules is None:
|
|
85
|
+
self._state.data_model.issue_lists.append(imported.issues)
|
|
86
|
+
raise NeatSessionError(
|
|
87
|
+
"Could not import the referenced views and containers. "
|
|
88
|
+
"See `neat.inspect.issues()` for more information."
|
|
89
|
+
)
|
|
90
|
+
verified = VerifyDMSRules("continue", validate=False).transform(imported.rules)
|
|
91
|
+
if verified.rules is None:
|
|
92
|
+
self._state.data_model.issue_lists.append(verified.issues)
|
|
93
|
+
raise NeatSessionError(
|
|
94
|
+
"Could not verify the referenced views and containers. "
|
|
95
|
+
"See `neat.inspect.issues()` for more information."
|
|
96
|
+
)
|
|
97
|
+
if copy_.containers is None:
|
|
98
|
+
copy_.containers = verified.rules.containers
|
|
99
|
+
else:
|
|
100
|
+
existing_containers = {c.container for c in copy_.containers}
|
|
101
|
+
copy_.containers.extend(
|
|
102
|
+
[c for c in verified.rules.containers or [] if c.container not in existing_containers]
|
|
103
|
+
)
|
|
104
|
+
existing_views = {v.view for v in copy_.views}
|
|
105
|
+
copy_.views.extend([v for v in verified.rules.views if v.view not in existing_views])
|
|
106
|
+
end = datetime.now(timezone.utc)
|
|
107
|
+
|
|
108
|
+
change = Change.from_rules_activity(
|
|
109
|
+
copy_,
|
|
110
|
+
ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
|
|
111
|
+
start,
|
|
112
|
+
end,
|
|
113
|
+
(f"Included referenced views and containers in the data model {rules.metadata.as_data_model_id()}"),
|
|
114
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
115
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
self._state.data_model.write(copy_, change)
|
|
119
|
+
|
|
120
|
+
if not use_parent_property_name:
|
|
121
|
+
return
|
|
122
|
+
|
|
123
|
+
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
124
|
+
start = datetime.now(timezone.utc)
|
|
125
|
+
transformer = AsParentPropertyId(self._client)
|
|
126
|
+
output = transformer.transform(rules)
|
|
127
|
+
end = datetime.now(timezone.utc)
|
|
128
|
+
|
|
129
|
+
change = Change.from_rules_activity(
|
|
130
|
+
output,
|
|
131
|
+
transformer.agent,
|
|
132
|
+
start,
|
|
133
|
+
end,
|
|
134
|
+
"Renaming property names to parent name",
|
|
135
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
136
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
self._state.data_model.write(output.rules, change)
|
|
@@ -8,11 +8,13 @@ from rdflib import URIRef
|
|
|
8
8
|
|
|
9
9
|
from cognite.neat._client import NeatClient
|
|
10
10
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
11
|
-
from cognite.neat._graph.transformers import
|
|
11
|
+
from cognite.neat._graph.transformers import RelationshipAsEdgeTransformer
|
|
12
12
|
from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
|
|
13
13
|
from cognite.neat._rules._shared import InputRules, ReadRules
|
|
14
14
|
from cognite.neat._rules.importers import DMSImporter
|
|
15
15
|
from cognite.neat._rules.models import DMSRules
|
|
16
|
+
from cognite.neat._rules.models.dms import DMSValidation
|
|
17
|
+
from cognite.neat._rules.models.entities import ClassEntity
|
|
16
18
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
17
19
|
from cognite.neat._rules.transformers import (
|
|
18
20
|
PrefixEntities,
|
|
@@ -112,20 +114,23 @@ class InstancePrepareAPI:
|
|
|
112
114
|
raise NeatSessionError(f"Property {property_} is not defined for type {type_}. Cannot make connection")
|
|
113
115
|
return type_uri[0], property_uri[0]
|
|
114
116
|
|
|
115
|
-
def
|
|
117
|
+
def relationships_as_edges(self, min_relationship_types: int = 1, limit_per_type: int | None = None) -> None:
|
|
116
118
|
"""This assumes that you have read a classic CDF knowledge graph including relationships.
|
|
117
119
|
|
|
118
|
-
This
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
properties are replaced by a schema that contains the properties as attributes.
|
|
120
|
+
This method converts relationships into edges in the graph. This is useful as the
|
|
121
|
+
edges will be picked up as part of the schema connected to Assets, Events, Files, Sequences,
|
|
122
|
+
and TimeSeries in the InferenceImporter.
|
|
122
123
|
|
|
123
124
|
Args:
|
|
124
|
-
|
|
125
|
-
to
|
|
125
|
+
min_relationship_types: The minimum number of relationship types that must exists to convert those
|
|
126
|
+
relationships to edges. For example, if there is only 5 relationships between Assets and TimeSeries,
|
|
127
|
+
and limit is 10, those relationships will not be converted to edges.
|
|
128
|
+
limit_per_type: The number of conversions to perform per relationship type. For example, if there are 10
|
|
129
|
+
relationships between Assets and TimeSeries, and limit_per_type is 1, only 1 of those relationships
|
|
130
|
+
will be converted to an edge. If None, all relationships will be converted.
|
|
126
131
|
|
|
127
132
|
"""
|
|
128
|
-
transformer =
|
|
133
|
+
transformer = RelationshipAsEdgeTransformer(min_relationship_types, limit_per_type)
|
|
129
134
|
self._state.instances.store.transform(transformer)
|
|
130
135
|
|
|
131
136
|
|
|
@@ -316,7 +321,7 @@ class DataModelPrepareAPI:
|
|
|
316
321
|
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
317
322
|
|
|
318
323
|
dms_ref: DMSRules | None = None
|
|
319
|
-
view_ids, container_ids = rules.imported_views_and_containers_ids(
|
|
324
|
+
view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
|
|
320
325
|
if view_ids or container_ids:
|
|
321
326
|
if self._client is None:
|
|
322
327
|
raise NeatSessionError(
|
|
@@ -324,7 +329,7 @@ class DataModelPrepareAPI:
|
|
|
324
329
|
"NEAT needs a client to lookup the definitions. "
|
|
325
330
|
"Please set the client in the session, NeatSession(client=client)."
|
|
326
331
|
)
|
|
327
|
-
schema = self._client.schema.retrieve(
|
|
332
|
+
schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
|
|
328
333
|
|
|
329
334
|
importer = DMSImporter(schema)
|
|
330
335
|
reference_rules = importer.to_rules().rules
|
|
@@ -409,7 +414,7 @@ class DataModelPrepareAPI:
|
|
|
409
414
|
start = datetime.now(timezone.utc)
|
|
410
415
|
|
|
411
416
|
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
412
|
-
view_ids, container_ids = rules.imported_views_and_containers_ids(
|
|
417
|
+
view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
|
|
413
418
|
if not (view_ids or container_ids):
|
|
414
419
|
print(
|
|
415
420
|
f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
|
|
@@ -422,7 +427,7 @@ class DataModelPrepareAPI:
|
|
|
422
427
|
"NEAT needs a client to lookup the definitions. "
|
|
423
428
|
"Please set the client in the session, NeatSession(client=client)."
|
|
424
429
|
)
|
|
425
|
-
schema = self._client.schema.retrieve(
|
|
430
|
+
schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
|
|
426
431
|
copy_ = rules.model_copy(deep=True)
|
|
427
432
|
copy_.metadata.version = f"{rules.metadata.version}_completed"
|
|
428
433
|
importer = DMSImporter(schema)
|
|
@@ -433,7 +438,7 @@ class DataModelPrepareAPI:
|
|
|
433
438
|
"Could not import the referenced views and containers. "
|
|
434
439
|
"See `neat.inspect.issues()` for more information."
|
|
435
440
|
)
|
|
436
|
-
verified = VerifyDMSRules("continue",
|
|
441
|
+
verified = VerifyDMSRules("continue", validate=False).transform(imported.rules)
|
|
437
442
|
if verified.rules is None:
|
|
438
443
|
self._state.data_model.issue_lists.append(verified.issues)
|
|
439
444
|
raise NeatSessionError(
|
|
@@ -462,3 +467,33 @@ class DataModelPrepareAPI:
|
|
|
462
467
|
)
|
|
463
468
|
|
|
464
469
|
self._state.data_model.write(copy_, change)
|
|
470
|
+
|
|
471
|
+
def add_implements_to_classes(self, suffix: Literal["Edge"], implements: str = "Edge") -> None:
|
|
472
|
+
"""All classes with the suffix will have the implements property set to the given value.
|
|
473
|
+
|
|
474
|
+
Args:
|
|
475
|
+
suffix: The suffix of the classes to add the implements property to.
|
|
476
|
+
implements: The value of the implements property to set.
|
|
477
|
+
|
|
478
|
+
"""
|
|
479
|
+
source_id, rules = self._state.data_model.last_verified_information_rules
|
|
480
|
+
start = datetime.now(timezone.utc)
|
|
481
|
+
|
|
482
|
+
output = rules.model_copy(deep=True)
|
|
483
|
+
for class_ in output.classes:
|
|
484
|
+
if class_.class_.suffix.endswith(suffix):
|
|
485
|
+
class_.implements = [ClassEntity(prefix=class_.class_.prefix, suffix=implements)]
|
|
486
|
+
output.metadata.version = f"{rules.metadata.version}.implements_{implements}"
|
|
487
|
+
end = datetime.now(timezone.utc)
|
|
488
|
+
|
|
489
|
+
change = Change.from_rules_activity(
|
|
490
|
+
output,
|
|
491
|
+
ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
|
|
492
|
+
start,
|
|
493
|
+
end,
|
|
494
|
+
(f"Added implements property to classes with suffix {suffix}"),
|
|
495
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
496
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
self._state.data_model.write(output, change)
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -20,7 +20,7 @@ from cognite.neat._store._provenance import Entity as ProvenanceEntity
|
|
|
20
20
|
from cognite.neat._utils.reader import GitHubReader, NeatReader, PathReader
|
|
21
21
|
|
|
22
22
|
from ._state import SessionState
|
|
23
|
-
from ._wizard import NeatObjectType, RDFFileType, object_wizard, rdf_dm_wizard
|
|
23
|
+
from ._wizard import NeatObjectType, RDFFileType, XMLFileType, object_wizard, rdf_dm_wizard, xml_format_wizard
|
|
24
24
|
from .engine import import_engine
|
|
25
25
|
from .exceptions import NeatSessionError, session_class_wrapper
|
|
26
26
|
|
|
@@ -35,6 +35,7 @@ class ReadAPI:
|
|
|
35
35
|
self.excel = ExcelReadAPI(state, client, verbose)
|
|
36
36
|
self.csv = CSVReadAPI(state, client, verbose)
|
|
37
37
|
self.yaml = YamlReadAPI(state, client, verbose)
|
|
38
|
+
self.xml = XMLReadAPI(state, client, verbose)
|
|
38
39
|
|
|
39
40
|
|
|
40
41
|
@session_class_wrapper
|
|
@@ -118,7 +119,7 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
118
119
|
raise ValueError("No client provided. Please provide a client to read a data model.")
|
|
119
120
|
return self._client
|
|
120
121
|
|
|
121
|
-
def graph(self, root_asset_external_id: str) -> None:
|
|
122
|
+
def graph(self, root_asset_external_id: str, limit_per_type: int | None = None) -> None:
|
|
122
123
|
"""Reads the classic knowledge graph from CDF.
|
|
123
124
|
|
|
124
125
|
The Classic Graph consists of the following core resource type.
|
|
@@ -153,9 +154,12 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
153
154
|
|
|
154
155
|
Args:
|
|
155
156
|
root_asset_external_id: The external id of the root asset
|
|
157
|
+
limit_per_type: The maximum number of nodes to extract per core node type. If None, all nodes are extracted.
|
|
156
158
|
|
|
157
159
|
"""
|
|
158
|
-
extractor = extractors.ClassicGraphExtractor(
|
|
160
|
+
extractor = extractors.ClassicGraphExtractor(
|
|
161
|
+
self._get_client, root_asset_external_id=root_asset_external_id, limit_per_type=limit_per_type
|
|
162
|
+
)
|
|
159
163
|
|
|
160
164
|
self._state.instances.store.write(extractor)
|
|
161
165
|
if self._verbose:
|
|
@@ -248,7 +252,7 @@ class CSVReadAPI(BaseReadAPI):
|
|
|
248
252
|
else:
|
|
249
253
|
raise NeatValueError("Only file paths are supported for CSV files")
|
|
250
254
|
engine = import_engine()
|
|
251
|
-
engine.set.
|
|
255
|
+
engine.set.format = "csv"
|
|
252
256
|
engine.set.file = path
|
|
253
257
|
engine.set.type = type
|
|
254
258
|
engine.set.primary_key = primary_key
|
|
@@ -257,6 +261,48 @@ class CSVReadAPI(BaseReadAPI):
|
|
|
257
261
|
self._state.instances.store.write(extractor)
|
|
258
262
|
|
|
259
263
|
|
|
264
|
+
@session_class_wrapper
|
|
265
|
+
class XMLReadAPI(BaseReadAPI):
|
|
266
|
+
def __call__(
|
|
267
|
+
self,
|
|
268
|
+
io: Any,
|
|
269
|
+
format: XMLFileType | None = None,
|
|
270
|
+
) -> None:
|
|
271
|
+
reader = NeatReader.create(io)
|
|
272
|
+
if isinstance(reader, GitHubReader):
|
|
273
|
+
path = Path(tempfile.gettempdir()).resolve() / reader.name
|
|
274
|
+
path.write_text(reader.read_text())
|
|
275
|
+
elif isinstance(reader, PathReader):
|
|
276
|
+
path = reader.path
|
|
277
|
+
else:
|
|
278
|
+
raise NeatValueError("Only file paths are supported for XML files")
|
|
279
|
+
if format is None:
|
|
280
|
+
format = xml_format_wizard()
|
|
281
|
+
|
|
282
|
+
if format.lower() == "dexpi":
|
|
283
|
+
return self.dexpi(path)
|
|
284
|
+
|
|
285
|
+
if format.lower() == "aml":
|
|
286
|
+
return self.aml(path)
|
|
287
|
+
|
|
288
|
+
else:
|
|
289
|
+
raise NeatValueError("Only support XML files of DEXPI format at the moment.")
|
|
290
|
+
|
|
291
|
+
def dexpi(self, path):
|
|
292
|
+
engine = import_engine()
|
|
293
|
+
engine.set.format = "dexpi"
|
|
294
|
+
engine.set.file = path
|
|
295
|
+
extractor = engine.create_extractor()
|
|
296
|
+
self._state.instances.store.write(extractor)
|
|
297
|
+
|
|
298
|
+
def aml(self, path):
|
|
299
|
+
engine = import_engine()
|
|
300
|
+
engine.set.format = "aml"
|
|
301
|
+
engine.set.file = path
|
|
302
|
+
extractor = engine.create_extractor()
|
|
303
|
+
self._state.instances.store.write(extractor)
|
|
304
|
+
|
|
305
|
+
|
|
260
306
|
@session_class_wrapper
|
|
261
307
|
class RDFReadAPI(BaseReadAPI):
|
|
262
308
|
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
cognite/neat/_session/_set.py
CHANGED
cognite/neat/_session/_to.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from collections.abc import Collection
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
from typing import Any, Literal, overload
|
|
3
4
|
|
|
@@ -9,6 +10,7 @@ from cognite.neat._issues import IssueList, catch_warnings
|
|
|
9
10
|
from cognite.neat._rules import exporters
|
|
10
11
|
from cognite.neat._rules._constants import PATTERNS
|
|
11
12
|
from cognite.neat._rules._shared import VerifiedRules
|
|
13
|
+
from cognite.neat._rules.exporters._rules2dms import Component
|
|
12
14
|
from cognite.neat._utils.upload import UploadResultCore, UploadResultList
|
|
13
15
|
|
|
14
16
|
from ._state import SessionState
|
|
@@ -51,7 +53,24 @@ class ToAPI:
|
|
|
51
53
|
@overload
|
|
52
54
|
def yaml(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None: ...
|
|
53
55
|
|
|
54
|
-
def yaml(
|
|
56
|
+
def yaml(
|
|
57
|
+
self, io: Any | None = None, format: Literal["neat", "toolkit"] = "neat", skip_system_spaces: bool = True
|
|
58
|
+
) -> str | None:
|
|
59
|
+
"""Export the verified data model to YAML.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
io: The file path or file-like object to write the YAML file to. Defaults to None.
|
|
63
|
+
format: The format of the YAML file. Defaults to "neat".
|
|
64
|
+
skip_system_spaces: If True, system spaces will be skipped. Defaults to True.
|
|
65
|
+
|
|
66
|
+
... note::
|
|
67
|
+
|
|
68
|
+
- "neat": This is the format Neat uses to store the data model.
|
|
69
|
+
- "toolkit": This is the format used by Cognite Toolkit, that matches the CDF API.
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
str | None: If io is None, the YAML string will be returned. Otherwise, None will be returned.
|
|
73
|
+
"""
|
|
55
74
|
if format == "neat":
|
|
56
75
|
exporter = exporters.YAMLExporter()
|
|
57
76
|
last_verified = self._state.data_model.last_verified_rule[1]
|
|
@@ -66,9 +85,12 @@ class ToAPI:
|
|
|
66
85
|
"This is required for the 'toolkit' format."
|
|
67
86
|
)
|
|
68
87
|
dms_rule = self._state.data_model.last_verified_dms_rules[1]
|
|
69
|
-
|
|
88
|
+
user_path = Path(io)
|
|
89
|
+
if user_path.suffix == "" and not user_path.exists():
|
|
90
|
+
user_path.mkdir(parents=True)
|
|
91
|
+
exporters.DMSExporter(remove_cdf_spaces=skip_system_spaces).export_to_file(dms_rule, user_path)
|
|
70
92
|
else:
|
|
71
|
-
raise NeatSessionError("Please provide a valid format.
|
|
93
|
+
raise NeatSessionError("Please provide a valid format. 'neat' or 'toolkit'")
|
|
72
94
|
|
|
73
95
|
return None
|
|
74
96
|
|
|
@@ -106,36 +128,40 @@ class CDFToAPI:
|
|
|
106
128
|
|
|
107
129
|
def data_model(
|
|
108
130
|
self,
|
|
109
|
-
|
|
131
|
+
existing: Literal["fail", "skip", "update", "force", "recreate"] = "update",
|
|
110
132
|
dry_run: bool = False,
|
|
111
|
-
|
|
133
|
+
drop_data: bool = False,
|
|
134
|
+
components: Component | Collection[Component] | None = None,
|
|
112
135
|
):
|
|
113
136
|
"""Export the verified DMS data model to CDF.
|
|
114
137
|
|
|
115
138
|
Args:
|
|
116
|
-
|
|
139
|
+
existing: What to do if the component already exists. Defaults to "update".
|
|
140
|
+
See the note below for more information about the options.
|
|
117
141
|
dry_run: If True, no changes will be made to CDF. Defaults to False.
|
|
118
|
-
|
|
142
|
+
drop_data: If existing is 'force' or 'recreate' and the operation will lead to data loss,
|
|
143
|
+
the component will be skipped unless drop_data is True. Defaults to False.
|
|
144
|
+
Note this only applies to spaces and containers if they contain data.
|
|
145
|
+
components: The components to export. If None, all components will be exported. Defaults to None.
|
|
119
146
|
|
|
120
147
|
... note::
|
|
121
148
|
|
|
122
149
|
- "fail": If any component already exists, the export will fail.
|
|
123
150
|
- "skip": If any component already exists, it will be skipped.
|
|
124
151
|
- "update": If any component already exists, it will be updated.
|
|
125
|
-
- "force": If any component already exists, it will be deleted and recreated.
|
|
152
|
+
- "force": If any component already exists, and the update fails, it will be deleted and recreated.
|
|
153
|
+
- "recreate": All components will be deleted and recreated.
|
|
126
154
|
|
|
127
155
|
"""
|
|
128
156
|
|
|
129
|
-
exporter = exporters.DMSExporter(
|
|
157
|
+
exporter = exporters.DMSExporter(existing=existing, export_components=components, drop_data=drop_data)
|
|
130
158
|
|
|
131
159
|
if not self._client:
|
|
132
160
|
raise NeatSessionError("No client provided!")
|
|
133
161
|
|
|
134
162
|
conversion_issues = IssueList(action="to.cdf.data_model")
|
|
135
163
|
with catch_warnings(conversion_issues):
|
|
136
|
-
result = exporter.export_to_cdf(
|
|
137
|
-
self._state.data_model.last_verified_dms_rules[1], self._client, dry_run, fallback_one_by_one
|
|
138
|
-
)
|
|
164
|
+
result = exporter.export_to_cdf(self._state.data_model.last_verified_dms_rules[1], self._client, dry_run)
|
|
139
165
|
result.insert(0, UploadResultCore(name="schema", issues=conversion_issues))
|
|
140
166
|
self._state.data_model.outcome.append(result)
|
|
141
167
|
print("You can inspect the details with the .inspect.data_model.outcome(...) method.")
|
cognite/neat/_session/_wizard.py
CHANGED
|
@@ -7,12 +7,17 @@ from cognite.neat._rules._constants import PATTERNS
|
|
|
7
7
|
|
|
8
8
|
RDFFileType = Literal["Ontology", "IMF Types", "Inference"]
|
|
9
9
|
NeatObjectType = Literal["Data Model", "Instances"]
|
|
10
|
+
XMLFileType = Literal["dexpi", "aml"]
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def object_wizard(message: str = "Select object") -> NeatObjectType:
|
|
13
14
|
return _selection(message, get_args(NeatObjectType))
|
|
14
15
|
|
|
15
16
|
|
|
17
|
+
def xml_format_wizard(message: str = "Select XML format") -> XMLFileType:
|
|
18
|
+
return _selection(message, get_args(XMLFileType))
|
|
19
|
+
|
|
20
|
+
|
|
16
21
|
def rdf_dm_wizard(message: str = "Select source:") -> RDFFileType:
|
|
17
22
|
return _selection(message, get_args(RDFFileType))
|
|
18
23
|
|
|
@@ -9,14 +9,15 @@ class Extractor(Protocol):
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class ConfigAPI(Protocol):
|
|
12
|
-
|
|
12
|
+
format: str | None = None
|
|
13
13
|
file: Any | None
|
|
14
14
|
type: str | None
|
|
15
15
|
primary_key: str | None
|
|
16
|
+
mapping: Any | None = None
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
class NeatEngine(Protocol):
|
|
19
|
-
version: str = "
|
|
20
|
+
version: str = "2.0.0"
|
|
20
21
|
|
|
21
22
|
@property
|
|
22
23
|
def set(self) -> ConfigAPI: ...
|