cognite-neat 0.108.0__py3-none-any.whl → 0.109.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_constants.py +1 -1
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +8 -4
- cognite/neat/_graph/queries/_base.py +4 -0
- cognite/neat/_graph/transformers/__init__.py +3 -3
- cognite/neat/_graph/transformers/_base.py +4 -4
- cognite/neat/_graph/transformers/_classic_cdf.py +13 -13
- cognite/neat/_graph/transformers/_prune_graph.py +3 -3
- cognite/neat/_graph/transformers/_rdfpath.py +3 -4
- cognite/neat/_graph/transformers/_value_type.py +23 -16
- cognite/neat/_issues/errors/__init__.py +2 -0
- cognite/neat/_issues/errors/_external.py +8 -0
- cognite/neat/_issues/warnings/_resources.py +1 -1
- cognite/neat/_rules/exporters/_rules2yaml.py +1 -1
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +179 -118
- cognite/neat/_rules/models/_base_rules.py +9 -8
- cognite/neat/_rules/models/dms/_exporter.py +5 -4
- cognite/neat/_rules/transformers/__init__.py +4 -3
- cognite/neat/_rules/transformers/_base.py +6 -1
- cognite/neat/_rules/transformers/_converters.py +436 -361
- cognite/neat/_rules/transformers/_mapping.py +4 -4
- cognite/neat/_session/_base.py +71 -69
- cognite/neat/_session/_create.py +133 -0
- cognite/neat/_session/_drop.py +55 -1
- cognite/neat/_session/_fix.py +28 -0
- cognite/neat/_session/_inspect.py +19 -5
- cognite/neat/_session/_mapping.py +8 -8
- cognite/neat/_session/_prepare.py +3 -247
- cognite/neat/_session/_read.py +78 -4
- cognite/neat/_session/_set.py +34 -12
- cognite/neat/_session/_show.py +14 -41
- cognite/neat/_session/_state.py +48 -51
- cognite/neat/_session/_to.py +7 -3
- cognite/neat/_session/exceptions.py +7 -1
- cognite/neat/_store/_graph_store.py +14 -13
- cognite/neat/_store/_provenance.py +36 -20
- cognite/neat/_store/_rules_store.py +172 -293
- cognite/neat/_store/exceptions.py +40 -4
- cognite/neat/_utils/auth.py +4 -2
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.108.0.dist-info → cognite_neat-0.109.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.108.0.dist-info → cognite_neat-0.109.0.dist-info}/RECORD +44 -42
- {cognite_neat-0.108.0.dist-info → cognite_neat-0.109.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.108.0.dist-info → cognite_neat-0.109.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.108.0.dist-info → cognite_neat-0.109.0.dist-info}/entry_points.txt +0 -0
|
@@ -3,12 +3,12 @@ from collections import defaultdict
|
|
|
3
3
|
from collections.abc import Callable, Hashable
|
|
4
4
|
from dataclasses import dataclass
|
|
5
5
|
from datetime import datetime, timezone
|
|
6
|
-
from functools import partial
|
|
7
6
|
from pathlib import Path
|
|
8
|
-
from typing import Any
|
|
7
|
+
from typing import Any
|
|
9
8
|
|
|
10
9
|
import rdflib
|
|
11
10
|
from cognite.client import data_modeling as dm
|
|
11
|
+
from rdflib import URIRef
|
|
12
12
|
|
|
13
13
|
from cognite.neat._client import NeatClient
|
|
14
14
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
@@ -20,42 +20,41 @@ from cognite.neat._rules.exporters import BaseExporter
|
|
|
20
20
|
from cognite.neat._rules.exporters._base import CDFExporter, T_Export
|
|
21
21
|
from cognite.neat._rules.importers import BaseImporter
|
|
22
22
|
from cognite.neat._rules.models import DMSRules, InformationRules
|
|
23
|
-
from cognite.neat._rules.
|
|
24
|
-
from cognite.neat._rules.transformers import RulesTransformer
|
|
23
|
+
from cognite.neat._rules.transformers import DMSToInformation, VerifiedRulesTransformer, VerifyAnyRules
|
|
25
24
|
from cognite.neat._utils.upload import UploadResultList
|
|
26
25
|
|
|
27
|
-
from ._provenance import
|
|
28
|
-
from .exceptions import EmptyStore,
|
|
26
|
+
from ._provenance import UNKNOWN_AGENT, Activity, Change, Entity, Provenance
|
|
27
|
+
from .exceptions import EmptyStore, InvalidActivityInput
|
|
29
28
|
|
|
30
29
|
|
|
31
30
|
@dataclass(frozen=True)
|
|
32
|
-
class
|
|
33
|
-
|
|
31
|
+
class RulesEntity(Entity):
|
|
32
|
+
information: InformationRules
|
|
33
|
+
dms: DMSRules | None = None
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def has_dms(self) -> bool:
|
|
37
|
+
return self.dms is not None
|
|
34
38
|
|
|
35
39
|
@property
|
|
36
40
|
def display_name(self) -> str:
|
|
37
|
-
if self.
|
|
38
|
-
return
|
|
39
|
-
|
|
40
|
-
if self.result.rules is None:
|
|
41
|
-
return "FailedRead"
|
|
42
|
-
return self.result.rules.display_type_name()
|
|
43
|
-
else:
|
|
44
|
-
return self.result.display_type_name()
|
|
41
|
+
if self.dms is not None:
|
|
42
|
+
return self.dms.display_name
|
|
43
|
+
return self.information.display_name
|
|
45
44
|
|
|
46
45
|
|
|
47
46
|
@dataclass(frozen=True)
|
|
48
47
|
class OutcomeEntity(Entity):
|
|
49
|
-
result: UploadResultList | Path | str |
|
|
48
|
+
result: UploadResultList | Path | str | URIRef
|
|
50
49
|
|
|
51
50
|
|
|
52
51
|
class NeatRulesStore:
|
|
53
52
|
def __init__(self) -> None:
|
|
54
|
-
self.provenance = Provenance()
|
|
55
|
-
self.exports_by_source_entity_id: dict[rdflib.URIRef, list[Change]] = defaultdict(list)
|
|
56
|
-
self.pruned_by_source_entity_id: dict[rdflib.URIRef, list[Provenance]] = defaultdict(list)
|
|
53
|
+
self.provenance = Provenance[RulesEntity]()
|
|
54
|
+
self.exports_by_source_entity_id: dict[rdflib.URIRef, list[Change[OutcomeEntity]]] = defaultdict(list)
|
|
57
55
|
self._last_outcome: UploadResultList | None = None
|
|
58
56
|
self._iteration_by_id: dict[Hashable, int] = {}
|
|
57
|
+
self._last_issues: IssueList | None = None
|
|
59
58
|
|
|
60
59
|
def calculate_provenance_hash(self, shorten: bool = True) -> str:
|
|
61
60
|
sha256_hash = hashlib.sha256()
|
|
@@ -67,159 +66,182 @@ class NeatRulesStore:
|
|
|
67
66
|
return calculated_hash[:8]
|
|
68
67
|
return calculated_hash
|
|
69
68
|
|
|
70
|
-
def
|
|
71
|
-
|
|
69
|
+
def import_rules(
|
|
70
|
+
self, importer: BaseImporter, validate: bool = True, client: NeatClient | None = None
|
|
71
|
+
) -> IssueList:
|
|
72
|
+
if self.empty:
|
|
73
|
+
return self._import_rules(importer, validate, client)
|
|
74
|
+
else:
|
|
75
|
+
# Importing can be used as a manual transformation.
|
|
76
|
+
return self._manual_transform(importer, validate, client)
|
|
77
|
+
|
|
78
|
+
def _import_rules(
|
|
79
|
+
self, importer: BaseImporter, validate: bool = True, client: NeatClient | None = None
|
|
80
|
+
) -> IssueList:
|
|
81
|
+
def action() -> tuple[InformationRules, DMSRules | None]:
|
|
82
|
+
read_rules = importer.to_rules()
|
|
83
|
+
verified = VerifyAnyRules(validate, client).transform(read_rules) # type: ignore[arg-type]
|
|
84
|
+
if isinstance(verified, InformationRules):
|
|
85
|
+
return verified, None
|
|
86
|
+
elif isinstance(verified, DMSRules):
|
|
87
|
+
return DMSToInformation().transform(verified), verified
|
|
88
|
+
else:
|
|
89
|
+
# Bug in the code
|
|
90
|
+
raise ValueError(f"Invalid output from importer: {type(verified)}")
|
|
72
91
|
|
|
73
|
-
|
|
74
|
-
was_attributed_to=UNKNOWN_AGENT,
|
|
75
|
-
id_=importer.source_uri,
|
|
76
|
-
)
|
|
92
|
+
return self.import_action(action, importer)
|
|
77
93
|
|
|
78
|
-
|
|
94
|
+
def _manual_transform(
|
|
95
|
+
self, importer: BaseImporter, validate: bool = True, client: NeatClient | None = None
|
|
96
|
+
) -> IssueList:
|
|
97
|
+
raise NotImplementedError("Manual transformation is not yet implemented.")
|
|
79
98
|
|
|
80
99
|
def import_graph(self, extractor: KnowledgeGraphExtractor) -> IssueList:
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
100
|
+
def action() -> tuple[InformationRules, DMSRules | None]:
|
|
101
|
+
info = extractor.get_information_rules()
|
|
102
|
+
dms: DMSRules | None = None
|
|
103
|
+
if isinstance(extractor, DMSGraphExtractor):
|
|
104
|
+
dms = extractor.get_dms_rules()
|
|
105
|
+
return info, dms
|
|
106
|
+
|
|
107
|
+
return self.import_action(action, extractor)
|
|
108
|
+
|
|
109
|
+
def import_action(
|
|
110
|
+
self,
|
|
111
|
+
action: Callable[[], tuple[InformationRules, DMSRules | None]],
|
|
112
|
+
agent_tool: BaseImporter | KnowledgeGraphExtractor,
|
|
113
|
+
) -> IssueList:
|
|
114
|
+
if self.provenance:
|
|
115
|
+
raise NeatValueError(f"Data model already exists. Cannot import {agent_tool.source_uri}.")
|
|
116
|
+
return self.do_activity(action, agent_tool)
|
|
117
|
+
|
|
118
|
+
def transform(self, *transformer: VerifiedRulesTransformer) -> IssueList:
|
|
93
119
|
if not self.provenance:
|
|
94
120
|
raise EmptyStore()
|
|
95
121
|
|
|
96
122
|
all_issues = IssueList()
|
|
97
123
|
for item in transformer:
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
item.agent,
|
|
111
|
-
source_entity,
|
|
112
|
-
item.description,
|
|
113
|
-
)[1]
|
|
114
|
-
all_issues.extend(transform_issues)
|
|
124
|
+
|
|
125
|
+
def action(transformer_item=item) -> tuple[InformationRules, DMSRules | None]:
|
|
126
|
+
last_change = self.provenance[-1]
|
|
127
|
+
source_entity = last_change.target_entity
|
|
128
|
+
transformer_input = self._get_transformer_input(source_entity, transformer_item)
|
|
129
|
+
transformer_output = transformer_item.transform(transformer_input)
|
|
130
|
+
if isinstance(transformer_output, InformationRules):
|
|
131
|
+
return transformer_output, None
|
|
132
|
+
return last_change.target_entity.information, transformer_output
|
|
133
|
+
|
|
134
|
+
issues = self.do_activity(action, item)
|
|
135
|
+
all_issues.extend(issues)
|
|
115
136
|
return all_issues
|
|
116
137
|
|
|
117
138
|
def export(self, exporter: BaseExporter[T_VerifiedRules, T_Export]) -> T_Export:
|
|
118
|
-
|
|
119
|
-
source_entity = last_change.target_entity
|
|
120
|
-
if not isinstance(source_entity, ModelEntity):
|
|
121
|
-
# Todo: Provenance should be of an entity type
|
|
122
|
-
raise ValueError("Bug in neat: The last entity in the provenance is not a model entity.")
|
|
123
|
-
expected_types = exporter.source_types()
|
|
124
|
-
if not any(isinstance(source_entity.result, type_) for type_ in expected_types):
|
|
125
|
-
raise InvalidInputOperation(expected=expected_types, got=type(source_entity.result))
|
|
139
|
+
return self._export_activity(exporter.export, exporter, DEFAULT_NAMESPACE["export-result"])
|
|
126
140
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
started_at_time=start,
|
|
138
|
-
used=source_entity,
|
|
139
|
-
)
|
|
140
|
-
target_entity = OutcomeEntity(
|
|
141
|
-
was_attributed_to=agent,
|
|
142
|
-
was_generated_by=activity,
|
|
143
|
-
result=type(result).__name__,
|
|
144
|
-
issues=issue_list,
|
|
145
|
-
id_=target_id,
|
|
146
|
-
)
|
|
147
|
-
change = Change(
|
|
148
|
-
agent=agent,
|
|
149
|
-
activity=activity,
|
|
150
|
-
target_entity=target_entity,
|
|
151
|
-
description=exporter.description,
|
|
152
|
-
source_entity=source_entity,
|
|
141
|
+
def export_to_file(self, exporter: BaseExporter, path: Path) -> None:
|
|
142
|
+
def export_action(input_: VerifiedRules) -> Path:
|
|
143
|
+
exporter.export_to_file(input_, path)
|
|
144
|
+
return path
|
|
145
|
+
|
|
146
|
+
self._export_activity(export_action, exporter, DEFAULT_NAMESPACE[path.name])
|
|
147
|
+
|
|
148
|
+
def export_to_cdf(self, exporter: CDFExporter, client: NeatClient, dry_run: bool) -> UploadResultList:
|
|
149
|
+
return self._export_activity(
|
|
150
|
+
exporter.export_to_cdf, exporter, DEFAULT_NAMESPACE["upload-result"], client, dry_run
|
|
153
151
|
)
|
|
154
|
-
self.exports_by_source_entity_id[source_entity.id_].append(change)
|
|
155
|
-
return result
|
|
156
152
|
|
|
157
|
-
def
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
153
|
+
def do_activity(
|
|
154
|
+
self,
|
|
155
|
+
action: Callable[[], tuple[InformationRules, DMSRules | None]],
|
|
156
|
+
agent_tool: BaseImporter | VerifiedRulesTransformer | KnowledgeGraphExtractor,
|
|
157
|
+
) -> IssueList:
|
|
158
|
+
if isinstance(agent_tool, BaseImporter | KnowledgeGraphExtractor):
|
|
159
|
+
source_entity = Entity.create_with_defaults(
|
|
160
|
+
was_attributed_to=UNKNOWN_AGENT,
|
|
161
|
+
id_=agent_tool.source_uri,
|
|
162
|
+
)
|
|
163
|
+
else:
|
|
164
|
+
# This is a transformer
|
|
165
|
+
source_entity = self.provenance[-1].target_entity
|
|
166
|
+
|
|
168
167
|
start = datetime.now(timezone.utc)
|
|
168
|
+
result: tuple[InformationRules, DMSRules | None] | None = None
|
|
169
169
|
with catch_issues() as issue_list:
|
|
170
|
-
|
|
170
|
+
result = action()
|
|
171
|
+
|
|
171
172
|
end = datetime.now(timezone.utc)
|
|
173
|
+
self._last_issues = issue_list
|
|
172
174
|
|
|
175
|
+
agent = agent_tool.agent
|
|
173
176
|
activity = Activity(
|
|
174
177
|
was_associated_with=agent,
|
|
175
178
|
ended_at_time=end,
|
|
176
179
|
started_at_time=start,
|
|
177
180
|
used=source_entity,
|
|
178
181
|
)
|
|
179
|
-
|
|
182
|
+
if result is None:
|
|
183
|
+
return issue_list
|
|
184
|
+
info, dms = result
|
|
185
|
+
|
|
186
|
+
target_entity = RulesEntity(
|
|
180
187
|
was_attributed_to=agent,
|
|
181
188
|
was_generated_by=activity,
|
|
182
|
-
|
|
189
|
+
information=info,
|
|
190
|
+
dms=dms,
|
|
183
191
|
issues=issue_list,
|
|
184
|
-
|
|
192
|
+
# here id can be bumped in case id already exists
|
|
193
|
+
id_=self._create_id(info, dms),
|
|
185
194
|
)
|
|
186
195
|
change = Change(
|
|
187
196
|
agent=agent,
|
|
188
197
|
activity=activity,
|
|
189
198
|
target_entity=target_entity,
|
|
190
|
-
description=
|
|
199
|
+
description=agent_tool.description,
|
|
191
200
|
source_entity=source_entity,
|
|
192
201
|
)
|
|
193
|
-
self.
|
|
202
|
+
self.provenance.append(change)
|
|
203
|
+
return issue_list
|
|
194
204
|
|
|
195
|
-
def
|
|
205
|
+
def _export_activity(self, action: Callable, exporter: BaseExporter, target_id: URIRef, *exporter_args: Any) -> Any:
|
|
206
|
+
if self.empty:
|
|
207
|
+
raise EmptyStore()
|
|
196
208
|
last_change = self.provenance[-1]
|
|
197
209
|
source_entity = last_change.target_entity
|
|
198
|
-
if not isinstance(source_entity, ModelEntity):
|
|
199
|
-
# Todo: Provenance should be of an entity type
|
|
200
|
-
raise ValueError("Bug in neat: The last entity in the provenance is not a model entity.")
|
|
201
210
|
expected_types = exporter.source_types()
|
|
202
|
-
|
|
203
|
-
|
|
211
|
+
|
|
212
|
+
if source_entity.dms is not None and isinstance(source_entity.dms, expected_types):
|
|
213
|
+
input_ = source_entity.dms
|
|
214
|
+
elif isinstance(source_entity.information, expected_types):
|
|
215
|
+
input_ = source_entity.information
|
|
216
|
+
else:
|
|
217
|
+
available: list[type] = [InformationRules]
|
|
218
|
+
if source_entity.dms is not None:
|
|
219
|
+
available.append(DMSRules)
|
|
220
|
+
raise InvalidActivityInput(expected=expected_types, have=tuple(available))
|
|
204
221
|
|
|
205
222
|
agent = exporter.agent
|
|
206
223
|
start = datetime.now(timezone.utc)
|
|
207
|
-
target_id = DEFAULT_NAMESPACE["upload-result"]
|
|
208
|
-
result: UploadResultList | None = None
|
|
209
224
|
with catch_issues() as issue_list:
|
|
210
|
-
|
|
211
|
-
|
|
225
|
+
# Validate the type of the result
|
|
226
|
+
result = action(input_, *exporter_args)
|
|
212
227
|
|
|
228
|
+
end = datetime.now(timezone.utc)
|
|
229
|
+
self._last_issues = issue_list
|
|
213
230
|
activity = Activity(
|
|
214
231
|
was_associated_with=agent,
|
|
215
232
|
ended_at_time=end,
|
|
216
233
|
started_at_time=start,
|
|
217
234
|
used=source_entity,
|
|
218
235
|
)
|
|
236
|
+
if isinstance(result, UploadResultList | Path | URIRef):
|
|
237
|
+
outcome_result: UploadResultList | Path | URIRef | str = result
|
|
238
|
+
else:
|
|
239
|
+
outcome_result = type(result).__name__
|
|
240
|
+
|
|
219
241
|
target_entity = OutcomeEntity(
|
|
220
242
|
was_attributed_to=agent,
|
|
221
243
|
was_generated_by=activity,
|
|
222
|
-
result=
|
|
244
|
+
result=outcome_result,
|
|
223
245
|
issues=issue_list,
|
|
224
246
|
id_=target_id,
|
|
225
247
|
)
|
|
@@ -231,46 +253,24 @@ class NeatRulesStore:
|
|
|
231
253
|
source_entity=source_entity,
|
|
232
254
|
)
|
|
233
255
|
self.exports_by_source_entity_id[source_entity.id_].append(change)
|
|
234
|
-
|
|
256
|
+
if isinstance(result, UploadResultList):
|
|
257
|
+
self._last_outcome = result
|
|
235
258
|
return result
|
|
236
259
|
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
pruned_candidates.append(change)
|
|
252
|
-
else:
|
|
253
|
-
break
|
|
254
|
-
else:
|
|
255
|
-
raise NeatValueError("No compatible entity found in the provenance.")
|
|
256
|
-
if not pruned_candidates:
|
|
257
|
-
return []
|
|
258
|
-
self.provenance = self.provenance[: -len(pruned_candidates)]
|
|
259
|
-
pruned_candidates.reverse()
|
|
260
|
-
self.pruned_by_source_entity_id[self.provenance[-1].target_entity.id_].append(Provenance(pruned_candidates))
|
|
261
|
-
return pruned_candidates
|
|
262
|
-
|
|
263
|
-
def _export(self, action: Callable[[Any], Any], agent: Agent, description: str) -> Any:
|
|
264
|
-
last_entity: ModelEntity | None = None
|
|
265
|
-
for change in reversed(self.provenance):
|
|
266
|
-
if isinstance(change.target_entity, ModelEntity) and isinstance(change.target_entity.result, DMSRules):
|
|
267
|
-
last_entity = change.target_entity
|
|
268
|
-
break
|
|
269
|
-
if last_entity is None:
|
|
270
|
-
raise NeatValueError("No verified DMS rules found in the provenance.")
|
|
271
|
-
rules = last_entity.result
|
|
272
|
-
result, _ = self._do_activity(lambda: action(rules), agent, last_entity, description)
|
|
273
|
-
return result
|
|
260
|
+
@staticmethod
|
|
261
|
+
def _get_transformer_input(
|
|
262
|
+
source_entity: RulesEntity, transformer: VerifiedRulesTransformer
|
|
263
|
+
) -> InformationRules | DMSRules:
|
|
264
|
+
# Case 1: We only have information rules
|
|
265
|
+
if source_entity.dms is None:
|
|
266
|
+
if transformer.is_valid_input(source_entity.information):
|
|
267
|
+
return source_entity.information
|
|
268
|
+
raise InvalidActivityInput(expected=(DMSRules,), have=(InformationRules,))
|
|
269
|
+
# Case 2: We have both information and dms rules and the transformer is compatible with dms rules
|
|
270
|
+
elif isinstance(source_entity.dms, DMSRules) and transformer.is_valid_input(source_entity.dms):
|
|
271
|
+
return source_entity.dms
|
|
272
|
+
# Case 3: We have both information and dms rules and the transformer is compatible with information rules
|
|
273
|
+
raise InvalidActivityInput(expected=(InformationRules,), have=(DMSRules,))
|
|
274
274
|
|
|
275
275
|
def _update_source_entity(self, source_entity: Entity, result: Rules, issue_list: IssueList) -> Entity:
|
|
276
276
|
"""Update source entity to keep the unbroken provenance chain of changes."""
|
|
@@ -321,52 +321,6 @@ class NeatRulesStore:
|
|
|
321
321
|
|
|
322
322
|
return update_source_entity or source_entity
|
|
323
323
|
|
|
324
|
-
def _do_activity(
|
|
325
|
-
self, action: Callable[[], Rules | None], agent: Agent, source_entity: Entity, description: str
|
|
326
|
-
) -> tuple[Any, IssueList]:
|
|
327
|
-
start = datetime.now(timezone.utc)
|
|
328
|
-
result: Rules | None = None
|
|
329
|
-
with catch_issues() as issue_list:
|
|
330
|
-
result = action()
|
|
331
|
-
end = datetime.now(timezone.utc)
|
|
332
|
-
|
|
333
|
-
# This handles import activity that needs to be properly registered
|
|
334
|
-
# hence we check if class of action is subclass of BaseImporter
|
|
335
|
-
# and only if the store is not empty !
|
|
336
|
-
if (
|
|
337
|
-
hasattr(action, "__self__")
|
|
338
|
-
and issubclass(action.__self__.__class__, BaseImporter)
|
|
339
|
-
and source_entity.was_attributed_to == UNKNOWN_AGENT
|
|
340
|
-
and result
|
|
341
|
-
and not self.empty
|
|
342
|
-
):
|
|
343
|
-
source_entity = self._update_source_entity(source_entity, result, issue_list)
|
|
344
|
-
|
|
345
|
-
activity = Activity(
|
|
346
|
-
was_associated_with=agent,
|
|
347
|
-
ended_at_time=end,
|
|
348
|
-
started_at_time=start,
|
|
349
|
-
used=source_entity,
|
|
350
|
-
)
|
|
351
|
-
target_entity = ModelEntity(
|
|
352
|
-
was_attributed_to=agent,
|
|
353
|
-
was_generated_by=activity,
|
|
354
|
-
result=result,
|
|
355
|
-
issues=issue_list,
|
|
356
|
-
# here id can be bumped in case id already exists
|
|
357
|
-
id_=self._create_id(result),
|
|
358
|
-
)
|
|
359
|
-
change = Change(
|
|
360
|
-
agent=agent,
|
|
361
|
-
activity=activity,
|
|
362
|
-
target_entity=target_entity,
|
|
363
|
-
description=description,
|
|
364
|
-
source_entity=source_entity,
|
|
365
|
-
)
|
|
366
|
-
|
|
367
|
-
self.provenance.append(change)
|
|
368
|
-
return result, issue_list
|
|
369
|
-
|
|
370
324
|
def _get_source_id(self, result: Rules) -> rdflib.URIRef | None:
|
|
371
325
|
"""Return the source of the result."""
|
|
372
326
|
|
|
@@ -385,30 +339,11 @@ class NeatRulesStore:
|
|
|
385
339
|
return result.metadata.as_data_model_id()
|
|
386
340
|
return None
|
|
387
341
|
|
|
388
|
-
def _create_id(self,
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
# Case 1: Unsuccessful activity -> target entity will be EMPTY_ENTITY
|
|
392
|
-
if result is None:
|
|
393
|
-
identifier = EMPTY_ENTITY.id_
|
|
394
|
-
|
|
395
|
-
# Case 2: Result ReadRules
|
|
396
|
-
elif isinstance(result, ReadRules):
|
|
397
|
-
# Case 2.1: ReadRules with no rules -> target entity will be EMPTY_ENTITY
|
|
398
|
-
if result.rules is None:
|
|
399
|
-
identifier = EMPTY_ENTITY.id_
|
|
400
|
-
|
|
401
|
-
# Case 2.2: ReadRules with rules identified by metadata.identifier
|
|
402
|
-
else:
|
|
403
|
-
identifier = result.rules.metadata.identifier
|
|
404
|
-
|
|
405
|
-
# Case 3: Result VerifiedRules, identifier will be metadata.identifier
|
|
406
|
-
elif isinstance(result, VerifiedRules):
|
|
407
|
-
identifier = result.metadata.identifier
|
|
408
|
-
|
|
409
|
-
# Case 4: Defaults to unknown entity
|
|
342
|
+
def _create_id(self, info: InformationRules, dms: DMSRules | None) -> rdflib.URIRef:
|
|
343
|
+
if dms is None:
|
|
344
|
+
identifier = info.metadata.identifier
|
|
410
345
|
else:
|
|
411
|
-
identifier =
|
|
346
|
+
identifier = dms.metadata.identifier
|
|
412
347
|
|
|
413
348
|
# Here we check if the identifier is already in the iteration dictionary
|
|
414
349
|
# to track specific changes to the same entity, if it is we increment the iteration
|
|
@@ -421,79 +356,23 @@ class NeatRulesStore:
|
|
|
421
356
|
self._iteration_by_id[identifier] += 1
|
|
422
357
|
return identifier + f"/Iteration_{self._iteration_by_id[identifier]}"
|
|
423
358
|
|
|
424
|
-
def get_last_entity(self) -> ModelEntity:
|
|
425
|
-
if not self.provenance:
|
|
426
|
-
raise NeatValueError("No entity found in the provenance.")
|
|
427
|
-
return cast(ModelEntity, self.provenance[-1].target_entity)
|
|
428
|
-
|
|
429
|
-
def get_last_successful_entity(self) -> ModelEntity:
|
|
430
|
-
for change in reversed(self.provenance):
|
|
431
|
-
if isinstance(change.target_entity, ModelEntity) and change.target_entity.result:
|
|
432
|
-
return change.target_entity
|
|
433
|
-
raise NeatValueError("No successful entity found in the provenance.")
|
|
434
|
-
|
|
435
|
-
@property
|
|
436
|
-
def has_unverified_rules(self) -> bool:
|
|
437
|
-
return any(
|
|
438
|
-
isinstance(change.target_entity, ModelEntity)
|
|
439
|
-
and isinstance(change.target_entity.result, ReadRules)
|
|
440
|
-
and change.target_entity.result.rules is not None
|
|
441
|
-
for change in self.provenance
|
|
442
|
-
)
|
|
443
|
-
|
|
444
|
-
@property
|
|
445
|
-
def has_verified_rules(self) -> bool:
|
|
446
|
-
return any(
|
|
447
|
-
isinstance(change.target_entity, ModelEntity)
|
|
448
|
-
and isinstance(change.target_entity.result, DMSRules | InformationRules)
|
|
449
|
-
for change in self.provenance
|
|
450
|
-
)
|
|
451
|
-
|
|
452
|
-
@property
|
|
453
|
-
def last_unverified_rule(self) -> InputRules:
|
|
454
|
-
for change in reversed(self.provenance):
|
|
455
|
-
if (
|
|
456
|
-
isinstance(change.target_entity, ModelEntity)
|
|
457
|
-
and isinstance(change.target_entity.result, ReadRules)
|
|
458
|
-
and change.target_entity.result.rules is not None
|
|
459
|
-
):
|
|
460
|
-
return change.target_entity.result.rules
|
|
461
|
-
|
|
462
|
-
raise NeatValueError("No unverified rule found in the provenance.")
|
|
463
|
-
|
|
464
|
-
@property
|
|
465
|
-
def last_verified_rule(self) -> DMSRules | InformationRules:
|
|
466
|
-
for change in reversed(self.provenance):
|
|
467
|
-
if isinstance(change.target_entity, ModelEntity) and isinstance(
|
|
468
|
-
change.target_entity.result, DMSRules | InformationRules
|
|
469
|
-
):
|
|
470
|
-
return change.target_entity.result
|
|
471
|
-
raise NeatValueError("No verified rule found in the provenance.")
|
|
472
|
-
|
|
473
359
|
@property
|
|
474
360
|
def last_verified_dms_rules(self) -> DMSRules:
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
361
|
+
if not self.provenance:
|
|
362
|
+
raise EmptyStore()
|
|
363
|
+
if self.provenance[-1].target_entity.dms is None:
|
|
364
|
+
raise NeatValueError("No verified DMS rules found in the provenance.")
|
|
365
|
+
return self.provenance[-1].target_entity.dms
|
|
479
366
|
|
|
480
367
|
@property
|
|
481
368
|
def last_verified_information_rules(self) -> InformationRules:
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
):
|
|
486
|
-
return change.target_entity.result
|
|
487
|
-
raise NeatValueError("No verified information rules found in the provenance.")
|
|
369
|
+
if not self.provenance:
|
|
370
|
+
raise EmptyStore()
|
|
371
|
+
return self.provenance[-1].target_entity.information
|
|
488
372
|
|
|
489
373
|
@property
|
|
490
|
-
def last_issues(self) -> IssueList:
|
|
491
|
-
|
|
492
|
-
raise NeatValueError("No issues found in the provenance.")
|
|
493
|
-
last_change = self.provenance[-1]
|
|
494
|
-
if last_change.target_entity.issues:
|
|
495
|
-
return last_change.target_entity.issues
|
|
496
|
-
return last_change.source_entity.issues
|
|
374
|
+
def last_issues(self) -> IssueList | None:
|
|
375
|
+
return self._last_issues
|
|
497
376
|
|
|
498
377
|
@property
|
|
499
378
|
def last_outcome(self) -> UploadResultList:
|
|
@@ -2,19 +2,55 @@
|
|
|
2
2
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
|
|
5
|
+
from cognite.neat._graph.extractors import KnowledgeGraphExtractor
|
|
6
|
+
from cognite.neat._issues import IssueList
|
|
7
|
+
from cognite.neat._rules.importers import BaseImporter
|
|
8
|
+
from cognite.neat._rules.transformers import VerifiedRulesTransformer
|
|
9
|
+
|
|
10
|
+
from ._provenance import Activity
|
|
11
|
+
|
|
5
12
|
|
|
6
13
|
class NeatStoreError(Exception):
|
|
7
14
|
"""Base class for all exceptions in the store module"""
|
|
8
15
|
|
|
9
|
-
|
|
16
|
+
def __str__(self):
|
|
17
|
+
return type(self).__name__
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ActivityFailed(NeatStoreError):
|
|
21
|
+
"""Raised when an activity fails"""
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
activity: Activity,
|
|
26
|
+
issue_list: IssueList,
|
|
27
|
+
tool: BaseImporter | VerifiedRulesTransformer | KnowledgeGraphExtractor,
|
|
28
|
+
) -> None:
|
|
29
|
+
self.activity = activity
|
|
30
|
+
self.issue_list = issue_list
|
|
31
|
+
self.tool = tool
|
|
32
|
+
|
|
33
|
+
def __str__(self):
|
|
34
|
+
return self.tool.description
|
|
10
35
|
|
|
11
36
|
|
|
12
37
|
@dataclass
|
|
13
|
-
class
|
|
14
|
-
"""Raised when an invalid
|
|
38
|
+
class InvalidActivityInput(NeatStoreError, RuntimeError):
|
|
39
|
+
"""Raised when an invalid activity is attempted"""
|
|
15
40
|
|
|
16
41
|
expected: tuple[type, ...]
|
|
17
|
-
|
|
42
|
+
have: tuple[type, ...]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class InvalidActivityOutput(NeatStoreError):
|
|
46
|
+
"""Raised when an activity has an invalid output"""
|
|
47
|
+
|
|
48
|
+
def __init__(self, activity: Activity, output: str) -> None:
|
|
49
|
+
self.activity = activity
|
|
50
|
+
self.output = output
|
|
51
|
+
|
|
52
|
+
def __str__(self):
|
|
53
|
+
return f"{super().__str__()}: {self.activity.id_} -> {self.output}"
|
|
18
54
|
|
|
19
55
|
|
|
20
56
|
class EmptyStore(NeatStoreError, RuntimeError):
|