cognite-neat 0.76.1__py3-none-any.whl → 0.76.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/routers/core.py +1 -1
- cognite/neat/app/api/routers/rules.py +1 -1
- cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
- cognite/neat/rules/_shared.py +1 -1
- cognite/neat/rules/analysis/_information_rules.py +3 -3
- cognite/neat/rules/exporters/_base.py +1 -1
- cognite/neat/rules/exporters/_rules2dms.py +8 -49
- cognite/neat/rules/exporters/_rules2excel.py +71 -40
- cognite/neat/rules/exporters/_rules2ontology.py +2 -2
- cognite/neat/rules/exporters/_rules2yaml.py +1 -1
- cognite/neat/rules/exporters/_validation.py +2 -2
- cognite/neat/rules/importers/_base.py +1 -1
- cognite/neat/rules/importers/_dms2rules.py +93 -108
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +1 -1
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +2 -3
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -2
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2rules.py +1 -1
- cognite/neat/rules/importers/_spreadsheet2rules.py +87 -62
- cognite/neat/rules/importers/_yaml2rules.py +3 -3
- cognite/neat/rules/issues/base.py +5 -0
- cognite/neat/rules/issues/dms.py +65 -0
- cognite/neat/rules/models/__init__.py +27 -0
- cognite/neat/rules/models/dms/__init__.py +18 -0
- cognite/neat/rules/models/dms/_converter.py +140 -0
- cognite/neat/rules/models/dms/_exporter.py +405 -0
- cognite/neat/rules/models/dms/_rules.py +379 -0
- cognite/neat/rules/models/{rules/_dms_rules_write.py → dms/_rules_input.py} +42 -33
- cognite/neat/rules/models/{rules/_dms_schema.py → dms/_schema.py} +36 -4
- cognite/neat/rules/models/dms/_serializer.py +126 -0
- cognite/neat/rules/models/dms/_validation.py +288 -0
- cognite/neat/rules/models/{rules/_domain_rules.py → domain.py} +1 -0
- cognite/neat/rules/models/information/__init__.py +3 -0
- cognite/neat/rules/models/information/_converter.py +195 -0
- cognite/neat/rules/models/{rules/_information_rules.py → information/_rules.py} +35 -202
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -3
- cognite/neat/workflows/steps/lib/current/rules_importer.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_validator.py +1 -2
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/METADATA +1 -1
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/RECORD +51 -44
- cognite/neat/rules/models/rules/__init__.py +0 -14
- cognite/neat/rules/models/rules/_dms_architect_rules.py +0 -1255
- /cognite/neat/rules/models/{rules/_base.py → _base.py} +0 -0
- /cognite/neat/rules/models/{rdfpath.py → _rdfpath.py} +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/__init__.py +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/_base.py +0 -0
- /cognite/neat/rules/models/{rules/_types → _types}/_field.py +0 -0
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.76.1.dist-info → cognite_neat-0.76.3.dist-info}/entry_points.txt +0 -0
|
@@ -2,7 +2,7 @@ from collections import Counter
|
|
|
2
2
|
from collections.abc import Sequence
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Literal, cast, overload
|
|
5
|
+
from typing import Any, Literal, cast, overload
|
|
6
6
|
|
|
7
7
|
from cognite.client import CogniteClient
|
|
8
8
|
from cognite.client import data_modeling as dm
|
|
@@ -20,7 +20,22 @@ from cognite.client.utils import ms_to_datetime
|
|
|
20
20
|
from cognite.neat.rules import issues
|
|
21
21
|
from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
|
|
22
22
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
23
|
+
from cognite.neat.rules.models import (
|
|
24
|
+
DataModelType,
|
|
25
|
+
DMSRules,
|
|
26
|
+
DMSSchema,
|
|
27
|
+
ExtensionCategory,
|
|
28
|
+
RoleTypes,
|
|
29
|
+
SchemaCompleteness,
|
|
30
|
+
SheetList,
|
|
31
|
+
)
|
|
23
32
|
from cognite.neat.rules.models.data_types import DataType
|
|
33
|
+
from cognite.neat.rules.models.dms import (
|
|
34
|
+
DMSContainer,
|
|
35
|
+
DMSMetadata,
|
|
36
|
+
DMSProperty,
|
|
37
|
+
DMSView,
|
|
38
|
+
)
|
|
24
39
|
from cognite.neat.rules.models.entities import (
|
|
25
40
|
ClassEntity,
|
|
26
41
|
ContainerEntity,
|
|
@@ -29,15 +44,6 @@ from cognite.neat.rules.models.entities import (
|
|
|
29
44
|
ViewEntity,
|
|
30
45
|
ViewPropertyEntity,
|
|
31
46
|
)
|
|
32
|
-
from cognite.neat.rules.models.rules import DMSRules, DMSSchema, RoleTypes
|
|
33
|
-
from cognite.neat.rules.models.rules._base import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
34
|
-
from cognite.neat.rules.models.rules._dms_architect_rules import (
|
|
35
|
-
DMSContainer,
|
|
36
|
-
DMSMetadata,
|
|
37
|
-
DMSProperty,
|
|
38
|
-
DMSView,
|
|
39
|
-
SheetList,
|
|
40
|
-
)
|
|
41
47
|
|
|
42
48
|
|
|
43
49
|
class DMSImporter(BaseImporter):
|
|
@@ -47,10 +53,17 @@ class DMSImporter(BaseImporter):
|
|
|
47
53
|
read_issues: Sequence[ValidationIssue] | None = None,
|
|
48
54
|
metadata: DMSMetadata | None = None,
|
|
49
55
|
):
|
|
50
|
-
|
|
56
|
+
# Calling this root schema to distinguish it from
|
|
57
|
+
# * User Schema
|
|
58
|
+
# * Reference Schema
|
|
59
|
+
self.root_schema = schema
|
|
51
60
|
self.metadata = metadata
|
|
52
61
|
self.issue_list = IssueList(read_issues)
|
|
53
|
-
self.
|
|
62
|
+
self._all_containers_by_id = {container.as_id(): container for container in schema.containers}
|
|
63
|
+
if self.root_schema.reference:
|
|
64
|
+
self._all_containers_by_id.update(
|
|
65
|
+
{container.as_id(): container for container in self.root_schema.reference.containers}
|
|
66
|
+
)
|
|
54
67
|
|
|
55
68
|
@classmethod
|
|
56
69
|
def from_data_model_id(cls, client: CogniteClient, data_model_id: DataModelIdentifier) -> "DMSImporter":
|
|
@@ -134,130 +147,102 @@ class DMSImporter(BaseImporter):
|
|
|
134
147
|
# In case there were errors during the import, the to_rules method will return None
|
|
135
148
|
return self._return_or_raise(self.issue_list, errors)
|
|
136
149
|
|
|
137
|
-
if len(self.
|
|
150
|
+
if len(self.root_schema.data_models) == 0:
|
|
138
151
|
self.issue_list.append(issues.importing.NoDataModelError("No data model found."))
|
|
139
152
|
return self._return_or_raise(self.issue_list, errors)
|
|
140
153
|
|
|
141
|
-
|
|
154
|
+
with _handle_issues(
|
|
155
|
+
self.issue_list,
|
|
156
|
+
) as future:
|
|
157
|
+
schema_completeness = SchemaCompleteness.complete
|
|
158
|
+
data_model_type = DataModelType.enterprise
|
|
159
|
+
reference: DMSRules | None = None
|
|
160
|
+
if ref_schema := self.root_schema.reference:
|
|
161
|
+
# Reference should always be an enterprise model.
|
|
162
|
+
reference = DMSRules(
|
|
163
|
+
**self._create_rule_components(
|
|
164
|
+
ref_schema, self._create_default_metadata(ref_schema.views), DataModelType.enterprise
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
schema_completeness = SchemaCompleteness.extended
|
|
168
|
+
data_model_type = DataModelType.solution
|
|
169
|
+
|
|
170
|
+
user_rules = DMSRules(
|
|
171
|
+
**self._create_rule_components(self.root_schema, self.metadata, data_model_type, schema_completeness),
|
|
172
|
+
reference=reference,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
if future.result == "failure" or self.issue_list.has_errors:
|
|
176
|
+
return self._return_or_raise(self.issue_list, errors)
|
|
177
|
+
|
|
178
|
+
return self._to_output(user_rules, self.issue_list, errors, role)
|
|
179
|
+
|
|
180
|
+
def _create_rule_components(
|
|
181
|
+
self,
|
|
182
|
+
schema: DMSSchema,
|
|
183
|
+
metadata: DMSMetadata | None = None,
|
|
184
|
+
data_model_type: DataModelType | None = None,
|
|
185
|
+
schema_completeness: SchemaCompleteness | None = None,
|
|
186
|
+
) -> dict[str, Any]:
|
|
187
|
+
if len(schema.data_models) > 2:
|
|
142
188
|
# Creating a DataModelEntity to convert the data model id to a string.
|
|
143
189
|
self.issue_list.append(
|
|
144
190
|
issues.importing.MultipleDataModelsWarning(
|
|
145
|
-
[str(DataModelEntity.from_id(model.as_id())) for model in
|
|
191
|
+
[str(DataModelEntity.from_id(model.as_id())) for model in schema.data_models]
|
|
146
192
|
)
|
|
147
193
|
)
|
|
148
194
|
|
|
149
|
-
data_model =
|
|
195
|
+
data_model = schema.data_models[0]
|
|
150
196
|
|
|
151
197
|
properties = SheetList[DMSProperty]()
|
|
152
|
-
|
|
153
|
-
for view in self.schema.views:
|
|
198
|
+
for view in schema.views:
|
|
154
199
|
view_id = view.as_id()
|
|
155
200
|
view_entity = ViewEntity.from_id(view_id)
|
|
156
201
|
class_entity = view_entity.as_class()
|
|
157
202
|
for prop_id, prop in (view.properties or {}).items():
|
|
158
203
|
dms_property = self._create_dms_property(prop_id, prop, view_entity, class_entity)
|
|
159
204
|
if dms_property is not None:
|
|
160
|
-
|
|
161
|
-
ref_properties.append(dms_property)
|
|
162
|
-
else:
|
|
163
|
-
properties.append(dms_property)
|
|
205
|
+
properties.append(dms_property)
|
|
164
206
|
|
|
165
207
|
data_model_view_ids: set[dm.ViewId] = {
|
|
166
208
|
view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
|
|
167
209
|
}
|
|
168
210
|
|
|
169
|
-
metadata =
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
self.issue_list,
|
|
176
|
-
) as future:
|
|
177
|
-
user_rules = DMSRules(
|
|
178
|
-
metadata=metadata,
|
|
179
|
-
properties=properties,
|
|
180
|
-
containers=SheetList[DMSContainer](
|
|
181
|
-
data=[
|
|
182
|
-
DMSContainer.from_container(container)
|
|
183
|
-
for container in self.schema.containers
|
|
184
|
-
if container.as_id() not in self.schema.frozen_ids
|
|
185
|
-
]
|
|
186
|
-
),
|
|
187
|
-
views=SheetList[DMSView](
|
|
188
|
-
data=[
|
|
189
|
-
DMSView.from_view(view, in_model=view.as_id() in data_model_view_ids)
|
|
190
|
-
for view in self.schema.views
|
|
191
|
-
if view.as_id() not in self.schema.frozen_ids
|
|
192
|
-
]
|
|
193
|
-
),
|
|
194
|
-
reference=self._create_reference_rules(ref_properties),
|
|
195
|
-
)
|
|
196
|
-
|
|
197
|
-
if future.result == "failure" or self.issue_list.has_errors:
|
|
198
|
-
return self._return_or_raise(self.issue_list, errors)
|
|
199
|
-
|
|
200
|
-
return self._to_output(user_rules, self.issue_list, errors, role)
|
|
201
|
-
|
|
202
|
-
def _create_reference_rules(self, properties: SheetList[DMSProperty]) -> DMSRules | None:
|
|
203
|
-
if not properties:
|
|
204
|
-
return None
|
|
205
|
-
|
|
206
|
-
if len(self.schema.data_models) == 2:
|
|
207
|
-
data_model = self.schema.data_models[1]
|
|
208
|
-
data_model_view_ids: set[dm.ViewId] = {
|
|
209
|
-
view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
|
|
210
|
-
}
|
|
211
|
-
metadata = self._create_metadata_from_model(data_model)
|
|
212
|
-
else:
|
|
213
|
-
data_model_view_ids = set()
|
|
214
|
-
now = datetime.now().replace(microsecond=0)
|
|
215
|
-
space = Counter(prop.view.space for prop in properties).most_common(1)[0][0]
|
|
216
|
-
metadata = DMSMetadata(
|
|
217
|
-
schema_=SchemaCompleteness.complete,
|
|
218
|
-
extension=ExtensionCategory.addition,
|
|
219
|
-
space=space,
|
|
220
|
-
external_id="Unknown",
|
|
221
|
-
version="0.1.0",
|
|
222
|
-
creator=["Unknown"],
|
|
223
|
-
created=now,
|
|
224
|
-
updated=now,
|
|
225
|
-
)
|
|
226
|
-
|
|
227
|
-
metadata.data_model_type = DataModelType.enterprise
|
|
228
|
-
return DMSRules(
|
|
211
|
+
metadata = metadata or DMSMetadata.from_data_model(data_model)
|
|
212
|
+
if data_model_type is not None:
|
|
213
|
+
metadata.data_model_type = data_model_type
|
|
214
|
+
if schema_completeness is not None:
|
|
215
|
+
metadata.schema_ = schema_completeness
|
|
216
|
+
return dict(
|
|
229
217
|
metadata=metadata,
|
|
230
218
|
properties=properties,
|
|
231
|
-
views=SheetList[DMSView](
|
|
232
|
-
data=[
|
|
233
|
-
DMSView.from_view(view, in_model=not data_model_view_ids or (view.as_id() in data_model_view_ids))
|
|
234
|
-
for view in self.schema.views
|
|
235
|
-
if view.as_id() in self.schema.frozen_ids
|
|
236
|
-
]
|
|
237
|
-
),
|
|
238
219
|
containers=SheetList[DMSContainer](
|
|
239
|
-
data=[
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
]
|
|
220
|
+
data=[DMSContainer.from_container(container) for container in schema.containers]
|
|
221
|
+
),
|
|
222
|
+
views=SheetList[DMSView](
|
|
223
|
+
data=[DMSView.from_view(view, in_model=view.as_id() in data_model_view_ids) for view in schema.views]
|
|
244
224
|
),
|
|
245
|
-
reference=None,
|
|
246
225
|
)
|
|
247
226
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
227
|
+
@classmethod
|
|
228
|
+
def _create_default_metadata(cls, views: Sequence[dm.View | dm.ViewApply]) -> DMSMetadata:
|
|
229
|
+
now = datetime.now().replace(microsecond=0)
|
|
230
|
+
space = Counter(view.space for view in views).most_common(1)[0][0]
|
|
231
|
+
return DMSMetadata(
|
|
232
|
+
schema_=SchemaCompleteness.complete,
|
|
233
|
+
extension=ExtensionCategory.addition,
|
|
234
|
+
space=space,
|
|
235
|
+
external_id="Unknown",
|
|
236
|
+
version="0.1.0",
|
|
237
|
+
creator=["Unknown"],
|
|
238
|
+
created=now,
|
|
239
|
+
updated=now,
|
|
240
|
+
)
|
|
256
241
|
|
|
257
242
|
def _create_dms_property(
|
|
258
243
|
self, prop_id: str, prop: ViewPropertyApply, view_entity: ViewEntity, class_entity: ClassEntity
|
|
259
244
|
) -> DMSProperty | None:
|
|
260
|
-
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self.
|
|
245
|
+
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
|
|
261
246
|
self.issue_list.append(
|
|
262
247
|
issues.importing.MissingContainerWarning(
|
|
263
248
|
view_id=str(view_entity),
|
|
@@ -268,7 +253,7 @@ class DMSImporter(BaseImporter):
|
|
|
268
253
|
return None
|
|
269
254
|
if (
|
|
270
255
|
isinstance(prop, dm.MappedPropertyApply)
|
|
271
|
-
and prop.container_property_identifier not in self.
|
|
256
|
+
and prop.container_property_identifier not in self._all_containers_by_id[prop.container].properties
|
|
272
257
|
):
|
|
273
258
|
self.issue_list.append(
|
|
274
259
|
issues.importing.MissingContainerPropertyWarning(
|
|
@@ -315,7 +300,7 @@ class DMSImporter(BaseImporter):
|
|
|
315
300
|
|
|
316
301
|
def _container_prop_unsafe(self, prop: dm.MappedPropertyApply) -> dm.ContainerProperty:
|
|
317
302
|
"""This method assumes you have already checked that the container with property exists."""
|
|
318
|
-
return self.
|
|
303
|
+
return self._all_containers_by_id[prop.container].properties[prop.container_property_identifier]
|
|
319
304
|
|
|
320
305
|
def _get_relation_type(self, prop: ViewPropertyApply) -> Literal["edge", "reverse", "direct"] | None:
|
|
321
306
|
if isinstance(prop, SingleEdgeConnectionApply | MultiEdgeConnectionApply) and prop.direction == "outwards":
|
|
@@ -380,7 +365,7 @@ class DMSImporter(BaseImporter):
|
|
|
380
365
|
def _get_index(self, prop: ViewPropertyApply, prop_id) -> list[str] | None:
|
|
381
366
|
if not isinstance(prop, dm.MappedPropertyApply):
|
|
382
367
|
return None
|
|
383
|
-
container = self.
|
|
368
|
+
container = self._all_containers_by_id[prop.container]
|
|
384
369
|
index: list[str] = []
|
|
385
370
|
for index_name, index_obj in (container.indexes or {}).items():
|
|
386
371
|
if isinstance(index_obj, BTreeIndex | InvertedIndex) and prop_id in index_obj.properties:
|
|
@@ -390,7 +375,7 @@ class DMSImporter(BaseImporter):
|
|
|
390
375
|
def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[str] | None:
|
|
391
376
|
if not isinstance(prop, dm.MappedPropertyApply):
|
|
392
377
|
return None
|
|
393
|
-
container = self.
|
|
378
|
+
container = self._all_containers_by_id[prop.container]
|
|
394
379
|
unique_constraints: list[str] = []
|
|
395
380
|
for constraint_name, constraint_obj in (container.constraints or {}).items():
|
|
396
381
|
if isinstance(constraint_obj, dm.RequiresConstraint):
|
|
@@ -23,7 +23,7 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
23
23
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
24
24
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
25
25
|
from cognite.neat.rules.models.entities import ClassEntity, ParentClassEntity
|
|
26
|
-
from cognite.neat.rules.models.
|
|
26
|
+
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
class _DTDLConverter:
|
|
@@ -12,9 +12,8 @@ from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
|
|
|
12
12
|
from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
|
|
13
13
|
from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
14
14
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
15
|
-
from cognite.neat.rules.models
|
|
16
|
-
from cognite.neat.rules.models.
|
|
17
|
-
from cognite.neat.rules.models.rules._information_rules import InformationClass, InformationProperty
|
|
15
|
+
from cognite.neat.rules.models import InformationRules, RoleTypes, SchemaCompleteness, SheetList
|
|
16
|
+
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
18
17
|
from cognite.neat.utils.text import to_pascal
|
|
19
18
|
|
|
20
19
|
|
|
@@ -4,8 +4,8 @@ import re
|
|
|
4
4
|
from rdflib import Graph, Namespace
|
|
5
5
|
|
|
6
6
|
from cognite.neat.constants import DEFAULT_NAMESPACE
|
|
7
|
-
from cognite.neat.rules.models
|
|
8
|
-
from cognite.neat.rules.models.
|
|
7
|
+
from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
|
|
8
|
+
from cognite.neat.rules.models._types._base import (
|
|
9
9
|
PREFIX_COMPLIANCE_REGEX,
|
|
10
10
|
VERSION_COMPLIANCE_REGEX,
|
|
11
11
|
)
|
|
@@ -4,7 +4,7 @@ import numpy as np
|
|
|
4
4
|
import pandas as pd
|
|
5
5
|
from rdflib import Graph
|
|
6
6
|
|
|
7
|
-
from cognite.neat.rules.models.
|
|
7
|
+
from cognite.neat.rules.models._base import MatchType
|
|
8
8
|
from cognite.neat.utils.utils import remove_namespace
|
|
9
9
|
|
|
10
10
|
from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
|
|
@@ -10,8 +10,8 @@ from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
|
|
|
10
10
|
|
|
11
11
|
from cognite.neat.rules.importers._base import BaseImporter, Rules
|
|
12
12
|
from cognite.neat.rules.issues import IssueList
|
|
13
|
+
from cognite.neat.rules.models import InformationRules, RoleTypes
|
|
13
14
|
from cognite.neat.rules.models.data_types import _XSD_TYPES
|
|
14
|
-
from cognite.neat.rules.models.rules import InformationRules, RoleTypes
|
|
15
15
|
|
|
16
16
|
from ._owl2classes import parse_owl_classes
|
|
17
17
|
from ._owl2metadata import parse_owl_metadata
|
|
@@ -13,9 +13,15 @@ from pandas import ExcelFile
|
|
|
13
13
|
|
|
14
14
|
from cognite.neat.rules import issues
|
|
15
15
|
from cognite.neat.rules.issues import IssueList
|
|
16
|
-
from cognite.neat.rules.models
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
from cognite.neat.rules.models import (
|
|
17
|
+
RULES_PER_ROLE,
|
|
18
|
+
DMSRules,
|
|
19
|
+
DomainRules,
|
|
20
|
+
InformationRules,
|
|
21
|
+
RoleTypes,
|
|
22
|
+
SchemaCompleteness,
|
|
23
|
+
)
|
|
24
|
+
from cognite.neat.rules.models.dms import DMSRulesInput
|
|
19
25
|
from cognite.neat.utils.auxiliary import local_import
|
|
20
26
|
from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
|
|
21
27
|
|
|
@@ -81,71 +87,95 @@ class MetadataRaw(UserDict):
|
|
|
81
87
|
class ReadResult:
|
|
82
88
|
sheets: dict[str, dict | list]
|
|
83
89
|
read_info_by_sheet: dict[str, SpreadsheetRead]
|
|
84
|
-
|
|
85
|
-
|
|
90
|
+
metadata: MetadataRaw
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def role(self) -> RoleTypes:
|
|
94
|
+
return self.metadata.role
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def schema(self) -> SchemaCompleteness | None:
|
|
98
|
+
return self.metadata.schema
|
|
86
99
|
|
|
87
100
|
|
|
88
101
|
class SpreadsheetReader:
|
|
89
|
-
def __init__(
|
|
102
|
+
def __init__(
|
|
103
|
+
self,
|
|
104
|
+
issue_list: IssueList,
|
|
105
|
+
required: bool = True,
|
|
106
|
+
metadata: MetadataRaw | None = None,
|
|
107
|
+
sheet_prefix: Literal["", "Last", "Ref"] = "",
|
|
108
|
+
):
|
|
90
109
|
self.issue_list = issue_list
|
|
91
|
-
self.
|
|
110
|
+
self.required = required
|
|
111
|
+
self.metadata = metadata
|
|
112
|
+
self._sheet_prefix = sheet_prefix
|
|
92
113
|
|
|
93
114
|
@property
|
|
94
115
|
def metadata_sheet_name(self) -> str:
|
|
95
|
-
|
|
96
|
-
return self.to_reference_sheet(metadata_name) if self._is_reference else metadata_name
|
|
116
|
+
return f"{self._sheet_prefix}Metadata"
|
|
97
117
|
|
|
98
118
|
def sheet_names(self, role: RoleTypes) -> set[str]:
|
|
99
119
|
names = MANDATORY_SHEETS_BY_ROLE[role]
|
|
100
|
-
return {self.
|
|
101
|
-
|
|
102
|
-
@classmethod
|
|
103
|
-
def to_reference_sheet(cls, sheet_name: str) -> str:
|
|
104
|
-
return f"Ref{sheet_name}"
|
|
120
|
+
return {f"{self._sheet_prefix}{sheet_name}" for sheet_name in names if sheet_name != "Metadata"}
|
|
105
121
|
|
|
106
122
|
def read(self, filepath: Path) -> None | ReadResult:
|
|
107
123
|
with pd.ExcelFile(filepath) as excel_file:
|
|
108
|
-
|
|
124
|
+
metadata: MetadataRaw | None
|
|
125
|
+
if self.metadata is not None:
|
|
126
|
+
metadata = self.metadata
|
|
127
|
+
else:
|
|
128
|
+
metadata = self._read_metadata(excel_file, filepath)
|
|
129
|
+
if metadata is None:
|
|
130
|
+
# The reading of metadata failed, so we can't continue
|
|
131
|
+
return None
|
|
132
|
+
|
|
133
|
+
sheets, read_info_by_sheet = self._read_sheets(excel_file, metadata.role)
|
|
134
|
+
if sheets is None or self.issue_list.has_errors:
|
|
135
|
+
return None
|
|
136
|
+
sheets["Metadata"] = dict(metadata)
|
|
137
|
+
|
|
138
|
+
return ReadResult(sheets, read_info_by_sheet, metadata)
|
|
139
|
+
|
|
140
|
+
def _read_metadata(self, excel_file: ExcelFile, filepath: Path) -> MetadataRaw | None:
|
|
141
|
+
if self.metadata_sheet_name not in excel_file.sheet_names:
|
|
142
|
+
if self.required:
|
|
109
143
|
self.issue_list.append(
|
|
110
144
|
issues.spreadsheet_file.MetadataSheetMissingOrFailedError(
|
|
111
145
|
filepath, sheet_name=self.metadata_sheet_name
|
|
112
146
|
)
|
|
113
147
|
)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
|
|
148
|
+
return None
|
|
117
149
|
|
|
118
|
-
|
|
119
|
-
return None
|
|
150
|
+
metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
|
|
120
151
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
return ReadResult(sheets, read_info_by_sheet, metadata.role, metadata.schema)
|
|
152
|
+
if not metadata.is_valid(self.issue_list, filepath):
|
|
153
|
+
return None
|
|
154
|
+
return metadata
|
|
126
155
|
|
|
127
156
|
def _read_sheets(
|
|
128
|
-
self,
|
|
157
|
+
self, excel_file: ExcelFile, read_role: RoleTypes
|
|
129
158
|
) -> tuple[dict[str, dict | list] | None, dict[str, SpreadsheetRead]]:
|
|
130
159
|
read_info_by_sheet: dict[str, SpreadsheetRead] = defaultdict(SpreadsheetRead)
|
|
131
160
|
|
|
132
|
-
sheets: dict[str, dict | list] = {
|
|
161
|
+
sheets: dict[str, dict | list] = {}
|
|
133
162
|
|
|
134
|
-
expected_sheet_names = self.sheet_names(
|
|
163
|
+
expected_sheet_names = self.sheet_names(read_role)
|
|
135
164
|
|
|
136
165
|
if missing_sheets := expected_sheet_names.difference(set(excel_file.sheet_names)):
|
|
137
|
-
self.
|
|
138
|
-
|
|
139
|
-
|
|
166
|
+
if self.required:
|
|
167
|
+
self.issue_list.append(
|
|
168
|
+
issues.spreadsheet_file.SheetMissingError(cast(Path, excel_file.io), list(missing_sheets))
|
|
169
|
+
)
|
|
140
170
|
return None, read_info_by_sheet
|
|
141
171
|
|
|
142
172
|
for source_sheet_name, target_sheet_name, headers_input in SOURCE_SHEET__TARGET_FIELD__HEADERS:
|
|
143
|
-
source_sheet_name = self.
|
|
173
|
+
source_sheet_name = f"{self._sheet_prefix}{source_sheet_name}"
|
|
144
174
|
|
|
145
175
|
if source_sheet_name not in excel_file.sheet_names:
|
|
146
176
|
continue
|
|
147
177
|
if isinstance(headers_input, dict):
|
|
148
|
-
headers = headers_input[
|
|
178
|
+
headers = headers_input[read_role]
|
|
149
179
|
else:
|
|
150
180
|
headers = headers_input
|
|
151
181
|
|
|
@@ -182,42 +212,37 @@ class ExcelImporter(BaseImporter):
|
|
|
182
212
|
issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
|
|
183
213
|
return self._return_or_raise(issue_list, errors)
|
|
184
214
|
|
|
185
|
-
|
|
186
|
-
if
|
|
215
|
+
user_read = SpreadsheetReader(issue_list).read(self.filepath)
|
|
216
|
+
if user_read is None or issue_list.has_errors:
|
|
187
217
|
return self._return_or_raise(issue_list, errors)
|
|
188
218
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
219
|
+
last_read: ReadResult | None = None
|
|
220
|
+
reference_read: ReadResult | None = None
|
|
221
|
+
if user_read.schema == SchemaCompleteness.extended:
|
|
222
|
+
# Last does not have its own metadata sheet. It is the same as the user's metadata sheet.
|
|
223
|
+
last_read = SpreadsheetReader(
|
|
224
|
+
issue_list, required=False, metadata=user_read.metadata, sheet_prefix="Last"
|
|
225
|
+
).read(self.filepath)
|
|
226
|
+
reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(self.filepath)
|
|
196
227
|
if issue_list.has_errors:
|
|
197
228
|
return self._return_or_raise(issue_list, errors)
|
|
198
229
|
|
|
199
|
-
if
|
|
230
|
+
if reference_read and user_read.role != reference_read.role:
|
|
200
231
|
issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
|
|
201
232
|
return self._return_or_raise(issue_list, errors)
|
|
202
233
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
read_info_by_sheet.update(
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
original_role = reference_result.role
|
|
216
|
-
read_info_by_sheet = reference_result.read_info_by_sheet
|
|
217
|
-
else:
|
|
218
|
-
raise ValueError(
|
|
219
|
-
"No rules were generated. This should have been caught earlier. " f"Bug in {type(self).__name__}."
|
|
220
|
-
)
|
|
234
|
+
sheets = user_read.sheets
|
|
235
|
+
original_role = user_read.role
|
|
236
|
+
read_info_by_sheet = user_read.read_info_by_sheet
|
|
237
|
+
if last_read:
|
|
238
|
+
sheets["last"] = last_read.sheets
|
|
239
|
+
read_info_by_sheet.update(last_read.read_info_by_sheet)
|
|
240
|
+
if reference_read:
|
|
241
|
+
# The last rules will also be validated against the reference rules
|
|
242
|
+
sheets["last"]["reference"] = reference_read.sheets # type: ignore[call-overload]
|
|
243
|
+
if reference_read:
|
|
244
|
+
sheets["reference"] = reference_read.sheets
|
|
245
|
+
read_info_by_sheet.update(reference_read.read_info_by_sheet)
|
|
221
246
|
|
|
222
247
|
rules_cls = RULES_PER_ROLE[original_role]
|
|
223
248
|
with _handle_issues(
|
|
@@ -227,7 +252,7 @@ class ExcelImporter(BaseImporter):
|
|
|
227
252
|
) as future:
|
|
228
253
|
rules: Rules
|
|
229
254
|
if rules_cls is DMSRules:
|
|
230
|
-
rules =
|
|
255
|
+
rules = DMSRulesInput.load(sheets).as_rules()
|
|
231
256
|
else:
|
|
232
257
|
rules = rules_cls.model_validate(sheets) # type: ignore[attr-defined]
|
|
233
258
|
|
|
@@ -5,8 +5,8 @@ import yaml
|
|
|
5
5
|
|
|
6
6
|
from cognite.neat.rules import issues
|
|
7
7
|
from cognite.neat.rules.issues import IssueList, NeatValidationError, ValidationIssue
|
|
8
|
-
from cognite.neat.rules.models
|
|
9
|
-
from cognite.neat.rules.models.
|
|
8
|
+
from cognite.neat.rules.models import RULES_PER_ROLE, DMSRules, RoleTypes
|
|
9
|
+
from cognite.neat.rules.models.dms import DMSRulesInput
|
|
10
10
|
|
|
11
11
|
from ._base import BaseImporter, Rules, _handle_issues
|
|
12
12
|
|
|
@@ -98,7 +98,7 @@ class YAMLImporter(BaseImporter):
|
|
|
98
98
|
with _handle_issues(issue_list) as future:
|
|
99
99
|
rules: Rules
|
|
100
100
|
if rules_model is DMSRules:
|
|
101
|
-
rules =
|
|
101
|
+
rules = DMSRulesInput.load(self.raw_data).as_rules()
|
|
102
102
|
else:
|
|
103
103
|
rules = rules_model.model_validate(self.raw_data)
|
|
104
104
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import sys
|
|
2
|
+
import warnings
|
|
2
3
|
from abc import ABC, abstractmethod
|
|
3
4
|
from collections import UserList
|
|
4
5
|
from collections.abc import Sequence
|
|
@@ -182,6 +183,10 @@ class IssueList(UserList[ValidationIssue]):
|
|
|
182
183
|
[ValueError(issue.message()) for issue in self if isinstance(issue, NeatValidationError)],
|
|
183
184
|
)
|
|
184
185
|
|
|
186
|
+
def trigger_warnings(self) -> None:
|
|
187
|
+
for warning in [issue for issue in self if isinstance(issue, ValidationWarning)]:
|
|
188
|
+
warnings.warn(warning, stacklevel=2)
|
|
189
|
+
|
|
185
190
|
def to_pandas(self) -> pd.DataFrame:
|
|
186
191
|
return pd.DataFrame([issue.dump() for issue in self])
|
|
187
192
|
|