cognite-neat 0.88.2__py3-none-any.whl → 0.88.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_version.py +1 -1
- cognite/neat/graph/__init__.py +0 -3
- cognite/neat/graph/loaders/_base.py +3 -3
- cognite/neat/graph/loaders/_rdf2asset.py +24 -25
- cognite/neat/graph/loaders/_rdf2dms.py +20 -15
- cognite/neat/issues/__init__.py +1 -3
- cognite/neat/issues/_base.py +259 -70
- cognite/neat/issues/errors/__init__.py +72 -0
- cognite/neat/issues/errors/_external.py +67 -0
- cognite/neat/issues/errors/_general.py +28 -0
- cognite/neat/issues/errors/_properties.py +62 -0
- cognite/neat/issues/errors/_resources.py +111 -0
- cognite/neat/issues/errors/_workflow.py +36 -0
- cognite/neat/issues/formatters.py +1 -1
- cognite/neat/issues/warnings/__init__.py +66 -0
- cognite/neat/issues/warnings/_external.py +40 -0
- cognite/neat/issues/warnings/_general.py +29 -0
- cognite/neat/issues/warnings/_models.py +92 -0
- cognite/neat/issues/warnings/_properties.py +44 -0
- cognite/neat/issues/warnings/_resources.py +55 -0
- cognite/neat/issues/warnings/user_modeling.py +113 -0
- cognite/neat/rules/_shared.py +10 -2
- cognite/neat/rules/exporters/_base.py +6 -6
- cognite/neat/rules/exporters/_rules2dms.py +18 -11
- cognite/neat/rules/exporters/_rules2excel.py +4 -4
- cognite/neat/rules/exporters/_rules2ontology.py +74 -51
- cognite/neat/rules/exporters/_rules2yaml.py +3 -3
- cognite/neat/rules/exporters/_validation.py +11 -96
- cognite/neat/rules/importers/_base.py +8 -12
- cognite/neat/rules/importers/_dms2rules.py +21 -24
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +22 -17
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +26 -19
- cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +1 -1
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +9 -7
- cognite/neat/rules/importers/_rdf/_inference2rules.py +8 -8
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +4 -4
- cognite/neat/rules/importers/_rdf/_shared.py +3 -3
- cognite/neat/rules/importers/_spreadsheet2rules.py +35 -22
- cognite/neat/rules/importers/_yaml2rules.py +23 -22
- cognite/neat/rules/models/_constants.py +2 -1
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/_types/_field.py +5 -10
- cognite/neat/rules/models/asset/_rules.py +1 -3
- cognite/neat/rules/models/asset/_validation.py +13 -9
- cognite/neat/rules/models/dms/_converter.py +2 -4
- cognite/neat/rules/models/dms/_exporter.py +30 -8
- cognite/neat/rules/models/dms/_rules.py +23 -7
- cognite/neat/rules/models/dms/_schema.py +87 -78
- cognite/neat/rules/models/dms/_validation.py +104 -65
- cognite/neat/rules/models/information/_converter.py +2 -2
- cognite/neat/rules/models/information/_rules.py +7 -8
- cognite/neat/rules/models/information/_validation.py +47 -24
- cognite/neat/rules/transformers/_base.py +15 -0
- cognite/neat/utils/auxiliary.py +2 -35
- cognite/neat/utils/text.py +17 -0
- cognite/neat/workflows/base.py +4 -4
- cognite/neat/workflows/cdf_store.py +3 -3
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
- cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
- cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -10
- cognite/neat/workflows/steps/lib/current/rules_importer.py +6 -6
- cognite/neat/workflows/steps/lib/current/rules_validator.py +5 -6
- cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
- cognite/neat/workflows/steps_registry.py +4 -5
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/RECORD +78 -84
- cognite/neat/exceptions.py +0 -145
- cognite/neat/graph/exceptions.py +0 -90
- cognite/neat/issues/errors/external.py +0 -21
- cognite/neat/issues/errors/properties.py +0 -75
- cognite/neat/issues/errors/resources.py +0 -123
- cognite/neat/issues/neat_warnings/__init__.py +0 -2
- cognite/neat/issues/neat_warnings/identifier.py +0 -27
- cognite/neat/issues/neat_warnings/models.py +0 -22
- cognite/neat/issues/neat_warnings/properties.py +0 -77
- cognite/neat/issues/neat_warnings/resources.py +0 -125
- cognite/neat/rules/issues/__init__.py +0 -22
- cognite/neat/rules/issues/base.py +0 -63
- cognite/neat/rules/issues/dms.py +0 -549
- cognite/neat/rules/issues/fileread.py +0 -197
- cognite/neat/rules/issues/ontology.py +0 -298
- cognite/neat/rules/issues/spreadsheet.py +0 -563
- cognite/neat/rules/issues/spreadsheet_file.py +0 -151
- cognite/neat/rules/issues/tables.py +0 -72
- cognite/neat/workflows/_exceptions.py +0 -41
- /cognite/neat/{issues/errors/schema.py → rules/transformers/__init__.py} +0 -0
- /cognite/neat/{graph/stores → store}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/_base.py +0 -0
- /cognite/neat/{graph/stores → store}/_provenance.py +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/entry_points.txt +0 -0
|
@@ -19,14 +19,13 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
19
19
|
from cognite.client.utils import ms_to_datetime
|
|
20
20
|
|
|
21
21
|
from cognite.neat.issues import IssueList, NeatIssue
|
|
22
|
-
from cognite.neat.issues.errors
|
|
23
|
-
from cognite.neat.issues.
|
|
22
|
+
from cognite.neat.issues.errors import FileTypeUnexpectedError, ResourceMissingIdentifierError, ResourceRetrievalError
|
|
23
|
+
from cognite.neat.issues.warnings import (
|
|
24
|
+
PropertyNotFoundWarning,
|
|
24
25
|
PropertyTypeNotSupportedWarning,
|
|
25
|
-
|
|
26
|
+
ResourceNotFoundWarning,
|
|
26
27
|
)
|
|
27
|
-
from cognite.neat.
|
|
28
|
-
from cognite.neat.rules import issues
|
|
29
|
-
from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
|
|
28
|
+
from cognite.neat.rules.importers._base import BaseImporter, VerifiedRules, _handle_issues
|
|
30
29
|
from cognite.neat.rules.models import (
|
|
31
30
|
DataModelType,
|
|
32
31
|
DMSRules,
|
|
@@ -109,9 +108,9 @@ class DMSImporter(BaseImporter):
|
|
|
109
108
|
return cls(
|
|
110
109
|
DMSSchema(),
|
|
111
110
|
[
|
|
112
|
-
|
|
111
|
+
ResourceRetrievalError(
|
|
113
112
|
dm.DataModelId.load(reference_model_id), # type: ignore[arg-type]
|
|
114
|
-
"
|
|
113
|
+
"data model",
|
|
115
114
|
"Data Model is missing in CDF",
|
|
116
115
|
)
|
|
117
116
|
],
|
|
@@ -124,8 +123,8 @@ class DMSImporter(BaseImporter):
|
|
|
124
123
|
return cls(
|
|
125
124
|
DMSSchema(),
|
|
126
125
|
[
|
|
127
|
-
|
|
128
|
-
dm.DataModelId.load(reference_model_id), "
|
|
126
|
+
ResourceRetrievalError(
|
|
127
|
+
dm.DataModelId.load(reference_model_id), "data model", "Data Model is missing in CDF"
|
|
129
128
|
)
|
|
130
129
|
],
|
|
131
130
|
)
|
|
@@ -198,29 +197,29 @@ class DMSImporter(BaseImporter):
|
|
|
198
197
|
@classmethod
|
|
199
198
|
def from_zip_file(cls, zip_file: str | Path) -> "DMSImporter":
|
|
200
199
|
if Path(zip_file).suffix != ".zip":
|
|
201
|
-
return cls(DMSSchema(), [
|
|
200
|
+
return cls(DMSSchema(), [FileTypeUnexpectedError(Path(zip_file), frozenset([".zip"]))])
|
|
202
201
|
issue_list = IssueList()
|
|
203
202
|
with _handle_issues(issue_list) as _:
|
|
204
203
|
schema = DMSSchema.from_zip(zip_file)
|
|
205
204
|
return cls(schema, issue_list)
|
|
206
205
|
|
|
207
206
|
@overload
|
|
208
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
207
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
209
208
|
|
|
210
209
|
@overload
|
|
211
210
|
def to_rules(
|
|
212
211
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
213
|
-
) -> tuple[
|
|
212
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
214
213
|
|
|
215
214
|
def to_rules(
|
|
216
215
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
217
|
-
) -> tuple[
|
|
216
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
218
217
|
if self.issue_list.has_errors:
|
|
219
218
|
# In case there were errors during the import, the to_rules method will return None
|
|
220
219
|
return self._return_or_raise(self.issue_list, errors)
|
|
221
220
|
|
|
222
221
|
if not self.root_schema.data_model:
|
|
223
|
-
self.issue_list.append(
|
|
222
|
+
self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.root_schema).__name__))
|
|
224
223
|
return self._return_or_raise(self.issue_list, errors)
|
|
225
224
|
model = self.root_schema.data_model
|
|
226
225
|
with _handle_issues(
|
|
@@ -321,11 +320,11 @@ class DMSImporter(BaseImporter):
|
|
|
321
320
|
) -> DMSProperty | None:
|
|
322
321
|
if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
|
|
323
322
|
self.issue_list.append(
|
|
324
|
-
|
|
323
|
+
ResourceNotFoundWarning[dm.ContainerId, dm.PropertyId](
|
|
325
324
|
dm.ContainerId.load(prop.container),
|
|
326
|
-
"
|
|
325
|
+
"container",
|
|
327
326
|
view_entity.to_property_id(prop_id),
|
|
328
|
-
"
|
|
327
|
+
"view property",
|
|
329
328
|
)
|
|
330
329
|
)
|
|
331
330
|
return None
|
|
@@ -334,9 +333,7 @@ class DMSImporter(BaseImporter):
|
|
|
334
333
|
and prop.container_property_identifier not in self._all_containers_by_id[prop.container].properties
|
|
335
334
|
):
|
|
336
335
|
self.issue_list.append(
|
|
337
|
-
|
|
338
|
-
prop.container, "Container", view_entity.as_id(), "View", prop_id
|
|
339
|
-
),
|
|
336
|
+
PropertyNotFoundWarning(prop.container, "container", prop_id, view_entity.as_id(), "view"),
|
|
340
337
|
)
|
|
341
338
|
return None
|
|
342
339
|
if not isinstance(
|
|
@@ -348,7 +345,7 @@ class DMSImporter(BaseImporter):
|
|
|
348
345
|
| MultiReverseDirectRelationApply,
|
|
349
346
|
):
|
|
350
347
|
self.issue_list.append(
|
|
351
|
-
PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "
|
|
348
|
+
PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "view", prop_id, type(prop).__name__)
|
|
352
349
|
)
|
|
353
350
|
return None
|
|
354
351
|
|
|
@@ -414,7 +411,7 @@ class DMSImporter(BaseImporter):
|
|
|
414
411
|
return DataType.load(container_prop.type._type)
|
|
415
412
|
else:
|
|
416
413
|
self.issue_list.append(
|
|
417
|
-
PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "
|
|
414
|
+
PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "view", prop_id, type(prop).__name__)
|
|
418
415
|
)
|
|
419
416
|
return None
|
|
420
417
|
|
|
@@ -475,7 +472,7 @@ class DMSImporter(BaseImporter):
|
|
|
475
472
|
else:
|
|
476
473
|
self.issue_list.append(
|
|
477
474
|
PropertyTypeNotSupportedWarning[dm.ContainerId](
|
|
478
|
-
prop.container, "
|
|
475
|
+
prop.container, "container", prop_id, type(constraint_obj).__name__
|
|
479
476
|
)
|
|
480
477
|
)
|
|
481
478
|
return unique_constraints or None
|
|
@@ -2,10 +2,12 @@ from collections import Counter
|
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
3
|
|
|
4
4
|
from cognite.neat.issues import IssueList, NeatIssue
|
|
5
|
-
from cognite.neat.issues.errors
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
from cognite.neat.issues.errors import (
|
|
6
|
+
PropertyTypeNotSupportedError,
|
|
7
|
+
ResourceMissingIdentifierError,
|
|
8
|
+
ResourceNotFoundError,
|
|
9
|
+
)
|
|
10
|
+
from cognite.neat.issues.warnings import PropertyTypeNotSupportedWarning, ResourceTypeNotSupportedWarning
|
|
9
11
|
from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
10
12
|
DTMI,
|
|
11
13
|
Command,
|
|
@@ -78,8 +80,8 @@ class _DTDLConverter:
|
|
|
78
80
|
convert_method(item, parent)
|
|
79
81
|
else:
|
|
80
82
|
self.issues.append(
|
|
81
|
-
ResourceTypeNotSupportedWarning
|
|
82
|
-
item.
|
|
83
|
+
ResourceTypeNotSupportedWarning(
|
|
84
|
+
item.identifier_with_fallback,
|
|
83
85
|
item.type,
|
|
84
86
|
),
|
|
85
87
|
)
|
|
@@ -134,8 +136,11 @@ class _DTDLConverter:
|
|
|
134
136
|
|
|
135
137
|
def _missing_parent_warning(self, item: DTDLBaseWithName):
|
|
136
138
|
self.issues.append(
|
|
137
|
-
ResourceNotFoundError
|
|
138
|
-
|
|
139
|
+
ResourceNotFoundError(
|
|
140
|
+
"UNKNOWN",
|
|
141
|
+
"parent",
|
|
142
|
+
item.identifier_with_fallback,
|
|
143
|
+
item.type,
|
|
139
144
|
)
|
|
140
145
|
)
|
|
141
146
|
|
|
@@ -152,7 +157,7 @@ class _DTDLConverter:
|
|
|
152
157
|
if item.request is None:
|
|
153
158
|
self.issues.append(
|
|
154
159
|
ResourceTypeNotSupportedWarning[str](
|
|
155
|
-
item.
|
|
160
|
+
item.identifier_with_fallback,
|
|
156
161
|
f"{item.type}.request",
|
|
157
162
|
),
|
|
158
163
|
)
|
|
@@ -206,8 +211,8 @@ class _DTDLConverter:
|
|
|
206
211
|
else:
|
|
207
212
|
# Falling back to json
|
|
208
213
|
self.issues.append(
|
|
209
|
-
|
|
210
|
-
"
|
|
214
|
+
ResourceMissingIdentifierError(
|
|
215
|
+
"unknown",
|
|
211
216
|
item.target.model_dump(),
|
|
212
217
|
)
|
|
213
218
|
)
|
|
@@ -231,7 +236,7 @@ class _DTDLConverter:
|
|
|
231
236
|
def convert_object(self, item: Object, _: str | None) -> None:
|
|
232
237
|
if item.id_ is None:
|
|
233
238
|
self.issues.append(
|
|
234
|
-
|
|
239
|
+
ResourceMissingIdentifierError(
|
|
235
240
|
resource_type=item.type,
|
|
236
241
|
name=item.display_name,
|
|
237
242
|
)
|
|
@@ -272,8 +277,8 @@ class _DTDLConverter:
|
|
|
272
277
|
return _DATA_TYPE_BY_NAME[input_type.casefold()]()
|
|
273
278
|
elif isinstance(input_type, str):
|
|
274
279
|
self.issues.append(
|
|
275
|
-
PropertyTypeNotSupportedError
|
|
276
|
-
|
|
280
|
+
PropertyTypeNotSupportedError(
|
|
281
|
+
item.identifier_with_fallback,
|
|
277
282
|
item.type,
|
|
278
283
|
"schema",
|
|
279
284
|
input_type,
|
|
@@ -283,7 +288,7 @@ class _DTDLConverter:
|
|
|
283
288
|
elif isinstance(input_type, Object | Interface):
|
|
284
289
|
if input_type.id_ is None:
|
|
285
290
|
self.issues.append(
|
|
286
|
-
|
|
291
|
+
ResourceMissingIdentifierError(
|
|
287
292
|
input_type.type,
|
|
288
293
|
input_type.display_name,
|
|
289
294
|
)
|
|
@@ -296,8 +301,8 @@ class _DTDLConverter:
|
|
|
296
301
|
else:
|
|
297
302
|
self.issues.append(
|
|
298
303
|
PropertyTypeNotSupportedWarning(
|
|
299
|
-
item.
|
|
300
|
-
item.type,
|
|
304
|
+
item.identifier_with_fallback,
|
|
305
|
+
item.type, # type: ignore[arg-type]
|
|
301
306
|
"schema",
|
|
302
307
|
input_type.type if input_type else "missing",
|
|
303
308
|
)
|
|
@@ -7,15 +7,20 @@ from typing import Literal, overload
|
|
|
7
7
|
from pydantic import ValidationError
|
|
8
8
|
|
|
9
9
|
from cognite.neat.issues import IssueList, NeatIssue
|
|
10
|
-
from cognite.neat.
|
|
11
|
-
|
|
10
|
+
from cognite.neat.issues.warnings import (
|
|
11
|
+
FileItemNotSupportedWarning,
|
|
12
|
+
FileMissingRequiredFieldWarning,
|
|
13
|
+
FileReadWarning,
|
|
14
|
+
FileTypeUnexpectedWarning,
|
|
15
|
+
NeatValueWarning,
|
|
16
|
+
)
|
|
17
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
12
18
|
from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
|
|
13
19
|
from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
|
|
14
20
|
from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
15
|
-
from cognite.neat.rules.issues import ValidationIssue
|
|
16
21
|
from cognite.neat.rules.models import InformationRules, RoleTypes, SchemaCompleteness, SheetList
|
|
17
22
|
from cognite.neat.rules.models.information import InformationClass, InformationProperty
|
|
18
|
-
from cognite.neat.utils.text import to_pascal
|
|
23
|
+
from cognite.neat.utils.text import humanize_collection, to_pascal
|
|
19
24
|
|
|
20
25
|
|
|
21
26
|
class DTDLImporter(BaseImporter):
|
|
@@ -46,11 +51,11 @@ class DTDLImporter(BaseImporter):
|
|
|
46
51
|
self._schema_completeness = schema
|
|
47
52
|
|
|
48
53
|
@classmethod
|
|
49
|
-
def _from_file_content(cls, file_content: str, filepath: Path) -> Iterable[DTDLBase |
|
|
54
|
+
def _from_file_content(cls, file_content: str, filepath: Path) -> Iterable[DTDLBase | NeatIssue]:
|
|
50
55
|
raw = json.loads(file_content)
|
|
51
56
|
if isinstance(raw, dict):
|
|
52
57
|
if (context := raw.get("@context")) is None:
|
|
53
|
-
yield
|
|
58
|
+
yield FileMissingRequiredFieldWarning(filepath, "@context", "Missing '@context' key.")
|
|
54
59
|
return
|
|
55
60
|
raw_list = [raw]
|
|
56
61
|
elif isinstance(raw, list):
|
|
@@ -58,13 +63,11 @@ class DTDLImporter(BaseImporter):
|
|
|
58
63
|
(entry["@context"] for entry in raw if isinstance(entry, dict) and "@context" in entry), None
|
|
59
64
|
)
|
|
60
65
|
if context is None:
|
|
61
|
-
yield
|
|
66
|
+
yield FileMissingRequiredFieldWarning(filepath, "@context", "Missing '@context' key.")
|
|
62
67
|
return
|
|
63
68
|
raw_list = raw
|
|
64
69
|
else:
|
|
65
|
-
yield
|
|
66
|
-
filepath=filepath, reason="Content is not an object or array."
|
|
67
|
-
)
|
|
70
|
+
yield FileTypeUnexpectedWarning(filepath, frozenset(["dict", "list"]), "Content is not an object or array.")
|
|
68
71
|
return
|
|
69
72
|
|
|
70
73
|
if isinstance(context, list):
|
|
@@ -74,23 +77,27 @@ class DTDLImporter(BaseImporter):
|
|
|
74
77
|
try:
|
|
75
78
|
cls_by_type = DTDL_CLS_BY_TYPE_BY_SPEC[spec_version]
|
|
76
79
|
except KeyError:
|
|
77
|
-
yield
|
|
80
|
+
yield NeatValueWarning(
|
|
81
|
+
f"Unsupported DTDL spec version: {spec_version} in {filepath}. "
|
|
82
|
+
f"Supported versions are {humanize_collection(DTDL_CLS_BY_TYPE_BY_SPEC.keys())}."
|
|
83
|
+
" The file will be skipped."
|
|
84
|
+
)
|
|
78
85
|
return
|
|
79
86
|
|
|
80
87
|
for item in raw_list:
|
|
81
88
|
if not (type_ := item.get("@type")):
|
|
82
|
-
yield
|
|
89
|
+
yield FileMissingRequiredFieldWarning(filepath, "@type", "Missing '@type' key.")
|
|
83
90
|
continue
|
|
84
91
|
cls_ = cls_by_type.get(type_)
|
|
85
92
|
if cls_ is None:
|
|
86
|
-
yield
|
|
93
|
+
yield FileItemNotSupportedWarning(f"Unknown '@type' {type_}.", filepath=filepath)
|
|
87
94
|
continue
|
|
88
95
|
try:
|
|
89
96
|
yield cls_.model_validate(item)
|
|
90
97
|
except ValidationError as e:
|
|
91
|
-
yield
|
|
98
|
+
yield FileTypeUnexpectedWarning(filepath, frozenset([cls.__name__]), str(e))
|
|
92
99
|
except Exception as e:
|
|
93
|
-
yield
|
|
100
|
+
yield FileReadWarning(filepath=filepath, reason=str(e))
|
|
94
101
|
|
|
95
102
|
@classmethod
|
|
96
103
|
def from_directory(cls, directory: Path) -> "DTDLImporter":
|
|
@@ -112,23 +119,23 @@ class DTDLImporter(BaseImporter):
|
|
|
112
119
|
for filepath in z.namelist():
|
|
113
120
|
if filepath.endswith(".json"):
|
|
114
121
|
for item in cls._from_file_content(z.read(filepath).decode(), Path(filepath)):
|
|
115
|
-
if isinstance(item,
|
|
122
|
+
if isinstance(item, NeatIssue):
|
|
116
123
|
issues.append(item)
|
|
117
124
|
else:
|
|
118
125
|
items.append(item)
|
|
119
126
|
return cls(items, zip_file.stem, read_issues=issues)
|
|
120
127
|
|
|
121
128
|
@overload
|
|
122
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
129
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
123
130
|
|
|
124
131
|
@overload
|
|
125
132
|
def to_rules(
|
|
126
133
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
127
|
-
) -> tuple[
|
|
134
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
128
135
|
|
|
129
136
|
def to_rules(
|
|
130
137
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
131
|
-
) -> tuple[
|
|
138
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
132
139
|
converter = _DTDLConverter(self._read_issues)
|
|
133
140
|
|
|
134
141
|
converter.convert(self._items)
|
|
@@ -13,6 +13,7 @@ from abc import ABC
|
|
|
13
13
|
from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeAlias
|
|
14
14
|
|
|
15
15
|
from pydantic import BaseModel, Field, field_validator, model_serializer, model_validator
|
|
16
|
+
from pydantic.fields import FieldInfo
|
|
16
17
|
|
|
17
18
|
from cognite.neat.rules.models.entities import ClassEntity
|
|
18
19
|
|
|
@@ -134,6 +135,10 @@ class DTDLBase(BaseModel, ABC):
|
|
|
134
135
|
display_name: str | None = Field(None, alias="displayName")
|
|
135
136
|
description: str | None = None
|
|
136
137
|
|
|
138
|
+
@property
|
|
139
|
+
def identifier_with_fallback(self) -> str:
|
|
140
|
+
return (self.id_.model_dump() if self.id_ else self.display_name) or "MISSING"
|
|
141
|
+
|
|
137
142
|
|
|
138
143
|
PrimitiveSchema: TypeAlias = Literal[
|
|
139
144
|
"boolean", "date", "dateTime", "double", "duration", "float", "integer", "long", "string", "time"
|
|
@@ -317,6 +322,8 @@ class Interface(DTDLBase):
|
|
|
317
322
|
if not isinstance(value, list):
|
|
318
323
|
return value
|
|
319
324
|
context = info.data.get("@context", cls.default_context)
|
|
325
|
+
if isinstance(context, FieldInfo):
|
|
326
|
+
context = context.default
|
|
320
327
|
spec_version = context.rsplit(";", maxsplit=1)[1]
|
|
321
328
|
try:
|
|
322
329
|
cls_by_type = DTDL_CLS_BY_TYPE_BY_SPEC[spec_version]
|
|
@@ -53,7 +53,7 @@ def parse_imf_to_classes(graph: Graph, language: str = "en") -> list[dict]:
|
|
|
53
53
|
BIND(IF(!bound(?parent) && ?type = imf:AttributeType, imf:Attribute, ?parent) AS ?parentClass)
|
|
54
54
|
|
|
55
55
|
# Rebind the IRI of the IMF-type to the ?reference variable to align with dataframe column headers
|
|
56
|
-
# This is solely for readability, the ?imfClass could have been
|
|
56
|
+
# This is solely for readability, the ?imfClass could have been returned directly instead of ?reference
|
|
57
57
|
BIND(?imfClass AS ?reference)
|
|
58
58
|
|
|
59
59
|
FILTER (!isBlank(?class))
|
|
@@ -7,7 +7,7 @@ from typing import Literal, overload
|
|
|
7
7
|
from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SH, SKOS, Graph
|
|
8
8
|
|
|
9
9
|
from cognite.neat.issues import IssueList
|
|
10
|
-
from cognite.neat.rules.importers._base import BaseImporter,
|
|
10
|
+
from cognite.neat.rules.importers._base import BaseImporter, VerifiedRules
|
|
11
11
|
from cognite.neat.rules.models import InformationRules, RoleTypes
|
|
12
12
|
from cognite.neat.rules.models.data_types import _XSD_TYPES
|
|
13
13
|
|
|
@@ -36,22 +36,24 @@ class IMFImporter(BaseImporter):
|
|
|
36
36
|
"""
|
|
37
37
|
|
|
38
38
|
def __init__(self, filepath: Path):
|
|
39
|
-
self.
|
|
39
|
+
self.filepath = filepath
|
|
40
40
|
|
|
41
41
|
@overload
|
|
42
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
42
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
43
43
|
|
|
44
44
|
@overload
|
|
45
45
|
def to_rules(
|
|
46
46
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
47
|
-
) -> tuple[
|
|
47
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
48
48
|
|
|
49
49
|
def to_rules(
|
|
50
|
-
self,
|
|
51
|
-
|
|
50
|
+
self,
|
|
51
|
+
errors: Literal["raise", "continue"] = "continue",
|
|
52
|
+
role: RoleTypes | None = None,
|
|
53
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
52
54
|
graph = Graph()
|
|
53
55
|
try:
|
|
54
|
-
graph.parse(self.
|
|
56
|
+
graph.parse(self.filepath)
|
|
55
57
|
except Exception as e:
|
|
56
58
|
raise Exception(f"Could not parse owl file: {e}") from e
|
|
57
59
|
|
|
@@ -6,17 +6,17 @@ from typing import Literal, cast, overload
|
|
|
6
6
|
from rdflib import Graph, Namespace, URIRef
|
|
7
7
|
from rdflib import Literal as RdfLiteral
|
|
8
8
|
|
|
9
|
-
import cognite.neat.rules.issues as issues
|
|
10
9
|
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
11
|
-
from cognite.neat.graph.stores import NeatGraphStore
|
|
12
10
|
from cognite.neat.issues import IssueList
|
|
13
|
-
from cognite.neat.
|
|
11
|
+
from cognite.neat.issues.errors import FileReadError
|
|
12
|
+
from cognite.neat.rules.importers._base import BaseImporter, VerifiedRules, _handle_issues
|
|
14
13
|
from cognite.neat.rules.models import InformationRules, RoleTypes
|
|
15
14
|
from cognite.neat.rules.models._base import MatchType
|
|
16
15
|
from cognite.neat.rules.models.information import (
|
|
17
16
|
InformationMetadata,
|
|
18
17
|
InformationRulesInput,
|
|
19
18
|
)
|
|
19
|
+
from cognite.neat.store import NeatGraphStore
|
|
20
20
|
from cognite.neat.utils.rdf_ import get_namespace, remove_namespace_from_uri, uri_to_short_form
|
|
21
21
|
|
|
22
22
|
ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
|
|
@@ -106,8 +106,8 @@ class InferenceImporter(BaseImporter):
|
|
|
106
106
|
graph = Graph()
|
|
107
107
|
try:
|
|
108
108
|
graph.parse(filepath)
|
|
109
|
-
except Exception:
|
|
110
|
-
issue_list.append(
|
|
109
|
+
except Exception as e:
|
|
110
|
+
issue_list.append(FileReadError(filepath, str(e)))
|
|
111
111
|
|
|
112
112
|
return cls(
|
|
113
113
|
issue_list,
|
|
@@ -148,20 +148,20 @@ class InferenceImporter(BaseImporter):
|
|
|
148
148
|
raise NotImplementedError("JSON file format is not supported yet.")
|
|
149
149
|
|
|
150
150
|
@overload
|
|
151
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
151
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
152
152
|
|
|
153
153
|
@overload
|
|
154
154
|
def to_rules(
|
|
155
155
|
self,
|
|
156
156
|
errors: Literal["continue"] = "continue",
|
|
157
157
|
role: RoleTypes | None = None,
|
|
158
|
-
) -> tuple[
|
|
158
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
159
159
|
|
|
160
160
|
def to_rules(
|
|
161
161
|
self,
|
|
162
162
|
errors: Literal["raise", "continue"] = "continue",
|
|
163
163
|
role: RoleTypes | None = None,
|
|
164
|
-
) -> tuple[
|
|
164
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
165
165
|
"""
|
|
166
166
|
Creates `Rules` object from the data for target role.
|
|
167
167
|
"""
|
|
@@ -34,6 +34,7 @@ def parse_owl_classes(graph: Graph, language: str = "en") -> list[dict]:
|
|
|
34
34
|
FILTER (!bound(?parentClass) || !isBlank(?parentClass))
|
|
35
35
|
FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
|
|
36
36
|
FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
|
|
37
|
+
BIND(?class AS ?reference)
|
|
37
38
|
}
|
|
38
39
|
"""
|
|
39
40
|
|
|
@@ -40,6 +40,7 @@ def parse_owl_properties(graph: Graph, language: str = "en") -> list[dict]:
|
|
|
40
40
|
FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
|
|
41
41
|
BIND(IF(bound(?minCount), ?minCount, 0) AS ?minCount)
|
|
42
42
|
BIND(IF(bound(?maxCount), ?maxCount, 1) AS ?maxCount)
|
|
43
|
+
BIND(?property AS ?reference)
|
|
43
44
|
}
|
|
44
45
|
"""
|
|
45
46
|
|
|
@@ -7,7 +7,7 @@ from typing import Literal, overload
|
|
|
7
7
|
from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
|
|
8
8
|
|
|
9
9
|
from cognite.neat.issues import IssueList
|
|
10
|
-
from cognite.neat.rules.importers._base import BaseImporter,
|
|
10
|
+
from cognite.neat.rules.importers._base import BaseImporter, VerifiedRules
|
|
11
11
|
from cognite.neat.rules.importers._rdf._shared import make_components_compliant
|
|
12
12
|
from cognite.neat.rules.models import InformationRules, RoleTypes
|
|
13
13
|
|
|
@@ -38,18 +38,18 @@ class OWLImporter(BaseImporter):
|
|
|
38
38
|
self.owl_filepath = filepath
|
|
39
39
|
|
|
40
40
|
@overload
|
|
41
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
41
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
42
42
|
|
|
43
43
|
@overload
|
|
44
44
|
def to_rules(
|
|
45
45
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
46
|
-
) -> tuple[
|
|
46
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
47
47
|
|
|
48
48
|
def to_rules(
|
|
49
49
|
self,
|
|
50
50
|
errors: Literal["raise", "continue"] = "continue",
|
|
51
51
|
role: RoleTypes | None = None,
|
|
52
|
-
) -> tuple[
|
|
52
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
53
53
|
graph = Graph()
|
|
54
54
|
try:
|
|
55
55
|
graph.parse(self.owl_filepath)
|
|
@@ -23,13 +23,13 @@ def parse_raw_classes_dataframe(query_results: list[tuple]) -> pd.DataFrame:
|
|
|
23
23
|
"Comment",
|
|
24
24
|
],
|
|
25
25
|
)
|
|
26
|
+
|
|
26
27
|
if df.empty:
|
|
27
28
|
return df
|
|
28
29
|
|
|
29
30
|
# # remove NaNs
|
|
30
31
|
df.replace(np.nan, "", regex=True, inplace=True)
|
|
31
32
|
|
|
32
|
-
df.Reference = df.Class
|
|
33
33
|
df.Class = df.Class.apply(lambda x: remove_namespace_from_uri(x))
|
|
34
34
|
df["Match Type"] = len(df) * [MatchType.exact]
|
|
35
35
|
df["Comment"] = len(df) * [None]
|
|
@@ -202,7 +202,7 @@ def parse_raw_properties_dataframe(query_results: list[tuple]) -> pd.DataFrame:
|
|
|
202
202
|
return df
|
|
203
203
|
|
|
204
204
|
df.replace(np.nan, "", regex=True, inplace=True)
|
|
205
|
-
|
|
205
|
+
|
|
206
206
|
df.Class = df.Class.apply(lambda x: remove_namespace_from_uri(x))
|
|
207
207
|
df.Property = df.Property.apply(lambda x: remove_namespace_from_uri(x))
|
|
208
208
|
df["Value Type"] = df["Value Type"].apply(lambda x: remove_namespace_from_uri(x))
|
|
@@ -231,7 +231,7 @@ def clean_up_properties(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
231
231
|
"Min Count": property_grouped_df["Min Count"].unique()[0],
|
|
232
232
|
"Max Count": property_grouped_df["Max Count"].unique()[0],
|
|
233
233
|
"Default": property_grouped_df["Default"].unique()[0],
|
|
234
|
-
"Reference": property_grouped_df["Reference"].unique()[0]
|
|
234
|
+
"Reference": property_grouped_df["Reference"].unique()[0],
|
|
235
235
|
"Match Type": property_grouped_df["Match Type"].unique()[0],
|
|
236
236
|
"Comment": property_grouped_df["Comment"].unique()[0],
|
|
237
237
|
"_property_type": property_grouped_df["_property_type"].unique()[0],
|